Graylog2
Add search "EXPLAIN" endpoint
The data tiering feature needs a way to indicate to the user that a search request will be reaching into indices that are stored in the warm tier. Those requests are possibly slow and expensive, so we want to inform the user about this fact. This new endpoint can be queried from the UI with a regular search request, before it is executed. It returns which indices will be used for the search and whether those indices are stored in the warm tier or not. Furthermore it contains the timerange info for each index, so the frontend can give the user suggestions on how to shorten the query so it does not reach the warm tiered indices.
This kind of functionality fits perfectly into a more generic "explain" endpoint. So instead of a adding a special endpoint for just the warm tier index range info, we decided to build a more generic version which also contains a preview of the generated OS/ES query and a list of search validation errors.
Example response:
{
"search_id": "647f0565d060431199a12e96",
"search": {
"queries": {
"a1647eb6-a064-4fe6-b459-1e4267d3f659": {
"search_types": {
"22249f29-f042-4bd8-b745-252b00a35891": {
"query_string": "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}},{\"bool\":{\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"timestamp\":{\"from\":\"2023-09-11 20:55:50.185\",\"to\":\"2024-01-18 14:49:10.185\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"terms\":{\"streams\":[\"63d6d52ebf9c684b3da2deb3\",\"63a5ab32e71520111ed3ce06\",\"000000000000000000000001\"],\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"track_total_hits\":2147483647,\"aggregations\":{\"agg\":{\"filters\":{\"filters\":[{\"bool\":{\"should\":[{\"exists\":{\"field\":\"source\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"other_bucket\":true,\"other_bucket_key\":\"_other_\"},\"aggregations\":{\"agg\":{\"terms\":{\"script\":{\"source\":\"(doc.containsKey('source') && doc['source'].size() > 0\\n? doc['source'].size() > 1\\n ? doc['source']\\n : String.valueOf(doc['source'].value)\\n: \\\"(Empty Value)\\\")\\n\",\"lang\":\"painless\"},\"size\":10,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":[{\"_count\":\"desc\"},{\"_key\":\"asc\"}]}}}},\"timestamp-min\":{\"min\":{\"field\":\"timestamp\"}},\"timestamp-max\":{\"max\":{\"field\":\"timestamp\"}}}}",
"searched_index_ranges": [
{
"index_name": "graylog_0",
"begin": 0,
"end": 0,
"is_warm_tiered": false
},
{
"index_name": "bar_1512",
"begin": 1705589036047,
"end": 1705589284808,
"is_warm_tiered": false
},
{
"index_name": "bar_1513",
"begin": 0,
"end": 0,
"is_warm_tiered": false
},
{
"index_name": "bar_warm_1511",
"begin": 1705588785906,
"end": 1705589035782,
"is_warm_tiered": true
}
]
},
"5e9a9bfe-7a97-4835-86fd-896f40b20531": {
"query_string": "{\"from\":0,\"size\":0,\"query\":{\"bool\":{\"must\":[{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}},{\"bool\":{\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"timestamp\":{\"from\":\"2023-09-11 20:55:50.185\",\"to\":\"2024-01-18 14:49:10.185\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"terms\":{\"streams\":[\"63d6d52ebf9c684b3da2deb3\",\"63a5ab32e71520111ed3ce06\",\"000000000000000000000001\"],\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"track_total_hits\":2147483647,\"aggregations\":{\"agg\":{\"filters\":{\"filters\":[{\"bool\":{\"should\":[{\"exists\":{\"field\":\"source\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"other_bucket\":true,\"other_bucket_key\":\"_other_\"},\"aggregations\":{\"agg\":{\"terms\":{\"script\":{\"source\":\"(doc.containsKey('source') && doc['source'].size() > 0\\n? doc['source'].size() > 1\\n ? doc['source']\\n : String.valueOf(doc['source'].value)\\n: \\\"(Empty Value)\\\")\\n\",\"lang\":\"painless\"},\"size\":15,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":[{\"_count\":\"desc\"},{\"_key\":\"asc\"}]}}}},\"timestamp-min\":{\"min\":{\"field\":\"timestamp\"}},\"timestamp-max\":{\"max\":{\"field\":\"timestamp\"}}}}",
"searched_index_ranges": [
{
"index_name": "graylog_0",
"begin": 0,
"end": 0,
"is_warm_tiered": false
}
]
}
}
}
}
},
"search_errors": [
]
}
refs https://github.com/Graylog2/graylog-plugin-enterprise/issues/6453
/nocl
