opentelemetry-collector-contrib
opentelemetry-collector-contrib copied to clipboard
Published
20 hours ago •
open-telemetry
open-telemetry
Add shard count to the Elasticsearch index stats scrapper
Component(s)
receiver/elasticsearch
Is your feature request related to a problem? Please describe.
I was trying to be able to compare 3 dimensions of Elasticsearch index stats document count, shards size and shard count but this receiver only scapes 2 of the 3 from the /_stats endpoint.
Describe the solution you'd like
Add the shard count to the collected metrics for each index. The /users/_stats endpoint contains the total_count for shards.
{
"_shards": {
"total": 6,
"successful": 6,
"failed": 0
},
"_all": {
"primaries": {
"docs": {
"count": 599007114,
"deleted": 6255158,
"total_size_in_bytes": 106202644463
},
"shard_stats": {
"total_count": 3
},
"store": {
"size_in_bytes": 106202656601,
"total_data_set_size_in_bytes": 106202656601,
"reserved_in_bytes": 0
},
"indexing": {
"index_total": 15653444,
"index_time_in_millis": 5137004,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 75007,
"is_throttled": false,
"throttle_time_in_millis": 0,
"write_load": 0.002295616505724265
},
"get": {
"total": 1077709,
"time_in_millis": 217716,
"exists_total": 1077709,
"exists_time_in_millis": 217716,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 1,
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 1,
"scroll_time_in_millis": 1294,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 1052,
"total_time_in_millis": 17872746,
"total_docs": 568163196,
"total_size_in_bytes": 105534994910,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 14177420,
"total_auto_throttle_in_bytes": 45279418
},
"refresh": {
"total": 3070,
"total_time_in_millis": 100790,
"external_total": 1064,
"external_total_time_in_millis": 834,
"listeners": 0
},
"flush": {
"total": 904,
"periodic": 902,
"total_time_in_millis": 434681,
"total_time_excluding_waiting_on_lock_in_millis": 466358
},
"warmer": {
"current": 0,
"total": 1061,
"total_time_in_millis": 313
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 99,
"hit_count": 0,
"miss_count": 99,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0,
"global_ordinals": {
"build_time_in_millis": 0
}
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 99,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 75662136,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 0,
"size_in_bytes": 165,
"uncommitted_operations": 0,
"uncommitted_size_in_bytes": 165,
"earliest_last_modified_age": 408423717
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 309022
},
"bulk": {
"total_operations": 183844,
"total_time_in_millis": 5419278,
"total_size_in_bytes": 26490845523,
"avg_time_in_millis": 16,
"avg_size_in_bytes": 63504
},
"dense_vector": {
"value_count": 0
},
"sparse_vector": {
"value_count": 0
}
},
"total": {
"docs": {
"count": 1198014228,
"deleted": 12506844,
"total_size_in_bytes": 212356147240
},
"shard_stats": {
"total_count": 6
},
"store": {
"size_in_bytes": 212356170745,
"total_data_set_size_in_bytes": 212356170745,
"reserved_in_bytes": 0
},
"indexing": {
"index_total": 15653444,
"index_time_in_millis": 5137004,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 75007,
"is_throttled": false,
"throttle_time_in_millis": 0,
"write_load": 0.0012662688070682509
},
"get": {
"total": 1077709,
"time_in_millis": 217716,
"exists_total": 1077709,
"exists_time_in_millis": 217716,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 1,
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 1,
"scroll_time_in_millis": 1294,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 1053,
"total_time_in_millis": 17873029,
"total_docs": 568207232,
"total_size_in_bytes": 105545476271,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 14177420,
"total_auto_throttle_in_bytes": 108193978
},
"refresh": {
"total": 3091,
"total_time_in_millis": 101623,
"external_total": 1075,
"external_total_time_in_millis": 987,
"listeners": 0
},
"flush": {
"total": 910,
"periodic": 907,
"total_time_in_millis": 434756,
"total_time_excluding_waiting_on_lock_in_millis": 466433
},
"warmer": {
"current": 0,
"total": 1069,
"total_time_in_millis": 451
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 99,
"hit_count": 0,
"miss_count": 99,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0,
"global_ordinals": {
"build_time_in_millis": 0
}
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 191,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 151323528,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 0,
"size_in_bytes": 330,
"uncommitted_operations": 0,
"uncommitted_size_in_bytes": 330,
"earliest_last_modified_age": 408423717
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1678130
},
"bulk": {
"total_operations": 183844,
"total_time_in_millis": 5419278,
"total_size_in_bytes": 26490845523,
"avg_time_in_millis": 16,
"avg_size_in_bytes": 63504
},
"dense_vector": {
"value_count": 0
},
"sparse_vector": {
"value_count": 0
}
}
},
"indices": {
"users": {
"uuid": "6UqMYhVwS4OwyngTqKG8ow",
"health": "green",
"status": "open",
"primaries": {
"docs": {
"count": 599007114,
"deleted": 6255158,
"total_size_in_bytes": 106202644463
},
"shard_stats": {
"total_count": 3
},
"store": {
"size_in_bytes": 106202656601,
"total_data_set_size_in_bytes": 106202656601,
"reserved_in_bytes": 0
},
"indexing": {
"index_total": 15653444,
"index_time_in_millis": 5137004,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 75007,
"is_throttled": false,
"throttle_time_in_millis": 0,
"write_load": 0.002295616505724265
},
"get": {
"total": 1077709,
"time_in_millis": 217716,
"exists_total": 1077709,
"exists_time_in_millis": 217716,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 1,
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 1,
"scroll_time_in_millis": 1294,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 1052,
"total_time_in_millis": 17872746,
"total_docs": 568163196,
"total_size_in_bytes": 105534994910,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 14177420,
"total_auto_throttle_in_bytes": 45279418
},
"refresh": {
"total": 3070,
"total_time_in_millis": 100790,
"external_total": 1064,
"external_total_time_in_millis": 834,
"listeners": 0
},
"flush": {
"total": 904,
"periodic": 902,
"total_time_in_millis": 434681,
"total_time_excluding_waiting_on_lock_in_millis": 466358
},
"warmer": {
"current": 0,
"total": 1061,
"total_time_in_millis": 313
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 99,
"hit_count": 0,
"miss_count": 99,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0,
"global_ordinals": {
"build_time_in_millis": 0
}
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 99,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 75662136,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 0,
"size_in_bytes": 165,
"uncommitted_operations": 0,
"uncommitted_size_in_bytes": 165,
"earliest_last_modified_age": 408423717
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 309022
},
"bulk": {
"total_operations": 183844,
"total_time_in_millis": 5419278,
"total_size_in_bytes": 26490845523,
"avg_time_in_millis": 16,
"avg_size_in_bytes": 63504
},
"dense_vector": {
"value_count": 0
},
"sparse_vector": {
"value_count": 0
}
},
"total": {
"docs": {
"count": 1198014228,
"deleted": 12506844,
"total_size_in_bytes": 212356147240
},
"shard_stats": {
"total_count": 6
},
"store": {
"size_in_bytes": 212356170745,
"total_data_set_size_in_bytes": 212356170745,
"reserved_in_bytes": 0
},
"indexing": {
"index_total": 15653444,
"index_time_in_millis": 5137004,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 75007,
"is_throttled": false,
"throttle_time_in_millis": 0,
"write_load": 0.0012662688070682509
},
"get": {
"total": 1077709,
"time_in_millis": 217716,
"exists_total": 1077709,
"exists_time_in_millis": 217716,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 1,
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 1,
"scroll_time_in_millis": 1294,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 1053,
"total_time_in_millis": 17873029,
"total_docs": 568207232,
"total_size_in_bytes": 105545476271,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 14177420,
"total_auto_throttle_in_bytes": 108193978
},
"refresh": {
"total": 3091,
"total_time_in_millis": 101623,
"external_total": 1075,
"external_total_time_in_millis": 987,
"listeners": 0
},
"flush": {
"total": 910,
"periodic": 907,
"total_time_in_millis": 434756,
"total_time_excluding_waiting_on_lock_in_millis": 466433
},
"warmer": {
"current": 0,
"total": 1069,
"total_time_in_millis": 451
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 99,
"hit_count": 0,
"miss_count": 99,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0,
"global_ordinals": {
"build_time_in_millis": 0
}
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 191,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 151323528,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 0,
"size_in_bytes": 330,
"uncommitted_operations": 0,
"uncommitted_size_in_bytes": 330,
"earliest_last_modified_age": 408423717
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1678130
},
"bulk": {
"total_operations": 183844,
"total_time_in_millis": 5419278,
"total_size_in_bytes": 26490845523,
"avg_time_in_millis": 16,
"avg_size_in_bytes": 63504
},
"dense_vector": {
"value_count": 0
},
"sparse_vector": {
"value_count": 0
}
}
}
}
}
Describe alternatives you've considered
No response
Additional context
No response
