trickster
trickster copied to clipboard
Published
20 hours ago •
trickstercache
trickstercache
BUG: diffrent request /api/v1/query response same result
trickster version: v1.1.5 Use trickster in frontend of VictoriaMetrics, set vmalert datasource use trickster. when vmalert send sequence request diffrent /api/v1/query in one tcp connection, trickster response same cache reult.
+---------------------+ +---------------------+ +---------------------+
| | | | | |
| vmalert |-------------- trickster ---------------- vmselect |
| | | | | |
+---------------------+ +---------------------+ +---------------------+
trickster configure:
trickster.conf: |-
[main]
[frontend]
listen_port = 8480
connections_limit = 0
[caches]
[caches.default]
cache_type = "redis"
[caches.default.redis]
client_type = "cluster"
password = "bFdfkKWSbT"
endpoints = [ 'redis-cluster-0.redis-cluster-headless.victoria:6379', 'redis-cluster-1.redis-cluster-headless.victoria:6379', 'redis-cluster-2.redis-cluster-headless.victoria:6379' ]
[origins]
[origins.default]
origin_type = "prometheus"
origin_url = "http://vmselect-victoria-metrics-k8s-stack.victoria.svc:8481/select/0/prometheus"
health_check_upstream_path = "/select/0/prometheus/api/v1/query"
[metrics]
listen_port = 8481
[logging]
log_level = "debug"
trickster configure dump:
[main]
instance_id = 0
config_handler_path = "/trickster/config"
ping_handler_path = "/trickster/ping"
reload_handler_path = "/trickster/config/reload"
health_handler_path = "/trickster/health"
pprof_server = "both"
server_name = "trickster-6d669ccbfc-bv7n2"
[origins]
[origins.default]
origin_type = "prometheus"
origin_url = "http://vmselect-victoria-metrics-k8s-stack.victoria.svc:8481/select/0/prometheus"
timeout_secs = 180
keep_alive_timeout_secs = 300
max_idle_conns = 20
cache_name = "default"
cache_key_prefix = "vmselect-victoria-metrics-k8s-stack.victoria.svc:8481"
health_check_upstream_path = "/select/0/prometheus/api/v1/query"
health_check_verb = "GET"
health_check_query = "query=up"
timeseries_retention_factor = 1024
timeseries_eviction_method = "oldest"
backfill_tolerance_secs = 0
negative_cache_name = "default"
timeseries_ttl_secs = 21600
fastforward_ttl_secs = 15
max_ttl_secs = 86400
revalidation_factor = 2.0
max_object_size_bytes = 524288
compressable_types = ["text/html", "text/javascript", "text/css", "text/plain", "text/xml", "text/json", "application/json", "application/javascript", "application/xml"]
tracing_name = "default"
rule_name = ""
req_rewriter_name = ""
forwarded_headers = "standard"
is_default = true
fast_forward_disable = false
path_routing_disabled = false
require_tls = false
multipart_ranges_disabled = false
dearticulate_upstream_ranges = false
[origins.default.health_check_headers]
[origins.default.paths]
[origins.default.paths."/-GET-POST"]
path = "/"
match_type = "prefix"
handler = "proxy"
methods = ["GET", "POST"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/-GET-POST"]
path = "/api/v1/"
match_type = "prefix"
handler = "proxy"
methods = ["GET", "POST"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/alertmanagers-GET"]
path = "/api/v1/alertmanagers"
match_type = "exact"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/alertmanagers-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/alerts-GET"]
path = "/api/v1/alerts"
match_type = "exact"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/alerts-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/label/-GET"]
path = "/api/v1/label/"
match_type = "prefix"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/label/-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/labels-GET-POST"]
path = "/api/v1/labels"
match_type = "exact"
handler = "proxycache"
methods = ["GET", "POST"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/labels-GET-POST".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/query-GET-POST"]
path = "/api/v1/query"
match_type = "exact"
handler = "query"
methods = ["GET", "POST"]
cache_key_params = ["query", "time"]
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/query-GET-POST".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/query_range-GET-POST"]
path = "/api/v1/query_range"
match_type = "exact"
handler = "query_range"
methods = ["GET", "POST"]
cache_key_params = ["query", "step"]
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/query_range-GET-POST".response_headers]
Cache-Control = "s-maxage=21600"
[origins.default.paths."/api/v1/rules-GET"]
path = "/api/v1/rules"
match_type = "exact"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/rules-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/series-GET-POST"]
path = "/api/v1/series"
match_type = "exact"
handler = "series"
methods = ["GET", "POST"]
cache_key_params = ["match[]", "start", "end"]
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/series-GET-POST".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/status-GET"]
path = "/api/v1/status"
match_type = "prefix"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/status-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/targets-GET"]
path = "/api/v1/targets"
match_type = "exact"
handler = "proxycache"
methods = ["GET"]
cache_key_params = []
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/targets-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.paths."/api/v1/targets/metadata-GET"]
path = "/api/v1/targets/metadata"
match_type = "exact"
handler = "proxycache"
methods = ["GET"]
cache_key_params = ["match_target", "metric", "limit"]
cache_key_headers = []
cache_key_form_fields = []
response_code = 0
response_body = ""
collapsed_forwarding = ""
req_rewriter_name = ""
no_metrics = false
[origins.default.paths."/api/v1/targets/metadata-GET".response_headers]
Cache-Control = "s-maxage=30"
[origins.default.tls]
full_chain_cert_path = ""
private_key_path = ""
insecure_skip_verify = false
client_cert_path = ""
client_key_path = ""
[caches]
[caches.default]
cache_type = "redis"
[caches.default.index]
reap_interval_secs = 3
flush_interval_secs = 5
max_size_bytes = 536870912
max_size_backoff_bytes = 16777216
max_size_objects = 0
max_size_backoff_objects = 100
[caches.default.redis]
client_type = "cluster"
protocol = "tcp"
endpoint = "redis:6379"
endpoints = ["redis-cluster-0.redis-cluster-headless.victoria:6379", "redis-cluster-1.redis-cluster-headless.victoria:6379", "redis-cluster-2.redis-cluster-headless.victoria:6379"]
password = "*****"
sentinel_master = ""
db = 0
max_retries = 0
min_retry_backoff_ms = 0
max_retry_backoff_ms = 0
dial_timeout_ms = 0
read_timeout_ms = 0
write_timeout_ms = 0
pool_size = 0
min_idle_conns = 0
max_conn_age_ms = 0
pool_timeout_ms = 0
idle_timeout_ms = 0
idle_check_frequency_ms = 0
[caches.default.filesystem]
cache_path = "/tmp/trickster"
[caches.default.bbolt]
filename = "trickster.db"
bucket = "trickster"
[caches.default.badger]
directory = "/tmp/trickster"
value_directory = "/tmp/trickster"
[frontend]
listen_address = ""
listen_port = 8480
tls_listen_address = ""
tls_listen_port = 8483
connections_limit = 0
[logging]
log_file = ""
log_level = "debug"
[metrics]
listen_address = ""
listen_port = 8481
[tracing]
[tracing.default]
tracer_type = "none"
service_name = "trickster"
collector_url = ""
collector_user = ""
collector_pass = ""
sample_rate = 1.0
[tracing.default.stdout]
pretty_print = false
[tracing.default.jaeger]
endpoint_type = ""
[negative_caches]
[negative_caches.default]
[reloading]
listen_address = "127.0.0.1"
listen_port = 8484
handler_path = "/trickster/config/reload"
drain_timeout_secs = 30
rate_limit_secs = 3
tcpdump http flow:
POST /api/v1/query?query=kube_hpa_status_condition%7Bcondition%3D%22AbleToScale%22%2C+status%3D%22false%22%7D+%2A+on%28hpa%2Cnamespace%29++group_left%28label_app%29+kube_hpa_labels+%3D%3D+1&step=15s&time=1635906330 HTTP/1.1
Host: trickster.victoria.svc:8480
User-Agent: Go-http-client/1.1
Content-Length: 0
Content-Type: application/json; charset=utf-8
Accept-Encoding: gzip
HTTP/1.1 200 OK
Access-Control-Allow-Origin: *
Cache-Control: s-maxage=30
Content-Encoding: gzip
Content-Type: application/json; charset=utf-8
Vary: Accept-Encoding
X-Server-Hostname: vmselect-victoria-metrics-k8s-stack-2
X-Trickster-Result: engine=ObjectProxyCache; status=hit
Date: Wed, 03 Nov 2021 02:25:37 GMT
Content-Length: 346
{"status":"success","isPartial":false,"data":{"resultType":"vector","result":[{"metric":{"backend":"bizmap-backend.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"4168.1701488"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"325.85644152"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8scs1-online-sh5-victoria"},"value":[1635906315,"282.73285584"]},{"metric":{"backend":"dopen.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"21234.966215028002"]},{"metric":{"backend":"dopen.xxx.com/fuwu/b/","cluster":"k8s-online-victoria"},"value":[1635906315,"1774.2817677599999"]},{"metric":{"backend":"jscracker.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"828222.4420423079"]},{"metric":{"backend":"saas-fe-zeus-common.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"129.67017696"]}]}}POST /api/v1/query?query=absent%28up%7Bjob%3D%22kube-controller-manager%22%7D+%3D%3D+1%29&step=15s&time=1635906330 HTTP/1.1
Host: trickster.victoria.svc:8480
User-Agent: Go-http-client/1.1
Content-Length: 0
Content-Type: application/json; charset=utf-8
Accept-Encoding: gzip
HTTP/1.1 200 OK
Access-Control-Allow-Origin: *
Cache-Control: s-maxage=30
Content-Encoding: gzip
Content-Type: application/json; charset=utf-8
Vary: Accept-Encoding
X-Server-Hostname: vmselect-victoria-metrics-k8s-stack-2
X-Trickster-Result: engine=ObjectProxyCache; status=hit
Date: Wed, 03 Nov 2021 02:25:37 GMT
Content-Length: 346
{"status":"success","isPartial":false,"data":{"resultType":"vector","result":[{"metric":{"backend":"bizmap-backend.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"4168.1701488"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"325.85644152"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8scs1-online-sh5-victoria"},"value":[1635906315,"282.73285584"]},{"metric":{"backend":"dopen.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"21234.966215028002"]},{"metric":{"backend":"dopen.xxx.com/fuwu/b/","cluster":"k8s-online-victoria"},"value":[1635906315,"1774.2817677599999"]},{"metric":{"backend":"jscracker.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"828222.4420423079"]},{"metric":{"backend":"saas-fe-zeus-common.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"129.67017696"]}]}}
