Question: Some of metrics can not be sent to Sumologic after about one week.

[hmiyakoshi0803]

I have a problem with prometheus-sumo-scraper. If I run it for one week, I can't have some metrics in Sumologic, it seems prometheus-sumo-scraper can't send the metric data or sumologic can't receive the metrics. I've looked for some clues in sumologic-prometheus-scraper logs, but there aren't anything.

As work around, If I reboot the pod of sumologic-prometheus-scraper, all of metrics can be sent correctly, but after about one week, I can't see some of the metrics again.

For example, I can see below metrics now. "kube_pod_status_phase", "kube_pod_container_status_ready", "kube_pod_container_status_restarts_total", "kube_pod_container_status_terminated_reason", "kube_pod_container_status_waiting", "kube_deployment_spec_replicas", "kube_deployment_status_replicas_unavailable"

But I can't see below metrics after one week. "kube_node_status_capacity_cpu_cores", "kube_node_status_capacity_memory_bytes", "kube_node_status_capacity_pods", "kube_node_status_condition"

My config.josn is here

    {
      "global": {
        "sumo_http_url": "https://collectors.jp.sumologic.com/receiver/v1/http/***********************",
        "source_host": "sumologic-prometheus-scraper",
        "run_interval_seconds": 120,
        "metadata": "type=metrics"
      },
      "targets": [
        {
          "name": "openshift-prod-node-exporter",
          "source_name": "node-exporter",
          "source_category": "prod/server/ocp/metrics",
          "url": { 
            "service": "node-exporter",
            "namespace": "openshift-monitoring",
            "protocol" : "https"
          },
          "token_file_path": "/var/run/secrets/kubernetes.io/serviceaccount/token",
          "verify": "False",
          "include_metrics": [
            "node_cpu_seconds_total",
            "node_memory_Shmem_bytes",
            "node_memory_Cached_bytes",
            "node_memory_Buffers_bytes",
            "node_memory_MemTotal_bytes",
            "node_memory_MemFree_bytes",
            "node_disk_reads_merged_total",
            "node_disk_writes_merged_total"
            ]
        },
        {
          "name": "openshift-prod-kube-state-metrics-nodes",
          "source_name": "kube-state-metrics",
          "source_category": "prod/containers/ocp/metrics",
          "url": { 
            "service": "kube-state-metrics",
            "namespace": "openshift-monitoring",
            "protocol" : "https"
          },
          "include_metrics": [
            "kube_pod_status_phase", 
            "kube_pod_container_status_ready",
            "kube_pod_container_status_restarts_total",
            "kube_pod_container_status_terminated_reason",
            "kube_pod_container_status_waiting",
            "kube_deployment_spec_replicas",
            "kube_deployment_status_replicas_unavailable"
          ],
          "token_file_path": "/var/run/secrets/kubernetes.io/serviceaccount/token",
          "verify": "False"
        },
        {
          "name": "openshift-prod-kube-state-metrics-containers",
          "source_name": "kube-state-metrics",
          "source_category": "prod/server/ocp/metrics",
          "url": { 
            "service": "kube-state-metrics",
            "namespace": "openshift-monitoring",
            "protocol" : "https"
          },
          "include_metrics": [
            "kube_node_status_capacity_cpu_cores",
            "kube_node_status_capacity_memory_bytes",
            "kube_node_status_capacity_pods",
            "kube_node_status_condition"
          ],
          "token_file_path": "/var/run/secrets/kubernetes.io/serviceaccount/token",
          "verify": "False"
        },
        {
          "name": "openshift-prod-kubelet",
          "source_name": "kubelet",
          "source_category": "prod/container/ocp/metrics",
          "url": { 
            "service": "kubelet",
            "namespace": "kube-system",
            "protocol" : "https",
            "path": "/metrics/cadvisor"
          },
          "include_metrics": [
            "container_cpu_usage_seconds_total",
            "container_cpu_cfs_throttled_seconds_total",
            "container_memory_usage_bytes",
            "container_memory_working_set_bytes",
            "container_fs_writes_bytes_total",
            "container_fs_reads_bytes_total",
            "container_network_transmit_bytes_total",
            "container_network_receive_bytes_total"
          ],
          "token_file_path": "/var/run/secrets/kubernetes.io/serviceaccount/token",
          "verify": "False"
        }
      ]
    }