grafana-ansible-collection

grafana-ansible-collection copied to clipboard

I am using the grafana.grafana.alloy role to provision a Ubuntu 22.04 machine.

The relevant part of my cadvisor configuration is the following:

prometheus.exporter.cadvisor "integrations_cadvisor" {
  docker_only = true
}

discovery.relabel "integrations_cadvisor" {
  targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets

  rule {
    target_label = "job"
    replacement  = "integrations/docker"
  }

  rule {
    target_label = "instance"
    replacement  = "localhost:9090"
  }
}

prometheus.scrape "integrations_cadvisor" {
  targets    = discovery.relabel.integrations_cadvisor.output
  forward_to = [prometheus.relabel.integrations_cadvisor.receiver]
  job_name   = "integrations/cadvisor"
}

prometheus.relabel "integrations_cadvisor" {
  forward_to = [prometheus.remote_write.prom.receiver]

  rule {
    source_labels = ["__name__"]
    regex         = "(prometheus_target_.*|prometheus_sd_discovered_targets|agent_build.*|agent_wal_samples_appended_total|process_start_time_seconds)"
    action        = "keep"
  }
}

The provisioning with ansible succeeds, however journalctl -u alloy -f displays the following error messages and I can't see my container metrics:

May 15 19:42:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:42:09.31076995Z level=error msg="Failed to create existing container: /docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:43:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:43:09.307001969Z level=error msg="Failed to create existing container: /docker.slice/docker-1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228.scope: failed to identify the read-write layer ID for container \"1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228\". - open /var/lib/docker/image/overlay2/layerdb/mounts/1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:43:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:43:09.30884948Z level=error msg="Failed to create existing container: /docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:43:10 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:43:10.216734299Z level=warn msg="Failed to process watch event {EventType:0 Name:/docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope WatchSource:0}: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Warningf
May 15 19:44:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:44:09.324291896Z level=error msg="Failed to create existing container: /docker.slice/docker-1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228.scope: failed to identify the read-write layer ID for container \"1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228\". - open /var/lib/docker/image/overlay2/layerdb/mounts/1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:44:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:44:09.330254379Z level=error msg="Failed to create existing container: /docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:44:34 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:44:34.921831182Z level=warn msg="Failed to process watch event {EventType:0 Name:/docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope WatchSource:0}: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Warningf
May 15 19:45:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:45:09.305998351Z level=error msg="Failed to create existing container: /docker.slice/docker-27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801.scope: failed to identify the read-write layer ID for container \"27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801\". - open /var/lib/docker/image/overlay2/layerdb/mounts/27643eca5bdc9bf8e9e2274d26d27ecaf92b63d6f2a995ac2ad581553e86d801/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf
May 15 19:45:09 hostname001 alloy-linux-amd64[378793]: ts=2024-05-15T19:45:09.309794122Z level=error msg="Failed to create existing container: /docker.slice/docker-1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228.scope: failed to identify the read-write layer ID for container \"1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228\". - open /var/lib/docker/image/overlay2/layerdb/mounts/1d27f9bae66677609186a1cd060e4d30fffbecf614412cdbfa9bb82b24b44228/mount-id: permission denied" component_path=/ component_id=prometheus.exporter.cadvisor.integrations_cadvisor func=Errorf

Since it looks like a permissions issue, I also tried provisioning with vars service_user: "root" and service_group: "root". Now the error messages don't show up but I can't still see any container_ metrics.

What am I doing wrong?