Add labels field to internal_metrics data stream

[b2ronn]

some documents (for example, with histograms) do not have label fields . For example, I have a label with the name of the team (labels.project). Each command can have several services, and without these labels it is impossible to make Document level security.

[simitt]

@b2ronn is this request covered by the open issue to add global_labels to metrics events (https://github.com/elastic/apm-server/issues/8139) or are you missing something else?

[b2ronn]

Yes and no labels are needed not only in transactions/metrics/logs, but also in calculated histograms(data_stream.dataset: apm.internal).

[b2ronn]

updated to 8.4.2. but in documents where histogram calculation global-labels are not added. the test application is deployed with these variables.

        - resources: {}
          terminationMessagePath: /dev/termination-log
          name: petclinic
          env:
            - name: ELASTIC_APM_SERVER_URL
              value: 'https://apm-server-apm-http:8200'
            - name: ELASTIC_APM_SERVICE_NAME
              value: petclinic-test-app
            - name: ELASTIC_APM_APPLICATION_PACKAGES
              value: org.springframework.samples.petclinic
            - name: ELASTIC_APM_ENVIRONMENT
              value: dev
            - name: JAVA_TOOL_OPTIONS
              value: '-javaagent:/elastic/apm/agent/elastic-apm-agent.jar'
            - name: KUBERNETES_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: ELASTIC_APM_VERIFY_SERVER_CERT
              value: 'false'
            - name: ELASTIC_APM_GLOBAL_LABELS
              value: project=epm-paas

  "_index": ".ds-metrics-apm.internal-default-2022.10.07-000137",
  "_id": "uaGNsYMB5zIH6lvifC4y",
  "_version": 1,
  "_score": 0,
  "_source": {
    "container": {
      "id": "056bed4f14b13a48a3e26e3e439a40f647d3e1bfc4550a27c5951b8fb6558924"
    },
    "kubernetes": {
      "pod": {
        "name": "petclinic-6b988755b6-t2lxx"
      }
    },
    "agent": {
      "name": "java"
    },
    "_doc_count": 2,
    "data_stream.namespace": "default",
    "data_stream.type": "metrics",
    "processor": {
      "name": "metric",
      "event": "metric"
    },
    "metricset.name": "transaction",
    "observer": {
      "hostname": "apm-server-78f7bcf5d9-rrxc9",
      "id": "bbfd70c0-1e29-41b0-b7d4-8beaf3ffe5d1",
      "type": "apm-server",
      "ephemeral_id": "e3a37872-16aa-46a2-b199-8a8063be28c3",
      "version": "8.4.2"
    },
    "@timestamp": "2022-10-07T08:25:00.000Z",
    "timeseries": {
      "instance": "petclinic-test-app:OperationHandler#handle:58034ebea5a955e3"
    },
    "ecs": {
      "version": "1.12.0"
    },
    "service": {
      "node": {
        "name": "056bed4f14b13a48a3e26e3e439a40f647d3e1bfc4550a27c5951b8fb6558924"
      },
      "environment": "dev",
      "name": "petclinic-test-app",
      "runtime": {
        "name": "Java",
        "version": "11.0.15"
      },
      "language": {
        "name": "Java",
        "version": "11.0.15"
      },
      "version": "2.7.0-SNAPSHOT"
    },
    "data_stream.dataset": "apm.internal",
    "host": {
      "os": {
        "platform": "Linux"
      }
    },
    "event": {
      "agent_id_status": "missing",
      "ingested": "2022-10-07T08:26:23Z",
      "outcome": "success"
    },
    "transaction": {
      "result": "HTTP 2xx",
      "root": true,
      "name": "OperationHandler#handle",
      "type": "request",
      "duration.histogram": {
        "counts": [
          1,
          1
        ],
        "values": [
          5855,
          7007
        ]
      }
    }
  },
  "fields": {
    "service.node.name": [
      "056bed4f14b13a48a3e26e3e439a40f647d3e1bfc4550a27c5951b8fb6558924"
    ],
    "service.language.name": [
      "Java"
    ],
    "transaction.result": [
      "HTTP 2xx"
    ],
    "container.id": [
      "056bed4f14b13a48a3e26e3e439a40f647d3e1bfc4550a27c5951b8fb6558924"
    ],
    "transaction.root": [
      true
    ],
    "processor.event": [
      "metric"
    ],
    "kubernetes.pod.name": [
      "petclinic-6b988755b6-t2lxx"
    ],
    "agent.name": [
      "java"
    ],
    "event.agent_id_status": [
      "missing"
    ],
    "event.outcome": [
      "success"
    ],
    "service.environment": [
      "dev"
    ],
    "_doc_count": [
      2
    ],
    "service.name": [
      "petclinic-test-app"
    ],
    "data_stream.namespace": [
      "default"
    ],
    "service.runtime.name": [
      "Java"
    ],
    "processor.name": [
      "metric"
    ],
    "service.runtime.version": [
      "11.0.15"
    ],
    "observer.hostname": [
      "apm-server-78f7bcf5d9-rrxc9"
    ],
    "data_stream.type": [
      "metrics"
    ],
    "transaction.type": [
      "request"
    ],
    "transaction.duration.histogram": [
      {
        "counts": [
          1,
          1
        ],
        "values": [
          5855,
          7007
        ]
      }
    ],
    "metricset.name": [
      "transaction"
    ],
    "observer.id": [
      "bbfd70c0-1e29-41b0-b7d4-8beaf3ffe5d1"
    ],
    "event.ingested": [
      "2022-10-07T08:26:23.000Z"
    ],
    "@timestamp": [
      "2022-10-07T08:25:00.000Z"
    ],
    "timeseries.instance": [
      "petclinic-test-app:OperationHandler#handle:58034ebea5a955e3"
    ],
    "observer.ephemeral_id": [
      "e3a37872-16aa-46a2-b199-8a8063be28c3"
    ],
    "service.version": [
      "2.7.0-SNAPSHOT"
    ],
    "observer.version": [
      "8.4.2"
    ],
    "host.os.platform": [
      "Linux"
    ],
    "ecs.version": [
      "1.12.0"
    ],
    "observer.type": [
      "apm-server"
    ],
    "data_stream.dataset": [
      "apm.internal"
    ],
    "service.language.version": [
      "11.0.15"
    ],
    "transaction.name": [
      "OperationHandler#handle"
    ]
  }
}`

[axw]

@b2ronn the enhancement to add global labels to transaction metrics will be going into 8.5.0.

[axw]

This was released in 8.5.0, please reopen if things aren't working as expected after upgrading.