beats
beats copied to clipboard
elastic
GCP: fix omitted labels
Proposed commit message
When parsing time series objects, we're omitting key label instance_id which was correct for services that has its own metadata logic, but we were missing crucial labels for services without additional metadata logic. It is now fixed with an explicit condition for instance_id.
Checklist
- [X] My code follows the style guidelines of this project
- [X] I have commented my code, particularly in hard-to-understand areas
- [X] I have made corresponding changes to the documentation
- [ ] I have made corresponding change to the default configuration files
- [X] I have added tests that prove my fix is effective or that my feature works
- [X] I have added an entry in
CHANGELOG.next.asciidocorCHANGELOG-developer.next.asciidoc.
- Closes #40033
Screenshots
Compute Service Document:
Spanner Service Before Change:
Spanner Service After Change:
Before Change
{
"_index": ".ds-metricbeat-8.15.0-2024.07.03-000001",
"_id": "ymisd5AB56NXeJqdMPru",
"_version": 1,
"_score": 0,
"_source": {
"@timestamp": "2024-07-03T08:13:00.000Z",
"agent": {
"name": "EPINPUNW06A8",
"type": "metricbeat",
"version": "8.15.0",
"ephemeral_id": "2e83ec6e-b29b-407b-99fa-e8383f908e74",
"id": "f49fbc8d-cc6a-4629-89b1-454f33f48d82"
},
"metricset": {
"name": "metrics",
"period": 60000
},
"gcp": {
"labels": {},
"metrics": {
"instance": {
"uptime": {
"sec": 60
},
"cpu": {
"usage": {
"pct": 0.006929705028543509
},
"reserved_cores": {
"value": 0.2
},
"usage_time": {
"sec": 0.08315646034252211
}
}
}
}
},
"service": {
"type": "gcp"
},
"cloud": {
"availability_zone": "us-central1-a",
"region": "us-central1",
"instance": {
"name": "demo-crest-group-j76r",
"id": "2314488029850962513"
},
"machine": {
"type": "f1-micro"
},
"account": {
"id": "elastic-obs-integrations-dev",
"name": "elastic-obs-integrations-dev"
},
"provider": "gcp"
},
"host": {
"hostname": "epinpunw06a8",
"architecture": "x86_64",
"os": {
"platform": "ubuntu",
"version": "22.04.3 LTS (Jammy Jellyfish)",
"family": "debian",
"name": "Ubuntu",
"kernel": "5.15.146.1-microsoft-standard-WSL2",
"codename": "jammy",
"type": "linux"
},
"containerized": false,
"ip": [
"172.29.134.71",
"fe80::215:5dff:fe11:8275"
],
"name": "epinpunw06a8",
"id": "2fef4836d29e4bf6b57df5049e8f58e4",
"mac": [
"00-15-5D-11-82-75"
]
},
"event": {
"dataset": "gcp.metrics",
"module": "gcp",
"duration": 2541915617
},
"ecs": {
"version": "8.0.0"
}
},
"fields": {
"beats_state.state.host.hostname": [
"epinpunw06a8"
],
"host.os.name.text": [
"Ubuntu"
],
"host.hostname": [
"epinpunw06a8"
],
"host.mac": [
"00-15-5D-11-82-75"
],
"cloud.availability_zone": [
"us-central1-a"
],
"service.type": [
"gcp"
],
"cloud.instance.id": [
"2314488029850962513"
],
"host.ip": [
"172.29.134.71",
"fe80::215:5dff:fe11:8275"
],
"agent.type": [
"metricbeat"
],
"gcp.metrics.instance.cpu.reserved_cores.value": [
0.2
],
"logstash_stats.timestamp": [
"2024-07-03T08:13:00.000Z"
],
"event.module": [
"gcp"
],
"host.os.version": [
"22.04.3 LTS (Jammy Jellyfish)"
],
"host.os.kernel": [
"5.15.146.1-microsoft-standard-WSL2"
],
"host.os.name": [
"Ubuntu"
],
"agent.name": [
"EPINPUNW06A8"
],
"host.name": [
"epinpunw06a8"
],
"beats_state.timestamp": [
"2024-07-03T08:13:00.000Z"
],
"host.id": [
"2fef4836d29e4bf6b57df5049e8f58e4"
],
"beats_state.state.host.name": [
"epinpunw06a8"
],
"timestamp": [
"2024-07-03T08:13:00.000Z"
],
"cloud.account.name": [
"elastic-obs-integrations-dev"
],
"cloud.region": [
"us-central1"
],
"host.os.type": [
"linux"
],
"kibana_stats.timestamp": [
"2024-07-03T08:13:00.000Z"
],
"metricset.period": [
60000
],
"host.os.codename": [
"jammy"
],
"gcp.metrics.instance.cpu.usage.pct": [
0.006929705028543509
],
"agent.hostname": [
"EPINPUNW06A8"
],
"gcp.metrics.instance.cpu.usage_time.sec": [
0.08315646034252211
],
"event.duration": [
2541915617
],
"host.architecture": [
"x86_64"
],
"metricset.name": [
"metrics"
],
"cloud.machine.type": [
"f1-micro"
],
"cloud.provider": [
"gcp"
],
"@timestamp": [
"2024-07-03T08:13:00.000Z"
],
"agent.id": [
"f49fbc8d-cc6a-4629-89b1-454f33f48d82"
],
"cloud.account.id": [
"elastic-obs-integrations-dev"
],
"ecs.version": [
"8.0.0"
],
"host.containerized": [
false
],
"host.os.platform": [
"ubuntu"
],
"agent.ephemeral_id": [
"2e83ec6e-b29b-407b-99fa-e8383f908e74"
],
"agent.version": [
"8.15.0"
],
"beats_state.state.host.architecture": [
"x86_64"
],
"host.os.family": [
"debian"
],
"gcp.metrics.instance.uptime.sec": [
60
],
"event.dataset": [
"gcp.metrics"
],
"cloud.instance.name": [
"demo-crest-group-j76r"
]
}
}
After Change:
{
"_index": ".ds-metricbeat-8.15.0-2024.07.03-000001",
"_id": "0c4gd5ABrcpmtJO-0-eX",
"_version": 1,
"_score": 0,
"_source": {
"@timestamp": "2024-07-03T05:41:00.000Z",
"ecs": {
"version": "8.0.0"
},
"metricset": {
"name": "metrics",
"period": 60000
},
"gcp": {
"metrics": {
"instance": {
"cpu": {
"usage": {
"pct": 0.006597337449646782
},
"reserved_cores": {
"value": 0.2
},
"usage_time": {
"sec": 0.07916804939576139
}
},
"uptime": {
"sec": 60
}
}
},
"labels": {}
},
"service": {
"type": "gcp"
},
"cloud": {
"region": "us-central1",
"instance": {
"id": "2314488029850962513",
"name": "demo-crest-group-j76r"
},
"machine": {
"type": "f1-micro"
},
"account": {
"name": "elastic-obs-integrations-dev",
"id": "elastic-obs-integrations-dev"
},
"provider": "gcp",
"availability_zone": "us-central1-a"
},
"host": {
"name": "epinpunw06a8",
"architecture": "x86_64",
"os": {
"codename": "jammy",
"type": "linux",
"platform": "ubuntu",
"version": "22.04.3 LTS (Jammy Jellyfish)",
"family": "debian",
"name": "Ubuntu",
"kernel": "5.15.146.1-microsoft-standard-WSL2"
},
"containerized": false,
"ip": [
"172.29.134.71",
"fe80::215:5dff:fe10:5cff"
],
"mac": [
"00-15-5D-10-5C-FF"
],
"hostname": "epinpunw06a8",
"id": "2fef4836d29e4bf6b57df5049e8f58e4"
},
"event": {
"dataset": "gcp.metrics",
"module": "gcp",
"duration": 2606976189
},
"agent": {
"ephemeral_id": "145cb68a-7481-4a5e-887b-41fe3f0af245",
"id": "f49fbc8d-cc6a-4629-89b1-454f33f48d82",
"name": "EPINPUNW06A8",
"type": "metricbeat",
"version": "8.15.0"
}
},
"fields": {
"beats_state.state.host.hostname": [
"epinpunw06a8"
],
"host.os.name.text": [
"Ubuntu"
],
"host.hostname": [
"epinpunw06a8"
],
"host.mac": [
"00-15-5D-10-5C-FF"
],
"cloud.availability_zone": [
"us-central1-a"
],
"service.type": [
"gcp"
],
"cloud.instance.id": [
"2314488029850962513"
],
"host.ip": [
"172.29.134.71",
"fe80::215:5dff:fe10:5cff"
],
"agent.type": [
"metricbeat"
],
"gcp.metrics.instance.cpu.reserved_cores.value": [
0.2
],
"logstash_stats.timestamp": [
"2024-07-03T05:41:00.000Z"
],
"event.module": [
"gcp"
],
"host.os.version": [
"22.04.3 LTS (Jammy Jellyfish)"
],
"host.os.kernel": [
"5.15.146.1-microsoft-standard-WSL2"
],
"host.os.name": [
"Ubuntu"
],
"agent.name": [
"EPINPUNW06A8"
],
"host.name": [
"epinpunw06a8"
],
"beats_state.timestamp": [
"2024-07-03T05:41:00.000Z"
],
"host.id": [
"2fef4836d29e4bf6b57df5049e8f58e4"
],
"beats_state.state.host.name": [
"epinpunw06a8"
],
"timestamp": [
"2024-07-03T05:41:00.000Z"
],
"cloud.account.name": [
"elastic-obs-integrations-dev"
],
"cloud.region": [
"us-central1"
],
"host.os.type": [
"linux"
],
"kibana_stats.timestamp": [
"2024-07-03T05:41:00.000Z"
],
"metricset.period": [
60000
],
"host.os.codename": [
"jammy"
],
"gcp.metrics.instance.cpu.usage.pct": [
0.006597337449646782
],
"agent.hostname": [
"EPINPUNW06A8"
],
"gcp.metrics.instance.cpu.usage_time.sec": [
0.07916804939576139
],
"event.duration": [
2606976189
],
"host.architecture": [
"x86_64"
],
"metricset.name": [
"metrics"
],
"cloud.machine.type": [
"f1-micro"
],
"cloud.provider": [
"gcp"
],
"@timestamp": [
"2024-07-03T05:41:00.000Z"
],
"agent.id": [
"f49fbc8d-cc6a-4629-89b1-454f33f48d82"
],
"cloud.account.id": [
"elastic-obs-integrations-dev"
],
"ecs.version": [
"8.0.0"
],
"host.containerized": [
false
],
"host.os.platform": [
"ubuntu"
],
"agent.ephemeral_id": [
"145cb68a-7481-4a5e-887b-41fe3f0af245"
],
"agent.version": [
"8.15.0"
],
"beats_state.state.host.architecture": [
"x86_64"
],
"host.os.family": [
"debian"
],
"gcp.metrics.instance.uptime.sec": [
60
],
"event.dataset": [
"gcp.metrics"
],
"cloud.instance.name": [
"demo-crest-group-j76r"
]
}
}
![botelastic[bot] avatar](https://avatars.githubusercontent.com/u/6764390?v=4 "Published by a pub.dev verified publisher"){kind=small}
This pull request does not have a backport label. If this is a bug or security fix, could you label this PR @Linu-Elias? 🙏. For such, you'll need to label your PR with:
- The upcoming major version of the Elastic Stack
- The upcoming minor version of the Elastic Stack (if you're not pushing a breaking change)
To fixup this pull request, you need to add the backport labels for the needed branches, such as:
backport-v8./d.0is the label to automatically backport to the8./dbranch./dis the digit
![mergify[bot] avatar](https://avatars.githubusercontent.com/in/10562?v=4 "Published by a pub.dev verified publisher"){kind=small}
@Linu-Elias do we have a before & after document for compute for example? We have to check that events are not being affected by this change. I see that in the PR description you added a document but not sure if that's before or after the change.
This pull request is now in conflicts. Could you fix it? 🙏 To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/
git fetch upstream
git checkout -b gcp-label-fix upstream/gcp-label-fix
git merge upstream/main
git push upstream gcp-label-fix
@Linu-Elias do we have a before & after document for
computefor example? We have to check that events are not being affected by this change. I see that in the PR description you added a document but not sure if that's before or after the change.
@gpop63 I have added an example of compute before and after the change. I don't any difference in both the documents hence it will not break the existing flow.
This pull request is now in conflicts. Could you fix it? 🙏 To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/
git fetch upstream
git checkout -b gcp-label-fix upstream/gcp-label-fix
git merge upstream/main
git push upstream gcp-label-fix
I did some tests and it seems fine to me. Services that have no metadata will include cloud.instance.id and services that already have metadata will not be affected.
spanner (service without metadata) diff
{
- "@timestamp": "2024-07-09T12:28:46.200Z",
+ "@timestamp": "2024-07-09T12:29:46.200Z",
"cloud": {
+ "instance": {
+ "id": "spanner-1"
+ },
"account": {
"id": "elastic-obs-integrations-dev",
"name": "elastic-obs-integrations-dev"
},
"provider": "gcp"
},
"event": {
- "duration": 269604388,
+ "duration": 262185450,
"dataset": "gcp.metrics",
"module": "gcp"
},
"metricset": {
"name": "metrics",
"period": 60000
},
"agent": {
"id": "c3a8e245-a074-4133-8a29-30808077cace",
"name": "gab-MS-7C95",
"type": "metricbeat",
"version": "8.16.0",
- "ephemeral_id": "e6d85758-d871-4681-a7cd-0fe247ebf801"
+ "ephemeral_id": "eb9d711f-a1be-4dc6-8e94-9802fb4de460"
},
"ecs": {
"version": "8.0.0"
},
"host": {
// ... (host details unchanged)
},
"gcp": {
"labels": {
"resource": {
"location": "europe-west1",
"instance_config": "regional-europe-west1"
},
"metrics": {
"database": "db-1",
"region": "europe-west1"
}
},
"metrics": {
"instance": {
"leader_percentage_by_region": {
"value": 1
}
}
}
},
"service": {
"type": "gcp"
}
}
compute (service with metadata) diff
{
- "@timestamp": "2024-07-09T12:19:00.000Z",
+ "@timestamp": "2024-07-09T12:22:00.000Z",
"cloud": {
"account": {
"id": "elastic-obs-integrations-dev",
"name": "elastic-obs-integrations-dev"
},
"provider": "gcp",
"availability_zone": "asia-south1-c",
"region": "asia-south1",
"instance": {
"id": "6751872764960737742",
"name": "service-integration-dev-idc-1"
},
"machine": {
"type": "n1-standard-8"
}
},
"host": {
// ... (host details unchanged)
},
"event": {
"module": "gcp",
- "duration": 2486983199,
+ "duration": 2092800246,
"dataset": "gcp.metrics"
},
"metricset": {
"name": "metrics",
"period": 60000
},
"agent": {
"name": "gab-MS-7C95",
"type": "metricbeat",
"version": "8.16.0",
- "ephemeral_id": "9ee881eb-973f-4851-b12c-995ee83202f8",
+ "ephemeral_id": "23a53a7b-96dc-431c-97af-9afa30a372cd",
"id": "c3a8e245-a074-4133-8a29-30808077cace"
},
"ecs": {
"version": "8.0.0"
},
"gcp": {
"labels": {
"user": {
"division": "engineering",
"org": "obs",
"project": "agithomas",
"team": "service_integration"
}
},
"metrics": {
"instance": {
"cpu": {
"usage_time": {
- "sec": 27.490263244137168
+ "sec": 26.798008978366852
},
"usage": {
- "pct": 0.0572713817586191
+ "pct": 0.05582918537159761
},
"reserved_cores": {
"value": 8
}
},
"uptime": {
"sec": 60
}
}
}
},
"service": {
"type": "gcp"
}
}
@elastic/obs-ds-hosted-services hey guys, could someone take a look at this PR please?