Multiple traces will be generated for the same request in service mesh

[tuhao1020]

What happened?

I deployed 2 nodejs services, otel-demo-consumer and otel-demo-provider, in the service mesh, and then used the built-in gateway to access otel-demo-consumer, but I got three following parts:

gateway -> consumer, traceid: 63ae1fc32dba13ffab2b428fe8940866

consumer sidecar -> provider sidecar ,traceid: aa8abaa48130e037e53814d58514b5a3

consumer -> provider, traceid: 20a8f65ab5753a6b22e36b6eba5fc45b

Why doesn't a call chain contain all three parts?

consumer code

const opentelemetry = require("@opentelemetry/sdk-node");
const express = require("express");
const axios = require('axios');
const PORT = "8081";
const app = express();

app.get("/hello", (req, res) => {
    const span = opentelemetry.api.trace.getSpan(opentelemetry.contextBase.context.active());
    console.log( req.path + " traceId = " + span.spanContext().traceId)
    axios.get('http://otel-demo-provider:8080/hello').then(resp => {
        res.send(resp.data);
    })
});

app.get("/stu/:id", (req, res) => {
    const span = opentelemetry.api.trace.getSpan(opentelemetry.contextBase.context.active());
    console.log( req.path + " traceId = " + span.spanContext().traceId)
    axios.get('http://otel-demo-provider:8080/stu/' + req.params.id)
        .then(resp => {
            res.send(resp.data);
        })
        .catch(function (error) {
            // handle error
            //console.log(error);
            span.recordException(error);
            if (error.response) {
                span.setStatus({ code: error.response.status });
                span.setAttribute('http.err_msg', error.response.data);
                res.status(error.response.status).send(error.response.data)
            }
        });
});


app.listen(parseInt(PORT, 10), () => {
    console.log(`Listening for requests on http://localhost:${PORT}`);
});

OpenTelemetry Setup Code

// Require dependencies
const opentelemetry = require("@opentelemetry/sdk-node");
const { getNodeAutoInstrumentations } = require("@opentelemetry/auto-instrumentations-node");
const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api');
const { ZipkinExporter } = require("@opentelemetry/exporter-zipkin");
const { ConsoleSpanExporter, BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base");
const { TraceIdRatioBasedSampler } = require('@opentelemetry/core')

// For troubleshooting, set the log level to DiagLogLevel.DEBUG
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);

const sdk = new opentelemetry.NodeSDK({
  serviceName: "otel-demo-consumer",
  spanProcessor: new BatchSpanProcessor(new ZipkinExporter({
    url: 'http://zipkin.istio-system:9411/api/v2/spans'
  })),
  instrumentations: [getNodeAutoInstrumentations()],
  sampler: new TraceIdRatioBasedSampler(1),
});

sdk.start()

package.json

{
  "dependencies": {
    "@opentelemetry/api": "^1.1.0",
    "@opentelemetry/auto-instrumentations-node": "^0.31.0",
    "@opentelemetry/exporter-zipkin": "^1.4.0",
    "@opentelemetry/sdk-node": "^0.30.0",
    "@opentelemetry/sdk-trace-web": "^1.4.0",
    "axios": "^0.27.2",
    "express": "^4.18.1",
    "pg": "^8.7.3",
    "redis": "^4.2.0"
  }
}

Relevant log output

No response

[vmarchaud]

I don't think the node sdk by default register a way to propagate trace context. I would advise to use the https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-core/src/trace/W3CTraceContextPropagator.ts one but it depend of which one your service mesh is using

[tuhao1020]

@vmarchaud I am using istio, is there a propatator for B3 headers?

[legendecas]

https://github.com/open-telemetry/opentelemetry-js/tree/main/packages/opentelemetry-propagator-b3 The documentation of the B3 Propagator can be found here.

[github-actions[bot]]

This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days.

[github-actions[bot]]

This issue was closed because it has been stale for 14 days with no activity.