Unable to access to trace information using opentelemetry-js SDK + aws lambda layer

[jarpz]

trafficstars

What happened?

We are unable to access tracecontext like showed here: https://opentelemetry.io/docs/languages/js/propagation/#generic-example

We want tracecontext information to propagate manually to other services like SNS, SQS, KaFka.

Even we can see in Xray traces working correctly, due the example acces to other service like s3 and trace is visualized.

Steps to Reproduce

• Use only OTEL API
• Use aws lambda layer => "arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1"
• Execute lambda code to access to trace info using Otel SDK API.

Expected Result

Access to current trace information using OTEL API SDK

Actual Result

Empty trace information from OTEL API SDK

Additional Details

IaC code

resource "aws_iam_role" "lambda_role" {
  name = "CreateFnRole"
  assume_role_policy = jsonencode(
    {
      "Version" : "2012-10-17",
      "Statement" : [
        {
          "Action" : "sts:AssumeRole",
          "Principal" : {
            "Service" : "lambda.amazonaws.com"
          },
          "Effect" : "Allow",
          "Sid" : ""
        }
      ]
    })

  managed_policy_arns = [
    "arn:aws:iam::aws:policy/AmazonS3FullAccess",
    "arn:aws:iam::aws:policy/AWSLambda_FullAccess",
    "arn:aws:iam::aws:policy/CloudWatchFullAccess",
    "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess",
    "arn:aws:iam::aws:policy/AWSXrayFullAccess"
  ]

  inline_policy {
    name = "CreateFnPolicy"
    policy = jsonencode(
      {
        "Version" : "2012-10-17",
        "Statement" : [
          {
            "Action" : [
              "appconfig:*",
              "kms:*"
            ],
            "Resource" : "*",
            "Effect" : "Allow"
          }
        ]
      },
    )
  }
}

variable "service_name" {}
variable "env" {}

data "archive_file" "zip" {
  source_dir  = "../app/dist/functions/create"
  output_path = "../app/dist/create.zip"
  type        = "zip"
}

resource "aws_lambda_function" "create_fnc" {
  function_name = "${var.env}-${var.service_name}_create"

  runtime          = "nodejs18.x"
  filename         = data.archive_file.zip.output_path
  source_code_hash = data.archive_file.zip.output_base64sha256
  handler          = "handler.main"
  role             = aws_iam_role.lambda_role.arn
  memory_size      = 256
  tracing_config {
    mode = "Active"
  }

  environment {
    variables = {
      AWS_LAMBDA_EXEC_WRAPPER : "/opt/otel-handler"
      SERVICE_NAME          = var.service_name
      ENV                   = var.env
      OTEL_TRACES_EXPORTER  = "oltp"
      OTEL_METRICS_EXPORTER = "oltp"
      OTEL_LOG_LEVEL        = "ERROR"
      OTEL_TRACES_SAMPLER   = "xray"
      OTEL_PROPAGATORS      = "tracecontext, baggage, xray-lambda"
      OTEL_SERVICE_NAME     = var.service_name
    }
  }

  layers = [
    "arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-nodejs-amd64-ver-1-18-1:1"
  ]
}

output "create_fn_arn" {
  value = aws_lambda_function.create_fnc.arn
}

OpenTelemetry Setup Code

import "reflect-metadata"
import {Context} from 'aws-lambda';
import {Logger} from '@aws-lambda-powertools/logger';
import {LambdaInterface} from '@aws-lambda-powertools/commons/types';
import middy from '@middy/core';
import {ListBucketsCommand, S3Client} from '@aws-sdk/client-s3';
import {inject, injectable, singleton} from "tsyringe";
import api, {context, propagation} from '@opentelemetry/api';

const logger = new Logger();

const s3 = new S3Client({});

@injectable()
@singleton()
class Handler implements LambdaInterface {

    constructor() {
    }

    public async handler(_event: any, _context: Context): Promise<any> {
        logger.info(`==> currentTrace: ${currentSpan()} | ${JSON.stringify(_event)}`);

        const result = await s3.send(new ListBucketsCommand({}));

        logger.info(`context => ${JSON.stringify(api.context.active())} || ${JSON.stringify(currentContext())}`);
        if (api.context.active()) {
            const span = api.trace.getSpan(api.context.active());
            logger.info(`span => ${span}`);
            if (span) {
                logger.info(`QUE PASO??? ==> ${JSON.stringify(span)}`);
            }
        }

        return {
            result: 'hello world => ' + result.Buckets.length,
        };
    }
}

const main = middy()
        .handler(new Handler().handler)

module.exports = {main};

export function currentContext(): Record<string, any> {
    const output = {};

    propagation.inject(context.active(), output);

    return output;
}

export function currentSpan(): Record<string, string> | null {
    if (!api.context.active() || !api.trace.getSpan(api.context.active())) {
        return null;
    }

    let currentSpan = api.trace.getSpan(api.context.active());

    return {
        // @ts-ignore
        traceId: currentSpan.spanContext().traceId,
        // @ts-ignore
        spanId: currentSpan.spanContext().spanId,
        // @ts-ignore
        traceFlags: currentSpan.spanContext().traceFlags
    }
}

package.json

{
  "name": "hello_world",
  "version": "1.0.0",
  "description": "hello world sample for NodeJS",
  "main": "app.js",
  "repository": "https://github.com/aws/aws-sam-cli-app-templates/tree/master/nodejs18.x/hello-ts-pt",
  "author": "SAM CLI",
  "license": "MIT",
  "dependencies": {
    "@aws-lambda-powertools/commons": "^2.1.1",
    "@aws-lambda-powertools/logger": "^2.0.3",
    "@aws-lambda-powertools/metrics": "^2.0.3",
    "@aws-lambda-powertools/parameters": "^2.2.0",
    "@aws-lambda-powertools/tracer": "^2.0.3",
    "@aws-sdk/client-s3": "^3.592.0",
    "@middy/appconfig": "^5.4.0",
    "@opentelemetry/api": "^1.9.0",
    "esbuild": "^0.17.6",
    "reflect-metadata": "^0.1.13",
    "tiny-glob": "^0.2.9",
    "tsyringe": "4.8.0"
  },
  "scripts": {
    "unit": "jest",
    "lint": "eslint '*.ts' --quiet --fix",
    "compile": "tsc",
    "test": "npm run compile && npm run unit",
    "build": "esbuild src/functions/**/*.ts --bundle --minify --outdir=dist --outbase=src --sourcemap=inline --platform=node --target=node18.16.1 "
  },
  "devDependencies": {
    "@aws-sdk/client-appconfigdata": "^3.598.0",
    "@jest/globals": "^29.4.0",
    "@middy/core": "^5.4.0",
    "@middy/util": "^5.4.0",
    "@types/aws-lambda": "^8.10.109",
    "@types/jest": "^29.4.0",
    "@types/node": "^18.13.0",
    "@typescript-eslint/eslint-plugin": "^5.46.1",
    "@typescript-eslint/parser": "^5.46.1",
    "eslint": "^8.30.0",
    "eslint-config-prettier": "^8.3.0",
    "eslint-plugin-prettier": "^4.0.0",
    "jest": "^29.3.1",
    "prettier": "^2.5.1",
    "ts-jest": "^29.0.5",
    "ts-node": "^10.9.1",
    "typescript": "^4.9.4"
  }
}

Relevant log output

{
    "level": "INFO",
    "message": "==> 10 currentTrace: null | {\"go\":1}",
    "sampling_rate": 0,
    "service": "service_undefined",
    "timestamp": "2024-07-05T23:26:10.195Z",
    "xray_trace_id": "1-6688810f-a224dfbcdd26280c3fd8f8c6"
}

{
    "level": "INFO",
    "message": "context => {\"_currentContext\":{}} || {}",
    "sampling_rate": 0,
    "service": "service_undefined",
    "timestamp": "2024-07-05T23:26:10.358Z",
    "xray_trace_id": "1-6688810f-a224dfbcdd26280c3fd8f8c6"
}