Random JavaScript mapping "Runtime Error" on concurrent `request.headers` deletion

[yvele]

I have a pipeline function that runs for every root (top-level) resolvers.

Context: This pipeline function handles propagation of identity information across all pipeline functions and nested child resolvers. I cannot use stash as it is not accessible to nested child resolvers. As a workaround the function adds identity info to a custom x-identity HTTP request header as described here https://github.com/aws/aws-appsync-community/issues/9#issuecomment-539077400 and here https://stackoverflow.com/a/58093410/1480391.

Here is my pipeline function JavaScript mapping code:

import { util } from "@aws-appsync/utils";

export function request(context) {
  return {
    operation: "Invoke",
    payload : { context }
  };
}

export function response(context) {
  if (context.error || context.result.error) {
    util.error("my custom error");
  }

  // This triggers random "Runtime Error"
  delete context.request.headers["unwanted"];

  // Loops are more likely to trigger "Runtime Error"
  const headersWhiteList = [{ "wanted" : true }];
  for (const key of Object.keys(context.request.headers)) {
   if (!headersWhiteList[key]) {
     delete context.request.headers[key];
   }
  }

  // Removing identity claims also triggers random "Runtime Error"
  if (context.identity?.claims) {
   for (const key of Object.keys(context.identity.claims)) {
     delete context.identity.claims[key];
   }
  }

  // This alone does NOT trigger any error
  context.request.headers["x-identity"] = JSON.stringify(context.result?.data?.identity);
  return null;
}

When I run a single request with multiple root (top-level) resolvers:

query multiple {
  me {
    name
  }
  node0: node(id: "aaa") {
    __typename
  }
  node1: node(id: "bbb") {
    __typename
  }
  node2: node(id: "ccc") {
    __typename
  }
  [...]
}

I randomly get the following errors:

{
  "data": {
    "me": null,
    "node0": { "__typename": "FOO" },
    "node1": null,
    "node2": null,
    "node3": null,
    "node4": null,
    "node5": { "__typename": "BAR" }
  },
  "errors": [{
      "path": ["me"],
      "data": null,
      "errorType": "Code",
      "errorInfo": null,
      "locations": [{ "line": 2, "column": 3, "sourceName": null }],
      "message": "Runtime Error"
    }, {
      "path": ["node4"],
      "data": null,
      "errorType": "Code",
      "errorInfo": null,
      "locations": [{ "line": 17, "column": 3, "sourceName": null }],
      "message": "Runtime Error"
    }, {
      "path": ["node3"],
      "data": null,
      "errorType": "Code",
      "errorInfo": null,
      "locations": [{ "line": 14, "column": 3, "sourceName": null }],
      "message": "Runtime Error"
    }, {
      "path": ["node2"],
      "data": null,
      "errorType": "Code",
      "errorInfo": null,
      "locations": [{ "line": 11, "column": 3, "sourceName": null }],
      "message": "Runtime Error"
    }, {
      "path": ["node1"],
      "data": null,
      "errorType": "Code",
      "errorInfo": null,
      "locations": [{ "line": 8, "column": 3, "sourceName": null }],
      "message": "Runtime Error"
    }]
}

This is the X-Ray traces:

And some CloudWatch logs:

{
    "logType": "ResponseFunctionEvaluation",
    "path": [
        "node2"
    ],
    "fieldName": "node",
    "resolverArn": "arn:aws:appsync:eu-west-1:xxx:apis/xxx/types/Query/resolvers/node",
    "functionName": "MyFunction",
    "requestId": "xxx-xxx-xxx-xxx-xxx",
    "context": {
        "arguments": {
            "id": "ccc"
        },
        "result": {
            "data": {
                "identityMeta": {
                    "authenticationMode": "COGNITO",
                    "user": {
                        "id": "xxx",
                        "roleIds": ["xxx"]
                    }
                }
            }
        },
        "prev": {},
        "stash": {},
        "outErrors": []
    },
    "fieldInError": true,
    "errors": [
        "Runtime Error"
    ],
    "parentType": "Query",
    "graphQLAPIId": "xxx",
    "functionArn": "arn:aws:appsync:eu-west-1:xxx:apis/xxx/functions/xxx"
}

{
    "logType": "GraphQLFieldRuntimeError",
    "fieldInError": true,
    "path": [
        "me"
    ],
    "errors": [
        "Runtime Error"
    ],
    "requestId": "xxx-xxx-xxx-xxx-xxx",
    "graphQLAPIId": "xxx"
}

As you can see, debugging this is quite difficult: My only clue is this "Runtime Error" message 🤔

It seems to be caused by the concurrent deletion of request.headers and identity.claims.

I'm aware that only stash modification is documented https://docs.aws.amazon.com/appsync/latest/devguide/resolver-context-reference-js.html

You can remove items from the stash by modifying the code below:

delete ctx.stash.key

Adding items to request.headers and identity.claims works, I can also remove items, but I'm facing random errors on concurrent deletion.

Additionally, concurrent function execution doesn't seem to be document in AppSync, which is quite counterintuitive since JavaScript is typically single-threaded.

[yvele]

When I replace delete with undefined assignment it's working perfectly and I get no random error anymore 🤔

- delete context.request.headers["unwanted"];
+ context.request.headers["unwanted"] = undefined;
  
  const headersWhiteList = [{ "wanted" : true }];
  for (const key of Object.keys(context.request.headers)) {
    if (!headersWhiteList[key]) {
-     delete context.request.headers[key];
+     context.request.headers[key] = undefined;
    }
  }
  
  if (context.identity?.claims) {
    for (const key of Object.keys(context.identity.claims)) {
-     delete context.identity.claims[key];
+     context.identity.claims[key] = undefined;
    }
  }

Looks like the problem comes with concurrent delete execution

[gilbertw1]

Thanks for reporting this issue. We're looking into it.