workers-sdk
workers-sdk copied to clipboard
🐛 BUG: Async queue whose consumer is both a producer and consumer fails locally
Which Cloudflare product(s) does this pertain to?
Queues
What version(s) of the tool(s) are you using?
3.20
What version of Node are you using?
21.2.0
What operating system are you using?
Mac
Describe the Bug
- Given a worker whose consumer is both a producer and a consumer, given a local context.
- The consumer enqueues n batches of 100 to itself.
- When reaching 140 000 ms of execution time, processing seems to stop and queue no longer consumes.
- CPU & Memory explosion
Debug level logging still shows queue listening:
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000023}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000023, result: { id: '7c27473422a2460f' } }
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000024}
...
Furthermore. After closing the process, workerd hangs and consumes ~92% of CPU. This also happens without closing the process in the case of a larger number of messages. RAM consumption also goes up to, for example 30gB in the case of jobs with 200k messages.
Please provide a link to a minimal reproduction
https://gist.github.com/Cosmin-Parvulescu/f09d0dd2329e0e4c08899a62d2046b64
Please provide any relevant error logs
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084868903,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '4',
url: 'node-internal:internal_inspect',
startLine: 0,
startColumn: 0,
endLine: 2213,
endColumn: 0,
executionContextId: 390389025,
hash: '60394ebc017aeb5e153eadf9e818a61bcfb1f344f006d676b41ec29d0e9db715',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 89123,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:internal_inspect'
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '5',
url: 'node-internal:internal_buffer',
startLine: 0,
startColumn: 0,
endLine: 1709,
endColumn: 0,
executionContextId: 390389025,
hash: '9621ec975892d3bca9453d74ef53577eeba6a67b2ccaad23ad4ddb4f0a0786d1',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 61179,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:internal_buffer'
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '8',
url: 'node-internal:internal_errors',
startLine: 0,
startColumn: 0,
endLine: 443,
endColumn: 0,
executionContextId: 390389025,
hash: 'dffaa5e9f9588d77e0dc47fffd1b6bb2935ac00d81d4b789608ffc478102c56f',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 15653,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:internal_errors'
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '6',
url: 'node-internal:internal_types',
startLine: 0,
startColumn: 0,
endLine: 115,
endColumn: 0,
executionContextId: 390389025,
hash: '9cf7c9384c0629ca99014f24a79fdd954ca7b50ed6425159d029d249d5a1e923',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 5065,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:internal_types'
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '9',
url: 'node-internal:internal_utils',
startLine: 0,
startColumn: 0,
endLine: 207,
endColumn: 0,
executionContextId: 390389025,
hash: 'cc03bacc35d5a77d3d362adbabf94fb719629d131bcb82140d6494af9bf29e32',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 6853,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:internal_utils'
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Debugger.scriptParsed',
params: {
scriptId: '7',
url: 'node-internal:validators',
startLine: 0,
startColumn: 0,
endLine: 164,
endColumn: 0,
executionContextId: 390389025,
hash: 'a278e95ed0aac7941e9a22a500c741f35e38210962b1645d7365d20bbd5adc14',
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: true,
length: 6571,
stackTrace: { callFrames: [Array] },
scriptLanguage: 'JavaScript',
embedderName: 'node-internal:validators'
}
}
{ first: 'go', last: 'go', executionTime: 0 }
{ first: 'message-1', last: 'message-100', executionTime: 110 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084869013,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-101', last: 'message-200', executionTime: 111 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084869014,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000004}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000004, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-201', last: 'message-300', executionTime: 7419 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084876322,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-301', last: 'message-400', executionTime: 7975 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084876878,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-401', last: 'message-500', executionTime: 15092 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084883995,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000005}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000005, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-501', last: 'message-600', executionTime: 15927 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084884830,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084891328,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-601', last: 'message-700', executionTime: 22425 }
{ first: 'message-701', last: 'message-800', executionTime: 23863 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084892767,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000006}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000006, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-801', last: 'message-900', executionTime: 29764 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084898667,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-901', last: 'message-1000', executionTime: 31735 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084900638,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000007}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000007, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-1001', last: 'message-1100', executionTime: 37622 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084906525,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1101', last: 'message-1200', executionTime: 39506 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084908410,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1201', last: 'message-1300', executionTime: 44634 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084913537,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000008}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000008, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-1301', last: 'message-1400', executionTime: 46731 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084915634,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1401', last: 'message-1500', executionTime: 52355 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084921258,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1501', last: 'message-1600', executionTime: 54919 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084923822,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000009}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000009, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-1601', last: 'message-1700', executionTime: 59835 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084928738,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1701', last: 'message-1800', executionTime: 62231 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084931134,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000010}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000010, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-1801', last: 'message-1900', executionTime: 66923 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084935826,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-1901', last: 'message-2000', executionTime: 69399 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084938302,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2001', last: 'message-2100', executionTime: 74235 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084943138,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000011}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000011, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-2101', last: 'message-2200', executionTime: 77450 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084946353,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2201', last: 'message-2300', executionTime: 81282 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084950185,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2301', last: 'message-2400', executionTime: 85286 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084954189,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000012}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000012, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-2401', last: 'message-2500', executionTime: 89698 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084958601,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2501', last: 'message-2600', executionTime: 92745 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084961648,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000013}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000013, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-2601', last: 'message-2700', executionTime: 97545 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084966449,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2701', last: 'message-2800', executionTime: 100829 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084969732,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-2801', last: 'message-2900', executionTime: 105229 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084974132,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000014}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000014, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-2901', last: 'message-3000', executionTime: 108053 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084976956,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-3001', last: 'message-3100', executionTime: 112899 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084981802,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-3101', last: 'message-3200', executionTime: 115177 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084984080,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000015}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000015, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-3201', last: 'message-3300', executionTime: 120516 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084989419,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-3301', last: 'message-3400', executionTime: 122644 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084991547,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000016}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000016, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-3401', last: 'message-3500', executionTime: 128595 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084997498,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-3501', last: 'message-3600', executionTime: 130081 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703084998984,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000017}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000017, result: { id: 'bf721ca19109af5b' } }
{ first: 'message-3601', last: 'message-3700', executionTime: 136691 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703085005594,
stackTrace: { callFrames: [Array] }
}
}
{ first: 'message-3701', last: 'message-3800', executionTime: 138276 }
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE {
method: 'Runtime.consoleAPICalled',
params: {
type: 'log',
args: [ [Object] ],
executionContextId: 390389025,
timestamp: 1703085007179,
stackTrace: { callFrames: [Array] }
}
}
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000018}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000018, result: { id: 'bf721ca19109af5b' } }
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000019}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000019, result: { id: 'bf721ca19109af5b' } }
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000020}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000020, result: { id: 'bf721ca19109af5b' } }
[InspectorProxyWorker] SEND TO RUNTIME {"method":"Runtime.getIsolateId","id":100000021}
[InspectorProxyWorker] RUNTIME INCOMING MESSAGE { id: 100000021, result: { id: 'bf721ca19109af5b' } }
...