opossum-prometheus icon indicating copy to clipboard operation
opossum-prometheus copied to clipboard

Getting warning during circuit breaker usage "Warning: Possible perf_hooks memory leak detected."

Open abhinavgoel91 opened this issue 3 years ago • 2 comments

Node.js Version: v8.12.0

Operating System: Mac OS 10.14.6

Steps to Produce Error:

  1. Create a new directory mkdir test
  2. cd into the new directory cd test
  3. check if node version is 8.12.0 nvm ls
  4. run npm init npm init, use defaults
  5. Install expressjs npm install --save express
  6. Install opposum npm install --save opossum
  7. Install opposum prometheus npm install --save opossum-prometheus
  8. create a new file circuit_breaker_metrics.js touch circuit_breaker_metrics.js
  9. Add the following content to circuit_breaker_metrics.js -
'use strict'

const PrometheusMetrics = require('opossum-prometheus')
var prometheus = new PrometheusMetrics()

class CircuitBreakerMetrics {
  static registerOpossumCustomMetrics (funcName, circuit) {
    prometheus.add(circuit);
    ['success', 'timeout', 'reject', 'open', 'halfOpen', 'close', 'fallback'].forEach(function (event) {
      circuit.on(event, (result) => {
        console.log(`OPOSSUM_EVENT api: ${funcName} event: ${event} result: ${JSON.stringify(result)}`)
      })
    })
  }
}

module.exports = CircuitBreakerMetrics
  1. Create a new file circuit_breaker_test.js touch circuit_breaker_test.js
  2. Add the following code to circuit_breaker_test.js -
'use strict'

const CircuitBreaker = require('opossum')
var CircuitBreakerMetrics = require('./circuit_breaker_metrics')

async function randomErrorAsync () {
  if (Math.random() < 0.3) {
    console.error('Throwing random error')
    throw new Error('random error')
  } else {
    console.log('Returning a string')
    return 'abcd'
  }
}

let config = {
  timeout: 3600000, // really high timeout
  errorThresholdPercentage: 20, // When 20% of requests fail, trip the circuit
  resetTimeout: 30000, // After 30 seconds, try again.
  rollingCountTimeout: 60, // total number of seconds for which the stats are being maintained,
  rollingCountBuckets: 10 // buckets value is the number of slots in the window
}

var testBreaker = new CircuitBreaker(randomErrorAsync, config)

console.log(`testBreaker = ${testBreaker}`)
CircuitBreakerMetrics.registerOpossumCustomMetrics('circuit_breaker_test', testBreaker)

testBreaker.fallback(function (result) {
  console.error(`result = ${result === undefined ? undefined : JSON.stringify(result)}`)

  if (result === null || result === undefined || result.code === undefined) {
    console.error(`returning new Error('CIRCUIT_BRK')`)
    throw new Error('CIRCUIT_BRK')
  } else {
    console.error(`returning new Error('${result.code}')`)
    throw new Error(result.code)
  }
})

module.exports.testBreaker = testBreaker

module.exports.middleware = function (req, res, next) {
  try {
    testBreaker.fire()
      .then(function (result) {
        res.json({
          'result': result
        })
      })
      .catch(function (err) {
        console.error(err)
        res.json({
          'error': err.message
        })
      })
  } catch (ex) {
    res.json({
      'error': ex
    })
  }
}

process.on('warning', e => console.warn(e.stack))

  1. Create a new file server.js touch server.js
  2. Add the following code to server.js -
'use strict'

const express = require('express')
const app = express()
const port = 3000
const CircuitBreakerTest = require('./circuit_breaker_test')

app.get('/circuit_breaker_test', CircuitBreakerTest.middleware)

app.listen(port, () => {
  console.log(`Example app listening on port ${port}`)
})
  1. Run the app node server.js
  2. Make requests to the app curl -XGET 'http://localhost:3000/circuit_breaker_test'
  3. The code is designed to intentionally trip the circuit breaker by throwing error 30% of the time. Make repeated requests and trip the circuit breaker.
  4. After about 6 minutes, you'll see the issue logs. Here's the request logs -

$ abhinavgoel in ~ 
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"result":"abcd"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"result":"abcd"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"error":"CIRCUIT_BRK"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"error":"EOPENBREAKER"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"error":"EOPENBREAKER"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"error":"EOPENBREAKER"}%

$ abhinavgoel in ~
➜ curl -XGET 'http://localhost:3000/circuit_breaker_test'
{"result":"abcd"}%

  1. Here are the application logs -
$ abhinavgoel in ~/Documents/repos/test  C:1
➜ node server.js
testBreaker = [object Object]
Example app listening on port 3000
Returning a string
OPOSSUM_EVENT api: circuit_breaker_test event: success result: "abcd"
Returning a string
OPOSSUM_EVENT api: circuit_breaker_test event: success result: "abcd"
Throwing random error
OPOSSUM_EVENT api: circuit_breaker_test event: open result: undefined
result = {}
returning new Error('CIRCUIT_BRK')
Error: CIRCUIT_BRK
    at Function.<anonymous> (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:34:11)
    at fallback (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:756:10)
    at handleError (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:744:16)
    at promise.then.catch.error (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:630:17)
    at <anonymous>
    at runMicrotasksCallback (internal/process/next_tick.js:122:5)
    at _combinedTickCallback (internal/process/next_tick.js:132:7)
    at process._tickCallback (internal/process/next_tick.js:181:9)
OPOSSUM_EVENT api: circuit_breaker_test event: reject result: {"code":"EOPENBREAKER"}
result = {"code":"EOPENBREAKER"}
returning new Error('EOPENBREAKER')
Error: EOPENBREAKER
    at Function.<anonymous> (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:37:11)
    at fallback (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:756:10)
    at CircuitBreaker.call (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:574:14)
    at CircuitBreaker.fire (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:499:22)
    at module.exports.middleware (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:45:17)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at next (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:137:13)
    at Route.dispatch (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:112:3)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at /Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/index.js:281:22
OPOSSUM_EVENT api: circuit_breaker_test event: reject result: {"code":"EOPENBREAKER"}
result = {"code":"EOPENBREAKER"}
returning new Error('EOPENBREAKER')
Error: EOPENBREAKER
    at Function.<anonymous> (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:37:11)
    at fallback (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:756:10)
    at CircuitBreaker.call (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:574:14)
    at CircuitBreaker.fire (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:499:22)
    at module.exports.middleware (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:45:17)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at next (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:137:13)
    at Route.dispatch (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:112:3)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at /Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/index.js:281:22
OPOSSUM_EVENT api: circuit_breaker_test event: reject result: {"code":"EOPENBREAKER"}
result = {"code":"EOPENBREAKER"}
returning new Error('EOPENBREAKER')
Error: EOPENBREAKER
    at Function.<anonymous> (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:37:11)
    at fallback (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:756:10)
    at CircuitBreaker.call (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:574:14)
    at CircuitBreaker.fire (/Users/abhinavgoel/Documents/repos/test/node_modules/opossum/lib/circuit.js:499:22)
    at module.exports.middleware (/Users/abhinavgoel/Documents/repos/test/circuit_breaker_test.js:45:17)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at next (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:137:13)
    at Route.dispatch (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/route.js:112:3)
    at Layer.handle [as handle_request] (/Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/layer.js:95:5)
    at /Users/abhinavgoel/Documents/repos/test/node_modules/express/lib/router/index.js:281:22
OPOSSUM_EVENT api: circuit_breaker_test event: halfOpen result: 30000
Returning a string
OPOSSUM_EVENT api: circuit_breaker_test event: close result: undefined
OPOSSUM_EVENT api: circuit_breaker_test event: success result: "abcd"
(node:79589) Warning: Possible perf_hooks memory leak detected. There are 151 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
Warning: Possible perf_hooks memory leak detected. There are 151 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
    at Performance.[index-entry] (perf_hooks.js:444:15)
    at Performance.[insert-entry] (perf_hooks.js:288:22)
    at process.observersCallback (perf_hooks.js:597:28)
(node:79589) Warning: Possible perf_hooks memory leak detected. There are 152 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
Warning: Possible perf_hooks memory leak detected. There are 152 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
    at Performance.[index-entry] (perf_hooks.js:444:15)
    at Performance.[insert-entry] (perf_hooks.js:288:22)
    at process.observersCallback (perf_hooks.js:597:28)
(node:79589) Warning: Possible perf_hooks memory leak detected. There are 153 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
Warning: Possible perf_hooks memory leak detected. There are 153 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
    at Performance.[index-entry] (perf_hooks.js:444:15)
    at Performance.[insert-entry] (perf_hooks.js:288:22)
    at process.observersCallback (perf_hooks.js:597:28)
(node:79589) Warning: Possible perf_hooks memory leak detected. There are 154 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
Warning: Possible perf_hooks memory leak detected. There are 154 entries in the Performance Timeline. Use the clear methods to remove entries that are no longer needed or set performance.maxEntries equal to a higher value (currently the maxEntries is 150).
    at Performance.[index-entry] (perf_hooks.js:444:15)
    at Performance.[insert-entry] (perf_hooks.js:288:22)
    at process.observersCallback (perf_hooks.js:597:28)

The above warning is confusing me. Is it because of some issue opposum or opposum-prometheus ?

abhinavgoel91 avatar Feb 09 '22 09:02 abhinavgoel91