nuclei
nuclei copied to clipboard
Published
20 hours ago •
projectdiscovery
projectdiscovery
Running nuclei with multiple instances results in a memory leak
Running nuclei with multiple instances results in a memory leak
I run nuclei with multiple instances in a project discover nuclei In use github.com/syndtr/goleveldb/leveldb/memdb/memdb , although close is called But there is still a memory leak
[pprof](http://localhost:9999/ui/)
VIEW
SAMPLE
REFINE
CONFIG
[DOWNLOAD](http://localhost:9999/ui/download)
[zbclient_linux_amd64 inuse_space](http://localhost:9999/ui/source?f=github%5C.com%2Fhktalent%2Fscan4all%2Fprojectdiscovery%2Fnuclei_Yaml%5C.RunNuclei%7Cgithub%5C.com%2Fhktalent%2Fscan4all%2Fprojectdiscovery%2Fnuclei_Yaml%2Fnclruner%2Frunner%5C.New%7Cgithub%5C.com%2Fsyndtr%2Fgoleveldb%2Fleveldb%5C.OpenFile%7Cgithub%5C.com%2Fsyndtr%2Fgoleveldb%2Fleveldb%5C.Open%7Cgithub%5C.com%2Fsyndtr%2Fgoleveldb%2Fleveldb%5C.%5C%28%5C*DB%5C%29%5C.newMem%7Cgithub%5C.com%2Fsyndtr%2Fgoleveldb%2Fleveldb%2Fmemdb%5C.New#)
github.com/syndtr/goleveldb/leveldb/memdb.New
/Users/51pwn/MyWork/zbClient/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go
Total: 80.03MB 80.03MB (flat, cum) 51.24%
467 . . //
468 . . // The returned DB instance is safe for concurrent use.
469 . . func New(cmp comparer.BasicComparer, capacity int) *DB {
470 . . p := &DB{
471 . . cmp: cmp,
472 . . rnd: rand.New(rand.NewSource(0xdeadbeef)),
473 . . maxHeight: 1,
474 80.03MB 80.03MB kvData: make([]byte, 0, capacity),
475 . . nodeData: make([]int, 4+tMaxHeight),
476 . . }
477 . . p.nodeData[nHeight] = tMaxHeight
478 . . return p
479 . . }
github.com/syndtr/goleveldb/leveldb.(*DB).newMem
/Users/51pwn/MyWork/zbClient/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go
Total: 532.26kB 80.55MB (flat, cum) 51.57%
133 . . if db.frozenMem != nil {
134 . . return nil, errHasFrozenMem
135 . .
[profile.pb.gz](https://github.com/projectdiscovery/nuclei/files/9603608/profile.pb.gz)
}
136 . .
137 . . if db.journal == nil {
138 532.26kB 532.26kB db.journal = journal.NewWriter(w)
139 . . } else {
140 . . db.journal.Reset(w)
141 . . db.journalWriter.Close()
142 . . db.frozenJournalFd = db.journalFd
143 . . }
144 . . db.journalWriter = w
145 . . db.journalFd = fd
146 . . db.frozenMem = db.mem
147 . 80.03MB mem = db.mpoolGet(n)
148 . . mem.incref() // for self
149 . . mem.incref() // for caller
150 . . db.mem = mem
151 . . // The seq only incremented by the writer. And whoever called newMem
152 . . // should hold write lock, so no need additional synchronization here.
github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner.New
/Users/51pwn/MyWork/zbClient/vendor/github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner/runner.go
Total: 0 24.55MB (flat, cum) 15.72%
119 . . if err != nil {
120 . . return nil, err
121 . . }
122 . . }
123 . .
124 . . reportingOptions, err := createReportingOptions(options)
125 . . if err != nil {
126 . . return nil, err
127 . . }
128 . . if reportingOptions != nil && httpclient != nil {
129 . . reportingOptions.HttpClient = httpclient
130 . . }
131 . .
132 . . if reportingOptions != nil {
133 . 24.55MB client, err := reporting.New(reportingOptions, options.ReportingDB)
134 . . if err != nil {
135 . . return nil, errors.Wrap(err, "could not create issue reporting client")
136 . . }
137 . . runner.issuesClient = client
138 . . }
github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner.New
/Users/51pwn/MyWork/zbClient/vendor/github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner/runner.go
Total: 0 55.52MB (flat, cum) 35.55%
162 . . if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
163 . . os.Exit(0)
164 . . }
165 . .
166 . . // Initialize the input source
167 . 55.52MB hmapInput, err := hybrid.New(options)
168 . . if err != nil {
169 . . return nil, errors.Wrap(err, "could not create input provider")
170 . . }
171 . . runner.hmapInputProvider = hmapInput
172 . .
github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner.New
/Users/51pwn/MyWork/zbClient/vendor/github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner/runner.go
Total: 0 -512.02kB (flat, cum) 0.32%
183 . . if options.StatsJSON {
184 . . options.EnableProgressBar = true
185 . . }
186 . . // Creates the progress tracking object
187 . . var progressErr error
188 . -512.02kB runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
189 . . if progressErr != nil {
190 . . return nil, progressErr
191 . . }
192 . .
193 . . // create project file if requested or load the existing one
github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner.New
/Users/51pwn/MyWork/zbClient/vendor/github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nclruner/runner/runner.go
Total: 0 -3.52MB (flat, cum) 2.25%
232 . . opts.DebugRequest = runner.options.DebugRequests
233 . . opts.DebugResponse = runner.options.DebugResponse
234 . . if httpclient != nil {
235 . . opts.HTTPClient = httpclient
236 . . }
237 . -3.52MB interactshClient, err := interactsh.New(opts)
238 . . if err != nil {
239 . . gologger.Error().Msgf("Could not create interactsh client: %s", err)
240 . . } else {
241 . . runner.interactsh = interactshClient
242 . . }
github.com/syndtr/goleveldb/leveldb.Open
/Users/51pwn/MyWork/zbClient/vendor/github.com/syndtr/goleveldb/leveldb/db.go
Total: 0 80.57MB (flat, cum) 51.59%
167 . . // due to corruption. Corrupted DB can be recovered with Recover function.
168 . . //
169 . . // The returned DB instance is safe for concurrent use.
170 . . // The DB must be closed after use, by calling Close method.
171 . . func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) {
172 . -512.05kB s, err := newSession(stor, o)
173 . . if err != nil {
174 . . return
175 . . }
176 . . defer func() {
177 . . if err != nil {
178 . . s.close()
179 . . s.release()
180 . . }
181 . . }()
182 . .
183 . . err = s.recover()
184 . . if err != nil {
185 . . if !os.IsNotExist(err) || s.o.GetErrorIfMissing() || s.o.GetReadOnly() {
186 . . return
187 . . }
188 . 532.26kB err = s.create()
189 . . if err != nil {
190 . . return
191 . . }
192 . . } else if s.o.GetErrorIfExist() {
193 . . err = os.ErrExist
194 . . return
195 . . }
196 . .
197 . 80.55MB return openDB(s)
198 . . }
199 . .
200 . . // OpenFile opens or creates a DB for the given path.
201 . . // The DB will be created if not exist, unless ErrorIfMissing is true.
202 . . // Also, if ErrorIfExist is true and the DB exist OpenFile will returns
github.com/syndtr/goleveldb/leveldb.OpenFile
/Users/51pwn/MyWork/zbClient/vendor/github.com/syndtr/goleveldb/leveldb/db.go
Total: 0 80.57MB (flat, cum) 51.59%
210 . . // due to corruption. Corrupted DB can be recovered with Recover function.
211 . . //
212 . . // The returned DB instance is safe for concurrent use.
213 . . // The DB must be closed after use, by calling Close method.
214 . . func OpenFile(path string, o *opt.Options) (db *DB, err error) {
215 . . stor, err := storage.OpenFile(path, o.GetReadOnly())
216 . . if err != nil {
217 . . return
218 . . }
219 . 80.57MB db, err = Open(stor, o)
220 . . if err != nil {
221 . . stor.Close()
222 . . } else {
223 . . db.closer = stor
224 . . }
github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml.RunNuclei
/Users/51pwn/MyWork/zbClient/vendor/github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml/nuclei_yaml.go
Total: 0 76.55MB (flat, cum) 49.01%
56 . . options.Debug = x01.Debug
57 . . options.EnableProgressBar = x01.EnableProgressBar // 开启进度条
58 . . }
59 . . }
60 . . ////////////////////////////////////*/
61 . 516.64kB runner2.ParseOptions(options)
62 . . if nil != oOpts {
63 . . // 指定覆盖
64 . 512.50kB data, err := json.Marshal(oOpts)
65 . . if nil == err && 0 < len(data) {
66 . . err := json.Unmarshal(data, options)
67 . . if nil != err {
68 . . log.Println("oOpts err ", err)
69 . . }
70 . . }
71 . . }
72 . . //data, err := json.Marshal(options)
73 . . //if nil == err {
74 . . // fmt.Printf("%s", string(data))
75 . . //}
76 . 76.05MB nucleiRunner, err := runner2.New(options)
77 . . if err != nil {
78 . . //fmt.Println(options)
79 . . gologger.Fatal().Msgf("nucleiRunner Could not create runner: %s\n", err)
80 . . }
81 . . if nucleiRunner == nil {
82 . . return
83 . . }
84 . . //data, _ := json.Marshal(options)
85 . . //log.Printf("%+v", string(data))
86 . . outNuclei <- nucleiRunner
87 . -512.01kB if err := nucleiRunner.RunEnumeration(); err != nil {
88 . . if options.Validate {
89 . . gologger.Fatal().Msgf("Could not validate templates: %s\n", err)
90 . . } else {
91 . . gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
I observed：
hmapInput, err := hybrid.New(options) runner.hmapInputProvider = hmapInput
func (r *Runner) Close() { ...
r.hmapInputProvider.Close()
hmap.go func (i *Input) Close() { i.hostMap.Close() if i.hostMapStream != nil { i.hostMapStream.Close() } }
There seems to be no problem, I wonder if you can help me out? @ehsandeep @Ice3man543
@XTeam-Wing I submitted a PR a long time ago and never waited for the merge