FLBPluginExitCtx returns incorrect context with multiinstance

[gufranmirza]

I have modified the out_multiinstance It returns incorrect id in the FLBPluginExitCtx

package main

import (
	"C"
	"fmt"
	"log"
	"time"
	"unsafe"

	"github.com/fluent/fluent-bit-go/output"
)

//export FLBPluginRegister
func FLBPluginRegister(def unsafe.Pointer) int {
	return output.FLBPluginRegister(def, "multiinstance", "Testing multiple instances.")
}

//export FLBPluginInit
func FLBPluginInit(plugin unsafe.Pointer) int {
	id := output.FLBPluginConfigKey(plugin, "id")
	log.Printf("[multiinstance] id = %q", id)
	// Set the context to point to any Go variable
	output.FLBPluginSetContext(plugin, id)

	return output.FLB_OK
}

//export FLBPluginFlush
func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int {
	log.Print("[multiinstance] Flush called for unknown instance")
	return output.FLB_OK
}

//export FLBPluginFlushCtx
func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int {
	// Type assert context back into the original type for the Go variable
	id := output.FLBPluginGetContext(ctx).(string)
	log.Printf("[multiinstance] Flush called for id: %s", id)

	dec := output.NewDecoder(data, int(length))

	count := 0
	for {
		ret, ts, record := output.GetRecord(dec)
		if ret != 0 {
			break
		}

		var timestamp time.Time
		switch t := ts.(type) {
		case output.FLBTime:
			timestamp = ts.(output.FLBTime).Time
		case uint64:
			timestamp = time.Unix(int64(t), 0)
		default:
			fmt.Println("time provided invalid, defaulting to now.")
			timestamp = time.Now()
		}

		// Print record keys and values
		fmt.Printf("[%d] %s: [%s, {", count, C.GoString(tag), timestamp.String())

		for k, v := range record {
			fmt.Printf("\"%s\": %v, ", k, v)
		}
		fmt.Printf("}\n")
		count++
	}

	return output.FLB_OK
}

//export FLBPluginExit
func FLBPluginExit() int {
	log.Print("[multiinstance] Exit called for unknown instance")
	return output.FLB_OK
}

//export FLBPluginExitCtx
func FLBPluginExitCtx(ctx unsafe.Pointer) int {
      
       // It returnes incorrect id, 
	if id, ok := output.FLBPluginGetContext(ctx).(string); ok {
		log.Printf("[out_thing] Exiting for id: %s", id)
	}

	return output.FLB_OK
}

func main() {
}

Logs

2022/01/11 11:19:03 [multiinstance] Flush called for id: dummy_metrics
[0] dummy.local: [2022-01-11 11:18:58.867516 +0000 UTC, {"message": [100 117 109 109 121], }
[1] dummy.local: [2022-01-11 11:18:59.8657335 +0000 UTC, {"message": [100 117 109 109 121], }
[2] dummy.local: [2022-01-11 11:19:00.8668099 +0000 UTC, {"message": [100 117 109 109 121], }
[3] dummy.local: [2022-01-11 11:19:01.8334735 +0000 UTC, {"message": [100 117 109 109 121], }
[4] dummy.local: [2022-01-11 11:19:02.8304936 +0000 UTC, {"message": [100 117 109 109 121], }
^C[engine] caught signal (SIGINT)
[2022/01/11 11:19:06] [ info] [input] pausing cpu.0
2022/01/11 11:19:06 [out_thing] Exiting for id: dummy_metrics
2022/01/11 11:19:06 [out_thing] Exiting for id: dummy_metrics

[corydodt]

Confirmed this; ran into it yesterday.

I need to use FLBPluginExitCtx to call .Close() on a bunch of connections to GCS writers (bucket object outputs). Because of this bug there's no way to tell them apart on exit, so I'm forced to use globals and FLBPluginExit() with no context.

[l2dy]

Please report this to https://github.com/fluent/fluent-bit. It is unlikely to be a problem on the Go side.

[corydodt]

I would but I don't know how to construct a test case for this that would make sense to the fluent-bit team. I don't even know where the implementation code that calls this callback lives?

[l2dy]

Search for cb_exit_ctx in Fluent Bit source code, it was refactored in https://github.com/fluent/fluent-bit/commit/83686a59438d3479ed3874e098496bdd6a0c6ce1.

[jan-zajic]

Hi @l2dy , @corydodt and others. I created PR # 6469 in fluent-bit repository that fix these problems. Tested with our plugins code in my company..

[cosmo0920]

#56 is merged. Closing.

[cosmo0920]

For actual fix, please stay tuned for https://github.com/fluent/fluent-bit/pull/6469.