sql driver segfaults when mixing COPY FROM parquet and regular SQL inserts

[vbmithr]

I don’t have a trivial repro ready, but I have noticed a segfault when alternating COPY FROM and regular inserts from the same SQL conn

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0xf0 pc=0x17a7b4f]

goroutine 56 [running]:
github.com/datafuselabs/databend-go.(*APIClient).getPaginationConfig(...)
	/home/vb/go/pkg/mod/github.com/datafuselabs/[email protected]/client.go:386
github.com/datafuselabs/databend-go.(*APIClient).StartQuery(0x0, {0x301fb78, 0xc001b38b70}, {0xc002148a10?, 0x30636137598b5801?}, {0xc000af76c0?, 0x5d42e5?, 0x47d105?})
	/home/vb/go/pkg/mod/github.com/datafuselabs/[email protected]/client.go:540 +0x6f
github.com/datafuselabs/databend-go.(*APIClient).QuerySync(0x0, {0x301fb78, 0xc001b38b70}, {0xc002148a10?, 0x5fbad7?}, {0xc000af76c0?, 0xc0011f47f8?, 0x28beec2?})
	/home/vb/go/pkg/mod/github.com/datafuselabs/[email protected]/client.go:450 +0x65
github.com/datafuselabs/databend-go.(*DatabendConn).exec(0xc001b3b720, {0x301f6b0?, 0x47b1460?}, {0xc002148a10, 0x70}, {0xc000af76c0, 0x0, 0x0})
	/home/vb/go/pkg/mod/github.com/datafuselabs/[email protected]/connection.go:44 +0x8d
github.com/datafuselabs/databend-go.(*DatabendConn).ExecContext(0x15?, {0x301f6b0?, 0x47b1460?}, {0xc002148a10?, 0x4?}, {0x47b1460?, 0x23be500?, 0xc001c1a560?})
	/home/vb/go/pkg/mod/github.com/datafuselabs/[email protected]/connection.go:178 +0x15c
database/sql.ctxDriverExec({0x301f6b0?, 0x47b1460?}, {0x7f8e9c101130?, 0xc001b3b720?}, {0x0?, 0x0?}, {0xc002148a10?, 0x0?}, {0x47b1460, 0x0, ...})
	/usr/lib/go/src/database/sql/ctxutil.go:31 +0xd7
database/sql.(*DB).execDC.func2()
	/usr/lib/go/src/database/sql/sql.go:1713 +0x15c
database/sql.withLock({0x300de38, 0xc00086e180}, 0xc000af79f8)
	/usr/lib/go/src/database/sql/sql.go:3572 +0x71
database/sql.(*DB).execDC(0x1?, {0x301f6b0, 0x47b1460}, 0xc00086e180, 0xc0?, {0xc002148a10, 0x70}, {0x0, 0x0, 0x0})
	/usr/lib/go/src/database/sql/sql.go:1708 +0x216
database/sql.(*DB).exec(0xc000a4ba00, {0x301f6b0, 0x47b1460}, {0xc002148a10, 0x70}, {0x0, 0x0, 0x0}, 0xb0?)
	/usr/lib/go/src/database/sql/sql.go:1693 +0xcf
database/sql.(*DB).ExecContext.func1(0x70?)
	/usr/lib/go/src/database/sql/sql.go:1672 +0x4f
database/sql.(*DB).retry(0xc000af7be8?, 0xc000af7be8)
	/usr/lib/go/src/database/sql/sql.go:1576 +0x42
database/sql.(*DB).ExecContext(0x2958138?, {0x301f6b0?, 0x47b1460?}, {0xc002148a10?, 0x2?}, {0x0?, 0xc000af7df0?, 0x4?})
	/usr/lib/go/src/database/sql/sql.go:1671 +0xc8

[hantmac]

@vbmithr Thanks for your report and sorry for delay responding . Would you like to give some example code to reproduce this?

[vbmithr]

Ok let met try.

[vbmithr]

Seems related to #142 The repro was that, in order to use prepare (you mention it on the main doc, it is supported or not?, see "Batch Insert" in the main doc page).

So when I use a unique sql connection (*sql.DB) and alternating ExecContext with tx/prepare, I got a segfault. If I create a dedicated connection for my tx/prepare, it works.

But please clarify if tx/prepare works or not, because you mention it in your doc.

[hantmac]

Seems related to #142 The repro was that, in order to use prepare (you mention it on the main doc, it is supported or not?, see "Batch Insert" in the main doc page).

So when I use a unique sql connection (*sql.DB) and alternating ExecContext with tx/prepare, I got a segfault. If I create a dedicated connection for my tx/prepare, it works.

But please clarify if tx/prepare works or not, because you mention it in your doc.

@vbmithr I got it. Now the prepare for batch insert only support Insert into and not support replace into.

[vbmithr]

So, the exact thing that I was that created a segfault when done from the same conn was to alternate

	if _, err := s.sqlconn.ExecContext(ctx, fmt.Sprintf("COPY INTO \"%s\" FROM %s FILE_FORMAT = (TYPE = PARQUET) PURGE = true FORCE = true", s.topic, stage.String())); err != nil {
		return err
	}

and

newsyms := s.getnewsyms()
	if len(newsyms) > 0 {
		for _, s := range newsyms {
			slog.Debug("Found sym", "s", s)
		}
		slog.Debug("Uploading symbols", "len", len(newsyms))
		// Send syms in DB
		ff := func(tx *sql.Tx) (*struct{}, error) {
			batch, err := tx.PrepareContext(ctx, "INSERT INTO xchsym VALUES")
			if err != nil {
				return &struct{}{}, err
			}
			for _, xs := range newsyms {
				if xs.sym != "" {
					if _, err := batch.Exec(uint8(xs.xch), xs.sym); err != nil {
						return &struct{}{}, err
					}
				}
			}
			return &struct{}{}, nil
		}
		_, err := withTX(ctx, s.sqlconn2, ff, nil)
		if err != nil {
			return err
		}
		// Insert newsyms into cursyms and clear newsyms
		for _, xs := range newsyms {
			s.cursyms[*xs] = struct{}{}
		}
		s.symsMutex.Lock()
		clear(s.newsyms)
		s.symsMutex.Unlock()
	}
	return nil
}

func withTX[T any](ctx context.Context, h *sql.DB, f func(tx *sql.Tx) (*T, error), opts *sql.TxOptions) (*T, error) {
	tx, err := h.BeginTx(ctx, opts)
	if err != nil {
		return nil, err
	}
	t, err := f(tx)
	if err != nil {
		return nil, errors.Join(err, tx.Rollback())
	}

	if err := tx.Commit(); err != nil {
		return t, err
	}

	return t, nil
}

I just copied some code verbatim, would need a simple repro to illustrate the problem better.

[hantmac]

hi @vbmithr if you want to batch insert data into databend, you can try the following example code:

package main

import (
	"database/sql"
	"fmt"

	_ "github.com/datafuselabs/databend-go"
)

func main() {
	conn, err := sql.Open("databend", "http://databend:databend@localhost:8009/default?sslmode=disable")
	tx, err := conn.Begin()
	if err != nil {
		fmt.Println(err)
	}
	batch, err := tx.Prepare(fmt.Sprintf("INSERT INTO %s VALUES", "test"))
	for i := 0; i < 10; i++ {
		_, err = batch.Exec(
			"1234",
			"2345",
			"3.1415",
			"test",
			"test2",
			"[4, 5, 6]",
			"[1, 2, 3]",
			"2021-01-01",
			"2021-01-01 00:00:00",
		)
	}
	err = tx.Commit()
}