(*WriteBatchIterator).decodeVarint unfortunately doesn’t correctly detect invalid varint sequences and can consume the entire buffer

[odeke-em]

Courtesy of the Cosmos Network Security, I discovered this code while auditing supply chain dependencies. If we examine this code in https://github.com/tecbot/gorocksdb/blob/37ba422d5c751e427bef06cd17fb6d3e69818240/write_batch.go#L265-L283

and then extract it and run the following https://go.dev/play/p/MXpanwJKofY or inlined below:

package main

import (
	"errors"
	"fmt"
	"io"
)

type wbi struct {
	data []byte
	err  error
}

func (iter *wbi) decodeVarint() uint64 {
	var n int
	var x uint64
	for shift := uint(0); shift < 64 && n < len(iter.data); shift += 7 {
		b := uint64(iter.data[n])
		n++
		x |= (b & 0x7F) << shift
		if (b & 0x80) == 0 {
			iter.data = iter.data[n:]
			return x
		}
	}
	if n == len(iter.data) {
		iter.err = io.ErrShortBuffer
	} else {
		iter.err = errors.New("malformed varint")
	}
	return 0
}

func main() {
	// The 10th byte here is invalid and should be reported.
	// Please see https://github.com/golang/go/issues/41185
	b := []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}
	wb := &wbi{data: b}
	v := wb.decodeVarint()
	println(v)
	fmt.Printf("%v\n", wb.err) // This should have reported an error.
}

which prints

18446744073709551575
<nil>

Fix

The fix entails just using the Go standard library's code in which I fixed this bug in March 2021 in the Go standard library per https://github.com/golang/go/commit/aafad20b617ee63d58fcd4f6e0d98fe27760678c

with a demo at https://go.dev/play/p/kYGBjzh24Qx which should properly print malformed varint

package main

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
)

type wbi struct {
	data []byte
	err  error
}

func (iter *wbi) decodeVarint() uint64 {
	var n int
	var x uint64
	for shift := uint(0); shift < 64 && n < len(iter.data); shift += 7 {
		b := uint64(iter.data[n])
		n++
		x |= (b & 0x7F) << shift
		if (b & 0x80) == 0 {
			iter.data = iter.data[n:]
			return x
		}
	}
	if n == len(iter.data) {
		iter.err = io.ErrShortBuffer
	} else {
		iter.err = errors.New("malformed varint")
	}
	return 0
}

func (iter *wbi) decodeVarint_good() (x uint64) {
	v, n := binary.Varint(iter.data)
	if n == len(iter.data) {
		iter.err = io.ErrShortBuffer
	} else if n < 0 {
		iter.err = errors.New("malformed varint")
	} else {
		x = uint64(v)
	}
	return x
}

func main() {
	// The 10th byte here is invalid and should be reported.
	// Please see https://github.com/golang/go/issues/41185
	b := []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}
	wb := &wbi{data: b}
	v := wb.decodeVarint_good()
	println(v)
	fmt.Printf("%v\n", wb.err) // This should have reported an error.
}

which correctly prints

0
malformed varint

[anilcse]

Not sure if the repo is still active, last commit was in 2019