pgzip icon indicating copy to clipboard operation
pgzip copied to clipboard

goroutine deadlock if Read or WriteTo is called after WriteTo end of stream

Open cyphar opened this issue 4 years ago • 1 comments

I found this when playing around with the reproducer for #38. It seems as though if you do an io.Copy of a stream (which uses z.WriteTo), followed by ReadAll (which uses z.Read) you end up with a goroutine deadlock. https://play.golang.org/p/x6u6JSoKd2t

package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/klauspost/pgzip"
)

// echo hello | gzip -c | xxd -i
var gzipData = []byte{
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xcb, 0x48,
	0xcd, 0xc9, 0xc9, 0xe7, 0x02, 0x00, 0x20, 0x30, 0x3a, 0x36, 0x06, 0x00,
	0x00, 0x00,
}

func main() {
	buf := bytes.NewBuffer(gzipData)

	rdr, err := pgzip.NewReader(buf)
	if err != nil {
		panic(err)
	}

	n, err := io.Copy(ioutil.Discard, rdr)
	fmt.Printf("io.Copy at start of stream: n=%v, err=%v\n", n, err)

	b, err := ioutil.ReadAll(rdr)
	if err != nil {
		panic(err)
	}
	fmt.Printf("read %q from stream\n", string(b))
}
io.Copy at start of stream: n=6, err=<nil>
fatal error: all goroutines are asleep - deadlock!

goroutine 1 [chan send]:
github.com/klauspost/pgzip.(*Reader).Read(0xc00006ea80, 0xc000120000, 0x200, 0x200, 0xc000120000, 0x0, 0x0)
	/tmp/gopath805285542/pkg/mod/github.com/klauspost/[email protected]/gunzip.go:473 +0xfe
bytes.(*Buffer).ReadFrom(0xc000043e80, 0x5055a0, 0xc00006ea80, 0xc000062a90, 0xc00011e000, 0x29)
	/usr/local/go-faketime/src/bytes/buffer.go:204 +0xb1
io/ioutil.readAll(0x5055a0, 0xc00006ea80, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go-faketime/src/io/ioutil/ioutil.go:36 +0xe5
io/ioutil.ReadAll(...)
	/usr/local/go-faketime/src/io/ioutil/ioutil.go:45
main.main()
	/tmp/sandbox128643491/prog.go:30 +0x1fc

Program exited: status 2.

cyphar avatar Feb 19 '21 06:02 cyphar

Turns out this also happens if you use io.Copy twice, though the stack trace is slightly different. https://play.golang.org/p/k-yT6L2pzj1

package main

import (
	"fmt"
	"bytes"
	"io"
	"io/ioutil"
	
	"github.com/klauspost/pgzip"
)

// echo hello | gzip -c | xxd -i
var gzipData = []byte{
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xcb, 0x48,
	0xcd, 0xc9, 0xc9, 0xe7, 0x02, 0x00, 0x20, 0x30, 0x3a, 0x36, 0x06, 0x00,
	0x00, 0x00,
}

func main() {
	buf := bytes.NewBuffer(gzipData)
	
	rdr, err := pgzip.NewReader(buf)
	if err != nil {
		panic(err)
	}
	
	n, err := io.Copy(ioutil.Discard, rdr)
	fmt.Printf("io.Copy at start of stream: n=%v, err=%v\n", n, err)
	
	n, err = io.Copy(ioutil.Discard, rdr)
	fmt.Printf("io.Copy at end of stream: n=%v, err=%v\n", n, err)
}
io.Copy at start of stream: n=6, err=<nil>
fatal error: all goroutines are asleep - deadlock!

goroutine 1 [chan send]:
github.com/klauspost/pgzip.(*Reader).WriteTo(0xc00006ea80, 0x5055e0, 0x5bdde0, 0x7fdd60437598, 0xc00006ea80, 0x1)
	/tmp/gopath370259506/pkg/mod/github.com/klauspost/[email protected]/gunzip.go:547 +0x1c5
io.copyBuffer(0x5055e0, 0x5bdde0, 0x5054c0, 0xc00006ea80, 0x0, 0x0, 0x0, 0x2b, 0x0, 0x0)
	/usr/local/go-faketime/src/io/io.go:391 +0x351
io.Copy(...)
	/usr/local/go-faketime/src/io/io.go:368
main.main()
	/tmp/sandbox071150175/prog.go:30 +0x21e

cyphar avatar Feb 19 '21 07:02 cyphar