Panic during test run

[tomponline]

lxc info cluster:c2 --project foo
+ set +x
+ grep -q Location: node2
+ timeout --foreground 120 /home/runner/go/bin/lxc info cluster:c2 --project foo --verbose
+ lxc delete -f c1 c2 --project foo
+ set +x
+ timeout --foreground 120 /home/runner/go/bin/lxc delete -f c1 c2 --project foo --verbose
                                           
time="2023-11-21T13:41:18Z" level=info msg="Deleting instance" created="2023-11-21 13:41:17.63546786 +0000 UTC" ephemeral=false instance=c1 instanceType=container project=foo used="1970-01-01 00:00:00 +0000 UTC"
time="2023-11-21T13:41:18Z" level=info msg="Deleted instance" created="2023-11-21 13:41:17.63546786 +0000 UTC" ephemeral=false instance=c1 instanceType=container project=foo used="1970-01-01 00:00:00 +0000 UTC"
time="2023-11-21T13:41:18Z" level=info msg="Deleting instance" created="2023-11-21 13:41:18.008007205 +0000 UTC" ephemeral=false instance=c2 instanceType=container project=foo used="1970-01-01 00:00:00 +0000 UTC"
time="2023-11-21T13:41:18Z" level=info msg="Deleted instance" created="2023-11-21 13:41:18.008007205 +0000 UTC" ephemeral=false instance=c2 instanceType=container project=foo used="1970-01-01 00:00:00 +0000 UTC"
+ lxc project delete foo
+ set +x
+ timeout --foreground 120 /home/runner/go/bin/lxc project delete foo --verbose
Project foo deleted
+ LXD_DIR=/home/runner/work/lxd/lxd/test/tmp.9hT/6m2 lxd shutdown
time="2023-11-21T13:41:18Z" level=info msg="Asked to shutdown by API" force=false
time="2023-11-21T13:41:18Z" level=info msg="Starting shutdown sequence" signal="power failure"
fatal error: sync: unlock of unlocked mutex

goroutine 749 [running]:
sync.fatal({0x1e6c9b3?, 0xc000b99f20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/panic.go:1061 +0x18
sync.(*Mutex).unlockSlow(0xc00049c310, 0xffffffff)
	/opt/hostedtoolcache/go/1.21.4/x64/src/sync/mutex.go:229 +0x35
sync.(*Mutex).Unlock(0x100000000?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/sync/mutex.go:223 +0x25
github.com/canonical/lxd/lxd/task.(*Group).Start.func1(0xc000af7b80?)
	/home/runner/work/lxd/lxd/lxd/task/group.go:71 +0xe3
created by github.com/canonical/lxd/lxd/task.(*Group).Start in goroutine 288
	/home/runner/work/lxd/lxd/lxd/task/group.go:60 +0x2e5

goroutine 1 [select]:
main.(*cmdDaemon).Run(0xc000013d28, 0x0?, {0xc00048dd10, 0x0, 0x0?})
	/home/runner/work/lxd/lxd/lxd/main_daemon.go:83 +0x605
github.com/spf13/cobra.(*Command).execute(0xc000005200, {0xc000040090, 0x3, 0x3})
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:983 +0xabc
github.com/spf13/cobra.(*Command).ExecuteC(0xc000005200)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1115 +0x3ff
github.com/spf13/cobra.(*Command).Execute(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1039
main.main()
	/home/runner/work/lxd/lxd/lxd/main.go:217 +0x1e05

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/asm_amd64.s:1650 +0x1

goroutine 18 [syscall]:
os/signal.signal_recv()
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/sigqueue.go:152 +0x29
os/signal.loop()
	/opt/hostedtoolcache/go/1.21.4/x64/src/os/signal/signal_unix.go:23 +0x13
created by os/signal.Notify.func1.1 in goroutine 1
	/opt/hostedtoolcache/go/1.21.4/x64/src/os/signal/signal.go:151 +0x1f

goroutine 34 [select]:
database/sql.(*DB).connectionOpener(0xc000561860, {0x21e48b8, 0xc00051c410})
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:1218 +0x87
created by database/sql.OpenDB in goroutine 1
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:791 +0x165

goroutine 23 [select]:
database/sql.(*DB).connectionOpener(0xc000520680, {0x21e48b8, 0xc000385900})
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:1218 +0x87
created by database/sql.OpenDB in goroutine 1
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:791 +0x165

goroutine 22 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8edb8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000297080?, 0xc0004bfd00?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000297080)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000297080)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_unix.go:172 +0x29
net.(*UnixListener).accept(0x4ab946?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock_posix.go:172 +0x16
net.(*UnixListener).Accept(0xc000513380)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock.go:260 +0x30
net/http.(*Server).Serve(0xc00024e1e0, {0x21e0848, 0xc000513380})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3056 +0x364
github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:433 +0x1f
gopkg.in/tomb%2ev2.(*Tomb).run(0xc0003858b0, 0xc00051c5a0?)
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 1
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5

goroutine 21 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8eeb0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000296e80?, 0xc002945200?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000296e80)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000296e80)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_unix.go:172 +0x29
net.(*UnixListener).accept(0x1dde0a0?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock_posix.go:172 +0x16
net.(*UnixListener).Accept(0xc0005132f0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock.go:260 +0x30
github.com/canonical/lxd/lxd/endpoints/listeners.(*StarttlsListener).Accept(0xc000513320)
	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/starttls.go:36 +0x50
net/http.(*Server).Serve(0xc00024e000, {0x21e10e8, 0xc000513320})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3056 +0x364
github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:433 +0x1f
gopkg.in/tomb%2ev2.(*Tomb).run(0xc0003858b0, 0xc000548700?)
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 1
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5

goroutine 727 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e218, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000aa2280?, 0xc000b48000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000aa2280, {0xc000b48000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc000aa2280, {0xc000b48000?, 0x7ff020bafea8?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0001542b0, {0xc000b48000?, 0xc0004baea0?, 0xc0004baeb0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
io.copyBuffer({0x21d26a0, 0xc0000cd880}, {0x21d1980, 0xc0001542b0}, {0x0, 0x0, 0x0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:430 +0x1a6
io.Copy(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:389
github.com/canonical/lxd/lxd/cluster.dqliteProxy.func2()
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1169 +0x7b
created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 685
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1168 +0x61e

goroutine 154 [syscall]:
syscall.Syscall(0x10?, 0x1a844e0?, 0x41f001?, 0x41ebba?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/syscall/syscall_linux.go:69 +0x25
golang.org/x/sys/unix.read(0x1b32920?, {0xc0004d5000?, 0x1e25ab3?, 0x9?})
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_linux.go:1376 +0x45
golang.org/x/sys/unix.Read(...)
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/syscall_unix.go:158
main.deviceNetlinkListener.func1(0xc000427440?, 0xc00011d920?, 0xc00011d980?, 0xc00011d9e0?)
	/home/runner/work/lxd/lxd/lxd/devices.go:100 +0x98
created by main.deviceNetlinkListener in goroutine 191
	/home/runner/work/lxd/lxd/lxd/devices.go:97 +0x1b3

goroutine 197 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e9d8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000297c00?, 0x10?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000297c00)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000297c00)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_unix.go:172 +0x29
net.(*UnixListener).accept(0x0?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock_posix.go:172 +0x16
net.(*UnixListener).Accept(0xc000557470)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/unixsock.go:260 +0x30
github.com/canonical/lxd/lxd/seccomp.NewSeccompServer.func1()
	/home/runner/work/lxd/lxd/lxd/seccomp/seccomp.go:1072 +0x59
created by github.com/canonical/lxd/lxd/seccomp.NewSeccompServer in goroutine 1
	/home/runner/work/lxd/lxd/lxd/seccomp/seccomp.go:1070 +0x1f5

goroutine 193 [syscall]:
syscall.Syscall(0x0?, 0x0?, 0x0?, 0x1000000000000?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/syscall/syscall_linux.go:69 +0x25
golang.org/x/sys/unix.read(0x0?, {0xc00010c600?, 0xc000609448?, 0x44e7b1?})
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_linux.go:1376 +0x45
golang.org/x/sys/unix.Read(...)
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/syscall_unix.go:158
github.com/canonical/lxd/lxd/fsmonitor/drivers.(*fanotify).getEvents(0xc00049f680, {0x21e48b8, 0xc000385810}, 0x2efdfd0?)
	/home/runner/work/lxd/lxd/lxd/fsmonitor/drivers/driver_fanotify.go:86 +0x78
created by github.com/canonical/lxd/lxd/fsmonitor/drivers.(*fanotify).load in goroutine 1
	/home/runner/work/lxd/lxd/lxd/fsmonitor/drivers/driver_fanotify.go:71 +0x306

goroutine 191 [select]:
main.deviceEventListener(0xc000562050)
	/home/runner/work/lxd/lxd/lxd/devices.go:669 +0x20f
created by main.(*Daemon).init in goroutine 1
	/home/runner/work/lxd/lxd/lxd/daemon.go:1463 +0x4a39

goroutine 724 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e5f8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0001b9700?, 0xc000b0e000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0001b9700, {0xc000b0e000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0001b9700, {0xc000b0e000?, 0x7ff020baf428?, 0x7ff06b17a108?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00006bd78, {0xc000b0e000?, 0x1a84820?, 0xc000a5e301?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
io.copyBuffer({0x21d26a0, 0xc000a5e380}, {0x21d1980, 0xc00006bd78}, {0x0, 0x0, 0x0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:430 +0x1a6
io.Copy(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:389
github.com/canonical/lxd/lxd/cluster.dqliteProxy.func2()
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1169 +0x7b
created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 673
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1168 +0x61e

goroutine 3324 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e8e0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc002893380?, 0xc0020d6f00?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002893380, {0xc0020d6f00, 0x500, 0x500})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc002893380, {0xc0020d6f00?, 0xc0020d6f05?, 0x20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000155e18, {0xc0020d6f00?, 0xc0007d8680?, 0xc0021ef338?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc002721ea8, {0xc0020d6f00?, 0xc002721ea8?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0021ef428, {0x21d4080, 0xc002721ea8})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0021ef180, {0x21d29e0?, 0xc000155e18}, 0x500?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc0021ef180, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc0021ef180, {0xc0027bc000, 0x1000, 0x4104a5?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc0027acde0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc0027acde0, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc002443e40, 0xc0027bc006?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc002443e40)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc002443e40)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
	/home/runner/work/lxd/lxd/lxd/events/connections.go:99 +0x5b
created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 3319
	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1f9

goroutine 723 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8ebc8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0001b9500?, 0xc000ad6000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0001b9500, {0xc000ad6000, 0xa000, 0xa000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0001b9500, {0xc000ad6000?, 0xc000ad6005?, 0x3101?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00006bd58, {0xc000ad6000?, 0x0?, 0xc000a5e538?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc001f2d440, {0xc000ad6000?, 0xc001f2d440?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc000a5e628, {0x21d4080, 0xc001f2d440})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000a5e380, {0x21d29e0?, 0xc00006bd58}, 0xa000?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000a5e380, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000a5e380, {0xc000b80000, 0x8000, 0xc000953c20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
io.copyBuffer({0x21d19a0, 0xc00006bd78}, {0x21d26c0, 0xc000a5e380}, {0x0, 0x0, 0x0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:430 +0x1a6
io.Copy(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:389
github.com/canonical/lxd/lxd/cluster.dqliteProxy.func1()
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1164 +0x7b
created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 673
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1163 +0x588

goroutine 685 [select]:
github.com/canonical/lxd/lxd/cluster.dqliteProxy({0x1e14ce4, 0x4}, 0xc0005a8420, {0x21efbb8, 0xc0000cd880}, {0x21efb60, 0xc0001542b0?})
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1175 +0x6c8
created by github.com/canonical/lxd/lxd/cluster.(*Gateway).init.(*Gateway).raftDial.func3 in goroutine 17
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:427 +0x2c5

goroutine 3319 [select]:
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc0028c6b40, {0x21e48b8?, 0xc0028e60a0?}, 0x0)
	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x3a5
github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc0005426e0)
	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 3315
	/home/runner/work/lxd/lxd/lxd/events/events.go:105 +0x510

goroutine 673 [select]:
github.com/canonical/lxd/lxd/cluster.dqliteProxy({0x1e18d26, 0x6}, 0xc0005a8420, {0x21efbb8, 0xc000a5e380}, {0x21efb60, 0xc00006bd78?})
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1175 +0x6c8
created by github.com/canonical/lxd/lxd/cluster.runDqliteProxy in goroutine 720
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1137 +0x125

goroutine 726 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e6f0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000296f80?, 0xc0009aeb00?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000296f80, {0xc0009aeb00, 0x580, 0x580})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc000296f80, {0xc0009aeb00?, 0xc000151410?, 0xc0004b9a18?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0005a64d8, {0xc0009aeb00?, 0x0?, 0x3fb6?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc0008de870, {0xc0009aeb00?, 0xc0008de870?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0000cdb28, {0x21d4080, 0xc0008de870})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0000cd880, {0x21d29e0?, 0xc0005a64d8}, 0xe3?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc0000cd880, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc0000cd880, {0xc000b5c000, 0x8000, 0x1029160?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
io.copyBuffer({0x21d19a0, 0xc0001542b0}, {0x21d26c0, 0xc0000cd880}, {0x0, 0x0, 0x0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:430 +0x1a6
io.Copy(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/io/io.go:389
github.com/canonical/lxd/lxd/cluster.dqliteProxy.func1()
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1164 +0x7b
created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 685
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1163 +0x588

goroutine 1901 [select]:
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb8
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1895
	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x43b

goroutine 6689 [runnable]:
github.com/mattn/go-sqlite3._Cfunc_free(0x7ff018020690)
	_cgo_gotypes.go:396 +0x3f
github.com/mattn/go-sqlite3.(*SQLiteConn).prepare.(*SQLiteConn).prepare.func1.func3()
	/home/runner/go/pkg/mod/github.com/mattn/[email protected]/sqlite3.go:1800 +0x35
github.com/mattn/go-sqlite3.(*SQLiteConn).prepare(0xc00053d020, {0x0?, 0x0?}, {0x1e19458?, 0xc0006b8f80?})
	/home/runner/go/pkg/mod/github.com/mattn/[email protected]/sqlite3.go:1813 +0x1ba
github.com/mattn/go-sqlite3.(*SQLiteConn).exec(0x1a8ab60?, {0x21e47a0, 0x355d900}, {0x1e19458?, 0x1b34660?}, {0x0, 0x0, 0x0})
	/home/runner/go/pkg/mod/github.com/mattn/[email protected]/sqlite3.go:853 +0x8d
github.com/mattn/go-sqlite3.(*SQLiteTx).Commit(0xc001fd8258)
	/home/runner/go/pkg/mod/github.com/mattn/[email protected]/sqlite3.go:496 +0x45
database/sql.(*Tx).Commit.func1()
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:2275 +0x29
database/sql.withLock({0x21dc918, 0xc0005321b0}, 0xc0006b9170)
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:3502 +0x82
database/sql.(*Tx).Commit(0xc002b15900)
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:2274 +0xfd
github.com/canonical/lxd/lxd/db/query.Transaction({0x21e4768?, 0x355d900?}, 0x7ff02034ca20?, 0xc0006b9270)
	/home/runner/work/lxd/lxd/lxd/db/query/transaction.go:34 +0x1b1
github.com/canonical/lxd/lxd/db.(*Node).Transaction(0xc000511278, {0x21e4768, 0x355d900}, 0xc0006b92c0)
	/home/runner/work/lxd/lxd/lxd/db/db.go:102 +0x74
github.com/canonical/lxd/lxd/cluster.Enabled(0x21d1c60?)
	/home/runner/work/lxd/lxd/lxd/cluster/membership.go:1087 +0x47
main.handoverMemberRole(0xc002562b60, 0x1e5d83c?)
	/home/runner/work/lxd/lxd/lxd/api_cluster.go:2568 +0x39
main.(*Daemon).Stop(0xc00049c240, {0x21e48b8, 0xc002176d70}, {0x21db768?, 0x21cd148?})
	/home/runner/work/lxd/lxd/lxd/daemon.go:1666 +0x1e6
main.internalShutdown.func1({0x21e0ae8, 0xc0025628c0})
	/home/runner/work/lxd/lxd/lxd/api_internal.go:256 +0x99
github.com/canonical/lxd/lxd/response.(*manualResponse).Render(0x1f7b250?, {0x21e0ae8?, 0xc0025628c0?})
	/home/runner/work/lxd/lxd/lxd/response/response.go:546 +0x26
main.(*Daemon).createCmd.func1({0x21e0ae8, 0xc0025628c0}, 0xc0001d3000)
	/home/runner/work/lxd/lxd/lxd/daemon.go:628 +0x15d1
net/http.HandlerFunc.ServeHTTP(0xc0001d2f00?, {0x21e0ae8?, 0xc0025628c0?}, 0x0?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000536b40, {0x21e0ae8, 0xc0025628c0}, 0xc0001d2d00)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1c5
main.(*lxdHttpServer).ServeHTTP(0xc000051cb0, {0x21e0ae8, 0xc0025628c0}, 0xc0001d2d00)
	/home/runner/work/lxd/lxd/lxd/api.go:379 +0xd0
net/http.serverHandler.ServeHTTP({0xc002da74d0?}, {0x21e0ae8?, 0xc0025628c0?}, 0x6?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc000da5cb0, {0x21e4880, 0xc002e21110})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 21
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3086 +0x5cb

goroutine 376 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8ead0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000788800?, 0xe?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000788800)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000788800)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc0007ac440)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/tcpsock_posix.go:152 +0x1e
net.(*TCPListener).Accept(0xc0007ac440)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/tcpsock.go:315 +0x30
github.com/canonical/lxd/lxd/endpoints/listeners.(*FancyTLSListener).Accept(0xc00055ce60)
	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/fancytls.go:37 +0x4d
net/http.(*Server).Serve(0xc00024e000, {0x21e1088, 0xc00055ce60})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3056 +0x364
github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:433 +0x1f
gopkg.in/tomb%2ev2.(*Tomb).run(0xc0003858b0, 0x0?)
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 366
	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5

goroutine 6810 [runnable]:
database/sql.(*Rows).initContextClose.func1()
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:2949
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/asm_amd64.s:1650 +0x1
created by database/sql.(*Rows).initContextClose in goroutine 6689
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:2949 +0x185

goroutine 6762 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e120, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc002945200?, 0xc000539000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002945200, {0xc000539000, 0x1000, 0x1000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc002945200, {0xc000539000?, 0xc0006d6f10?, 0x10290f3?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00052c260, {0xc000539000?, 0x0?, 0x1029160?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
bufio.(*Reader).Read(0xc0007f95c0, {0xc002da74e1, 0x1, 0x102c202?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:244 +0x197
github.com/canonical/lxd/lxd/endpoints/listeners.BufferedUnixConn.Read(...)
	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/starttls.go:94
net/http.(*connReader).backgroundRead(0xc002da74d0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:683 +0x37
created by net/http.(*connReader).startBackgroundRead in goroutine 6689
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:679 +0xba

goroutine 720 [chan receive]:
github.com/canonical/lxd/lxd/cluster.runDqliteProxy(0xc0005a8420, {0xc00062dd56, 0x6}, 0xc0004ce140?)
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1131 +0x45
created by github.com/canonical/lxd/lxd/cluster.(*Gateway).init in goroutine 288
	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:800 +0x5af

goroutine 1698 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e7e8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00216d680?, 0xc0006fd000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00216d680, {0xc0006fd000, 0x3800, 0x3800})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc00216d680, {0xc0006fd000?, 0x0?, 0xc002c3a9b8?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000154e28, {0xc0006fd000?, 0xc0006ad728?, 0x46024b?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc002144738, {0xc0006fd000?, 0xc002144738?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0021157a8, {0x21d4080, 0xc002144738})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc002115500, {0x21d29e0?, 0xc000154e28}, 0x35b4?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc002115500, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc002115500, {0xc0021be000, 0x1000, 0xc0006adc88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc0021b93e0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc0021b93e0, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc0020954a0, 0x0?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0020954a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc0020954a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadMessage(0x1b39d20?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x5c
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1705
	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4b9

goroutine 1610 [select]:
github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc0007fdae0, {0x21e4880?, 0xc0020a1860?})
	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
main.eventsSocket(0xc0020d10a0, 0xc0020dc100, {0x21e0ae8, 0xc0020d0ee0})
	/home/runner/work/lxd/lxd/lxd/events.go:165 +0xd2f
main.(*eventsServe).Render(0x1f7af78?, {0x21e0ae8?, 0xc0020d0ee0?})
	/home/runner/work/lxd/lxd/lxd/events.go:37 +0x30
main.(*Daemon).createCmd.func1({0x21e0ae8, 0xc0020d0ee0}, 0xc0007abf00)
	/home/runner/work/lxd/lxd/lxd/daemon.go:628 +0x15d1
net/http.HandlerFunc.ServeHTTP(0xc0007abe00?, {0x21e0ae8?, 0xc0020d0ee0?}, 0x1e13993?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000536b40, {0x21e0ae8, 0xc0020d0ee0}, 0xc0007abd00)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1c5
main.(*lxdHttpServer).ServeHTTP(0xc000051cb0, {0x21e0ae8, 0xc0020d0ee0}, 0xc0007abd00)
	/home/runner/work/lxd/lxd/lxd/api.go:379 +0xd0
net/http.serverHandler.ServeHTTP({0xc0020a1140?}, {0x21e0ae8?, 0xc0020d0ee0?}, 0x6?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc001ff7320, {0x21e4880, 0xc001ffaea0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 376
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3086 +0x5cb

goroutine 1697 [select]:
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb8
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1705
	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x43b

goroutine 976 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e500, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0003fda80?, 0xc00075b400?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003fda80, {0xc00075b400, 0x500, 0x500})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0003fda80, {0xc00075b400?, 0xc00075b405?, 0x20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00052c630, {0xc00075b400?, 0x9?, 0xc000728fb8?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc0008dfcf8, {0xc00075b400?, 0xc0008dfcf8?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0007290a8, {0x21d4080, 0xc0008dfcf8})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000728e00, {0x21d29e0?, 0xc00052c630}, 0x500?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000728e00, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000728e00, {0xc001f9e000, 0x1000, 0x1d08100?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc001f9c540)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc001f9c540, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc001fa6160, 0xc001f9e006?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc001fa6160)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc001fa6160)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadJSON(0x0?, {0x1b18340, 0xc000bcc6c0})
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/json.go:50 +0x25
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
	/home/runner/work/lxd/lxd/lxd/events/connections.go:88 +0xc9
created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 972
	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1f9

goroutine 906 [select]:
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb8
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 900
	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x43b

goroutine 915 [select]:
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb8
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 899
	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x43b

goroutine 916 [IO wait]:
internal/poll.runtime_pollWait(0x7ff0200ab3e8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000297480?, 0xc002500000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000297480, {0xc002500000, 0x4000, 0x4000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc000297480, {0xc002500000?, 0x0?, 0xc002c3bf18?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00006bdf0, {0xc002500000?, 0xc0006a8728?, 0x46024b?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc002144f00, {0xc002500000?, 0xc002144f00?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc000a5ed28, {0x21d4080, 0xc002144f00})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000a5ea80, {0x21d29e0?, 0xc00006bdf0}, 0x3ed4?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000a5ea80, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000a5ea80, {0xc001f12000, 0x1000, 0xc0006a8c88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc001f10840)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc001f10840, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc0008e6580, 0x0?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0008e6580)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc0008e6580)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadMessage(0x1b39d20?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x5c
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 899
	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4b9

goroutine 930 [select]:
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc000aff1a0, {0x21e48b8?, 0xc000959ae0?}, 0xc001ebd780)
	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x3a5
github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc000382e60)
	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 952
	/home/runner/work/lxd/lxd/lxd/events/events.go:105 +0x510

goroutine 1638 [select]:
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc0022042a0, {0x21e48b8?, 0xc001f7e6e0?}, 0xc001f73ab0)
	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x3a5
github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc0007fdae0)
	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 1610
	/home/runner/work/lxd/lxd/lxd/events/events.go:105 +0x510

goroutine 1642 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e028, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc001f99a00?, 0xc0020d6000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f99a00, {0xc0020d6000, 0x500, 0x500})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc001f99a00, {0xc0020d6000?, 0xc0020d6005?, 0x20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc001fd81f0, {0xc0020d6000?, 0xc001f3b1a0?, 0xc000a5f338?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc002204360, {0xc0020d6000?, 0xc002204360?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc000a5f428, {0x21d4080, 0xc002204360})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000a5f180, {0x21d29e0?, 0xc001fd81f0}, 0x500?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000a5f180, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000a5f180, {0xc00207a000, 0x1000, 0x1d08100?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc002073680)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc002073680, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc0020949a0, 0xc00207a006?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0020949a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc0020949a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadJSON(0x0?, {0x1b18340, 0xc0022082a0})
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/json.go:50 +0x25
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
	/home/runner/work/lxd/lxd/lxd/events/connections.go:88 +0xc9
created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 1638
	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1f9

goroutine 981 [IO wait]:
internal/poll.runtime_pollWait(0x7ff021d8e408, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0003fda00?, 0xc00075af00?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003fda00, {0xc00075af00, 0x500, 0x500})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0003fda00, {0xc00075af00?, 0xc00075af05?, 0x20?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00052c628, {0xc00075af00?, 0x9?, 0xc000728c38?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc0008dfc38, {0xc00075af00?, 0xc0008dfc38?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc000728d28, {0x21d4080, 0xc0008dfc38})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000728a80, {0x21d29e0?, 0xc00052c628}, 0x500?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000728a80, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000728a80, {0xc000bfc000, 0x1000, 0x1d08100?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc0007f8ae0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc0007f8ae0, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc000a1bb80, 0xc000bfc006?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc000a1bb80)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc000a1bb80)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadJSON(0xc0006c60c0?, {0x1b18340, 0xc001f9c8a0})
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/json.go:50 +0x25
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
	/home/runner/work/lxd/lxd/lxd/events/connections.go:88 +0xc9
created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 930
	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1f9

goroutine 972 [select]:
github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc0008dfc68, {0x21e48b8?, 0xc000b0aff0?}, 0xc001f9a680)
	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x3a5
github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc0003ad5e0)
	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 953
	/home/runner/work/lxd/lxd/lxd/events/events.go:105 +0x510

goroutine 907 [IO wait]:
internal/poll.runtime_pollWait(0x7ff0200ab4e0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0003fd580?, 0xc002314000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003fd580, {0xc002314000, 0x3800, 0x3800})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0003fd580, {0xc002314000?, 0x0?, 0xc002c3ac88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00052c5a0, {0xc002314000?, 0xc000b65728?, 0x46024b?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc002144750, {0xc002314000?, 0xc002144750?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0007289a8, {0x21d4080, 0xc002144750})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000728700, {0x21d29e0?, 0xc00052c5a0}, 0x3285?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000728700, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000728700, {0xc000b59000, 0x1000, 0xc000b65c88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc000b73f80)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc000b73f80, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc000a1b4a0, 0x0?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc000a1b4a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc000a1b4a0)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadMessage(0x1b39d20?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x5c
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 900
	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4b9

goroutine 3315 [select]:
github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc0005426e0, {0x21e4880?, 0xc0027d14a0?})
	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
main.eventsSocket(0xc00249fce0, 0xc00279b600, {0x21e0ae8, 0xc00249fb20})
	/home/runner/work/lxd/lxd/lxd/events.go:165 +0xd2f
main.(*eventsServe).Render(0x1f7af78?, {0x21e0ae8?, 0xc00249fb20?})
	/home/runner/work/lxd/lxd/lxd/events.go:37 +0x30
main.(*Daemon).createCmd.func1({0x21e0ae8, 0xc00249fb20}, 0xc00279b400)
	/home/runner/work/lxd/lxd/lxd/daemon.go:628 +0x15d1
net/http.HandlerFunc.ServeHTTP(0xc00279b300?, {0x21e0ae8?, 0xc00249fb20?}, 0x1e13993?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000536b40, {0x21e0ae8, 0xc00249fb20}, 0xc00279b200)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1c5
main.(*lxdHttpServer).ServeHTTP(0xc000051cb0, {0x21e0ae8, 0xc00249fb20}, 0xc00279b200)
	/home/runner/work/lxd/lxd/lxd/api.go:379 +0xd0
net/http.serverHandler.ServeHTTP({0xc0027d0d80?}, {0x21e0ae8?, 0xc00249fb20?}, 0x6?)
Error: Put "http://unix.socket/internal/shutdown?force=false": EOF
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc0028d1d40, {0x21e4880, 0xc0028de930})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 376
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3086 +0x5cb

goroutine 952 [select]:
github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc000382e60, {0x21e4880?, 0xc001f8cd20?})
	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
main.eventsSocket(0xc0006c56c0, 0xc000a1f000, {0x21e0ae8, 0xc0006c5500})
	/home/runner/work/lxd/lxd/lxd/events.go:165 +0xd2f
main.(*eventsServe).Render(0x1f7af78?, {0x21e0ae8?, 0xc0006c5500?})
	/home/runner/work/lxd/lxd/lxd/events.go:37 +0x30
main.(*Daemon).createCmd.func1({0x21e0ae8, 0xc0006c5500}, 0xc000a1ee00)
	/home/runner/work/lxd/lxd/lxd/daemon.go:628 +0x15d1
net/http.HandlerFunc.ServeHTTP(0xc000a1ed00?, {0x21e0ae8?, 0xc0006c5500?}, 0x1e13993?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000536b40, {0x21e0ae8, 0xc0006c5500}, 0xc000a1ec00)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1c5
main.(*lxdHttpServer).ServeHTTP(0xc000051cb0, {0x21e0ae8, 0xc0006c5500}, 0xc000a1ec00)
	/home/runner/work/lxd/lxd/lxd/api.go:379 +0xd0
net/http.serverHandler.ServeHTTP({0xc001f8c630?}, {0x21e0ae8?, 0xc0006c5500?}, 0x6?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc0008dd170, {0x21e4880, 0xc0008dbec0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 376
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3086 +0x5cb

goroutine 953 [select]:
github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc0003ad5e0, {0x21e4880?, 0xc001f8dcb0?})
	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
main.eventsSocket(0xc0006c5a40, 0xc000a1f600, {0x21e0ae8, 0xc0006c5880})
	/home/runner/work/lxd/lxd/lxd/events.go:165 +0xd2f
main.(*eventsServe).Render(0x1f7af78?, {0x21e0ae8?, 0xc0006c5880?})
	/home/runner/work/lxd/lxd/lxd/events.go:37 +0x30
main.(*Daemon).createCmd.func1({0x21e0ae8, 0xc0006c5880}, 0xc000a1f400)
	/home/runner/work/lxd/lxd/lxd/daemon.go:628 +0x15d1
net/http.HandlerFunc.ServeHTTP(0xc000a1f300?, {0x21e0ae8?, 0xc0006c5880?}, 0x1e13993?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000536b40, {0x21e0ae8, 0xc0006c5880}, 0xc000a1f200)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1c5
main.(*lxdHttpServer).ServeHTTP(0xc000051cb0, {0x21e0ae8, 0xc0006c5880}, 0xc000a1f200)
	/home/runner/work/lxd/lxd/lxd/api.go:379 +0xd0
net/http.serverHandler.ServeHTTP({0xc001f8d5c0?}, {0x21e0ae8?, 0xc0006c5880?}, 0x6?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc0008dd200, {0x21e4880, 0xc0008dbf50})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 376
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/http/server.go:3086 +0x5cb

goroutine 6809 [runnable]:
database/sql.(*DB).beginDC.func2()
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:1887
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/asm_amd64.s:1650 +0x1
created by database/sql.(*DB).beginDC in goroutine 6689
	/opt/hostedtoolcache/go/1.21.4/x64/src/database/sql/sql.go:1887 +0x20d

goroutine 1902 [IO wait]:
internal/poll.runtime_pollWait(0x7ff0200ab1f8, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0021de500?, 0xc002486000?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0021de500, {0xc002486000, 0x4000, 0x4000})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0021de500, {0xc002486000?, 0x0?, 0xc0001f93d8?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000154fc8, {0xc002486000?, 0xc001f3f728?, 0x46024b?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc001f2d2f0, {0xc002486000?, 0xc001f2d2f0?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0021ee2a8, {0x21d4080, 0xc001f2d2f0})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0021ee000, {0x21d29e0?, 0xc000154fc8}, 0x3ee0?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc0021ee000, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc0021ee000, {0xc0021f8000, 0x1000, 0xc001f3fc88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc0021f3c80)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc0021f3c80, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc0021ec160, 0x0?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0021ec160)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc0021ec160)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadMessage(0x1b39d20?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x5c
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1895
	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4b9

goroutine 1653 [IO wait]:
internal/poll.runtime_pollWait(0x7ff0200ab2f0, 0x72)
	/opt/hostedtoolcache/go/1.21.4/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc001f39800?, 0xc002031800?, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f39800, {0xc002031800, 0xc00, 0xc00})
	/opt/hostedtoolcache/go/1.21.4/x64/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc001f39800, {0xc002031800?, 0x0?, 0xc0001f9b28?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000154de0, {0xc002031800?, 0xc001ed5728?, 0x46024b?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc001f2d470, {0xc002031800?, 0xc001f2d470?, 0x0?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:805 +0x3b
bytes.(*Buffer).ReadFrom(0xc0021150a8, {0x21d4080, 0xc001f2d470})
	/opt/hostedtoolcache/go/1.21.4/x64/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc002114e00, {0x21d29e0?, 0xc000154de0}, 0xa1c?)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:827 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc002114e00, 0x0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:625 +0x250
crypto/tls.(*Conn).readRecord(...)
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc002114e00, {0xc0022a6000, 0x1000, 0xc001ed5c88?})
	/opt/hostedtoolcache/go/1.21.4/x64/src/crypto/tls/conn.go:1369 +0x158
bufio.(*Reader).fill(0xc002295da0)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc002295da0, 0x2)
	/opt/hostedtoolcache/go/1.21.4/x64/src/bufio/bufio.go:151 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc001fa6b00, 0x0?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc001fa6b00)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc001fa6b00)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
github.com/gorilla/websocket.(*Conn).ReadMessage(0x1b39d20?)
	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x5c
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1546
	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4b9

goroutine 1652 [select]:
github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb8
created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 1546
	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x43b
time="2023-11-21T13:41:18Z" level=info msg="Dqlite proxy stopped" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:55604"
time="2023-11-21T13:41:18Z" level=info msg="Dqlite proxy stopped" local="10.1.1.101:37348" name=raft remote="10.1.1.103:8443"
time="2023-11-21T13:41:18Z" level=info msg="Dqlite proxy stopped" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:55614"

Noticed from https://github.com/canonical/lxd/actions/runs/6944517919/job/18891832516?pr=12515

[MusicDin]

I'll just paste here some more action logs for later reference if needed:

• https://github.com/canonical/lxd/actions/runs/7057674211/job/19211771683?pr=12580#step:11:5768
• https://github.com/canonical/lxd/actions/runs/7033949894/job/19140894788?pr=12575#step:11:19298

[tomponline]

Still happening occaisionally:

2024-02-05T10:58:40.5276287Z time="2024-02-05T10:58:40Z" level=info msg="Initializing global database"
2024-02-05T10:58:40.5277453Z time="2024-02-05T10:58:40Z" level=info msg="Connecting to global database"
2024-02-05T10:58:40.5278335Z time="2024-02-05T10:58:40Z" level=info msg="Connected to global database"
2024-02-05T10:58:40.5500022Z time="2024-02-05T10:58:40Z" level=info msg="Initialized global database"
2024-02-05T10:58:40.5595254Z time="2024-02-05T10:58:40Z" level=info msg="Firewall loaded driver" driver=nftables
2024-02-05T10:58:40.5598610Z time="2024-02-05T10:58:40Z" level=info msg="Initializing storage pools"
2024-02-05T10:58:40.5846276Z time="2024-02-05T10:58:40Z" level=info msg="Initializing daemon storage mounts"
2024-02-05T10:58:40.5852230Z time="2024-02-05T10:58:40Z" level=info msg="Initializing networks"
2024-02-05T10:58:40.5856873Z time="2024-02-05T10:58:40Z" level=info msg="All networks initialized"
2024-02-05T10:58:40.5873547Z time="2024-02-05T10:58:40Z" level=info msg="Cleaning up leftover image files"
2024-02-05T10:58:40.5878607Z time="2024-02-05T10:58:40Z" level=info msg="Done cleaning up leftover image files"
2024-02-05T10:58:40.5879814Z time="2024-02-05T10:58:40Z" level=info msg="Starting device monitor"
2024-02-05T10:58:40.5881498Z time="2024-02-05T10:58:40Z" level=info msg="Initialized filesystem monitor" driver=fanotify path=/home/runner/work/lxd/lxd/test/tmp.CeP/dev
2024-02-05T10:58:40.5887939Z time="2024-02-05T10:58:40Z" level=info msg="Started seccomp handler" path=/home/runner/work/lxd/lxd/test/tmp.CeP/0gx/seccomp.socket
2024-02-05T10:58:40.5966638Z time="2024-02-05T10:58:40Z" level=info msg="Pruning expired images"
2024-02-05T10:58:40.5971662Z time="2024-02-05T10:58:40Z" level=info msg="Done pruning expired images"
2024-02-05T10:58:40.5983415Z time="2024-02-05T10:58:40Z" level=info msg="Pruning expired backups"
2024-02-05T10:58:40.5988229Z time="2024-02-05T10:58:40Z" level=info msg="Done pruning expired backups"
2024-02-05T10:58:40.6006617Z time="2024-02-05T10:58:40Z" level=info msg="Updating instance types"
2024-02-05T10:58:40.6030953Z time="2024-02-05T10:58:40Z" level=info msg="Expiring log files"
2024-02-05T10:58:40.6031948Z time="2024-02-05T10:58:40Z" level=info msg="Daemon started"
2024-02-05T10:58:40.6046017Z time="2024-02-05T10:58:40Z" level=info msg="Pruning resolved warnings"
2024-02-05T10:58:40.6060514Z time="2024-02-05T10:58:40Z" level=info msg="Updating images"
2024-02-05T10:58:40.6064855Z time="2024-02-05T10:58:40Z" level=info msg="Done pruning resolved warnings"
2024-02-05T10:58:40.6072030Z time="2024-02-05T10:58:40Z" level=info msg="Done expiring log files"
2024-02-05T10:58:40.6074524Z time="2024-02-05T10:58:40Z" level=info msg="Done updating images"
2024-02-05T10:58:40.6106690Z time="2024-02-05T10:58:40Z" level=warning msg="Failed to update instance types: Get \"https://images.linuxcontainers.org/meta/instance-types/.yaml\": lookup images.linuxcontainers.org on 127.0.0.53:53: read udp 127.0.0.1:54477->127.0.0.53:53: read: connection refused"
2024-02-05T10:58:40.6109126Z time="2024-02-05T10:58:40Z" level=error msg="Failed updating instance types" err="Get \"https://images.linuxcontainers.org/meta/instance-types/.yaml\": lookup images.linuxcontainers.org on 127.0.0.53:53: read udp 127.0.0.1:54477->127.0.0.53:53: read: connection refused"
2024-02-05T10:58:40.9389169Z ==> Setting trust password
2024-02-05T10:58:40.9394988Z + timeout --foreground 120 /home/runner/go/bin/lxc config set core.trust_password foo --verbose
2024-02-05T10:58:40.9905338Z + [ -n --verbose ]
2024-02-05T10:58:40.9905769Z + set -x
2024-02-05T10:58:40.9906131Z + [ lxd105733 =  ]
2024-02-05T10:58:40.9906385Z + [ false = true ]
2024-02-05T10:58:40.9908283Z + set -e
2024-02-05T10:58:40.9908633Z + [ 8443 != 8443 ]
2024-02-05T10:58:40.9909208Z + cat
2024-02-05T10:58:40.9916701Z + [ dir != ceph ]
2024-02-05T10:58:40.9917064Z + cat
2024-02-05T10:58:40.9923293Z + [ dir = zfs ]
2024-02-05T10:58:40.9923694Z + [ dir = lvm ]
2024-02-05T10:58:40.9924083Z + [ dir = btrfs ]
2024-02-05T10:58:40.9924554Z + lxd init --preseed
2024-02-05T10:58:41.0079920Z time="2024-02-05T10:58:41Z" level=info msg="Joining cluster" serverName=node3
2024-02-05T10:58:41.0081206Z time="2024-02-05T10:58:41Z" level=info msg="Update network address"
2024-02-05T10:58:41.0082663Z time="2024-02-05T10:58:41Z" level=info msg="Binding socket" socket="10.1.1.103:8443" type="REST API TCP socket"
2024-02-05T10:58:41.1424323Z time="2024-02-05T10:58:41Z" level=info msg="Stop database gateway"
2024-02-05T10:58:41.1861859Z time="2024-02-05T10:58:41Z" level=info msg="Starting database node" id=3 local="10.1.1.103:8443" role=voter
2024-02-05T10:58:41.2173311Z time="2024-02-05T10:58:41Z" level=info msg="Joining dqlite raft cluster" id=3 local="10.1.1.103:8443" role=voter
2024-02-05T10:58:41.2196436Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite connected outbound" local="10.1.1.103:59762" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:41.2208862Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:59762"
2024-02-05T10:58:41.2211075Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.103:59762" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:41.2214013Z time="2024-02-05T10:58:41Z" level=info msg="Adding node to cluster" id=3 local="10.1.1.103:8443" role=voter
2024-02-05T10:58:41.2243798Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite connected outbound" local="10.1.1.101:59752" name=raft remote="10.1.1.103:8443"
2024-02-05T10:58:41.2255610Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.103:8443" name=dqlite remote="10.1.1.1:59752"
2024-02-05T10:58:41.2257549Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.101:59752" name=raft remote="10.1.1.103:8443"
2024-02-05T10:58:41.2287301Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite connected outbound" local="10.1.1.103:59770" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:41.2298826Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:59770"
2024-02-05T10:58:41.2300964Z time="2024-02-05T10:58:41Z" level=info msg="Dqlite proxy started" local="10.1.1.103:59770" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:42.2234260Z time="2024-02-05T10:58:42Z" level=info msg="Migrate local data to cluster database"
2024-02-05T10:58:42.2267454Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite connected outbound" local="10.1.1.103:59774" name=dqlite remote="10.1.1.101:8443"
2024-02-05T10:58:42.2279946Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy started" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:59774"
2024-02-05T10:58:42.2365148Z time="2024-02-05T10:58:42Z" level=info msg="Sending member change notification heartbeat to all members" local="10.1.1.103:8443"
2024-02-05T10:58:42.2424678Z time="2024-02-05T10:58:42Z" level=info msg="Added member event listener client" local="10.1.1.103:8443" remote="10.1.1.101:8443"
2024-02-05T10:58:42.2444725Z time="2024-02-05T10:58:42Z" level=info msg="Added member event listener client" local="10.1.1.103:8443" remote="10.1.1.102:8443"
2024-02-05T10:58:42.2446559Z time="2024-02-05T10:58:42Z" level=info msg="Cluster member state has changed" local="10.1.1.101:8443"
2024-02-05T10:58:42.2450832Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy stopped" local="10.1.1.103:59762" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:42.2453176Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy stopped" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:59762"
2024-02-05T10:58:42.2459168Z time="2024-02-05T10:58:42Z" level=info msg="Cluster member state has changed" local="10.1.1.102:8443"
2024-02-05T10:58:42.2536325Z time="2024-02-05T10:58:42Z" level=info msg="Added member event listener client" local="10.1.1.101:8443" remote="10.1.1.103:8443"
2024-02-05T10:58:42.2571575Z time="2024-02-05T10:58:42Z" level=info msg="Added member event listener client" local="10.1.1.102:8443" remote="10.1.1.103:8443"
2024-02-05T10:58:42.2626413Z time="2024-02-05T10:58:42Z" level=info msg="Pruning expired images"
2024-02-05T10:58:42.2656560Z time="2024-02-05T10:58:42Z" level=info msg="Done pruning expired images"
2024-02-05T10:58:42.3644886Z time="2024-02-05T10:58:42Z" level=info msg="Initialized network" name=lxd10573 project=default
2024-02-05T10:58:42.3658001Z time="2024-02-05T10:58:42Z" level=info msg="All networks initialized"
2024-02-05T10:58:42.3765119Z time="2024-02-05T10:58:42Z" level=info msg="Found cluster member whose role needs to be changed" candidateAddress="10.1.1.102:8443" local="10.1.1.101:8443" newRole=voter
2024-02-05T10:58:42.3767463Z time="2024-02-05T10:58:42Z" level=info msg="Promoting member during rebalance" candidateAddress="10.1.1.102:8443"
2024-02-05T10:58:42.3817826Z time="2024-02-05T10:58:42Z" level=info msg="Changing local dqlite raft role" id=2 local="10.1.1.102:8443" role=voter
2024-02-05T10:58:42.3838067Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite connected outbound" local="10.1.1.102:50188" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:42.3850862Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy started" local="10.1.1.102:50188" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:42.3853723Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy started" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:50188"
2024-02-05T10:58:42.3891486Z time="2024-02-05T10:58:42Z" level=info msg="Sending member change notification heartbeat to all members" local="10.1.1.102:8443"
2024-02-05T10:58:42.4187395Z time="2024-02-05T10:58:42Z" level=info msg="Cluster member state has changed" local="10.1.1.103:8443"
2024-02-05T10:58:42.4190277Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy stopped" local="10.1.1.102:50188" name=raft remote="10.1.1.101:8443"
2024-02-05T10:58:42.4192235Z time="2024-02-05T10:58:42Z" level=info msg="Dqlite proxy stopped" local="10.1.1.101:8443" name=dqlite remote="10.1.1.1:50188"
2024-02-05T10:58:42.4321836Z + LXD_DIR=/home/runner/work/lxd/lxd/test/tmp.CeP/0gx lxd shutdown
2024-02-05T10:58:42.4453420Z time="2024-02-05T10:58:42Z" level=info msg="Asked to shutdown by API" force=false
2024-02-05T10:58:42.4454319Z time="2024-02-05T10:58:42Z" level=info msg="Starting shutdown sequence" signal="power failure"
2024-02-05T10:58:42.4454901Z fatal error: sync: unlock of unlocked mutex
2024-02-05T10:58:42.4474132Z 
2024-02-05T10:58:42.4474320Z goroutine 887 [running]:
2024-02-05T10:58:42.4474837Z sync.fatal({0x1b75be2?, 0xc001ea5f60?})
2024-02-05T10:58:42.4475528Z 	/home/runner/sdk/gotip/src/runtime/panic.go:995 +0x18
2024-02-05T10:58:42.4476280Z sync.(*Mutex).unlockSlow(0xc00045a0d0, 0xffffffff)
2024-02-05T10:58:42.4477019Z 	/home/runner/sdk/gotip/src/sync/mutex.go:229 +0x35
2024-02-05T10:58:42.4477661Z sync.(*Mutex).Unlock(...)
2024-02-05T10:58:42.4478170Z 	/home/runner/sdk/gotip/src/sync/mutex.go:223
2024-02-05T10:58:42.4478923Z github.com/canonical/lxd/lxd/task.(*Group).Start.func1(0x1)
2024-02-05T10:58:42.4479750Z 	/home/runner/work/lxd/lxd/lxd/task/group.go:71 +0xe3
2024-02-05T10:58:42.4480713Z created by github.com/canonical/lxd/lxd/task.(*Group).Start in goroutine 370
2024-02-05T10:58:42.4481686Z 	/home/runner/work/lxd/lxd/lxd/task/group.go:60 +0x2e5
2024-02-05T10:58:42.4482191Z 
2024-02-05T10:58:42.4482353Z goroutine 1 [select]:
2024-02-05T10:58:42.4483090Z main.(*cmdDaemon).Run(0xc0003beeb8, 0xc00031f9a8?, {0xc000436690, 0x0, 0x0?})
2024-02-05T10:58:42.4484355Z 	/home/runner/work/lxd/lxd/lxd/main_daemon.go:83 +0x5d7
2024-02-05T10:58:42.4485332Z github.com/spf13/cobra.(*Command).execute(0xc00034c608, {0xc0000b2050, 0x3, 0x3})
2024-02-05T10:58:42.4486492Z 	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:983 +0xaca
2024-02-05T10:58:42.4487478Z github.com/spf13/cobra.(*Command).ExecuteC(0xc00034c608)
2024-02-05T10:58:42.4488455Z 	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1115 +0x3ff
2024-02-05T10:58:42.4489775Z github.com/spf13/cobra.(*Command).Execute(...)
2024-02-05T10:58:42.4490635Z 	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1039
2024-02-05T10:58:42.4491428Z main.main()
2024-02-05T10:58:42.4492151Z 	/home/runner/work/lxd/lxd/lxd/main.go:217 +0x1e05
2024-02-05T10:58:42.4492642Z 
2024-02-05T10:58:42.4492857Z goroutine 17 [syscall, locked to thread]:
2024-02-05T10:58:42.4493415Z runtime.goexit({})
2024-02-05T10:58:42.4493976Z 	/home/runner/sdk/gotip/src/runtime/asm_amd64.s:1695 +0x1
2024-02-05T10:58:42.4494514Z 
2024-02-05T10:58:42.4494675Z goroutine 34 [syscall]:
2024-02-05T10:58:42.4495118Z os/signal.signal_recv()
2024-02-05T10:58:42.4495709Z 	/home/runner/sdk/gotip/src/runtime/sigqueue.go:152 +0x29
2024-02-05T10:58:42.4496383Z os/signal.loop()
2024-02-05T10:58:42.4496972Z 	/home/runner/sdk/gotip/src/os/signal/signal_unix.go:23 +0x13
2024-02-05T10:58:42.4497805Z created by os/signal.Notify.func1.1 in goroutine 1
2024-02-05T10:58:42.4498597Z 	/home/runner/sdk/gotip/src/os/signal/signal.go:151 +0x1f
2024-02-05T10:58:42.4499115Z 
2024-02-05T10:58:42.4499265Z goroutine 35 [select]:
2024-02-05T10:58:42.4499963Z database/sql.(*DB).connectionOpener(0xc0004ef930, {0x1e8a2e8, 0xc00049a410})
2024-02-05T10:58:42.4500946Z 	/home/runner/sdk/gotip/src/database/sql/sql.go:1246 +0x87
2024-02-05T10:58:42.4501722Z created by database/sql.OpenDB in goroutine 1
2024-02-05T10:58:42.4502482Z 	/home/runner/sdk/gotip/src/database/sql/sql.go:824 +0x14c
2024-02-05T10:58:42.4503036Z 
2024-02-05T10:58:42.4503188Z goroutine 24 [select]:
2024-02-05T10:58:42.4503885Z database/sql.(*DB).connectionOpener(0xc00069b6c0, {0x1e8a2e8, 0xc0006a2e60})
2024-02-05T10:58:42.4504848Z 	/home/runner/sdk/gotip/src/database/sql/sql.go:1246 +0x87
2024-02-05T10:58:42.4505620Z created by database/sql.OpenDB in goroutine 1
2024-02-05T10:58:42.4506377Z 	/home/runner/sdk/gotip/src/database/sql/sql.go:824 +0x14c
2024-02-05T10:58:42.4506916Z 
2024-02-05T10:58:42.4507071Z goroutine 22 [IO wait]:
2024-02-05T10:58:42.4507637Z internal/poll.runtime_pollWait(0x7fd8506a7e70, 0x72)
2024-02-05T10:58:42.4508434Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4509230Z internal/poll.(*pollDesc).wait(0x8?, 0xc000a43b80?, 0x0)
2024-02-05T10:58:42.4510183Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4511042Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4511809Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4512620Z internal/poll.(*FD).Accept(0xc00024e000)
2024-02-05T10:58:42.4513392Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:611 +0x2ac
2024-02-05T10:58:42.4514153Z net.(*netFD).accept(0xc00024e000)
2024-02-05T10:58:42.4514756Z 	/home/runner/sdk/gotip/src/net/fd_unix.go:172 +0x29
2024-02-05T10:58:42.4515405Z net.(*UnixListener).accept(0x10?)
2024-02-05T10:58:42.4516086Z 	/home/runner/sdk/gotip/src/net/unixsock_posix.go:172 +0x16
2024-02-05T10:58:42.4516831Z net.(*UnixListener).Accept(0xc000382ea0)
2024-02-05T10:58:42.4517501Z 	/home/runner/sdk/gotip/src/net/unixsock.go:260 +0x30
2024-02-05T10:58:42.4518594Z github.com/canonical/lxd/lxd/endpoints/listeners.(*StarttlsListener).Accept(0xc000383110)
2024-02-05T10:58:42.4519882Z 	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/starttls.go:36 +0x50
2024-02-05T10:58:42.4520900Z net/http.(*Server).Serve(0xc000332000, {0x1e87440, 0xc000383110})
2024-02-05T10:58:42.4521777Z 	/home/runner/sdk/gotip/src/net/http/server.go:3255 +0x33e
2024-02-05T10:58:42.4522945Z github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
2024-02-05T10:58:42.4524953Z 	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:409 +0x1f
2024-02-05T10:58:42.4525763Z gopkg.in/tomb%2ev2.(*Tomb).run(0xc0006a2dc0, 0xc000600200?)
2024-02-05T10:58:42.4526537Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
2024-02-05T10:58:42.4527185Z created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 1
2024-02-05T10:58:42.4527899Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5
2024-02-05T10:58:42.4528357Z 
2024-02-05T10:58:42.4528459Z goroutine 23 [IO wait]:
2024-02-05T10:58:42.4528799Z internal/poll.runtime_pollWait(0x7fd8506a7d78, 0x72)
2024-02-05T10:58:42.4529666Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4530134Z internal/poll.(*pollDesc).wait(0x12?, 0x10?, 0x0)
2024-02-05T10:58:42.4530645Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4531141Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4531592Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4532051Z internal/poll.(*FD).Accept(0xc00024e200)
2024-02-05T10:58:42.4532496Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:611 +0x2ac
2024-02-05T10:58:42.4532940Z net.(*netFD).accept(0xc00024e200)
2024-02-05T10:58:42.4533293Z 	/home/runner/sdk/gotip/src/net/fd_unix.go:172 +0x29
2024-02-05T10:58:42.4533689Z net.(*UnixListener).accept(0xc0003b6c01?)
2024-02-05T10:58:42.4534118Z 	/home/runner/sdk/gotip/src/net/unixsock_posix.go:172 +0x16
2024-02-05T10:58:42.4534540Z net.(*UnixListener).Accept(0xc000386060)
2024-02-05T10:58:42.4534936Z 	/home/runner/sdk/gotip/src/net/unixsock.go:260 +0x30
2024-02-05T10:58:42.4535412Z net/http.(*Server).Serve(0xc0003320f0, {0x1e86c00, 0xc000386060})
2024-02-05T10:58:42.4535914Z 	/home/runner/sdk/gotip/src/net/http/server.go:3255 +0x33e
2024-02-05T10:58:42.4536456Z github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
2024-02-05T10:58:42.4537032Z 	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:409 +0x1f
2024-02-05T10:58:42.4537556Z gopkg.in/tomb%2ev2.(*Tomb).run(0xc0006a2dc0, 0xc000552730?)
2024-02-05T10:58:42.4538288Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
2024-02-05T10:58:42.4538917Z created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 1
2024-02-05T10:58:42.4539607Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5
2024-02-05T10:58:42.4540063Z 
2024-02-05T10:58:42.4540160Z goroutine 621 [IO wait]:
2024-02-05T10:58:42.4540495Z internal/poll.runtime_pollWait(0x7fd8506a78a0, 0x72)
2024-02-05T10:58:42.4540960Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4541455Z internal/poll.(*pollDesc).wait(0xc00089f780?, 0xc000b26000?, 0x0)
2024-02-05T10:58:42.4542035Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4542524Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4542969Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4543528Z internal/poll.(*FD).Read(0xc00089f780, {0xc000b26000, 0x4a80, 0x4a80})
2024-02-05T10:58:42.4544082Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4544665Z net.(*netFD).Read(0xc00089f780, {0xc000b26000?, 0x7fd8504c1a88?, 0xc00082fef0?})
2024-02-05T10:58:42.4545183Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4545679Z net.(*conn).Read(0xc0004fe528, {0xc000b26000?, 0xc0004e2b88?, 0x41ac3b?})
2024-02-05T10:58:42.4546161Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4546691Z crypto/tls.(*atLeastReader).Read(0xc00082fef0, {0xc000b26000?, 0x0?, 0xc00082fef0?})
2024-02-05T10:58:42.4547265Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4547760Z bytes.(*Buffer).ReadFrom(0xc0001cbeb0, {0x1e7b2e0, 0xc00082fef0})
2024-02-05T10:58:42.4548238Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4548994Z crypto/tls.(*Conn).readFromUntil(0xc0001cbc08, {0x1e79c40, 0xc0004fe528}, 0xc0004e2bd0?)
2024-02-05T10:58:42.4549578Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4550027Z crypto/tls.(*Conn).readRecordOrCCS(0xc0001cbc08, 0x0)
2024-02-05T10:58:42.4550481Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4550889Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4551244Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4551904Z crypto/tls.(*Conn).Read(0xc0001cbc08, {0xc0009ec000, 0x8000, 0xc0007e3770?})
2024-02-05T10:58:42.4552452Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4553139Z io.copyBuffer({0x1e78d20, 0xc000716368}, {0x1e79ec0, 0xc0001cbc08}, {0x0, 0x0, 0x0})
2024-02-05T10:58:42.4553670Z 	/home/runner/sdk/gotip/src/io/io.go:429 +0x191
2024-02-05T10:58:42.4554014Z io.Copy(...)
2024-02-05T10:58:42.4554266Z 	/home/runner/sdk/gotip/src/io/io.go:388
2024-02-05T10:58:42.4554683Z github.com/canonical/lxd/lxd/cluster.dqliteProxy.func1()
2024-02-05T10:58:42.4555191Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1164 +0x85
2024-02-05T10:58:42.4555788Z created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 619
2024-02-05T10:58:42.4556392Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1163 +0x56e
2024-02-05T10:58:42.4556725Z 
2024-02-05T10:58:42.4556819Z goroutine 89 [syscall]:
2024-02-05T10:58:42.4557126Z syscall.Syscall(0x0, 0x1a, 0xc0005d2000, 0x1000)
2024-02-05T10:58:42.4557578Z 	/home/runner/sdk/gotip/src/syscall/syscall_linux.go:69 +0x25
2024-02-05T10:58:42.4558148Z golang.org/x/sys/unix.read(0x1895d40?, {0xc0005d2000?, 0x1b33f24?, 0x9?})
2024-02-05T10:58:42.4558817Z 	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_linux.go:1391 +0x45
2024-02-05T10:58:42.4559335Z golang.org/x/sys/unix.Read(...)
2024-02-05T10:58:42.4559798Z 	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/syscall_unix.go:158
2024-02-05T10:58:42.4560494Z main.deviceNetlinkListener.func1(0xc0000b4a80, 0xc0005c8240, 0xc0005c82a0, 0xc0005c8300)
2024-02-05T10:58:42.4561086Z 	/home/runner/work/lxd/lxd/lxd/devices.go:100 +0x98
2024-02-05T10:58:42.4561532Z created by main.deviceNetlinkListener in goroutine 199
2024-02-05T10:58:42.4561988Z 	/home/runner/work/lxd/lxd/lxd/devices.go:97 +0x1b3
2024-02-05T10:58:42.4562265Z 
2024-02-05T10:58:42.4562364Z goroutine 199 [select]:
2024-02-05T10:58:42.4562642Z main.deviceEventListener(0xc0004962d0)
2024-02-05T10:58:42.4563029Z 	/home/runner/work/lxd/lxd/lxd/devices.go:669 +0x20f
2024-02-05T10:58:42.4563438Z created by main.(*Daemon).init in goroutine 1
2024-02-05T10:58:42.4563845Z 	/home/runner/work/lxd/lxd/lxd/daemon.go:1395 +0x4185
2024-02-05T10:58:42.4564128Z 
2024-02-05T10:58:42.4564221Z goroutine 201 [syscall]:
2024-02-05T10:58:42.4564529Z syscall.Syscall(0x0, 0x19, 0xc0000ad800, 0x100)
2024-02-05T10:58:42.4565021Z 	/home/runner/sdk/gotip/src/syscall/syscall_linux.go:69 +0x25
2024-02-05T10:58:42.4565543Z golang.org/x/sys/unix.read(0x0?, {0xc0000ad800?, 0x0?, 0x0?})
2024-02-05T10:58:42.4566158Z 	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_linux.go:1391 +0x45
2024-02-05T10:58:42.4566671Z golang.org/x/sys/unix.Read(...)
2024-02-05T10:58:42.4567121Z 	/home/runner/go/pkg/mod/golang.org/x/[email protected]/unix/syscall_unix.go:158
2024-02-05T10:58:42.4567950Z github.com/canonical/lxd/lxd/fsmonitor/drivers.(*fanotify).getEvents(0xc0007036c0, {0x1e8a2e8, 0xc000362e10}, 0x1b)
2024-02-05T10:58:42.4568802Z 	/home/runner/work/lxd/lxd/lxd/fsmonitor/drivers/driver_fanotify.go:86 +0x78
2024-02-05T10:58:42.4569750Z created by github.com/canonical/lxd/lxd/fsmonitor/drivers.(*fanotify).load in goroutine 1
2024-02-05T10:58:42.4570513Z 	/home/runner/work/lxd/lxd/lxd/fsmonitor/drivers/driver_fanotify.go:71 +0x2fe
2024-02-05T10:58:42.4570940Z 
2024-02-05T10:58:42.4571042Z goroutine 75 [IO wait]:
2024-02-05T10:58:42.4571372Z internal/poll.runtime_pollWait(0x7fd8506a7998, 0x72)
2024-02-05T10:58:42.4572004Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4572447Z internal/poll.(*pollDesc).wait(0x1c?, 0x0?, 0x0)
2024-02-05T10:58:42.4572960Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4573451Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4573903Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4574355Z internal/poll.(*FD).Accept(0xc000334200)
2024-02-05T10:58:42.4574799Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:611 +0x2ac
2024-02-05T10:58:42.4575239Z net.(*netFD).accept(0xc000334200)
2024-02-05T10:58:42.4575596Z 	/home/runner/sdk/gotip/src/net/fd_unix.go:172 +0x29
2024-02-05T10:58:42.4576106Z net.(*UnixListener).accept(0x0?)
2024-02-05T10:58:42.4576517Z 	/home/runner/sdk/gotip/src/net/unixsock_posix.go:172 +0x16
2024-02-05T10:58:42.4576946Z net.(*UnixListener).Accept(0xc0004eb620)
2024-02-05T10:58:42.4577338Z 	/home/runner/sdk/gotip/src/net/unixsock.go:260 +0x30
2024-02-05T10:58:42.4577838Z github.com/canonical/lxd/lxd/seccomp.NewSeccompServer.func1()
2024-02-05T10:58:42.4578377Z 	/home/runner/work/lxd/lxd/lxd/seccomp/seccomp.go:1054 +0x59
2024-02-05T10:58:42.4578986Z created by github.com/canonical/lxd/lxd/seccomp.NewSeccompServer in goroutine 1
2024-02-05T10:58:42.4579612Z 	/home/runner/work/lxd/lxd/lxd/seccomp/seccomp.go:1052 +0x1f5
2024-02-05T10:58:42.4579935Z 
2024-02-05T10:58:42.4580036Z goroutine 624 [IO wait]:
2024-02-05T10:58:42.4580362Z internal/poll.runtime_pollWait(0x7fd8506a76b0, 0x72)
2024-02-05T10:58:42.4580824Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4581328Z internal/poll.(*pollDesc).wait(0xc00089f980?, 0xc0004cf080?, 0x0)
2024-02-05T10:58:42.4581897Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4582393Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4582836Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4583383Z internal/poll.(*FD).Read(0xc00089f980, {0xc0004cf080, 0x580, 0x580})
2024-02-05T10:58:42.4583929Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4584505Z net.(*netFD).Read(0xc00089f980, {0xc0004cf080?, 0x7fd8506a04a8?, 0xc0000cb9f8?})
2024-02-05T10:58:42.4585031Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4585521Z net.(*conn).Read(0xc0000723f8, {0xc0004cf080?, 0xc00071cb88?, 0x41ac3b?})
2024-02-05T10:58:42.4586002Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4586534Z crypto/tls.(*atLeastReader).Read(0xc0000cb9f8, {0xc0004cf080?, 0x0?, 0xc0000cb9f8?})
2024-02-05T10:58:42.4587096Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4587582Z bytes.(*Buffer).ReadFrom(0xc0001537b0, {0x1e7b2e0, 0xc0000cb9f8})
2024-02-05T10:58:42.4588054Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4588605Z crypto/tls.(*Conn).readFromUntil(0xc000153508, {0x1e79c40, 0xc0000723f8}, 0xc00071cbd0?)
2024-02-05T10:58:42.4589347Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4589793Z crypto/tls.(*Conn).readRecordOrCCS(0xc000153508, 0x0)
2024-02-05T10:58:42.4590233Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4590636Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4590995Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4591496Z crypto/tls.(*Conn).Read(0xc000153508, {0xc000c00000, 0x8000, 0xc000b5c230?})
2024-02-05T10:58:42.4592040Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4592610Z io.copyBuffer({0x1e78d20, 0xc0004fe550}, {0x1e79ec0, 0xc000153508}, {0x0, 0x0, 0x0})
2024-02-05T10:58:42.4593148Z 	/home/runner/sdk/gotip/src/io/io.go:429 +0x191
2024-02-05T10:58:42.4593483Z io.Copy(...)
2024-02-05T10:58:42.4593728Z 	/home/runner/sdk/gotip/src/io/io.go:388
2024-02-05T10:58:42.4594146Z github.com/canonical/lxd/lxd/cluster.dqliteProxy.func1()
2024-02-05T10:58:42.4594645Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1164 +0x85
2024-02-05T10:58:42.4595380Z created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 694
2024-02-05T10:58:42.4595982Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1163 +0x56e
2024-02-05T10:58:42.4596303Z 
2024-02-05T10:58:42.4596397Z goroutine 330 [IO wait]:
2024-02-05T10:58:42.4596729Z internal/poll.runtime_pollWait(0x7fd8506a7a90, 0x72)
2024-02-05T10:58:42.4597184Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4597613Z internal/poll.(*pollDesc).wait(0x1f?, 0xe?, 0x0)
2024-02-05T10:58:42.4598121Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4598607Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4599167Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4599631Z internal/poll.(*FD).Accept(0xc00024e700)
2024-02-05T10:58:42.4600077Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:611 +0x2ac
2024-02-05T10:58:42.4600511Z net.(*netFD).accept(0xc00024e700)
2024-02-05T10:58:42.4600873Z 	/home/runner/sdk/gotip/src/net/fd_unix.go:172 +0x29
2024-02-05T10:58:42.4601263Z net.(*TCPListener).accept(0xc00005a0e0)
2024-02-05T10:58:42.4601672Z 	/home/runner/sdk/gotip/src/net/tcpsock_posix.go:159 +0x1e
2024-02-05T10:58:42.4602090Z net.(*TCPListener).Accept(0xc00005a0e0)
2024-02-05T10:58:42.4602469Z 	/home/runner/sdk/gotip/src/net/tcpsock.go:327 +0x30
2024-02-05T10:58:42.4603087Z github.com/canonical/lxd/lxd/endpoints/listeners.(*FancyTLSListener).Accept(0xc000b72410)
2024-02-05T10:58:42.4603827Z 	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/fancytls.go:37 +0x4d
2024-02-05T10:58:42.4604417Z net/http.(*Server).Serve(0xc000332000, {0x1e873e0, 0xc000b72410})
2024-02-05T10:58:42.4604923Z 	/home/runner/sdk/gotip/src/net/http/server.go:3255 +0x33e
2024-02-05T10:58:42.4605449Z github.com/canonical/lxd/lxd/endpoints.(*Endpoints).serve.func1()
2024-02-05T10:58:42.4606008Z 	/home/runner/work/lxd/lxd/lxd/endpoints/endpoints.go:409 +0x1f
2024-02-05T10:58:42.4606532Z gopkg.in/tomb%2ev2.(*Tomb).run(0xc0006a2dc0, 0xc000624790?)
2024-02-05T10:58:42.4607271Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:163 +0x2b
2024-02-05T10:58:42.4607911Z created by gopkg.in/tomb%2ev2.(*Tomb).Go in goroutine 282
2024-02-05T10:58:42.4608613Z 	/home/runner/go/pkg/mod/gopkg.in/[email protected]/tomb.go:159 +0xe5
2024-02-05T10:58:42.4609258Z 
2024-02-05T10:58:42.4609363Z goroutine 622 [IO wait]:
2024-02-05T10:58:42.4609707Z internal/poll.runtime_pollWait(0x7fd8506a75b8, 0x72)
2024-02-05T10:58:42.4610165Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4610664Z internal/poll.(*pollDesc).wait(0xc00041dd00?, 0xc0009e4000?, 0x0)
2024-02-05T10:58:42.4611247Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4611730Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4612184Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4612743Z internal/poll.(*FD).Read(0xc00041dd00, {0xc0009e4000, 0x8000, 0x8000})
2024-02-05T10:58:42.4613289Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4613837Z net.(*netFD).Read(0xc00041dd00, {0xc0009e4000?, 0x8000?, 0xc000480808?})
2024-02-05T10:58:42.4614327Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4614810Z net.(*conn).Read(0xc000716368, {0xc0009e4000?, 0x17fd900?, 0x417401?})
2024-02-05T10:58:42.4615271Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4615798Z io.copyBuffer({0x1e79ea0, 0xc0001cbc08}, {0x1e78d00, 0xc000716368}, {0x0, 0x0, 0x0})
2024-02-05T10:58:42.4616330Z 	/home/runner/sdk/gotip/src/io/io.go:429 +0x191
2024-02-05T10:58:42.4616672Z io.Copy(...)
2024-02-05T10:58:42.4616913Z 	/home/runner/sdk/gotip/src/io/io.go:388
2024-02-05T10:58:42.4617332Z github.com/canonical/lxd/lxd/cluster.dqliteProxy.func2()
2024-02-05T10:58:42.4617848Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1169 +0x85
2024-02-05T10:58:42.4618609Z created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 619
2024-02-05T10:58:42.4619217Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1168 +0x5f2
2024-02-05T10:58:42.4619549Z 
2024-02-05T10:58:42.4619644Z goroutine 694 [select]:
2024-02-05T10:58:42.4620307Z github.com/canonical/lxd/lxd/cluster.dqliteProxy({0x1b237e0, 0x4}, 0xc000508240, {0x1e94cb8, 0xc000153508}, {0x1e94c60, 0xc0004fe550})
2024-02-05T10:58:42.4621115Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1175 +0x69c
2024-02-05T10:58:42.4621833Z created by github.com/canonical/lxd/lxd/cluster.(*Gateway).init.(*Gateway).raftDial.func3 in goroutine 17
2024-02-05T10:58:42.4622688Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:427 +0x2a5
2024-02-05T10:58:42.4623014Z 
2024-02-05T10:58:42.4623109Z goroutine 625 [IO wait]:
2024-02-05T10:58:42.4623442Z internal/poll.runtime_pollWait(0x7fd8506a71d8, 0x72)
2024-02-05T10:58:42.4623900Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4624401Z internal/poll.(*pollDesc).wait(0xc00089fc80?, 0xc0009f4000?, 0x0)
2024-02-05T10:58:42.4624976Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4625465Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4625905Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4626456Z internal/poll.(*FD).Read(0xc00089fc80, {0xc0009f4000, 0x8000, 0x8000})
2024-02-05T10:58:42.4627002Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4627516Z net.(*netFD).Read(0xc00089fc80, {0xc0009f4000?, 0x8000?, 0x0?})
2024-02-05T10:58:42.4627969Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4628486Z net.(*conn).Read(0xc0004fe550, {0xc0009f4000?, 0xc0004e5ea8?, 0xc0004e5e98?})
2024-02-05T10:58:42.4628979Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4629497Z io.copyBuffer({0x1e79ea0, 0xc000153508}, {0x1e78d00, 0xc0004fe550}, {0x0, 0x0, 0x0})
2024-02-05T10:58:42.4630027Z 	/home/runner/sdk/gotip/src/io/io.go:429 +0x191
2024-02-05T10:58:42.4630366Z io.Copy(...)
2024-02-05T10:58:42.4630609Z 	/home/runner/sdk/gotip/src/io/io.go:388
2024-02-05T10:58:42.4631030Z github.com/canonical/lxd/lxd/cluster.dqliteProxy.func2()
2024-02-05T10:58:42.4631537Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1169 +0x85
2024-02-05T10:58:42.4632139Z created by github.com/canonical/lxd/lxd/cluster.dqliteProxy in goroutine 694
2024-02-05T10:58:42.4632742Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1168 +0x5f2
2024-02-05T10:58:42.4633066Z 
2024-02-05T10:58:42.4633163Z goroutine 931 [select]:
2024-02-05T10:58:42.4633831Z github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc0004b7b00, {0x1e8a2e8?, 0xc000552690?}, 0xc000a45230)
2024-02-05T10:58:42.4634658Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x37d
2024-02-05T10:58:42.4635259Z github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc000641220)
2024-02-05T10:58:42.4635827Z 	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
2024-02-05T10:58:42.4636687Z created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 904
2024-02-05T10:58:42.4637310Z 	/home/runner/work/lxd/lxd/lxd/events/events.go:97 +0x47b
2024-02-05T10:58:42.4637610Z 
2024-02-05T10:58:42.4637709Z goroutine 863 [select]:
2024-02-05T10:58:42.4638238Z github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc00066b4a0, {0x1e8a2b0?, 0xc000baaf00?})
2024-02-05T10:58:42.4638909Z 	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
2024-02-05T10:58:42.4639435Z main.eventsSocket(0xc0001d9880, 0xc000baed80, {0x1e86ed0, 0xc0001d96c0})
2024-02-05T10:58:42.4639931Z 	/home/runner/work/lxd/lxd/lxd/events.go:148 +0xa05
2024-02-05T10:58:42.4640415Z main.(*eventsServe).Render(0x1c5e280?, {0x1e86ed0?, 0xc0001d96c0?})
2024-02-05T10:58:42.4640884Z 	/home/runner/work/lxd/lxd/lxd/events.go:36 +0x30
2024-02-05T10:58:42.4641368Z main.(*Daemon).createCmd.func1({0x1e86ed0, 0xc0001d96c0}, 0xc000baed80)
2024-02-05T10:58:42.4642045Z 	/home/runner/work/lxd/lxd/lxd/daemon.go:640 +0x167e
2024-02-05T10:58:42.4642595Z net/http.HandlerFunc.ServeHTTP(0xc000bae240?, {0x1e86ed0?, 0xc0001d96c0?}, 0x2?)
2024-02-05T10:58:42.4643177Z 	/home/runner/sdk/gotip/src/net/http/server.go:2166 +0x29
2024-02-05T10:58:42.4643797Z github.com/gorilla/mux.(*Router).ServeHTTP(0xc0000000c0, {0x1e86ed0, 0xc0001d96c0}, 0xc000bae120)
2024-02-05T10:58:42.4644500Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1e2
2024-02-05T10:58:42.4645151Z main.(*lxdHttpServer).ServeHTTP(0xc0006762d0, {0x1e86ed0, 0xc0001d96c0}, 0xc000bae120)
2024-02-05T10:58:42.4645683Z 	/home/runner/work/lxd/lxd/lxd/api.go:185 +0xd0
2024-02-05T10:58:42.4646350Z net/http.serverHandler.ServeHTTP({0xc000baa810?}, {0x1e86ed0?, 0xc0001d96c0?}, 0x6?)
2024-02-05T10:58:42.4646955Z 	/home/runner/sdk/gotip/src/net/http/server.go:3137 +0x8e
2024-02-05T10:58:42.4647441Z net/http.(*conn).serve(0xc000b98990, {0x1e8a2b0, 0xc000baa180})
2024-02-05T10:58:42.4647953Z 	/home/runner/sdk/gotip/src/net/http/server.go:2039 +0x5e8
2024-02-05T10:58:42.4648425Z created by net/http.(*Server).Serve in goroutine 330
2024-02-05T10:58:42.4648884Z 	/home/runner/sdk/gotip/src/net/http/server.go:3285 +0x4b4
2024-02-05T10:58:42.4649620Z 
2024-02-05T10:58:42.4649734Z goroutine 664 [chan receive]:
2024-02-05T10:58:42.4650314Z github.com/canonical/lxd/lxd/cluster.runDqliteProxy(0xc000508240, {0xc000473a20, 0x6}, 0xc000508480)
2024-02-05T10:58:42.4651018Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1131 +0x45
2024-02-05T10:58:42.4651630Z created by github.com/canonical/lxd/lxd/cluster.(*Gateway).init in goroutine 370
2024-02-05T10:58:42.4652243Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:800 +0x5a5
2024-02-05T10:58:42.4652566Z 
2024-02-05T10:58:42.4652668Z goroutine 619 [select]:
2024-02-05T10:58:42.4653345Z github.com/canonical/lxd/lxd/cluster.dqliteProxy({0x1b27640, 0x6}, 0xc000508240, {0x1e94cb8, 0xc0001cbc08}, {0x1e94c60, 0xc000716368})
2024-02-05T10:58:42.4654166Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1175 +0x69c
2024-02-05T10:58:42.4654793Z created by github.com/canonical/lxd/lxd/cluster.runDqliteProxy in goroutine 664
2024-02-05T10:58:42.4655410Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:1137 +0x105
2024-02-05T10:58:42.4655738Z 
2024-02-05T10:58:42.4655830Z goroutine 904 [select]:
2024-02-05T10:58:42.4656366Z github.com/canonical/lxd/lxd/events.(*listenerCommon).Wait(0xc000641220, {0x1e8a2b0?, 0xc001f1e900?})
2024-02-05T10:58:42.4657032Z 	/home/runner/work/lxd/lxd/lxd/events/common.go:52 +0x85
2024-02-05T10:58:42.4657547Z main.eventsSocket(0xc000697880, 0xc001f22480, {0x1e86ed0, 0xc0006976c0})
2024-02-05T10:58:42.4658054Z 	/home/runner/work/lxd/lxd/lxd/events.go:148 +0xa05
2024-02-05T10:58:42.4658537Z main.(*eventsServe).Render(0x1c5e280?, {0x1e86ed0?, 0xc0006976c0?})
2024-02-05T10:58:42.4658997Z 	/home/runner/work/lxd/lxd/lxd/events.go:36 +0x30
2024-02-05T10:58:42.4659484Z main.(*Daemon).createCmd.func1({0x1e86ed0, 0xc0006976c0}, 0xc001f22480)
2024-02-05T10:58:42.4659992Z 	/home/runner/work/lxd/lxd/lxd/daemon.go:640 +0x167e
2024-02-05T10:58:42.4660535Z net/http.HandlerFunc.ServeHTTP(0xc000a2f8c0?, {0x1e86ed0?, 0xc0006976c0?}, 0x2?)
2024-02-05T10:58:42.4661113Z 	/home/runner/sdk/gotip/src/net/http/server.go:2166 +0x29
2024-02-05T10:58:42.4661741Z github.com/gorilla/mux.(*Router).ServeHTTP(0xc0000000c0, {0x1e86ed0, 0xc0006976c0}, 0xc000a2f7a0)
2024-02-05T10:58:42.4662440Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1e2
2024-02-05T10:58:42.4663087Z main.(*lxdHttpServer).ServeHTTP(0xc0006762d0, {0x1e86ed0, 0xc0006976c0}, 0xc000a2f7a0)
2024-02-05T10:58:42.4663635Z 	/home/runner/work/lxd/lxd/lxd/api.go:185 +0xd0
2024-02-05T10:58:42.4664179Z net/http.serverHandler.ServeHTTP({0xc001f1e210?}, {0x1e86ed0?, 0xc0006976c0?}, 0x6?)
2024-02-05T10:58:42.4664771Z 	/home/runner/sdk/gotip/src/net/http/server.go:3137 +0x8e
2024-02-05T10:58:42.4665302Z net/http.(*conn).serve(0xc000b990e0, {0x1e8a2b0, 0xc000bab2c0})
2024-02-05T10:58:42.4665803Z 	/home/runner/sdk/gotip/src/net/http/server.go:2039 +0x5e8
2024-02-05T10:58:42.4666450Z created by net/http.(*Server).Serve in goroutine 330
2024-02-05T10:58:42.4666919Z 	/home/runner/sdk/gotip/src/net/http/server.go:3285 +0x4b4
2024-02-05T10:58:42.4667225Z 
2024-02-05T10:58:42.4667325Z goroutine 869 [IO wait]:
2024-02-05T10:58:42.4667646Z internal/poll.runtime_pollWait(0x7fd850381850, 0x72)
2024-02-05T10:58:42.4668100Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4668608Z internal/poll.(*pollDesc).wait(0xc000b41b80?, 0xc001ee2000?, 0x0)
2024-02-05T10:58:42.4669179Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4669666Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4670245Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4670798Z internal/poll.(*FD).Read(0xc000b41b80, {0xc001ee2000, 0x900, 0x900})
2024-02-05T10:58:42.4671340Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4671917Z net.(*netFD).Read(0xc000b41b80, {0xc001ee2000?, 0x7fd850430d68?, 0xc0008b8210?})
2024-02-05T10:58:42.4672446Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4672933Z net.(*conn).Read(0xc0004fe5e8, {0xc001ee2000?, 0xc0003028d8?, 0x41ac3b?})
2024-02-05T10:58:42.4673413Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4673940Z crypto/tls.(*atLeastReader).Read(0xc0008b8210, {0xc001ee2000?, 0x0?, 0xc0008b8210?})
2024-02-05T10:58:42.4674505Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4674993Z bytes.(*Buffer).ReadFrom(0xc0000c7430, {0x1e7b2e0, 0xc0008b8210})
2024-02-05T10:58:42.4675473Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4676035Z crypto/tls.(*Conn).readFromUntil(0xc0000c7188, {0x1e79c40, 0xc0004fe5e8}, 0xc000302920?)
2024-02-05T10:58:42.4676618Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4677068Z crypto/tls.(*Conn).readRecordOrCCS(0xc0000c7188, 0x0)
2024-02-05T10:58:42.4677519Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4677924Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4678290Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4678794Z crypto/tls.(*Conn).Read(0xc0000c7188, {0xc001e9a000, 0x1000, 0x10000717f80?})
2024-02-05T10:58:42.4679341Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4679750Z bufio.(*Reader).fill(0xc001e96fc0)
2024-02-05T10:58:42.4680118Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:110 +0x103
2024-02-05T10:58:42.4680504Z bufio.(*Reader).Peek(0xc001e96fc0, 0x2)
2024-02-05T10:58:42.4680891Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:148 +0x53
2024-02-05T10:58:42.4681398Z github.com/gorilla/websocket.(*Conn).read(0xc0009c49a0, 0x1a35f40?)
2024-02-05T10:58:42.4682027Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
2024-02-05T10:58:42.4682649Z github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0009c49a0)
2024-02-05T10:58:42.4683261Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
2024-02-05T10:58:42.4683841Z github.com/gorilla/websocket.(*Conn).NextReader(0xc0009c49a0)
2024-02-05T10:58:42.4684443Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
2024-02-05T10:58:42.4685048Z github.com/gorilla/websocket.(*Conn).ReadMessage(0x189a900?)
2024-02-05T10:58:42.4685636Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
2024-02-05T10:58:42.4686265Z github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
2024-02-05T10:58:42.4686791Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x56
2024-02-05T10:58:42.4687395Z created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 657
2024-02-05T10:58:42.4688004Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4c6
2024-02-05T10:58:42.4688320Z 
2024-02-05T10:58:42.4688415Z goroutine 883 [IO wait]:
2024-02-05T10:58:42.4688752Z internal/poll.runtime_pollWait(0x7fd8506a6fe8, 0x72)
2024-02-05T10:58:42.4689709Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4690227Z internal/poll.(*pollDesc).wait(0xc000a42b00?, 0xc000711080?, 0x0)
2024-02-05T10:58:42.4690814Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4691300Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4691749Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4692313Z internal/poll.(*FD).Read(0xc000a42b00, {0xc000711080, 0x580, 0x580})
2024-02-05T10:58:42.4692855Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4693586Z net.(*netFD).Read(0xc000a42b00, {0xc000711080?, 0x7fd8504e4628?, 0xc000b5df08?})
2024-02-05T10:58:42.4694126Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4694618Z net.(*conn).Read(0xc0004aa548, {0xc000711080?, 0xc0000848d8?, 0x41ac3b?})
2024-02-05T10:58:42.4695104Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4695638Z crypto/tls.(*atLeastReader).Read(0xc000b5df08, {0xc000711080?, 0x0?, 0xc000b5df08?})
2024-02-05T10:58:42.4696214Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4696700Z bytes.(*Buffer).ReadFrom(0xc0003fd7b0, {0x1e7b2e0, 0xc000b5df08})
2024-02-05T10:58:42.4697187Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4697753Z crypto/tls.(*Conn).readFromUntil(0xc0003fd508, {0x1e79c40, 0xc0004aa548}, 0xc000084920?)
2024-02-05T10:58:42.4698326Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4698786Z crypto/tls.(*Conn).readRecordOrCCS(0xc0003fd508, 0x0)
2024-02-05T10:58:42.4699247Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4699644Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4700008Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4700520Z crypto/tls.(*Conn).Read(0xc0003fd508, {0xc000b79000, 0x1000, 0x10000073f80?})
2024-02-05T10:58:42.4701068Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4701481Z bufio.(*Reader).fill(0xc001f82060)
2024-02-05T10:58:42.4701850Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:110 +0x103
2024-02-05T10:58:42.4702236Z bufio.(*Reader).Peek(0xc001f82060, 0x2)
2024-02-05T10:58:42.4702633Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:148 +0x53
2024-02-05T10:58:42.4703141Z github.com/gorilla/websocket.(*Conn).read(0xc0008a78c0, 0x1a35f40?)
2024-02-05T10:58:42.4703765Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
2024-02-05T10:58:42.4704382Z github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0008a78c0)
2024-02-05T10:58:42.4704991Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
2024-02-05T10:58:42.4705585Z github.com/gorilla/websocket.(*Conn).NextReader(0xc0008a78c0)
2024-02-05T10:58:42.4706181Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
2024-02-05T10:58:42.4706789Z github.com/gorilla/websocket.(*Conn).ReadMessage(0x189a900?)
2024-02-05T10:58:42.4707384Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1120 +0x13
2024-02-05T10:58:42.4708005Z github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func2()
2024-02-05T10:58:42.4708527Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:108 +0x56
2024-02-05T10:58:42.4709130Z created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 835
2024-02-05T10:58:42.4709741Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:106 +0x4c6
2024-02-05T10:58:42.4710056Z 
2024-02-05T10:58:42.4710147Z goroutine 897 [select]:
2024-02-05T10:58:42.4710816Z github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader(0xc00082ee58, {0x1e8a2e8?, 0xc001f90320?}, 0xc000b5bcf0)
2024-02-05T10:58:42.4711642Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:126 +0x37d
2024-02-05T10:58:42.4712238Z github.com/canonical/lxd/lxd/events.(*listenerCommon).start(0xc00066b4a0)
2024-02-05T10:58:42.4712805Z 	/home/runner/work/lxd/lxd/lxd/events/common.go:36 +0x1f5
2024-02-05T10:58:42.4713589Z created by github.com/canonical/lxd/lxd/events.(*Server).AddListener in goroutine 863
2024-02-05T10:58:42.4714200Z 	/home/runner/work/lxd/lxd/lxd/events/events.go:97 +0x47b
2024-02-05T10:58:42.4714511Z 
2024-02-05T10:58:42.4714605Z goroutine 949 [IO wait]:
2024-02-05T10:58:42.4714939Z internal/poll.runtime_pollWait(0x7fd8506a74c0, 0x72)
2024-02-05T10:58:42.4715391Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4715891Z internal/poll.(*pollDesc).wait(0xc00097f700?, 0xc0003a8a00?, 0x0)
2024-02-05T10:58:42.4716466Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4717193Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4717650Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4718200Z internal/poll.(*FD).Read(0xc00097f700, {0xc0003a8a00, 0x500, 0x500})
2024-02-05T10:58:42.4718735Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4719325Z net.(*netFD).Read(0xc00097f700, {0xc0003a8a00?, 0x7fd8504e4628?, 0xc000b5db18?})
2024-02-05T10:58:42.4719851Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4720343Z net.(*conn).Read(0xc0000725b8, {0xc0003a8a00?, 0xc00071a9a0?, 0x41ac3b?})
2024-02-05T10:58:42.4720821Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4721342Z crypto/tls.(*atLeastReader).Read(0xc000b5db18, {0xc0003a8a00?, 0x0?, 0xc000b5db18?})
2024-02-05T10:58:42.4721915Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4722410Z bytes.(*Buffer).ReadFrom(0xc000bac2b0, {0x1e7b2e0, 0xc000b5db18})
2024-02-05T10:58:42.4722895Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4723457Z crypto/tls.(*Conn).readFromUntil(0xc000bac008, {0x1e79c40, 0xc0000725b8}, 0xc00071a9e8?)
2024-02-05T10:58:42.4724036Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4724493Z crypto/tls.(*Conn).readRecordOrCCS(0xc000bac008, 0x0)
2024-02-05T10:58:42.4724961Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4725356Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4725718Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4726226Z crypto/tls.(*Conn).Read(0xc000bac008, {0xc00095e000, 0x1000, 0x10000000090?})
2024-02-05T10:58:42.4726765Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4727174Z bufio.(*Reader).fill(0xc000956600)
2024-02-05T10:58:42.4727541Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:110 +0x103
2024-02-05T10:58:42.4727928Z bufio.(*Reader).Peek(0xc000956600, 0x2)
2024-02-05T10:58:42.4728303Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:148 +0x53
2024-02-05T10:58:42.4728822Z github.com/gorilla/websocket.(*Conn).read(0xc000b969a0, 0xc00095e006?)
2024-02-05T10:58:42.4729663Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
2024-02-05T10:58:42.4730270Z github.com/gorilla/websocket.(*Conn).advanceFrame(0xc000b969a0)
2024-02-05T10:58:42.4730877Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
2024-02-05T10:58:42.4731472Z github.com/gorilla/websocket.(*Conn).NextReader(0xc000b969a0)
2024-02-05T10:58:42.4732073Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
2024-02-05T10:58:42.4732752Z github.com/gorilla/websocket.(*Conn).ReadJSON(0x0?, {0x187f4e0, 0xc001eacf00})
2024-02-05T10:58:42.4733412Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/json.go:50 +0x25
2024-02-05T10:58:42.4734093Z github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
2024-02-05T10:58:42.4734728Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:88 +0xc9
2024-02-05T10:58:42.4735435Z created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 897
2024-02-05T10:58:42.4736150Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1eb
2024-02-05T10:58:42.4736482Z 
2024-02-05T10:58:42.4736739Z goroutine 868 [select]:
2024-02-05T10:58:42.4737131Z github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
2024-02-05T10:58:42.4737658Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb4
2024-02-05T10:58:42.4738281Z created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 657
2024-02-05T10:58:42.4738897Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x448
2024-02-05T10:58:42.4739201Z 
2024-02-05T10:58:42.4739300Z goroutine 934 [IO wait]:
2024-02-05T10:58:42.4739629Z internal/poll.runtime_pollWait(0x7fd850381758, 0x72)
2024-02-05T10:58:42.4740085Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4740718Z internal/poll.(*pollDesc).wait(0xc00097f980?, 0xc001f20000?, 0x0)
2024-02-05T10:58:42.4741305Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4741804Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4742254Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4742820Z internal/poll.(*FD).Read(0xc00097f980, {0xc001f20000, 0x500, 0x500})
2024-02-05T10:58:42.4743363Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4743936Z net.(*netFD).Read(0xc00097f980, {0xc001f20000?, 0x7fd8504c1a88?, 0xc00082edb0?})
2024-02-05T10:58:42.4744473Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4744966Z net.(*conn).Read(0xc0000725f8, {0xc001f20000?, 0xc0003059a0?, 0x41ac3b?})
2024-02-05T10:58:42.4745449Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4745984Z crypto/tls.(*atLeastReader).Read(0xc00082edb0, {0xc001f20000?, 0x0?, 0xc00082edb0?})
2024-02-05T10:58:42.4746549Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:806 +0x3b
2024-02-05T10:58:42.4747070Z bytes.(*Buffer).ReadFrom(0xc000bac630, {0x1e7b2e0, 0xc00082edb0})
2024-02-05T10:58:42.4747554Z 	/home/runner/sdk/gotip/src/bytes/buffer.go:211 +0x98
2024-02-05T10:58:42.4748111Z crypto/tls.(*Conn).readFromUntil(0xc000bac388, {0x1e79c40, 0xc0000725f8}, 0xc0003059e8?)
2024-02-05T10:58:42.4748699Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:828 +0xde
2024-02-05T10:58:42.4749156Z crypto/tls.(*Conn).readRecordOrCCS(0xc000bac388, 0x0)
2024-02-05T10:58:42.4749606Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:626 +0x3cf
2024-02-05T10:58:42.4750012Z crypto/tls.(*Conn).readRecord(...)
2024-02-05T10:58:42.4750372Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:588
2024-02-05T10:58:42.4750899Z crypto/tls.(*Conn).Read(0xc000bac388, {0xc000263000, 0x1000, 0x10001c000b3b538?})
2024-02-05T10:58:42.4751461Z 	/home/runner/sdk/gotip/src/crypto/tls/conn.go:1370 +0x156
2024-02-05T10:58:42.4751873Z bufio.(*Reader).fill(0xc0007ca120)
2024-02-05T10:58:42.4752245Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:110 +0x103
2024-02-05T10:58:42.4752639Z bufio.(*Reader).Peek(0xc0007ca120, 0x2)
2024-02-05T10:58:42.4753026Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:148 +0x53
2024-02-05T10:58:42.4753544Z github.com/gorilla/websocket.(*Conn).read(0xc000b96b00, 0xc000263006?)
2024-02-05T10:58:42.4754181Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:378 +0x26
2024-02-05T10:58:42.4754796Z github.com/gorilla/websocket.(*Conn).advanceFrame(0xc000b96b00)
2024-02-05T10:58:42.4755409Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:824 +0x6d
2024-02-05T10:58:42.4755994Z github.com/gorilla/websocket.(*Conn).NextReader(0xc000b96b00)
2024-02-05T10:58:42.4756598Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1034 +0x13e
2024-02-05T10:58:42.4757306Z github.com/gorilla/websocket.(*Conn).ReadJSON(0xc000b3b7d0?, {0x187f4e0, 0xc001f82720})
2024-02-05T10:58:42.4757998Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/json.go:50 +0x25
2024-02-05T10:58:42.4758695Z github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader.func3()
2024-02-05T10:58:42.4759324Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:88 +0xc9
2024-02-05T10:58:42.4760037Z created by github.com/canonical/lxd/lxd/events.(*websockListenerConnection).Reader in goroutine 931
2024-02-05T10:58:42.4760884Z 	/home/runner/work/lxd/lxd/lxd/events/connections.go:82 +0x1eb
2024-02-05T10:58:42.4761225Z 
2024-02-05T10:58:42.4761315Z goroutine 818 [select]:
2024-02-05T10:58:42.4761704Z github.com/canonical/lxd/client.(*ProtocolLXD).getEvents.func1()
2024-02-05T10:58:42.4762219Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:78 +0xb4
2024-02-05T10:58:42.4762829Z created by github.com/canonical/lxd/client.(*ProtocolLXD).getEvents in goroutine 835
2024-02-05T10:58:42.4763438Z 	/home/runner/work/lxd/lxd/client/lxd_events.go:76 +0x448
2024-02-05T10:58:42.4763739Z 
2024-02-05T10:58:42.4763836Z goroutine 1140 [runnable]:
2024-02-05T10:58:42.4764251Z syscall.Syscall(0x0, 0x19, 0xc0008dc6a0, 0x8)
2024-02-05T10:58:42.4764712Z 	/home/runner/sdk/gotip/src/syscall/syscall_linux.go:69 +0x25
2024-02-05T10:58:42.4765264Z syscall.read(0xc0009e2f00?, {0xc0008dc6a0?, 0x531100?, 0x19?})
2024-02-05T10:58:42.4765815Z 	/home/runner/sdk/gotip/src/syscall/zsyscall_linux_amd64.go:736 +0x38
2024-02-05T10:58:42.4766278Z syscall.Read(...)
2024-02-05T10:58:42.4766600Z 	/home/runner/sdk/gotip/src/syscall/syscall_unix.go:181
2024-02-05T10:58:42.4767001Z internal/poll.ignoringEINTRIO(...)
2024-02-05T10:58:42.4767397Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:736
2024-02-05T10:58:42.4767891Z internal/poll.(*FD).Read(0xc0009e2f00, {0xc0008dc6a0, 0x8, 0x8})
2024-02-05T10:58:42.4768420Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:160 +0x2ae
2024-02-05T10:58:42.4769391Z net.(*netFD).Read(0xc0009e2f00, {0xc0008dc6a0?, 0xc001f04dc0?, 0x5f1a65?})
2024-02-05T10:58:42.4770052Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4770568Z net.(*conn).Read(0xc0007165a8, {0xc0008dc6a0?, 0xc001f04e08?, 0x605125?})
2024-02-05T10:58:42.4771067Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4771863Z github.com/canonical/go-dqlite/internal/protocol.(*Protocol).recvFill(0xc001ed4340, {0xc0008dc6a0, 0x8, 0x8})
2024-02-05T10:58:42.4772886Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/internal/protocol/protocol.go:226 +0x64
2024-02-05T10:58:42.4773899Z github.com/canonical/go-dqlite/internal/protocol.(*Protocol).recvPeek(0xc001ed4340, {0xc0008dc6a0, 0x8, 0x8})
2024-02-05T10:58:42.4774894Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/internal/protocol/protocol.go:210 +0x7d
2024-02-05T10:58:42.4775859Z github.com/canonical/go-dqlite/internal/protocol.(*Protocol).recvHeader(0xc0007165a8?, 0xc001f04ff8)
2024-02-05T10:58:42.4776809Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/internal/protocol/protocol.go:177 +0x2b
2024-02-05T10:58:42.4777741Z github.com/canonical/go-dqlite/internal/protocol.(*Protocol).recv(0xc001ed4340, 0xc001f04ff8)
2024-02-05T10:58:42.4778682Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/internal/protocol/protocol.go:165 +0x5e
2024-02-05T10:58:42.4779810Z github.com/canonical/go-dqlite/internal/protocol.(*Protocol).Call(0xc001ed4340, {0x1e8a358?, 0xc000248b60?}, 0xc001f05038, 0xc001f04ff8)
2024-02-05T10:58:42.4780897Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/internal/protocol/protocol.go:70 +0x4c5
2024-02-05T10:58:42.4781821Z github.com/canonical/go-dqlite/client.(*Client).Leader(0xc0007165b0, {0x1e8a358, 0xc000248b60})
2024-02-05T10:58:42.4782681Z 	/home/runner/go/pkg/mod/github.com/canonical/[email protected]/client/client.go:76 +0x1f5
2024-02-05T10:58:42.4783402Z github.com/canonical/lxd/lxd/cluster.(*Gateway).LeaderAddress(0xc00057a380)
2024-02-05T10:58:42.4784004Z 	/home/runner/work/lxd/lxd/lxd/cluster/gateway.go:633 +0xb85
2024-02-05T10:58:42.4784488Z main.handoverMemberRole(0xc00057b6c0, 0xc00057a380)
2024-02-05T10:58:42.4784947Z 	/home/runner/work/lxd/lxd/lxd/api_cluster.go:2536 +0x11c
2024-02-05T10:58:42.4785513Z main.(*Daemon).Stop(0xc00045a000, {0x1e8a2e8, 0xc00049bcc0}, {0x1e82068, 0x1e74430})
2024-02-05T10:58:42.4786063Z 	/home/runner/work/lxd/lxd/lxd/daemon.go:1586 +0x1ce
2024-02-05T10:58:42.4786697Z main.internalShutdown.func1({0x1e86ed0, 0xc00057b500})
2024-02-05T10:58:42.4787172Z 	/home/runner/work/lxd/lxd/lxd/api_internal.go:254 +0x99
2024-02-05T10:58:42.4787852Z github.com/canonical/lxd/lxd/response.(*manualResponse).Render(0x1c5e528?, {0x1e86ed0?, 0xc00057b500?})
2024-02-05T10:58:42.4788562Z 	/home/runner/work/lxd/lxd/lxd/response/response.go:553 +0x26
2024-02-05T10:58:42.4789107Z main.(*Daemon).createCmd.func1({0x1e86ed0, 0xc00057b500}, 0xc001ef07e0)
2024-02-05T10:58:42.4789608Z 	/home/runner/work/lxd/lxd/lxd/daemon.go:640 +0x167e
2024-02-05T10:58:42.4790166Z net/http.HandlerFunc.ServeHTTP(0xc001ef0120?, {0x1e86ed0?, 0xc00057b500?}, 0x10?)
2024-02-05T10:58:42.4790746Z 	/home/runner/sdk/gotip/src/net/http/server.go:2166 +0x29
2024-02-05T10:58:42.4791508Z github.com/gorilla/mux.(*Router).ServeHTTP(0xc0000000c0, {0x1e86ed0, 0xc00057b500}, 0xc001ef0000)
2024-02-05T10:58:42.4792223Z 	/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/mux.go:212 +0x1e2
2024-02-05T10:58:42.4792871Z main.(*lxdHttpServer).ServeHTTP(0xc0006762d0, {0x1e86ed0, 0xc00057b500}, 0xc001ef0000)
2024-02-05T10:58:42.4793418Z 	/home/runner/work/lxd/lxd/lxd/api.go:185 +0xd0
2024-02-05T10:58:42.4793960Z net/http.serverHandler.ServeHTTP({0xc001eb7dd0?}, {0x1e86ed0?, 0xc00057b500?}, 0x6?)
2024-02-05T10:58:42.4794553Z 	/home/runner/sdk/gotip/src/net/http/server.go:3137 +0x8e
2024-02-05T10:58:42.4795047Z net/http.(*conn).serve(0xc001f28a20, {0x1e8a2b0, 0xc001f35020})
2024-02-05T10:58:42.4795552Z 	/home/runner/sdk/gotip/src/net/http/server.go:2039 +0x5e8
2024-02-05T10:58:42.4796017Z created by net/http.(*Server).Serve in goroutine 22
2024-02-05T10:58:42.4796475Z 	/home/runner/sdk/gotip/src/net/http/server.go:3285 +0x4b4
2024-02-05T10:58:42.4796787Z 
2024-02-05T10:58:42.4796886Z goroutine 1171 [IO wait]:
2024-02-05T10:58:42.4797224Z internal/poll.runtime_pollWait(0x7fd8506a7b88, 0x72)
2024-02-05T10:58:42.4797678Z 	/home/runner/sdk/gotip/src/runtime/netpoll.go:345 +0x85
2024-02-05T10:58:42.4798179Z internal/poll.(*pollDesc).wait(0xc000a43b80?, 0xc001f4f000?, 0x0)
2024-02-05T10:58:42.4798762Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-02-05T10:58:42.4799243Z internal/poll.(*pollDesc).waitRead(...)
2024-02-05T10:58:42.4799693Z 	/home/runner/sdk/gotip/src/internal/poll/fd_poll_runtime.go:89
2024-02-05T10:58:42.4800253Z internal/poll.(*FD).Read(0xc000a43b80, {0xc001f4f000, 0x1000, 0x1000})
2024-02-05T10:58:42.4800804Z 	/home/runner/sdk/gotip/src/internal/poll/fd_unix.go:164 +0x27a
2024-02-05T10:58:42.4801393Z net.(*netFD).Read(0xc000a43b80, {0xc001f4f000?, 0xf5287a?, 0xc1684db9d9a8b5cb?})
2024-02-05T10:58:42.4801927Z 	/home/runner/sdk/gotip/src/net/fd_posix.go:55 +0x25
2024-02-05T10:58:42.4802419Z net.(*conn).Read(0xc0004aa810, {0xc001f4f000?, 0xf52900?, 0xc0004b7b08?})
2024-02-05T10:58:42.4802905Z 	/home/runner/sdk/gotip/src/net/net.go:179 +0x45
2024-02-05T10:58:42.4803369Z bufio.(*Reader).Read(0xc001f4b080, {0xc001eb7de1, 0x1, 0x1b2b079?})
2024-02-05T10:58:42.4803864Z 	/home/runner/sdk/gotip/src/bufio/bufio.go:241 +0x197
2024-02-05T10:58:42.4804436Z github.com/canonical/lxd/lxd/endpoints/listeners.BufferedUnixConn.Read(...)
2024-02-05T10:58:42.4805083Z 	/home/runner/work/lxd/lxd/lxd/endpoints/listeners/starttls.go:94
2024-02-05T10:58:42.4805590Z net/http.(*connReader).backgroundRead(0xc001eb7dd0)
2024-02-05T10:58:42.4806036Z 	/home/runner/sdk/gotip/src/net/http/server.go:681 +0x37
2024-02-05T10:58:42.4806577Z created by net/http.(*connReader).startBackgroundRead in goroutine 1140
2024-02-05T10:58:42.4807117Z 	/home/runner/sdk/gotip/src/net/http/server.go:677 +0xba
2024-02-05T10:58:42.4997168Z Error: Put "http://unix.socket/internal/shutdown?force=false": EOF

[MggMuggins]

I suspect this is a result of Daemon.stopClusterTasks running into the situation described so nicely in this stackoverflow answer. Because we are storing the task.Group by value in Daemon, replacing it after the timeout re-initializes the Mutex, leaving it unlocked. If some task exceeds the timeout at just the right time, it will lock the old mutex and then unlock the new one, generating a panic.

[MggMuggins]

Confirmed it with this patch: https://gist.github.com/MggMuggins/11bd05305a780f552bfa930b8d7ef8ac.

[tomponline]

Thanks!