metadata
metadata copied to clipboard
Published
20 hours ago •
kubeflow
kubeflow
Concurrent requests cause the backend server to crash.
/kind bug
What steps did you take and what happened: I wrote a program to concurrently send 100+ request to log artifacts. Concurrent requests cause the server to crash.
Seems the MLMD library crashes the server if there are two concurrent transactions competing.
What did you expect to happen: The backend sever should be able to handle concurrent requests.
Anything else you would like to add: [Miscellaneous information that will assist in solving the issue.]
- Metadata version: v0.1
Detailed Go crash logs
019-09-12 16:25:11.276879: F external/google_ml_metadata/ml_metadata/metadata_store/metadata_source.cc:87] Non-OK-status: metadata_source->Begin() status: Failed precondition: Transaction already open.
SIGABRT: abort
PC=0x7fff76e5a2c6 m=12 sigcode=0
goroutine 0 [idle]:
runtime: unknown pc 0x7fff76e5a2c6
stack: frame={sp:0x700005ef87b8, fp:0x0} stack=[0x700005e792b0,0x700005ef8eb0)
0000700005ef86b8: 0000000000000000 0000000000000000
0000700005ef86c8: 0000000000000000 0000000000000000
0000700005ef86d8: 0000000000000000 0000000000000000
0000700005ef86e8: 0000000000000000 0000000000000000
0000700005ef86f8: 0000000000000000 0000000005c38000
0000700005ef8708: 0000000000000000 0000000008c00000
0000700005ef8718: 0000000000000001 0000000008c00070
0000700005ef8728: 000000c0002478c8 ffffffffffffffff
0000700005ef8738: 000000000000000a 0000000000000009
0000700005ef8748: 0000000000000080 0000700005ef8820
0000700005ef8758: 00000000049276fc 0000700000000057
0000700005ef8768: 0000000008c00260 00000046000000cb
0000700005ef8778: 00000000052a07a3 000000570004398f
0000700005ef8788: 0000700005ef87f0 00007fffad5a7240
0000700005ef8798: 00007fffad5a6ca0 00007fff76da3bb8
0000700005ef87a8: 0000000000000000 0000700005ef87f0
0000700005ef87b8: <00007fff76f15bf1 0000700005ef9000
0000700005ef87c8: 0000700005ef8808 ffffffffffffffff
0000700005ef87d8: 000000000000000a 0000000000000009
0000700005ef87e8: 0000000000000080 0000700005ef8820
0000700005ef87f8: 00007fff76dc46a6 394a51120f3000c1
0000700005ef8808: 00000000ffffffdf ffffffff05ef8840
0000700005ef8818: 000000c0002478c8 0000700005ef8840
0000700005ef8828: 00000000049281ca 0000000005375120
0000700005ef8838: 0000700005ef88a0 0000700005ef8860
0000700005ef8848: 00000000049281ec 0000000005f00300
0000700005ef8858: 0000700005ef88a0 0000700005ef8b00
0000700005ef8868: 00000000047c7e25 0000700005ef88a0
0000700005ef8878: 0000700005ef88a0 0000000005f00300
0000700005ef8888: 0000700005ef89b8 0000700005ef8ab8
0000700005ef8898: 0000000005ef88d8 00000000053750e8
0000700005ef88a8: 0000000005374ca8 00007fffaccefaf0
runtime: unknown pc 0x7fff76e5a2c6
stack: frame={sp:0x700005ef87b8, fp:0x0} stack=[0x700005e792b0,0x700005ef8eb0)
0000700005ef86b8: 0000000000000000 0000000000000000
0000700005ef86c8: 0000000000000000 0000000000000000
0000700005ef86d8: 0000000000000000 0000000000000000
0000700005ef86e8: 0000000000000000 0000000000000000
0000700005ef86f8: 0000000000000000 0000000005c38000
0000700005ef8708: 0000000000000000 0000000008c00000
0000700005ef8718: 0000000000000001 0000000008c00070
0000700005ef8728: 000000c0002478c8 ffffffffffffffff
0000700005ef8738: 000000000000000a 0000000000000009
0000700005ef8748: 0000000000000080 0000700005ef8820
0000700005ef8758: 00000000049276fc 0000700000000057
0000700005ef8768: 0000000008c00260 00000046000000cb
0000700005ef8778: 00000000052a07a3 000000570004398f
0000700005ef8788: 0000700005ef87f0 00007fffad5a7240
0000700005ef8798: 00007fffad5a6ca0 00007fff76da3bb8
0000700005ef87a8: 0000000000000000 0000700005ef87f0
0000700005ef87b8: <00007fff76f15bf1 0000700005ef9000
0000700005ef87c8: 0000700005ef8808 ffffffffffffffff
0000700005ef87d8: 000000000000000a 0000000000000009
0000700005ef87e8: 0000000000000080 0000700005ef8820
0000700005ef87f8: 00007fff76dc46a6 394a51120f3000c1
0000700005ef8808: 00000000ffffffdf ffffffff05ef8840
0000700005ef8818: 000000c0002478c8 0000700005ef8840
0000700005ef8828: 00000000049281ca 0000000005375120
0000700005ef8838: 0000700005ef88a0 0000700005ef8860
0000700005ef8848: 00000000049281ec 0000000005f00300
0000700005ef8858: 0000700005ef88a0 0000700005ef8b00
0000700005ef8868: 00000000047c7e25 0000700005ef88a0
0000700005ef8878: 0000700005ef88a0 0000000005f00300
0000700005ef8888: 0000700005ef89b8 0000700005ef8ab8
0000700005ef8898: 0000000005ef88d8 00000000053750e8
0000700005ef88a8: 0000000005374ca8 00007fffaccefaf0
goroutine 101 [syscall]:
runtime.cgocall(0x4540090, 0xc0002478c8, 0x0)
GOROOT/src/runtime/cgocall.go:128 +0x5b fp=0xc000247880 sp=0xc000247848 pc=0x40058eb
ml_metadata/metadata_store/metadata_store_go_wrap._Cfunc__wrap_GetArtifactType_metadata_store_go_wrap_178937986352d5b7(0x8e002d0, 0xc00039a240, 0x33, 0x5f00290, 0x0, 0x0)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/metadata_store/darwin_amd64_stripped/metadata_store_go_wrap%darwin_amd64%cgo_codegen%/_cgo_gotypes.go:347 +0x4d fp=0xc0002478c8 sp=0xc000247880 pc=0x434b2dd
ml_metadata/metadata_store/metadata_store_go_wrap.GetArtifactType(0x4e2c2c0, 0xc0000390c0, 0xc00039a240, 0x33, 0x4e3d640, 0xc0004086d0, 0x0, 0x1dc)
metadata_store_go_wrap.go:199 +0xa2 fp=0xc000247940 sp=0xc0002478c8 pc=0x4349202
ml_metadata/metadata_store/mlmetadata.(*Store).callMetadataStoreWrapMethod(0xc00007cdc0, 0x4d88ce0, 0x4e38b40, 0xc000417020, 0x4e38b80, 0xc000417050, 0x0, 0x0)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/metadata_store/darwin_amd64_stripped/metadata_store_go%darwin_amd64%cgo_codegen%/metadata_store.cgo1.go:361 +0x147 fp=0xc0002479d0 sp=0xc000247940 pc=0x4461bc7
ml_metadata/metadata_store/mlmetadata.(*Store).GetArtifactType(0xc00007cdc0, 0xc00039a1cf, 0x31, 0xc000247a01, 0x43a9774, 0xc000524000)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/metadata_store/darwin_amd64_stripped/metadata_store_go%darwin_amd64%cgo_codegen%/metadata_store.cgo1.go:111 +0xe4 fp=0xc000247a38 sp=0xc0002479d0 pc=0x44601c4
github.com/kubeflow/metadata/service.(*Service).getArtifactType(0xc00007cdd0, 0xc00039a1c0, 0x40, 0x0, 0xc000247b60, 0x44451ce)
service/service.go:136 +0x78 fp=0xc000247a78 sp=0xc000247a38 pc=0x44f8908
github.com/kubeflow/metadata/service.(*Service).CreateArtifact(0xc00007cdd0, 0x4e3b240, 0xc000416f60, 0xc0000d6980, 0xc00007cdd0, 0xc000416f60, 0xc000523bd0)
service/service.go:251 +0xde fp=0xc000247b70 sp=0xc000247a78 pc=0x44f963e
github.com/kubeflow/metadata/api._MetadataService_CreateArtifact_Handler(0x4d46560, 0xc00007cdd0, 0x4e3b240, 0xc000416f60, 0xc000178cd0, 0x0, 0x4e3b240, 0xc000416f60, 0xc000524000, 0x1dc)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.go:1796 +0x23e fp=0xc000247be0 sp=0xc000247b70 pc=0x44e1bde
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300700, 0xc0004167b0, 0x53bce60, 0x0, 0x0, 0x0)
external/org_golang_google_grpc/server.go:971 +0x470 fp=0xc000247e78 sp=0xc000247be0 pc=0x4435900
google.golang.org/grpc.(*Server).handleStream(0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300700, 0x0)
external/org_golang_google_grpc/server.go:1250 +0xd25 fp=0xc000247f80 sp=0xc000247e78 pc=0x4439775
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0004c8020, 0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300700)
external/org_golang_google_grpc/server.go:690 +0x9f fp=0xc000247fb8 sp=0xc000247f80 pc=0x4444a2f
runtime.goexit()
GOROOT/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc000247fc0 sp=0xc000247fb8 pc=0x405a561
created by google.golang.org/grpc.(*Server).serveStreams.func1
external/org_golang_google_grpc/server.go:688 +0xa1
goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0d08, 0x72, 0x0)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc0003d9918, 0x72, 0x0, 0x0, 0x4d5dcf4)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0003d9900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc0003d9900, 0xc00049fc80, 0xc07b60f49c, 0xc0004ac000)
GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00017a008, 0x944f7b60f49c, 0x100000001, 0xc00049fcc0)
GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc00017a008, 0xc00049fce8, 0x40af7e6, 0x5d7ad3d7)
GOROOT/src/net/tcpsock.go:247 +0x48
net/http.tcpKeepAliveListener.Accept(0xc00017a008, 0xc00049fd38, 0x18, 0xc000000180, 0x42bdf44)
GOROOT/src/net/http/server.go:3264 +0x2f
net/http.(*Server).Serve(0xc00010b860, 0x4e3a500, 0xc00017a008, 0x0, 0x0)
GOROOT/src/net/http/server.go:2859 +0x22d
net/http.(*Server).ListenAndServe(0xc00010b860, 0xc00010b860, 0x4d6b371)
GOROOT/src/net/http/server.go:2797 +0xe4
net/http.ListenAndServe(...)
GOROOT/src/net/http/server.go:3037
main.main()
server/main.go:133 +0x61f
goroutine 19 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0x53d06c0)
external/com_github_golang_glog/glog.go:882 +0x8b
created by github.com/golang/glog.init.0
external/com_github_golang_glog/glog.go:410 +0x272
goroutine 26 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0dd8, 0x72, 0x0)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00046c018, 0x72, 0x0, 0x0, 0x4d5dcf4)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00046c000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc00046c000, 0xc00017a001, 0x0, 0x0)
GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000f4438, 0xc000075e30, 0xc000075e38, 0x18)
GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0000f4438, 0x4d887e8, 0xc0004bc600, 0x4e3ffa0, 0xc00017a018)
GOROOT/src/net/tcpsock.go:260 +0x48
google.golang.org/grpc.(*Server).Serve(0xc0004bc600, 0x4e39dc0, 0xc0000f4438, 0x0, 0x0)
external/org_golang_google_grpc/server.go:561 +0x1e9
main.main.func1(0xc000408628, 0x5, 0xc0004bc600)
server/main.go:119 +0x18f
created by main.main
server/main.go:113 +0x395
goroutine 27 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000d6780)
external/org_golang_google_grpc/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
external/org_golang_google_grpc/balancer_conn_wrappers.go:113 +0x14f
goroutine 28 [chan receive]:
github.com/kubeflow/metadata/api.RegisterMetadataServiceHandlerFromEndpoint.func1.1(0x4e3b180, 0xc00008eac0, 0xc00038e000, 0xc000408628, 0x5)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:577 +0x4c
created by github.com/kubeflow/metadata/api.RegisterMetadataServiceHandlerFromEndpoint.func1
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:576 +0x1b5
goroutine 8 [chan receive]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0001d6000)
external/org_golang_google_grpc/clientconn.go:1066 +0x9bb
created by google.golang.org/grpc.(*addrConn).connect
external/org_golang_google_grpc/clientconn.go:684 +0xb6
goroutine 51 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0000d67c0, 0x1, 0x0, 0x0, 0x0, 0x0)
external/org_golang_google_grpc/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000f6c00, 0x0, 0x0)
external/org_golang_google_grpc/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000204700)
external/org_golang_google_grpc/internal/transport/http2_client.go:330 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
external/org_golang_google_grpc/internal/transport/http2_client.go:328 +0xec1
goroutine 30 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0ea8, 0x72, 0xffffffffffffffff)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00044c018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00044c000, 0xc000400000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:169 +0x1f2
net.(*netFD).Read(0xc00044c000, 0xc000400000, 0x8000, 0x8000, 0x0, 0x800000601, 0x0)
GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0000f4440, 0xc000400000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
GOROOT/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc0000e8840, 0xc0002dc578, 0x9, 0x9, 0x400e169, 0xc0004e45c0, 0x20)
GOROOT/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x4e2a440, 0xc0000e8840, 0xc0002dc578, 0x9, 0x9, 0x9, 0x0, 0x0, 0x9)
GOROOT/src/io/io.go:310 +0x88
io.ReadFull(...)
GOROOT/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0002dc578, 0x9, 0x9, 0x4e2a440, 0xc0000e8840, 0x0, 0xc000000000, 0xc0004e45c0, 0x10)
external/org_golang_x_net/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0002dc540, 0xc0001102e0, 0xc0001102e0, 0x0, 0x0)
external/org_golang_x_net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000204700)
external/org_golang_google_grpc/internal/transport/http2_client.go:1223 +0x168
created by google.golang.org/grpc/internal/transport.newHTTP2Client
external/org_golang_google_grpc/internal/transport/http2_client.go:286 +0xd24
goroutine 39 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00008f380, 0x1, 0x0, 0x0, 0x0, 0x0)
external/org_golang_google_grpc/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000f6d20, 0x0, 0x0)
external/org_golang_google_grpc/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0004d0780)
external/org_golang_google_grpc/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
external/org_golang_google_grpc/internal/transport/http2_server.go:273 +0xfba
goroutine 40 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0004d0780)
external/org_golang_google_grpc/internal/transport/http2_server.go:921 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
external/org_golang_google_grpc/internal/transport/http2_server.go:282 +0xfdf
goroutine 41 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0c38, 0x72, 0xffffffffffffffff)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00044c098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00044c080, 0xc000500000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:169 +0x1f2
net.(*netFD).Read(0xc00044c080, 0xc000500000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00017a018, 0xc000500000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
GOROOT/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc0004eaa20, 0xc0003de118, 0x9, 0x9, 0xc087c5cc13, 0xc0004ba000, 0xc000215d88)
GOROOT/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x4e2a440, 0xc0004eaa20, 0xc0003de118, 0x9, 0x9, 0x9, 0xbf56d2d5d07d4488, 0x1734812a8, 0x53cf880)
GOROOT/src/io/io.go:310 +0x88
io.ReadFull(...)
GOROOT/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0003de118, 0x9, 0x9, 0x4e2a440, 0xc0004eaa20, 0x0, 0x0, 0xc00026e910, 0x7070e0910100402)
external/org_golang_x_net/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003de0e0, 0xc0000d24c0, 0xc0000d24c0, 0x0, 0x0)
external/org_golang_x_net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0004d0780, 0xc000105470, 0x4d88840)
external/org_golang_google_grpc/internal/transport/http2_server.go:429 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc0004bc600, 0x4e40e40, 0xc0004d0780)
external/org_golang_google_grpc/server.go:686 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0004bc600, 0x4e40e40, 0xc0004d0780)
external/org_golang_google_grpc/server.go:648 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
external/org_golang_google_grpc/server.go:647 +0x583
goroutine 31 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000300200, 0x8, 0xc0002ee370)
external/org_golang_google_grpc/internal/transport/transport.go:267 +0xcc
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
external/org_golang_google_grpc/internal/transport/transport.go:278
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0002ee370, 0x4d0ad00, 0xc0000c51a0, 0x0, 0xc00023d508, 0x0)
external/org_golang_google_grpc/stream.go:847 +0x70a
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0002ee370, 0x0, 0x0)
external/org_golang_google_grpc/stream.go:712 +0x46
google.golang.org/grpc.(*clientStream).withRetry(0xc00022a120, 0xc00023d510, 0xc00023d500, 0xc000233180, 0x0)
external/org_golang_google_grpc/stream.go:568 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00022a120, 0x4d0ad00, 0xc0000c51a0, 0x0, 0x0)
external/org_golang_google_grpc/stream.go:711 +0x100
google.golang.org/grpc.invoke(0x4e3b240, 0xc0000c4f30, 0x4d7113d, 0x23, 0x4d17c00, 0xc000310600, 0x4d0ad00, 0xc0000c51a0, 0xc00038e000, 0xc0000cc5a0, ...)
external/org_golang_google_grpc/call.go:73 +0x141
google.golang.org/grpc.(*ClientConn).Invoke(0xc00038e000, 0x4e3b240, 0xc0000c4f30, 0x4d7113d, 0x23, 0x4d17c00, 0xc000310600, 0x4d0ad00, 0xc0000c51a0, 0xc0000cc5a0, ...)
external/org_golang_google_grpc/call.go:37 +0x1b4
github.com/kubeflow/metadata/api.(*metadataServiceClient).CreateArtifact(0xc0000e6030, 0x4e3b240, 0xc0000c4f30, 0xc000310600, 0xc0000cc5a0, 0x2, 0x2, 0xc0000c4e40, 0xc0002ec6e0, 0xc0002ec6e0)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.go:1584 +0xd2
github.com/kubeflow/metadata/api.request_MetadataService_CreateArtifact_0(0x4e3b240, 0xc0000c4f30, 0x4e3cd00, 0x53b7080, 0x4e45480, 0xc0000e6030, 0xc000300000, 0xc0000c4ea0, 0x2, 0x4, ...)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:61 +0x3a8
github.com/kubeflow/metadata/api.RegisterMetadataServiceHandlerClient.func1(0x4e3a080, 0xc000226000, 0xc000300000, 0xc0000c4ea0)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:609 +0x1c4
github.com/grpc-ecosystem/grpc-gateway/runtime.(*ServeMux).ServeHTTP(0xc0000e85a0, 0x4e3a080, 0xc000226000, 0xc000300000)
external/com_github_grpc_ecosystem_grpc_gateway/runtime/mux.go:206 +0xd78
net/http.serverHandler.ServeHTTP(0xc00010b860, 0x4e3a080, 0xc000226000, 0xc000300000)
GOROOT/src/net/http/server.go:2774 +0xa8
net/http.(*conn).serve(0xc0002d63c0, 0x4e3b180, 0xc0004540c0)
GOROOT/src/net/http/server.go:1878 +0x851
created by net/http.(*Server).Serve
GOROOT/src/net/http/server.go:2884 +0x2f4
goroutine 82 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000300600, 0x8, 0xc0002ee420)
external/org_golang_google_grpc/internal/transport/transport.go:267 +0xcc
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
external/org_golang_google_grpc/internal/transport/transport.go:278
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0002ee420, 0x4d0ad00, 0xc0000c5950, 0x0, 0xc000263508, 0x0)
external/org_golang_google_grpc/stream.go:847 +0x70a
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0002ee420, 0x0, 0x0)
external/org_golang_google_grpc/stream.go:712 +0x46
google.golang.org/grpc.(*clientStream).withRetry(0xc00022a480, 0xc000263510, 0xc000263500, 0xc00051a000, 0x0)
external/org_golang_google_grpc/stream.go:568 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00022a480, 0x4d0ad00, 0xc0000c5950, 0x0, 0x0)
external/org_golang_google_grpc/stream.go:711 +0x100
google.golang.org/grpc.invoke(0x4e3b240, 0xc0000c57d0, 0x4d7113d, 0x23, 0x4d17c00, 0xc000310f80, 0x4d0ad00, 0xc0000c5950, 0xc00038e000, 0xc0000cca60, ...)
external/org_golang_google_grpc/call.go:73 +0x141
google.golang.org/grpc.(*ClientConn).Invoke(0xc00038e000, 0x4e3b240, 0xc0000c57d0, 0x4d7113d, 0x23, 0x4d17c00, 0xc000310f80, 0x4d0ad00, 0xc0000c5950, 0xc0000cca60, ...)
external/org_golang_google_grpc/call.go:37 +0x1b4
github.com/kubeflow/metadata/api.(*metadataServiceClient).CreateArtifact(0xc0000e6030, 0x4e3b240, 0xc0000c57d0, 0xc000310f80, 0xc0000cca60, 0x2, 0x2, 0xc0000c56e0, 0xc0002ed1e0, 0xc0002ed1e0)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.go:1584 +0xd2
github.com/kubeflow/metadata/api.request_MetadataService_CreateArtifact_0(0x4e3b240, 0xc0000c57d0, 0x4e3cd00, 0x53b7080, 0x4e45480, 0xc0000e6030, 0xc000300400, 0xc0000c5740, 0x2, 0x4, ...)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:61 +0x3a8
github.com/kubeflow/metadata/api.RegisterMetadataServiceHandlerClient.func1(0x4e3a080, 0xc0002260e0, 0xc000300400, 0xc0000c5740)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.gw.go:609 +0x1c4
github.com/grpc-ecosystem/grpc-gateway/runtime.(*ServeMux).ServeHTTP(0xc0000e85a0, 0x4e3a080, 0xc0002260e0, 0xc000300400)
external/com_github_grpc_ecosystem_grpc_gateway/runtime/mux.go:206 +0xd78
net/http.serverHandler.ServeHTTP(0xc00010b860, 0x4e3a080, 0xc0002260e0, 0xc000300400)
GOROOT/src/net/http/server.go:2774 +0xa8
net/http.(*conn).serve(0xc0003b41e0, 0x4e3b180, 0xc000454940)
GOROOT/src/net/http/server.go:1878 +0x851
created by net/http.(*Server).Serve
GOROOT/src/net/http/server.go:2884 +0x2f4
goroutine 98 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0b68, 0x72, 0xffffffffffffffff)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00046c098, 0x72, 0x0, 0x1, 0xffffffffffffffff)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00046c080, 0xc0004e0101, 0x1, 0x1, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:169 +0x1f2
net.(*netFD).Read(0xc00046c080, 0xc0004e0101, 0x1, 0x1, 0xc0004549d8, 0xc000353f68, 0x406debd)
GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0000e6038, 0xc0004e0101, 0x1, 0x1, 0x0, 0x0, 0x0)
GOROOT/src/net/net.go:177 +0x69
net/http.(*connReader).backgroundRead(0xc0004e00f0)
GOROOT/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
GOROOT/src/net/http/server.go:673 +0xca
goroutine 99 [runnable]:
reflect.(*rtype).Field(0x4cf6fc0, 0x0, 0xc00053a014, 0xc00024b2b8, 0x400e169, 0xc0004e45e0, 0x20, 0x18, 0x4cb9660, 0x0, ...)
GOROOT/src/reflect/type.go:924 +0xdb
github.com/golang/protobuf/proto.(*unmarshalInfo).computeUnmarshalInfo(0xc00053a000)
external/com_github_golang_protobuf/proto/table_unmarshal.go:290 +0x17f
github.com/golang/protobuf/proto.(*unmarshalInfo).unmarshal(0xc00053a000, 0xc000117d00, 0xc000536000, 0x2e8, 0x300, 0x400b266, 0x53ee588)
external/com_github_golang_protobuf/proto/table_unmarshal.go:136 +0xe07
github.com/golang/protobuf/proto.(*InternalMessageInfo).Unmarshal(0x53ce840, 0x4e38d80, 0xc000117d00, 0xc000536000, 0x2e8, 0x300, 0xc000536201, 0x9a2e008)
external/com_github_golang_protobuf/proto/table_unmarshal.go:63 +0x68
ml_metadata/proto/metadata_store_service_go_proto.(*GetArtifactsByIDResponse).XXX_Unmarshal(0xc000117d00, 0xc000536000, 0x2e8, 0x300, 0xc000117d00, 0xc000534001)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/proto/darwin_amd64_stripped/metadata_store_service_go_proto%/ml_metadata/proto/metadata_store_service_go_proto/metadata_store_service.pb.go:1373 +0x61
github.com/golang/protobuf/proto.Unmarshal(0xc000536000, 0x2e8, 0x300, 0x4e38d80, 0xc000117d00, 0x300, 0xc000534000)
external/com_github_golang_protobuf/proto/decode.go:337 +0x1bd
ml_metadata/metadata_store/mlmetadata.(*Store).callMetadataStoreWrapMethod(0xc00007cdc0, 0x4d88ce8, 0x4e38d40, 0xc000117cc0, 0x4e38d80, 0xc000117d00, 0x0, 0x0)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/metadata_store/darwin_amd64_stripped/metadata_store_go%darwin_amd64%cgo_codegen%/metadata_store.cgo1.go:365 +0x1d7
ml_metadata/metadata_store/mlmetadata.(*Store).GetArtifactsByID(0xc00007cdc0, 0xc000358dd8, 0x1, 0x1, 0x6, 0x0, 0x0, 0x0, 0x70)
bazel-out/darwin-fastbuild/bin/external/google_ml_metadata/ml_metadata/metadata_store/darwin_amd64_stripped/metadata_store_go%darwin_amd64%cgo_codegen%/metadata_store.cgo1.go:242 +0x106
github.com/kubeflow/metadata/service.(*Service).getStoredArtifact(0xc00007cdd0, 0xc0002e24c0, 0x32, 0x2, 0x2, 0xc0002e24c0)
service/service.go:229 +0x36d
github.com/kubeflow/metadata/service.(*Service).CreateArtifact(0xc00007cdd0, 0x4e3b240, 0xc000105620, 0xc0004e6140, 0xc00007cdd0, 0xc000105620, 0xc000079bd0)
service/service.go:268 +0x4fd
github.com/kubeflow/metadata/api._MetadataService_CreateArtifact_Handler(0x4d46560, 0xc00007cdd0, 0x4e3b240, 0xc000105620, 0xc0001c2550, 0x0, 0x4e3b240, 0xc000105620, 0xc000242380, 0x30a)
bazel-out/darwin-fastbuild/bin/api/darwin_amd64_stripped/api_go_proto%/github.com/kubeflow/metadata/api/service.pb.go:1796 +0x23e
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300300, 0xc0004167b0, 0x53bce60, 0x0, 0x0, 0x0)
external/org_golang_google_grpc/server.go:971 +0x470
google.golang.org/grpc.(*Server).handleStream(0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300300, 0x0)
external/org_golang_google_grpc/server.go:1250 +0xd25
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0004c8020, 0xc0004bc600, 0x4e40e40, 0xc0004d0780, 0xc000300300)
external/org_golang_google_grpc/server.go:690 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
external/org_golang_google_grpc/server.go:688 +0xa1
goroutine 100 [IO wait]:
internal/poll.runtime_pollWait(0x5cc0a98, 0x72, 0xffffffffffffffff)
GOROOT/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000170118, 0x72, 0x0, 0x1, 0xffffffffffffffff)
GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000170100, 0xc0004e0971, 0x1, 0x1, 0x0, 0x0, 0x0)
GOROOT/src/internal/poll/fd_unix.go:169 +0x1f2
net.(*netFD).Read(0xc000170100, 0xc0004e0971, 0x1, 0x1, 0x0, 0x0, 0x0)
GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0002ba028, 0xc0004e0971, 0x1, 0x1, 0x0, 0x0, 0x0)
GOROOT/src/net/net.go:177 +0x69
net/http.(*connReader).backgroundRead(0xc0004e0960)
GOROOT/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
GOROOT/src/net/http/server.go:673 +0xca
rax 0x0
rbx 0x700005ef9000
rcx 0x700005ef87b8
rdx 0x0
rdi 0x2303
rsi 0x6
rbp 0x700005ef87f0
rsp 0x700005ef87b8
r8 0x0
r9 0x0
r10 0x0
r11 0x206
r12 0x2303
r13 0xa
r14 0x6
r15 0x2d
rip 0x7fff76e5a2c6
rflags 0x206
cs 0x7
fs 0x0
gs 0x0
make: *** [run] Error 2
Issue-Label Bot is automatically applying the labels:
| Label | Probability |
|---|---|
| area/backend | 1.00 |
Please mark this comment with :thumbsup: or :thumbsdown: to give our bot feedback! Links: app homepage, dashboard and code for this bot.
![issue-label-bot[bot] avatar](https://avatars.githubusercontent.com/in/27079?v=4 "Published by a pub.dev verified publisher"){kind=small}