SIGSEV

Open krisdevopsbot opened this issue 1 year ago • 0 comments

Issue Description

goroutine 35983 [syscall]:
runtime.cgocall(0x5900642c14a0, 0xc002411958)
        /usr/lib/golang/src/runtime/cgocall.go:157 +0x4b fp=0xc002411930 sp=0xc0024118f8 pc=0x590063012d6b
github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools._Cfunc_subid_get_uid_ranges(0x79fd08039780, 0xc001e80048)
        _cgo_gotypes.go:123 +0x4f fp=0xc002411958 sp=0xc002411930 pc=0x5900633f7fcf
github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools.readSubid.func3(0xc000b67c50?, 0x4?)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go:55 +0x4a fp=0xc002411998 sp=0xc002411958 pc=0x5900633f880a
github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools.readSubid({0xc000b67c24, 0x7}, 0x1)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go:55 +0x1f8 fp=0xc002411ab8 sp=0xc002411998 pc=0x5900633f82d8
github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools.readSubuid(...)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go:82
github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools.NewIDMappings({0xc000b67c24, 0x7}, {0xc000b67c24, 0x7})
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/containers/storage/pkg/idtools/idtools.go:161 +0x45 fp=0xc002411b70 sp=0xc002411ab8 pc=0x5900633f6025
github.com/containers/podman/pkg/rootless.GetConfiguredMappings(0x1)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/rootless/rootless_linux.go:184 +0x105 fp=0xc002411c08 sp=0xc002411b70 pc=0x5900639580a5
github.com/containers/podman/pkg/util.GetKeepIDMapping(0xc00119e4e0)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/util/utils.go:247 +0x99 fp=0xc002411cd0 sp=0xc002411c08 pc=0x5900639717b9
github.com/containers/podman/pkg/specgen.SetupUserNS(0x590064aab280?, {{0xc00083a720, 0x7}, {0x0, 0x0}}, 0xc002411f98)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/specgen/namespaces.go:532 +0x72e fp=0xc002411da0 sp=0xc002411cd0 pc=0x590063c68b8e
github.com/containers/podman/pkg/specgen/generate.specConfigureNamespaces(0xc000996800, 0xc002411f98, 0xc00012e000, 0x0)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/specgen/generate/namespaces_linux.go:111 +0x81c fp=0xc002411e10 sp=0xc002411da0 pc=0x590064034fdc
github.com/containers/podman/pkg/specgen/generate.SpecGenToOCI({0x0?, 0x0?}, 0xc000996800, 0x0?, 0xc0005e3200, 0x0?, {0xc000005b00, 0x6, 0x6}, 0x0, ...)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/specgen/generate/oci_linux.go:298 +0x26c5 fp=0xc002412ae8 sp=0xc002411e10 pc=0x590064037f65
github.com/containers/podman/pkg/specgen/generate.MakeContainer({0x590064d33b68, 0xc0016b1890}, 0xc00012e000, 0xc000996800, 0x0, 0x0)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/specgen/generate/container_create.go:253 +0x1d78 fp=0xc002413150 sp=0xc002412ae8 pc=0x590064025598
github.com/containers/podman/pkg/api/handlers/libpod.CreateContainer({0x590064d32b50, 0xc002f8f620}, 0xc001ca3c00)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/api/handlers/libpod/containers_create.go:73 +0x4ab fp=0xc002413268 sp=0xc002413150 pc=0x59006423d50b
github.com/containers/podman/pkg/api/server.(*APIServer).apiWrapper(0xc0001d1600, 0x590064d1b378, {0x590064d32b50, 0xc002f8f620}, 0xc001ca3c00, 0x0)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/api/server/handler_api.go:64 +0xb39 fp=0xc0024135d0 sp=0xc002413268 pc=0x590064285859
github.com/containers/podman/pkg/api/server.(*APIServer).registerContainersHandlers.(*APIServer).APIHandler.func39({0x590064d32b50?, 0xc002f8f620?}, 0xc0016b1680?)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/api/server/handler_api.go:24 +0x37 fp=0xc002413610 sp=0xc0024135d0 pc=0x59006428ac97
net/http.HandlerFunc.ServeHTTP(0x5900643a10a6?, {0x590064d32b50?, 0xc002f8f620?}, 0xc00083a320?)
        /usr/lib/golang/src/net/http/server.go:2136 +0x29 fp=0xc002413638 sp=0xc002413610 pc=0x590063355e49
github.com/containers/podman/pkg/api/server.newServer.referenceIDHandler.func7.1({0x590064d32b50, 0xc002f8f620}, 0xc001ca3c00)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/api/server/handler_rid.go:38 +0x159 fp=0xc0024136b0 sp=0xc002413638 pc=0x59006429d579
net/http.HandlerFunc.ServeHTTP(0x590064d30b10?, {0x590064d32b50?, 0xc002f8f620?}, 0x590065b61060?)
        /usr/lib/golang/src/net/http/server.go:2136 +0x29 fp=0xc0024136d8 sp=0xc0024136b0 pc=0x590063355e49
github.com/containers/podman/vendor/github.com/gorilla/handlers.loggingHandler.ServeHTTP({{0x590064d23d48, 0x590065b5e620}, {0x590064d254c0, 0xc002285f98}, 0x590064d1b620}, {0x590064d30b10, 0xc00178c7e0}, 0xc001ca3c00)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/gorilla/handlers/logging.go:47 +0xef fp=0xc002413928 sp=0xc0024136d8 pc=0x59006420a38f
github.com/containers/podman/vendor/github.com/gorilla/handlers.(*loggingHandler).ServeHTTP(0xf8?, {0x590064d30b10?, 0xc00178c7e0?}, 0xc002413a10?)
        <autogenerated>:1 +0x7e fp=0xc0024139a0 sp=0xc002413928 pc=0x59006420be3e
github.com/containers/podman/pkg/api/server.newServer.panicHandler.func6.1({0x590064d30b10?, 0xc00178c7e0?}, 0xc0016b1890?)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/pkg/api/server/handler_panic.go:29 +0x76 fp=0xc0024139f8 sp=0xc0024139a0 pc=0x59006429d176
net/http.HandlerFunc.ServeHTTP(0xc001ca3b00?, {0x590064d30b10?, 0xc00178c7e0?}, 0x0?)
        /usr/lib/golang/src/net/http/server.go:2136 +0x29 fp=0xc002413a20 sp=0xc0024139f8 pc=0x590063355e49
github.com/containers/podman/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc0005b00c0, {0x590064d30b10, 0xc00178c7e0}, 0xc001ca3a00)
        /builddir/build/BUILD/containers-podman-1a6dca2/_build/src/github.com/containers/podman/vendor/github.com/gorilla/mux/mux.go:212 +0x1c5 fp=0xc002413b48 sp=0xc002413a20 pc=0x590063417205
net/http.serverHandler.ServeHTTP({0x590064d2bcd0?}, {0x590064d30b10?, 0xc00178c7e0?}, 0x6?)
        /usr/lib/golang/src/net/http/server.go:2938 +0x8e fp=0xc002413b78 sp=0xc002413b48 pc=0x590063358b2e
net/http.(*conn).serve(0xc001971320, {0x590064d33b68, 0xc0016c0e40})
        /usr/lib/golang/src/net/http/server.go:2009 +0x5f4 fp=0xc002413fb8 sp=0xc002413b78 pc=0x590063354a14
net/http.(*Server).Serve.func3()
        /usr/lib/golang/src/net/http/server.go:3086 +0x28 fp=0xc002413fe0 sp=0xc002413fb8 pc=0x590063359348
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002413fe8 sp=0xc002413fe0 pc=0x59006307d721
created by net/http.(*Server).Serve in goroutine 15
        /usr/lib/golang/src/net/http/server.go:3086 +0x5cb

Steps to reproduce the issue

Running 30~ ish nested container on gke with userns=keep-id

                "/usr/bin/podman",
                "run",
                "--detach",
                "--userns=keep-id",
                "--cgroups=enabled",
                "--umask=022",
                "--user=1000",
                "--mount",

Describe the results you received

Frequent (10% of pods of 1k pods) restarting due to the podman service segfaulting

Describe the results you expected

containers run.

podman info output

host:
  arch: amd64
  buildahVersion: 1.33.8
  cgroupControllers:
  - cpuset
  - cpu
  - io
  - memory
  - hugetlb
  - pids
  - rdma
  cgroupManager: cgroupfs
  cgroupVersion: v2
  conmon:
    package: conmon-2.1.10-1.module+el8.10.0+21962+8143777b.x86_64
    path: /usr/bin/conmon
    version: 'conmon version 2.1.10, commit: 8f1552a71b76511fb9b4a96282513d17686c56fc'
  cpuUtilization:
    idlePercent: 6
    systemPercent: 1.97
    userPercent: 92.03
  cpus: 32
  databaseBackend: sqlite
  distribution:
    distribution: rhel
    version: "8.10"
  eventLogger: file
  freeLocks: 1947
  idMappings:
    gidmap:
    - container_id: 0
      host_id: 1000
      size: 1
    - container_id: 1
      host_id: 1
      size: 999
    - container_id: 1000
      host_id: 1001
      size: 64535
    uidmap:
    - container_id: 0
      host_id: 1000
      size: 1
    - container_id: 1
      host_id: 1
      size: 999
    - container_id: 1000
      host_id: 1001
      size: 64535
  kernel: 5.15.146+
  linkmode: dynamic
  logDriver: k8s-file
  memFree: 24557101056
  memTotal: 188030095360
  networkBackend: cni
  networkBackendInfo:
    backend: cni
    dns:
      package: podman-plugins-4.9.4-4.module+el8.10.0+21995+81e8507c.x86_64
      path: /usr/libexec/cni/dnsname
      version: |-
        CNI dnsname plugin
        version: 1.4.0-dev
        commit: unknown
        CNI protocol versions supported: 0.1.0, 0.2.0, 0.3.0, 0.3.1, 0.4.0, 1.0.0
    package: containernetworking-plugins-1.4.0-2.module+el8.10.0+21962+8143777b.x86_64
    path: /usr/libexec/cni
  ociRuntime:
    name: crun
    package: crun-1.14.3-2.module+el8.10.0+21962+8143777b.x86_64
    path: /usr/bin/crun
    version: |-
      crun version 1.14.3
      commit: 1961d211ba98f532ea52d2e80f4c20359f241a98
      rundir: /tmp/podman-run-1000/crun
      spec: 1.0.0
      +SYSTEMD +SELINUX +APPARMOR +CAP +SECCOMP +EBPF +CRIU +YAJL
  os: linux
  pasta:
    executable: ""
    package: ""
    version: ""
  remoteSocket:
    exists: false
    path: /tmp/podman-run-1000/podman/podman.sock
  security:
    apparmorEnabled: false
    capabilities: CAP_NET_RAW,CAP_CHOWN,CAP_DAC_OVERRIDE,CAP_FOWNER,CAP_FSETID,CAP_KILL,CAP_NET_BIND_SERVICE,CAP_SETFCAP,CAP_SETGID,CAP_SETPCAP,CAP_SETUID,CAP_SYS_CHROOT
    rootless: true
    seccompEnabled: true
    seccompProfilePath: /usr/share/containers/seccomp.json
    selinuxEnabled: false
  serviceIsRemote: false
  slirp4netns:
    executable: /usr/bin/slirp4netns
    package: slirp4netns-1.2.3-1.module+el8.10.0+21962+8143777b.x86_64
    version: |-
      slirp4netns version 1.2.3
      commit: c22fde291bb35b354e6ca44d13be181c76a0a432
      libslirp: 4.4.0
      SLIRP_CONFIG_VERSION_MAX: 3
      libseccomp: 2.5.2
  swapFree: 0
  swapTotal: 0
  uptime: 3h 12m 57.00s (Approximately 0.12 days)
  variant: ""
plugins:
  authorization: null
  log:
  - k8s-file
  - none
  - passthrough
  - journald
  network:
  - bridge
  - macvlan
  - ipvlan
  volume:
  - local
registries:
  search:
  - registry.access.redhat.com
  - registry.redhat.io
  - docker.io
store:
  configFile: /home/runtime/.config/containers/storage.conf
  containerStore:
    number: 35
    paused: 0
    running: 33
    stopped: 2
  graphDriverName: overlay
  graphOptions: {}
  graphRoot: /home/runtime/.local/share/containers/storage
  graphRootAllocated: 418285613056
  graphRootUsed: 196143841280
  graphStatus:
    Backing Filesystem: overlayfs
    Native Overlay Diff: "false"
    Supports d_type: "true"
    Supports shifting: "true"
    Supports volatile: "true"
    Using metacopy: "false"
  imageCopyTmpDir: /tmp
  imageStore:
    number: 6
  runRoot: /tmp/containers-user-1000/containers
  transientStore: false
  volumePath: /home/runtime/.local/share/containers/storage/volumes
version:
  APIVersion: 4.9.4-rhel
  Built: 1718703254
  BuiltTime: Tue Jun 18 04:34:14 2024
  GitCommit: ""
  GoVersion: go1.21.7 (Red Hat 1.21.7-2.module+el8.10.0+21638+b01be198)
  Os: linux
  OsArch: linux/amd64
  Version: 4.9.4-rhel

Podman in a container

Yes

Privileged Or Rootless

Rootless

Upstream Latest Release

Additional environment details

Running on GKE

Additional information

Additional information like issue happens only occasionally or issue happens with a particular architecture or on a particular setting

Jul 12 '24 20:07 krisdevopsbot