panic: assignment to entry in nil map in `loadClusterInformationFromStore`

[aybabtme]

Expected Behavior

The program should return well formatted errors and shouldn't panic, especially not for bug-looking reasons like assigning in nil-maps.

Actual Behavior

panic: assignment to entry in nil map

goroutine 1 [running]:
go.temporal.io/server/temporal.loadClusterInformationFromStore({0x2a5fab0?, 0xc00005a080}, 0xc000578900, {0x2a697a8?, 0xc0005e6000}, {0x2a63ae0, 0xc000962a80})
        /home/runner/work/temporal/temporal/temporal/fx.go:733 +0x7ae
go.temporal.io/server/temporal.ApplyClusterMetadataConfigProvider({0x2a63ae0, 0xc000878f10}, 0xc000578900, {0x2a49ba0, 0x3cd9f28}, 0x2633f08, {0x0, 0x0})
        /home/runner/work/temporal/temporal/temporal/fx.go:686 +0x22eb
reflect.Value.call({0x21af600?, 0x2634540?, 0x40d645?}, {0x24c790e, 0x4}, {0xc00031d200, 0x5, 0x30?})
        /opt/hostedtoolcache/go/1.18.4/x64/src/reflect/value.go:556 +0x845
reflect.Value.Call({0x21af600?, 0x2634540?, 0x40d9a7?}, {0xc00031d200, 0x5, 0x5})
        /opt/hostedtoolcache/go/1.18.4/x64/src/reflect/value.go:339 +0xbf
go.uber.org/dig.defaultInvoker({0x21af600?, 0x2634540?, 0xc0000fae60?}, {0xc00031d200?, 0x5?, 0x2a6e588?})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/container.go:220 +0x28
go.uber.org/dig.(*constructorNode).Call(0xc00035a900, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/constructor.go:154 +0x297
go.uber.org/dig.paramSingle.Build({{0x0, 0x0}, 0x0, {0x2a71500, 0x234fe60}}, {0x2a6e588, 0xc000358000})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:296 +0x2f9
go.uber.org/dig.paramObjectField.Build(...)
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:480
go.uber.org/dig.paramObject.Build({{0x2a71500, 0x245c0e0}, {0xc000366f00, 0x14, 0x20}, {0x0, 0x0, 0x0}}, {0x2a6e588, 0xc000358000})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:407 +0x155
go.uber.org/dig.paramList.BuildList({{0x2a71500, 0x20d58a0}, {0xc000332830, 0x1, 0x1}}, {0x2a6e588, 0xc000358000})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:151 +0xb9
go.uber.org/dig.(*constructorNode).Call(0xc00035a600, {0x2a6e588, 0xc000358000})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/constructor.go:145 +0x132
go.uber.org/dig.paramGroupedSlice.callGroupProviders({{0x1f49fd8, 0x8}, {0x2a71500, 0x20036e0}, 0xc000339bc0}, {0x2a6e588?, 0xc0003580a0?})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:597 +0x192
go.uber.org/dig.paramGroupedSlice.Build({{0x1f49fd8, 0x8}, {0x2a71500, 0x20036e0}, 0xc000339bc0}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:624 +0x10c
go.uber.org/dig.paramObjectField.Build(...)
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:480
go.uber.org/dig.paramObject.Build({{0x2a71500, 0x222be80}, {0xc000339c20, 0x1, 0x1}, {0x0, 0x0, 0x0}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:407 +0x155
go.uber.org/dig.paramList.BuildList({{0x2a71500, 0x220fc80}, {0xc000358140, 0xa, 0xa}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:151 +0xb9
go.uber.org/dig.(*constructorNode).Call(0xc00035a0c0, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/constructor.go:145 +0x132
go.uber.org/dig.paramSingle.Build({{0x0, 0x0}, 0x0, {0x2a71500, 0x2164900}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:296 +0x2f9
go.uber.org/dig.paramList.BuildList({{0x2a71500, 0x2061de0}, {0xc000332430, 0x1, 0x1}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:151 +0xb9
go.uber.org/dig.(*constructorNode).Call(0xc00035a180, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/constructor.go:145 +0x132
go.uber.org/dig.paramSingle.Build({{0x0, 0x0}, 0x0, {0x2a71500, 0x2193d40}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:296 +0x2f9
go.uber.org/dig.paramList.BuildList({{0x2a71500, 0x207fea0}, {0xc00011d200, 0x2, 0x2}}, {0x2a6e588, 0xc0003580a0})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/param.go:151 +0xb9
go.uber.org/dig.(*Scope).Invoke(0xc0003580a0, {0x207fea0?, 0x2634588}, {0xc0007fee68?, 0x4eb245?, 0xc0007fee98?})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/invoke.go:85 +0x288
go.uber.org/fx.runInvoke({0x2a5e0b0?, 0xc0003580a0?}, {{0x207fea0, 0x2634588}, {0xc000344a00, 0x7, 0x8}})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/invoke.go:93 +0x1ff
go.uber.org/fx.(*module).executeInvoke(0xc0001f6090, {{0x207fea0, 0x2634588}, {0xc000344a00, 0x7, 0x8}})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/module.go:174 +0x133
go.uber.org/fx.(*module).executeInvokes(0xc0001f6090)
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/module.go:155 +0xf1
go.uber.org/fx.New({0xc0007ff338, 0xc, 0x7?})
        /home/runner/go/pkg/mod/go.uber.org/[email protected]/app.go:534 +0x887
go.temporal.io/server/temporal.NewServerFx({0xc000334000?, 0x2a63ae0?, 0xc000878f10?})
        /home/runner/work/temporal/temporal/temporal/fx.go:130 +0x716
go.temporal.io/server/temporal.NewServer(...)
        /home/runner/work/temporal/temporal/temporal/server.go:58
main.buildCLI.func2(0xc0004d1320?)
        /home/runner/work/temporal/temporal/cmd/server/main.go:166 +0x1538
github.com/urfave/cli/v2.(*Command).Run(0xc0004d1320, 0xc00040c4c0)
        /home/runner/go/pkg/mod/github.com/urfave/cli/[email protected]/command.go:163 +0x5bb
github.com/urfave/cli/v2.(*App).RunContext(0xc000103a00, {0x2a5fab0?, 0xc00005a068}, {0xc00004c1e0, 0x6, 0x6})
        /home/runner/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:313 +0xb48
github.com/urfave/cli/v2.(*App).Run(...)
        /home/runner/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:224
main.main()
        /home/runner/work/temporal/temporal/cmd/server/main.go:53 +0x45

Steps to Reproduce the Problem

Use this config file:

log:
    stdout: true
    level: info

persistence:
    defaultStore: mysql-default
    visibilityStore: mysql-visibility
    numHistoryShards: 4
    datastores:
        mysql-default:
            sql:
                pluginName: "mysql"
                databaseName: "{{ .Env.TEMPORAL_MYSQL_DEFAULT_DBNAME }}"
                connectAddr: "{{ .Env.TEMPORAL_MYSQL_DEFAULT_HOST }}:{{ .Env.TEMPORAL_MYSQL_DEFAULT_PORT }}"
                connectProtocol: "tcp"
                user: "{{ .Env.TEMPORAL_MYSQL_DEFAULT_USER }}"
                password: "{{ .Env.TEMPORAL_MYSQL_DEFAULT_PASSWORD }}"
                maxConns: 20
                maxIdleConns: 20
                maxConnLifetime: "1h"
        mysql-visibility:
            sql:
                pluginName: "mysql"
                databaseName: "{{ .Env.TEMPORAL_MYSQL_VISIBILITY_DBNAME }}"
                connectAddr: "{{ .Env.TEMPORAL_MYSQL_VISIBILITY_HOST }}:{{ .Env.TEMPORAL_MYSQL_VISIBILITY_PORT }}"
                connectProtocol: "tcp"
                user: "{{ .Env.TEMPORAL_MYSQL_VISIBILITY_USER }}"
                password: "{{ .Env.TEMPORAL_MYSQL_VISIBILITY_PASSWORD }}"
                maxConns: 2
                maxIdleConns: 2
                maxConnLifetime: "1h"

global:
    membership:
        maxJoinDuration: 30s
        broadcastAddress: "127.0.0.1"
    pprof:
        port: 7936
    metrics:
        statsd:
            hostPort: "{{ .Env.DD_AGENT_HOST }}:28125"
            prefix: "temporal"

services:
    frontend:
        rpc:
            grpcPort: 7233
            membershipPort: 6933
            bindOnLocalHost: true

    matching:
        rpc:
            grpcPort: 7235
            membershipPort: 6935
            bindOnLocalHost: true

    history:
        rpc:
            grpcPort: 7234
            membershipPort: 6934
            bindOnLocalHost: true

    worker:
        rpc:
            grpcPort: 7239
            membershipPort: 6939
            bindOnLocalHost: true

clusterMetadata:
    enableGlobalNamespace: false
    failoverVersionIncrement: 10
    masterClusterName: "active"
    currentClusterName: "active"
    clusterInformation:
        active:
            enabled: true
            initialFailoverVersion: 1
            rpcName: "frontend"
            rpcAddress: "localhost:7233"

dcRedirectionPolicy:
    policy: "noop"
    toDC: ""

archival:
    history:
        state: "enabled"
        enableRead: true
        provider:
            filestore:
            fileMode: "0666"
            dirMode: "0766"
            gstorage:
            credentialsPath: "/tmp/gcloud/keyfile.json"
    visibility:
        state: "enabled"
        enableRead: true
        provider:
            filestore:
            fileMode: "0666"
            dirMode: "0766"

namespaceDefaults:
    archival:
        history:
            state: "disabled"
            URI: "file:///tmp/temporal_archival/development"
        visibility:
            state: "disabled"
            URI: "file:///tmp/temporal_vis_archival/development"

publicClient:
    hostPort: "localhost:7233"

Specifications

• Version: 1.17.2
• Platform: x86_64 GNU/Linux