Users not created from mTLS certificates

[JakeSCahill]

What happened?

When you enable mTLS authentication in the chart, the Redpanda logs are flooded with errors, which suggest that users are not correctly created from the default self-signed certificates:

ERROR 2023-11-08 12:03:45,814 [shard 0] kafka - server.cc:156 - Error[applying protocol] remote address: 10.244.2.9:59544 - std::__1::system_error (error GnuTLS:-112, Certificate is required.)
WARN  2023-11-08 12:03:50,079 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found
WARN  2023-11-08 12:03:50,084 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found

What did you expect to happen?

These errors should not appear.

How can we reproduce it (as minimally and precisely as possible)?. Please include values file.

helm upgrade --install redpanda redpanda/redpanda \
  --namespace redpanda \
  --create-namespace \
  --set external.domain=${DOMAIN} \
  --set statefulset.initContainers.setDataDirOwnership.enabled=true --set statefulset.sideCars.controllers.enabled=true --set rbac.enabled=true --set statefulset.sideCars.controllers.image.tag=v23.2.14 --set statefulset.replicas=5 --set listeners.kafka.authenticationMethod=mtls_identity --set listeners.kafka.tls.requireClientAuth=true --set auth.sasl.enabled=true --set auth.sasl.secretRef=redpanda-superusers

$ helm get values <redpanda-release-name> -n <redpanda-release-namespace> --all
COMPUTED VALUES:
affinity: {}
auth:
  sasl:
    enabled: true
    mechanism: SCRAM-SHA-512
    secretRef: redpanda-superusers
    users: []
clusterDomain: cluster.local
commonLabels: {}
config:
  cluster:
    default_topic_replications: 3
  node:
    crash_loop_limit: 5
  pandaproxy_client: {}
  rpk: {}
  schema_registry_client: {}
  tunable:
    compacted_log_segment_size: 67108864
    group_topic_partitions: 16
    kafka_batch_max_bytes: 1048576
    kafka_connection_rate_limit: 1000
    log_segment_size: 134217728
    log_segment_size_max: 268435456
    log_segment_size_min: 16777216
    max_compacted_log_segment_size: 536870912
    topic_partitions_per_shard: 1000
connectors:
  auth:
    sasl:
      enabled: false
      mechanism: scram-sha-512
      secretRef: ""
      userName: ""
  commonLabels: {}
  connectors:
    additionalConfiguration: ""
    bootstrapServers: ""
    brokerTLS:
      ca:
        secretNameOverwrite: ""
        secretRef: ""
      cert:
        secretNameOverwrite: ""
        secretRef: ""
      enabled: false
      key:
        secretNameOverwrite: ""
        secretRef: ""
    groupID: connectors-cluster
    producerBatchSize: 131072
    producerLingerMS: 1
    restPort: 8083
    schemaRegistryURL: ""
    secretManager:
      connectorsPrefix: ""
      consolePrefix: ""
      enabled: false
      region: ""
    storage:
      remote:
        read:
          config: false
          offset: false
          status: false
        write:
          config: false
          offset: false
          status: false
      replicationFactor:
        config: -1
        offset: -1
        status: -1
      topic:
        config: _internal_connectors_configs
        offset: _internal_connectors_offsets
        status: _internal_connectors_status
  container:
    javaGCLogEnabled: "false"
    resources:
      javaMaxHeapSize: 2G
      limits:
        cpu: 1
        memory: 2350Mi
      request:
        cpu: 1
        memory: 2350Mi
    securityContext:
      allowPrivilegeEscalation: false
  deployment:
    annotations: {}
    budget:
      maxUnavailable: 1
    create: false
    extraEnv: []
    livenessProbe:
      failureThreshold: 3
      initialDelaySeconds: 10
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 1
    nodeAffinity: {}
    nodeSelector: {}
    podAffinity: {}
    podAntiAffinity:
      custom: {}
      topologyKey: kubernetes.io/hostname
      type: hard
      weight: 100
    priorityClassName: ""
    progressDeadlineSeconds: 600
    readinessProbe:
      failureThreshold: 2
      initialDelaySeconds: 60
      periodSeconds: 10
      successThreshold: 3
      timeoutSeconds: 5
    restartPolicy: Always
    revisionHistoryLimit: 10
    schedulerName: ""
    securityContext:
      fsGroup: 101
      fsGroupChangePolicy: OnRootMismatch
      runAsUser: 101
    strategy:
      type: RollingUpdate
    terminationGracePeriodSeconds: 30
    tolerations: []
    topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: topology.kubernetes.io/zone
      whenUnsatisfiable: ScheduleAnyway
    updateStrategy:
      type: RollingUpdate
  enabled: false
  fullnameOverride: ""
  global: {}
  image:
    pullPolicy: IfNotPresent
    repository: docker.redpanda.com/redpandadata/connectors
    tag: ""
  imagePullSecrets: []
  logging:
    level: warn
  monitoring:
    annotations: {}
    enabled: false
    labels: {}
    namespaceSelector:
      any: true
    scrapeInterval: 30s
  nameOverride: ""
  service:
    annotations: {}
    name: ""
    ports:
    - name: prometheus
      port: 9404
  serviceAccount:
    annotations: {}
    create: false
    name: ""
  storage:
    volume:
    - emptyDir:
        medium: Memory
        sizeLimit: 5Mi
      name: rp-connect-tmp
    volumeMounts:
    - mountPath: /tmp
      name: rp-connect-tmp
  test:
    create: false
  tolerations: []
console:
  affinity: {}
  annotations: {}
  autoscaling:
    enabled: false
    maxReplicas: 100
    minReplicas: 1
    targetCPUUtilizationPercentage: 80
  config: {}
  configmap:
    create: false
  console:
    config: {}
  deployment:
    create: false
  enabled: true
  enterprise:
    licenseSecretRef:
      key: ""
      name: ""
  extraContainers: []
  extraEnv: []
  extraEnvFrom: []
  extraVolumeMounts: []
  extraVolumes: []
  fullnameOverride: ""
  global: {}
  image:
    pullPolicy: IfNotPresent
    registry: docker.redpanda.com
    repository: redpandadata/console
    tag: ""
  imagePullSecrets: []
  ingress:
    annotations: {}
    className: ""
    enabled: false
    hosts:
    - host: chart-example.local
      paths:
      - path: /
        pathType: ImplementationSpecific
    tls: []
  initContainers:
    extraInitContainers: ""
  livenessProbe:
    failureThreshold: 3
    initialDelaySeconds: 0
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 1
  nameOverride: ""
  nodeSelector: {}
  podAnnotations: {}
  podLabels: {}
  podSecurityContext:
    fsGroup: 99
    runAsUser: 99
  priorityClassName: ""
  readinessProbe:
    failureThreshold: 3
    initialDelaySeconds: 10
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 1
  replicaCount: 1
  resources: {}
  secret:
    create: false
    enterprise: {}
    kafka: {}
    login:
      github: {}
      google: {}
      jwtSecret: ""
      oidc: {}
      okta: {}
    redpanda:
      adminApi: {}
  secretMounts: []
  securityContext:
    runAsNonRoot: true
  service:
    annotations: {}
    port: 8080
    type: ClusterIP
  serviceAccount:
    annotations: {}
    create: true
    name: ""
  tolerations: []
  topologySpreadConstraints: {}
enterprise:
  license: ""
  licenseSecretRef: {}
external:
  domain: customredpandadomain.local
  enabled: true
  service:
    enabled: true
  type: NodePort
fullnameOverride: ""
image:
  pullPolicy: IfNotPresent
  repository: docker.redpanda.com/redpandadata/redpanda
  tag: ""
imagePullSecrets: []
license_key: ""
license_secret_ref: {}
listeners:
  admin:
    external:
      default:
        advertisedPorts:
        - 31644
        port: 9645
        tls:
          cert: external
    port: 9644
    tls:
      cert: default
      requireClientAuth: false
  http:
    authenticationMethod: null
    enabled: true
    external:
      default:
        advertisedPorts:
        - 30082
        authenticationMethod: null
        port: 8083
        tls:
          cert: external
          requireClientAuth: false
    kafkaEndpoint: default
    port: 8082
    tls:
      cert: default
      requireClientAuth: false
  kafka:
    authenticationMethod: mtls_identity
    external:
      default:
        advertisedPorts:
        - 31092
        authenticationMethod: null
        port: 9094
        tls:
          cert: external
    port: 9093
    tls:
      cert: default
      requireClientAuth: true
  rpc:
    port: 33145
    tls:
      cert: default
      requireClientAuth: false
  schemaRegistry:
    authenticationMethod: null
    enabled: true
    external:
      default:
        advertisedPorts:
        - 30081
        authenticationMethod: null
        port: 8084
        tls:
          cert: external
          requireClientAuth: false
    kafkaEndpoint: default
    port: 8081
    tls:
      cert: default
      requireClientAuth: false
logging:
  logLevel: info
  usageStats:
    enabled: true
monitoring:
  enabled: false
  labels: {}
  scrapeInterval: 30s
  tlsConfig: {}
nameOverride: ""
nodeSelector: {}
post_install_job:
  affinity: {}
  enabled: true
post_upgrade_job:
  affinity: {}
  enabled: true
rackAwareness:
  enabled: false
  nodeAnnotation: topology.kubernetes.io/zone
rbac:
  annotations: {}
  enabled: true
resources:
  cpu:
    cores: 1
  memory:
    container:
      max: 2.5Gi
serviceAccount:
  annotations: {}
  create: false
  name: ""
statefulset:
  additionalRedpandaCmdFlags: []
  annotations: {}
  budget:
    maxUnavailable: 1
  extraVolumeMounts: ""
  extraVolumes: ""
  initContainerImage:
    repository: busybox
    tag: latest
  initContainers:
    configurator:
      extraVolumeMounts: ""
      resources: {}
    extraInitContainers: ""
    setDataDirOwnership:
      enabled: true
      extraVolumeMounts: ""
      resources: {}
    setTieredStorageCacheDirOwnership:
      extraVolumeMounts: ""
      resources: {}
    tuning:
      extraVolumeMounts: ""
      resources: {}
  livenessProbe:
    failureThreshold: 3
    initialDelaySeconds: 10
    periodSeconds: 10
  nodeSelector: {}
  podAffinity: {}
  podAntiAffinity:
    custom: {}
    topologyKey: kubernetes.io/hostname
    type: hard
    weight: 100
  priorityClassName: ""
  readinessProbe:
    failureThreshold: 3
    initialDelaySeconds: 1
    periodSeconds: 10
    successThreshold: 1
  replicas: 5
  securityContext:
    fsGroup: 101
    fsGroupChangePolicy: OnRootMismatch
    runAsUser: 101
  sideCars:
    configWatcher:
      enabled: true
      extraVolumeMounts: ""
      resources: {}
      securityContext: {}
    controllers:
      createRBAC: true
      enabled: true
      healthProbeAddress: :8085
      image:
        repository: docker.redpanda.com/redpandadata/redpanda-operator
        tag: v23.2.14
      metricsAddress: :9082
      resources: {}
      run:
      - all
      securityContext: {}
  startupProbe:
    failureThreshold: 120
    initialDelaySeconds: 1
    periodSeconds: 10
  terminationGracePeriodSeconds: 90
  tolerations: []
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
  updateStrategy:
    type: RollingUpdate
storage:
  hostPath: ""
  persistentVolume:
    annotations: {}
    enabled: true
    labels: {}
    size: 20Gi
    storageClass: ""
  tiered:
    config:
      cloud_storage_access_key: ""
      cloud_storage_api_endpoint: ""
      cloud_storage_azure_container: null
      cloud_storage_azure_shared_key: null
      cloud_storage_azure_storage_account: null
      cloud_storage_bucket: ""
      cloud_storage_cache_size: 5368709120
      cloud_storage_credentials_source: config_file
      cloud_storage_enable_remote_read: true
      cloud_storage_enable_remote_write: true
      cloud_storage_enabled: false
      cloud_storage_region: ""
      cloud_storage_secret_key: ""
    hostPath: ""
    mountType: emptyDir
    persistentVolume:
      annotations: {}
      labels: {}
      storageClass: ""
tls:
  certs:
    default:
      caEnabled: true
    external:
      caEnabled: true
  enabled: true
tolerations: []
tuning:
  tune_aio_events: true

Anything else we need to know?

No response

Which are the affected charts?

No response

Chart Version(s)

$ helm -n <redpanda-release-namespace> list 
# paste output here

Cloud provider

JIRA Link: K8S-74