helm-charts
helm-charts copied to clipboard
Published
20 hours ago •
redpanda-data
redpanda-data
Users not created from mTLS certificates
What happened?
When you enable mTLS authentication in the chart, the Redpanda logs are flooded with errors, which suggest that users are not correctly created from the default self-signed certificates:
ERROR 2023-11-08 12:03:45,814 [shard 0] kafka - server.cc:156 - Error[applying protocol] remote address: 10.244.2.9:59544 - std::__1::system_error (error GnuTLS:-112, Certificate is required.)
WARN 2023-11-08 12:03:50,079 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found
WARN 2023-11-08 12:03:50,084 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found
What did you expect to happen?
These errors should not appear.
How can we reproduce it (as minimally and precisely as possible)?. Please include values file.
helm upgrade --install redpanda redpanda/redpanda \
--namespace redpanda \
--create-namespace \
--set external.domain=${DOMAIN} \
--set statefulset.initContainers.setDataDirOwnership.enabled=true --set statefulset.sideCars.controllers.enabled=true --set rbac.enabled=true --set statefulset.sideCars.controllers.image.tag=v23.2.14 --set statefulset.replicas=5 --set listeners.kafka.authenticationMethod=mtls_identity --set listeners.kafka.tls.requireClientAuth=true --set auth.sasl.enabled=true --set auth.sasl.secretRef=redpanda-superusers
$ helm get values <redpanda-release-name> -n <redpanda-release-namespace> --all
COMPUTED VALUES:
affinity: {}
auth:
sasl:
enabled: true
mechanism: SCRAM-SHA-512
secretRef: redpanda-superusers
users: []
clusterDomain: cluster.local
commonLabels: {}
config:
cluster:
default_topic_replications: 3
node:
crash_loop_limit: 5
pandaproxy_client: {}
rpk: {}
schema_registry_client: {}
tunable:
compacted_log_segment_size: 67108864
group_topic_partitions: 16
kafka_batch_max_bytes: 1048576
kafka_connection_rate_limit: 1000
log_segment_size: 134217728
log_segment_size_max: 268435456
log_segment_size_min: 16777216
max_compacted_log_segment_size: 536870912
topic_partitions_per_shard: 1000
connectors:
auth:
sasl:
enabled: false
mechanism: scram-sha-512
secretRef: ""
userName: ""
commonLabels: {}
connectors:
additionalConfiguration: ""
bootstrapServers: ""
brokerTLS:
ca:
secretNameOverwrite: ""
secretRef: ""
cert:
secretNameOverwrite: ""
secretRef: ""
enabled: false
key:
secretNameOverwrite: ""
secretRef: ""
groupID: connectors-cluster
producerBatchSize: 131072
producerLingerMS: 1
restPort: 8083
schemaRegistryURL: ""
secretManager:
connectorsPrefix: ""
consolePrefix: ""
enabled: false
region: ""
storage:
remote:
read:
config: false
offset: false
status: false
write:
config: false
offset: false
status: false
replicationFactor:
config: -1
offset: -1
status: -1
topic:
config: _internal_connectors_configs
offset: _internal_connectors_offsets
status: _internal_connectors_status
container:
javaGCLogEnabled: "false"
resources:
javaMaxHeapSize: 2G
limits:
cpu: 1
memory: 2350Mi
request:
cpu: 1
memory: 2350Mi
securityContext:
allowPrivilegeEscalation: false
deployment:
annotations: {}
budget:
maxUnavailable: 1
create: false
extraEnv: []
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
nodeAffinity: {}
nodeSelector: {}
podAffinity: {}
podAntiAffinity:
custom: {}
topologyKey: kubernetes.io/hostname
type: hard
weight: 100
priorityClassName: ""
progressDeadlineSeconds: 600
readinessProbe:
failureThreshold: 2
initialDelaySeconds: 60
periodSeconds: 10
successThreshold: 3
timeoutSeconds: 5
restartPolicy: Always
revisionHistoryLimit: 10
schedulerName: ""
securityContext:
fsGroup: 101
fsGroupChangePolicy: OnRootMismatch
runAsUser: 101
strategy:
type: RollingUpdate
terminationGracePeriodSeconds: 30
tolerations: []
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
updateStrategy:
type: RollingUpdate
enabled: false
fullnameOverride: ""
global: {}
image:
pullPolicy: IfNotPresent
repository: docker.redpanda.com/redpandadata/connectors
tag: ""
imagePullSecrets: []
logging:
level: warn
monitoring:
annotations: {}
enabled: false
labels: {}
namespaceSelector:
any: true
scrapeInterval: 30s
nameOverride: ""
service:
annotations: {}
name: ""
ports:
- name: prometheus
port: 9404
serviceAccount:
annotations: {}
create: false
name: ""
storage:
volume:
- emptyDir:
medium: Memory
sizeLimit: 5Mi
name: rp-connect-tmp
volumeMounts:
- mountPath: /tmp
name: rp-connect-tmp
test:
create: false
tolerations: []
console:
affinity: {}
annotations: {}
autoscaling:
enabled: false
maxReplicas: 100
minReplicas: 1
targetCPUUtilizationPercentage: 80
config: {}
configmap:
create: false
console:
config: {}
deployment:
create: false
enabled: true
enterprise:
licenseSecretRef:
key: ""
name: ""
extraContainers: []
extraEnv: []
extraEnvFrom: []
extraVolumeMounts: []
extraVolumes: []
fullnameOverride: ""
global: {}
image:
pullPolicy: IfNotPresent
registry: docker.redpanda.com
repository: redpandadata/console
tag: ""
imagePullSecrets: []
ingress:
annotations: {}
className: ""
enabled: false
hosts:
- host: chart-example.local
paths:
- path: /
pathType: ImplementationSpecific
tls: []
initContainers:
extraInitContainers: ""
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 0
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
nameOverride: ""
nodeSelector: {}
podAnnotations: {}
podLabels: {}
podSecurityContext:
fsGroup: 99
runAsUser: 99
priorityClassName: ""
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
replicaCount: 1
resources: {}
secret:
create: false
enterprise: {}
kafka: {}
login:
github: {}
google: {}
jwtSecret: ""
oidc: {}
okta: {}
redpanda:
adminApi: {}
secretMounts: []
securityContext:
runAsNonRoot: true
service:
annotations: {}
port: 8080
type: ClusterIP
serviceAccount:
annotations: {}
create: true
name: ""
tolerations: []
topologySpreadConstraints: {}
enterprise:
license: ""
licenseSecretRef: {}
external:
domain: customredpandadomain.local
enabled: true
service:
enabled: true
type: NodePort
fullnameOverride: ""
image:
pullPolicy: IfNotPresent
repository: docker.redpanda.com/redpandadata/redpanda
tag: ""
imagePullSecrets: []
license_key: ""
license_secret_ref: {}
listeners:
admin:
external:
default:
advertisedPorts:
- 31644
port: 9645
tls:
cert: external
port: 9644
tls:
cert: default
requireClientAuth: false
http:
authenticationMethod: null
enabled: true
external:
default:
advertisedPorts:
- 30082
authenticationMethod: null
port: 8083
tls:
cert: external
requireClientAuth: false
kafkaEndpoint: default
port: 8082
tls:
cert: default
requireClientAuth: false
kafka:
authenticationMethod: mtls_identity
external:
default:
advertisedPorts:
- 31092
authenticationMethod: null
port: 9094
tls:
cert: external
port: 9093
tls:
cert: default
requireClientAuth: true
rpc:
port: 33145
tls:
cert: default
requireClientAuth: false
schemaRegistry:
authenticationMethod: null
enabled: true
external:
default:
advertisedPorts:
- 30081
authenticationMethod: null
port: 8084
tls:
cert: external
requireClientAuth: false
kafkaEndpoint: default
port: 8081
tls:
cert: default
requireClientAuth: false
logging:
logLevel: info
usageStats:
enabled: true
monitoring:
enabled: false
labels: {}
scrapeInterval: 30s
tlsConfig: {}
nameOverride: ""
nodeSelector: {}
post_install_job:
affinity: {}
enabled: true
post_upgrade_job:
affinity: {}
enabled: true
rackAwareness:
enabled: false
nodeAnnotation: topology.kubernetes.io/zone
rbac:
annotations: {}
enabled: true
resources:
cpu:
cores: 1
memory:
container:
max: 2.5Gi
serviceAccount:
annotations: {}
create: false
name: ""
statefulset:
additionalRedpandaCmdFlags: []
annotations: {}
budget:
maxUnavailable: 1
extraVolumeMounts: ""
extraVolumes: ""
initContainerImage:
repository: busybox
tag: latest
initContainers:
configurator:
extraVolumeMounts: ""
resources: {}
extraInitContainers: ""
setDataDirOwnership:
enabled: true
extraVolumeMounts: ""
resources: {}
setTieredStorageCacheDirOwnership:
extraVolumeMounts: ""
resources: {}
tuning:
extraVolumeMounts: ""
resources: {}
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
nodeSelector: {}
podAffinity: {}
podAntiAffinity:
custom: {}
topologyKey: kubernetes.io/hostname
type: hard
weight: 100
priorityClassName: ""
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
replicas: 5
securityContext:
fsGroup: 101
fsGroupChangePolicy: OnRootMismatch
runAsUser: 101
sideCars:
configWatcher:
enabled: true
extraVolumeMounts: ""
resources: {}
securityContext: {}
controllers:
createRBAC: true
enabled: true
healthProbeAddress: :8085
image:
repository: docker.redpanda.com/redpandadata/redpanda-operator
tag: v23.2.14
metricsAddress: :9082
resources: {}
run:
- all
securityContext: {}
startupProbe:
failureThreshold: 120
initialDelaySeconds: 1
periodSeconds: 10
terminationGracePeriodSeconds: 90
tolerations: []
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
updateStrategy:
type: RollingUpdate
storage:
hostPath: ""
persistentVolume:
annotations: {}
enabled: true
labels: {}
size: 20Gi
storageClass: ""
tiered:
config:
cloud_storage_access_key: ""
cloud_storage_api_endpoint: ""
cloud_storage_azure_container: null
cloud_storage_azure_shared_key: null
cloud_storage_azure_storage_account: null
cloud_storage_bucket: ""
cloud_storage_cache_size: 5368709120
cloud_storage_credentials_source: config_file
cloud_storage_enable_remote_read: true
cloud_storage_enable_remote_write: true
cloud_storage_enabled: false
cloud_storage_region: ""
cloud_storage_secret_key: ""
hostPath: ""
mountType: emptyDir
persistentVolume:
annotations: {}
labels: {}
storageClass: ""
tls:
certs:
default:
caEnabled: true
external:
caEnabled: true
enabled: true
tolerations: []
tuning:
tune_aio_events: true
Anything else we need to know?
No response
Which are the affected charts?
No response
Chart Version(s)
$ helm -n <redpanda-release-namespace> list
# paste output here
Cloud provider
JIRA Link: K8S-74
@JakeSCahill are you creating users from a list or is there a secret named 'redpanda-superusers'?
Tested this again. Can someone confirm if this config should be valid?
helm repo update
helm install redpanda redpanda/redpanda \
--version 5.9.0 \
--namespace jake \
--create-namespace \
--set external.domain=customredpandadomain.local \
--set statefulset.initContainers.setDataDirOwnership.enabled=true --set "auth.sasl.users[0].name=superuser" --set auth.sasl.enabled=true --set "auth.sasl.users[0].password=secretpassword" --set "auth.sasl.users[0].mechanism=SCRAM-SHA-512" --set listeners.kafka.authenticationMethod=mtls_identity --set listeners.kafka.tls.requireClientAuth=true
When I try to use the Kafka API, rpk seems not to have the correct certs:
kubectl exec redpanda-0 -n jake -- rpk topic create test -X user=superuser -X pass=secretpassword
Defaulted container "redpanda" out of: redpanda, config-watcher, tuning (init), set-datadir-ownership (init), redpanda-configurator (init)
unable to initialize kafka client: unable to read cert at "/etc/tls/certs/redpanda-client/tls.crt": open /etc/tls/certs/redpanda-client/tls.crt: no such file or directory
command terminated with exit code 1
And the Redpanda logs complain too:
INFO 2024-08-09 14:03:37,998 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/11}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 2
INFO 2024-08-09 14:03:37,998 [shard 0:main] raft - [group_id:13, {kafka/_internal_connectors_offsets/11}] vote_stm.cc:433 - became the leader term: 3
INFO 2024-08-09 14:03:38,000 [shard 0:main] raft - [group_id:44, {kafka/__consumer_offsets/11}] vote_stm.cc:433 - became the leader term: 4
INFO 2024-08-09 14:03:38,088 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/9}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 3
INFO 2024-08-09 14:03:38,088 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/17}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 2
WARN 2024-08-09 14:03:38,477 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
WARN 2024-08-09 14:03:38,480 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
WARN 2024-08-09 14:03:38,485 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
INFO 2024-08-09 14:03:38,491 [shard 0:main] cluster - health_monitor_backend.cc:424 - received node 1 health report, marking node as up
WARN 2024-08-09 14:03:38,492 [shard 0:main] cluster - metadata_dissemination_service.cc:436 - Error sending metadata update rpc::errc::exponential_backoff to 2
INFO 2024-08-09 14:03:38,492 [shard 0:main] cluster - partition_balancer_planner.cc:334 - node 2 is unresponsive, time since last status reply: 3042 ms
INFO 2024-08-09 14:03:38,492 [shard 0:main] cluster - partition_balancer_planner.cc:365 - node 2: no disk report
INFO 2024-08-09 14:03:38,724 [shard 0:main] cluster - members_manager.cc:411 - applying maintenance_mode_cmd, offset: 740, node id: 2, enabled: false
INFO 2024-08-09 14:03:38,724 [shard 0:main] cluster - members_table.cc:210 - marking node 2 not in maintenance state
redpanda configmap:
Name: redpanda
Namespace: jake
Labels: app.kubernetes.io/component=redpanda
app.kubernetes.io/instance=redpanda
app.kubernetes.io/managed-by=Helm
app.kubernetes.io/name=redpanda
helm.sh/chart=redpanda-5.9.0
Annotations: meta.helm.sh/release-name: redpanda
meta.helm.sh/release-namespace: jake
Data
====
bootstrap.yaml:
----
audit_enabled: false
compacted_log_segment_size: 67108864
default_topic_replications: 3
enable_rack_awareness: false
enable_sasl: true
group_topic_partitions: 16
kafka_batch_max_bytes: 1048576
kafka_connection_rate_limit: 1000
kafka_enable_authorization: true
log_segment_size: 134217728
log_segment_size_max: 268435456
log_segment_size_min: 16777216
max_compacted_log_segment_size: 536870912
storage_min_free_bytes: 1073741824
superusers:
- superuser
topic_partitions_per_shard: 1000
redpanda.yaml:
----
config_file: /etc/redpanda/redpanda.yaml
pandaproxy:
pandaproxy_api:
- address: 0.0.0.0
authentication_method: http_basic
name: internal
port: 8082
- address: 0.0.0.0
authentication_method: http_basic
name: default
port: 8083
pandaproxy_api_tls:
- cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
name: internal
require_client_auth: false
truststore_file: /etc/tls/certs/default/ca.crt
- cert_file: /etc/tls/certs/external/tls.crt
enabled: true
key_file: /etc/tls/certs/external/tls.key
name: default
require_client_auth: false
truststore_file: /etc/tls/certs/external/ca.crt
pandaproxy_client:
broker_tls:
cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
require_client_auth: true
truststore_file: /etc/tls/certs/default/ca.crt
brokers:
- address: redpanda-0.redpanda.jake.svc.cluster.local.
port: 9093
- address: redpanda-1.redpanda.jake.svc.cluster.local.
port: 9093
- address: redpanda-2.redpanda.jake.svc.cluster.local.
port: 9093
redpanda:
admin:
- address: 0.0.0.0
name: internal
port: 9644
- address: 0.0.0.0
name: default
port: 9645
admin_api_tls:
- cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
name: internal
require_client_auth: false
truststore_file: /etc/tls/certs/default/ca.crt
- cert_file: /etc/tls/certs/external/tls.crt
enabled: true
key_file: /etc/tls/certs/external/tls.key
name: default
require_client_auth: false
truststore_file: /etc/tls/certs/external/ca.crt
audit_enabled: false
compacted_log_segment_size: 67108864
crash_loop_limit: 5
default_topic_replications: 3
empty_seed_starts_cluster: false
enable_sasl: true
group_topic_partitions: 16
kafka_api:
- address: 0.0.0.0
authentication_method: mtls_identity
name: internal
port: 9093
- address: 0.0.0.0
authentication_method: sasl
name: default
port: 9094
kafka_api_tls:
- cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
name: internal
require_client_auth: true
truststore_file: /etc/tls/certs/default/ca.crt
- cert_file: /etc/tls/certs/external/tls.crt
enabled: true
key_file: /etc/tls/certs/external/tls.key
name: default
require_client_auth: false
truststore_file: /etc/tls/certs/external/ca.crt
kafka_batch_max_bytes: 1048576
kafka_connection_rate_limit: 1000
kafka_enable_authorization: true
log_segment_size: 134217728
log_segment_size_max: 268435456
log_segment_size_min: 16777216
max_compacted_log_segment_size: 536870912
rpc_server:
address: 0.0.0.0
port: 33145
rpc_server_tls:
cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
require_client_auth: false
truststore_file: /etc/tls/certs/default/ca.crt
seed_servers:
- host:
address: redpanda-0.redpanda.jake.svc.cluster.local.
port: 33145
- host:
address: redpanda-1.redpanda.jake.svc.cluster.local.
port: 33145
- host:
address: redpanda-2.redpanda.jake.svc.cluster.local.
port: 33145
storage_min_free_bytes: 1073741824
superusers:
- superuser
topic_partitions_per_shard: 1000
rpk:
additional_start_flags:
- --default-log-level=info
- --memory=2048M
- --reserve-memory=205M
- --smp=1
admin_api:
addresses:
- redpanda-0.redpanda.jake.svc.cluster.local.:9644
- redpanda-1.redpanda.jake.svc.cluster.local.:9644
- redpanda-2.redpanda.jake.svc.cluster.local.:9644
tls:
truststore_file: /etc/tls/certs/default/ca.crt
enable_memory_locking: false
kafka_api:
brokers:
- redpanda-0.redpanda.jake.svc.cluster.local.:9093
- redpanda-1.redpanda.jake.svc.cluster.local.:9093
- redpanda-2.redpanda.jake.svc.cluster.local.:9093
tls:
cert_file: /etc/tls/certs/redpanda-client/tls.crt
key_file: /etc/tls/certs/redpanda-client/tls.key
truststore_file: /etc/tls/certs/default/ca.crt
overprovisioned: false
tune_aio_events: true
schema_registry:
schema_registry_api:
- address: 0.0.0.0
authentication_method: http_basic
name: internal
port: 8081
- address: 0.0.0.0
authentication_method: http_basic
name: default
port: 8084
schema_registry_api_tls:
- cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
name: internal
require_client_auth: false
truststore_file: /etc/tls/certs/default/ca.crt
- cert_file: /etc/tls/certs/external/tls.crt
enabled: true
key_file: /etc/tls/certs/external/tls.key
name: default
require_client_auth: false
truststore_file: /etc/tls/certs/external/ca.crt
schema_registry_client:
broker_tls:
cert_file: /etc/tls/certs/default/tls.crt
enabled: true
key_file: /etc/tls/certs/default/tls.key
require_client_auth: true
truststore_file: /etc/tls/certs/default/ca.crt
brokers:
- address: redpanda-0.redpanda.jake.svc.cluster.local.
port: 9093
- address: redpanda-1.redpanda.jake.svc.cluster.local.
port: 9093
- address: redpanda-2.redpanda.jake.svc.cluster.local.
port: 9093
BinaryData
====
