Redis authentication error

[Jesse-Millwood]

After running docker compose up I receive this error:

zulip-1      |   File "/srv/zulip-venv-cache/41efcec5630d3f53c9f2bd1f3b5f6a9921c0197f/zulip-py3-venv/lib/python3.12/site-packages/redis/_parsers/resp2.py", line 38, in _read_response
zulip-1      |     raise error
zulip-1      | redis.exceptions.AuthenticationError: AUTH <password> called without any password configured for the default user. Are you sure your configuration is correct?
zulip-1      |   Applying zerver.0260_missed_message_addresses_from_redis_to_db...Zulip first start database initi failed in "initialize-database" exit code 1. Exiting.
zulip-1 exited with code 1

I have set all of the passwords.

I do see this earlier in the log though:

zulip-1      | Empty var for key "EMAIL_HOST".
zulip-1      | Setting key "EMAIL_HOST_USER", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "EMAIL_PORT", type "integer" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "EMAIL_USE_SSL", type "bool" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "EMAIL_USE_TLS", type "bool" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "EXTERNAL_HOST", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "MEMCACHED_LOCATION", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "RABBITMQ_HOST", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "RABBITMQ_USER", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "RATE_LIMITING", type "bool" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "REDIS_HOST", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "REDIS_PORT", type "integer" in file "/etc/zulip/settings.py".
zulip-1      | Setting key "ZULIP_ADMINISTRATOR", type "string" in file "/etc/zulip/settings.py".
zulip-1      | Zulip configuration succeeded.
zulip-1      | Auto backup enabled.

I have copied this almost verbatim over to run in HomeLabOs. The only difference really is that I'm mapping the volumes to specific local directories and I've added traefik labels.

[Jesse-Millwood]

For what it's worth, this is a redacted version of my compose file with homelab os template variables filled in:

---
version: '3'

networks:
  traefik_network:
    external:
      name: homelabos_traefik
  zulip:

services:
  database:
    image: zulip/zulip-postgresql:14
    restart: unless-stopped
    networks:
      - zulip
    environment:
      POSTGRES_DB: 'zulip'
      POSTGRES_USER: 'zulip'
      POSTGRES_PASSWORD: "<postgres-password>"
    volumes:
      - '/var/homelabos/zulip/postgresql:/var/lib/postgresql/data:rw'

  memcached:
    image: 'memcached:alpine'
    networks:
      - zulip
    restart: unless-stopped
    command:
      - "sh"
      - "-euc"
      - |
        echo 'mech_list: plain' > "$$SASL_CONF_PATH"
        echo "zulip@$$HOSTNAME:$$MEMCACHED_PASSWORD" > "$$MEMCACHED_SASL_PWDB"
        echo "zulip@localhost:$$MEMCACHED_PASSWORD" >> "$$MEMCACHED_SASL_PWDB"
        exec memcached -S
    environment:
      SASL_CONF_PATH: "/home/memcache/memcached.conf"
      MEMCACHED_SASL_PWDB: "/home/memcache/memcached-sasl-db"
      MEMCACHED_PASSWORD: "<memcached-password>"

  rabbitmq:
    image: rabbitmq:3.12.14
    networks:
      - zulip
    hostname: zulip-rabbit
    restart: unless-stopped
    environment:
        RABBITMQ_DEFAULT_USER: 'zulip'
        RABBITMQ_DEFAULT_PASS: "<rabbitmq-password>"
    volumes:
      - '/var/homelabos/zulip/rabbitmq:/var/lib/rabbitmq:rw'

  redis:
    image: "redis:alpine"
    networks:
      - zulip
    volumes:
      - '/var/homelabos/zulip/redis:/var/lib/redis:rw'
    command:
      - "sh"
      - "-euc"
      - |
        echo "requirepass '$$REDIS_PASSWORD'" > /etc/redis.conf
        exec redis-server /etc/redis.conf
    environment:
      REDIS_PASSWORD: "<redis-password>"

  zulip:
    image: zulip/docker-zulip:9.2-0
    networks:
      - traefik_network
      - zulip
    environment:
      DB_HOST: 'database'
      DB_HOST_PORT: '5432'
      DB_USER: 'zulip'
      SSL_CERTIFICATE_GENERATION: 'self-signed'
      DISABLE_HTTPS: 'True'
      SETTING_MEMCACHED_LOCATION: 'memcached:11211'
      SETTING_RABBITMQ_HOST: 'rabbitmq'
      SETTING_REDIS_HOST: 'redis'
      SECRETS_email_password: '123456789'
      SECRETS_rabbitmq_password: "<rabbitmq-password>"
      SECRETS_postgres_password: "<postgres-password>"
      SECRETS_memcached_password: "<memcached-password>"
      SECRETS_redis_password: "<redis-password>"
      SECRETS_secret_key: "<secrets-key>"
      SETTING_EXTERNAL_HOST: 'zulip.homelab.<my-domain>.com'
      SETTING_ZULIP_ADMINISTRATOR: '[email protected]'
      SETTING_EMAIL_HOST: ''  # e.g. smtp.example.com
      SETTING_EMAIL_HOST_USER: '[email protected]'
      SETTING_EMAIL_PORT: '587'
      # It seems that the email server needs to use ssl or tls and can't be used without it
      SETTING_EMAIL_USE_SSL: 'False'
      SETTING_EMAIL_USE_TLS: 'True'
      ZULIP_AUTH_BACKENDS: 'EmailAuthBackend'
      # Uncomment this when configuring the mobile push notifications service
      # SETTING_PUSH_NOTIFICATION_BOUNCER_URL: 'https://push.zulipchat.com'
    volumes:
      - '/var/homelabos/zulip/zulip:/data:rw'
    ulimits:
      nofile:
        soft: 40000
        hard: 50000
    labels:
      - "traefik.http.services.zulip.loadbalancer.server.scheme=http"
      - "traefik.http.services.zulip.loadbalancer.server.port=80"
      - "traefik.enable=true"
      - "traefik.docker.network=homelabos_traefik"
      - "traefik.http.routers.zulip-http.service=zulip"
      - "traefik.http.routers.zulip-http.rule=Host(`zulip.homelab.<my-domain>.com`)"
      - "traefik.http.routers.zulip-http.entrypoints=http"
      - "traefik.http.routers.zulip-http.middlewares=customFrameHomelab@file"
      - "traefik.http.routers.zulip.service=zulip"
      - "traefik.http.routers.zulip.rule=Host(`zulip.homelab.<my-domain>.com`)"
      - "traefik.http.routers.zulip.entrypoints=https"
      - "traefik.http.routers.zulip.middlewares=customFrameHomelab@file"
      - "traefik.http.routers.zulip.tls=true"
      - "traefik.http.routers.zulip.tls.certresolver=http"

[timabbott]

Have you checked whether the password you set is being correctly written into /etc/redis/zulip-redis.conf on the redis container and /etc/zulip/zulip-secrets.conf in the zulip container?

[Jesse-Millwood]

Have you checked whether the password you set is being correctly written into /etc/redis/zulip-redis.conf on the redis container and /etc/zulip/zulip-secrets.conf in the zulip container?

I was able to perform docker exec -it zulip-redis-1 sh to open a shell in the redis container spun up by docker compose up. I did not see an /etc/redis/zulip-redis.conf file but there is an /etc/redis.conf file that does have the requirepass line with the password that does match what I have in my docker compose yaml file. This configuration file seems to line up with what is happening in the docker compose file in this repo https://github.com/zulip/docker-zulip/blob/f445fc5fa1c3a595282226d5478c87e030ed5408/docker-compose.yml#L44.

I was then able to also drop to a shell in the zulip container before it died and print out the /etc/zulip/zulip-secrets.conf file. The redis_password line here matches the requirepass line in the redis container's /etc/redis.conf file.

[timabbott]

Puzzling. https://github.com/redis/redis/issues/13437 and my understanding of the Redis configuration system suggests that Redis should really be using the password there... Is it possible the password contains characters that redis can't parse? Maybe worth checking the logs there.

[Jesse-Millwood]

Puzzling. redis/redis#13437 and my understanding of the Redis configuration system suggests that Redis should really be using the password there... Is it possible the password contains characters that redis can't parse? Maybe worth checking the logs there.

The password that I generated is all alpha-numeric. I ended up opening a shell into the redis container with docker exec -it zulip-redis-1 sh and then used the redis-cli to start a cli session:

redis-cli -h 127.0.0.1 -p 6379
127.0.0.1:6379> AUTH <mypassword>
OK
127.0.0.1:6379> config get requirepass
1) "requirepass"
2) "<mypassword>"

So it seems like the password is fine for redis.

I feel like my compose file is the same as the one in this repo. Is there something else people do to deploy using the compose file in this repo?

I didn't see any logs written anywhere. I looked in /var/log in the container and it was empty.

[timabbott]

I'm pretty sure the compose file here just works.

Very puzzling. Well, a few thoughts:

• This particular trace will not be possible in Zulip 10.0, since we squashed the migration in question.
• You might be able to get past this failure by just adding a return statement at the start of move_missed_message_addresses_to_database in 0260_missed_message_addresses_from_redis_to_db.py; that's the only migration file that uses redis. I could imagine that there's some sort of initialization order failure here, though I don't think it's likely.
• I would double-check for typos and for having substituted properly. Like, if your redis password is abcd1234, you want abcd1234, not <abcd1234> to appear everywhere.

From a Zulip perspective, I think our fix is going to be something like #272 for making the secret generation/substitution automated.

[Jesse-Millwood]

I'm pretty sure the compose file here just works.

Very puzzling. Well, a few thoughts:

* This particular trace will not be possible in Zulip 10.0, since we squashed the migration in question.

* You might be able to get past this failure by just adding a `return` statement at the start of `move_missed_message_addresses_to_database` in `0260_missed_message_addresses_from_redis_to_db.py`; that's the only migration file that uses `redis`. I could imagine that there's some sort of initialization order failure here, though I don't think it's likely.

* I would double-check for typos and for having substituted properly. Like, if your redis password is `abcd1234`, you want `abcd1234`, not `<abcd1234>` to appear everywhere.

From a Zulip perspective, I think our fix is going to be something like #272 for making the secret generation/substitution automated.

Interesting, Thanks for looking into this. Maybe I'll wait for 10.0. I'm positive the passwords are the same here because HomelabOS uses templating so the password is inserted from one source. I might be doing something wrong too if I'm the only one having this issue.

[timabbott]

We did merge some PRs upgrading dependency versions not long ago, so maybe it's worth someone else testing that main in this repo works for them. Though I guess you're not using exactly the Kubernetes or docker-compose configs that we distribute?

[Jesse-Millwood]

We did merge some PRs upgrading dependency versions not long ago, so maybe it's worth someone else testing that main in this repo works for them. Though I guess you're not using exactly the Kubernetes or docker-compose configs that we distribute?

I did basically copy the docker compose file from this repo and just added the HomelabOS stuff, which is really just the templated fields and traefik labels. I posted my file earlier but it doesn't differ much beyond that. Maybe I'll try a simpler version outside of the HomelabOS environment. Thanks for your help, I'll report back here.

[hannes-sistemica]

Hi, I have the same issue. Tried the complete day, as I thought I had some issues with my env variables :( I tried all possible combinations, also checked redis (I can connect with given password from other tools), and I stopped postgres so that I had the chance to print out env vars of zulip container and it had all the right values as well. My compose is similar to Jesse, just extracted all env variables and added some traefik stuff. Btw. I am using Portainer for deployments. Here my compose:

services:
  database:
    image: "zulip/zulip-postgresql:14"
    restart: unless-stopped
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    volumes:
      - "postgresql-14:/var/lib/postgresql/data:rw"
    networks:
      - zulip-net
    deploy:
      placement:
        constraints:
          - node.labels.zulip == true
      mode: replicated
      replicas: 1

  memcached:
    image: "memcached:alpine"
    restart: unless-stopped
    command:
      - "sh"
      - "-euc"
      - |
        echo 'mech_list: plain' > "/home/memcache/memcached.conf"
        echo "zulip@$$HOSTNAME:${MEMCACHED_PASSWORD}" > "/home/memcache/memcached-sasl-db"
        echo "zulip@localhost:${MEMCACHED_PASSWORD}" >> "/home/memcache/memcached-sasl-db"
        exec memcached -S
    environment:
      MEMCACHED_PASSWORD: ${MEMCACHED_PASSWORD}
    networks:
      - zulip-net
    deploy:
      placement:
        constraints:
          - node.labels.zulip == true
      mode: replicated
      replicas: 1

  rabbitmq:
    image: "rabbitmq:3.12.14"
    restart: unless-stopped
    environment:
      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
    volumes:
      - "rabbitmq:/var/lib/rabbitmq:rw"
    networks:
      - zulip-net
    deploy:
      placement:
        constraints:
          - node.labels.zulip == true
      mode: replicated
      replicas: 1

  redis:
    image: "redis:alpine"
    restart: unless-stopped
    command:
      - "sh"
      - "-euc"
      - |
        echo "requirepass $REDIS_PASSWORD" > /etc/redis.conf
        exec redis-server /etc/redis.conf
    environment:
      REDIS_PASSWORD: ${REDIS_PASSWORD}
    volumes:
      - "redis:/data:rw"
    networks:
      - zulip-net
      - internal-backbone
    deploy:
      placement:
        constraints:
          - node.labels.zulip == true
      mode: replicated
      replicas: 1

  zulip:
    image: "zulip/docker-zulip:9.3-0"
    environment:
      # Basic settings
      DB_HOST: "database"
      DB_HOST_PORT: "5432"
      DB_USER: ${POSTGRES_USER}
      DB_DATABASE: ${POSTGRES_DB}
      
      # SSL Configuration
      SSL_CERTIFICATE_GENERATION: "self-signed"
      
      # Proxy Configuration
      SETTING_APPLICATION_SERVER_HTTP_ONLY: ${SETTING_APPLICATION_SERVER_HTTP_ONLY}
      SETTING_USE_X_FORWARDED_PROTO: ${SETTING_USE_X_FORWARDED_PROTO}
      SETTING_USE_X_FORWARDED_HOST: ${SETTING_USE_X_FORWARDED_HOST}
      SETTING_USE_X_FORWARDED_PORT: ${SETTING_USE_X_FORWARDED_PORT}
      SETTING_LOADBALANCER_IPS: ${SETTING_LOADBALANCER_IPS}
      
      # Service connections
      SETTING_MEMCACHED_LOCATION: "memcached:11211"
      SETTING_RABBITMQ_HOST: "rabbitmq"
      SETTING_REDIS_HOST: "redis"
      SETTING_REDIS_PORT: "6379"
      
      # Secrets and Passwords
      SECRETS_rabbitmq_password: ${RABBITMQ_DEFAULT_PASS}
      SECRETS_postgres_password: ${POSTGRES_PASSWORD}
      SECRETS_memcached_password: ${MEMCACHED_PASSWORD}
      SECRETS_redis_password: ${REDIS_PASSWORD}
      SECRETS_email_password: ${SECRETS_email_password}
      SECRETS_secret_key: ${SECRETS_secret_key}
      
      # Email settings
      SETTING_EXTERNAL_HOST: ${SETTING_EXTERNAL_HOST}
      SETTING_ZULIP_ADMINISTRATOR: ${SETTING_ZULIP_ADMINISTRATOR}
      SETTING_EMAIL_HOST: ${SETTING_EMAIL_HOST}
      SETTING_EMAIL_HOST_USER: ${SETTING_EMAIL_HOST_USER}
      SETTING_EMAIL_PORT: ${SETTING_EMAIL_PORT}
      SETTING_EMAIL_USE_SSL: ${SETTING_EMAIL_USE_SSL}
      SETTING_EMAIL_USE_TLS: ${SETTING_EMAIL_USE_TLS}
      
      # Auth settings
      ZULIP_AUTH_BACKENDS: ${ZULIP_AUTH_BACKENDS}
    volumes:
      - "zulip:/data:rw"
    networks:
      - zulip-net
      - traefik-public
    deploy:
      placement:
        constraints:
          - node.labels.zulip == true
      mode: replicated
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
        window: 120s
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=traefik-public"
        - "traefik.http.routers.zulip-https.rule=Host(`${SETTING_EXTERNAL_HOST}`)"
        - "traefik.http.routers.zulip-https.entrypoints=https"
        - "traefik.http.routers.zulip-https.tls=true"
        - "traefik.http.routers.zulip-https.tls.certresolver=${TRAEFIK_CERT_RESOLVER}"
        - "traefik.http.services.zulip-https.loadbalancer.server.port=443"
        - "traefik.http.middlewares.zulip-https-headers.headers.customrequestheaders.X-Forwarded-Proto=https"
        - "traefik.http.routers.zulip-https.middlewares=zulip-https-headers"
    ulimits:
      nofile:
        soft: 1000000
        hard: 1048576

networks:
  zulip-net:
    driver: overlay
    attachable: true
  traefik-public:
    external: true
  internal-backbone:
    external: true

volumes:
  zulip:
    driver: local
  postgresql-14:
    driver: local
  rabbitmq:
    driver: local
  redis:
    driver: local

[hannes-sistemica]

The password is also correctly in the /etc/zulip/settings.py file, so the issue might be with how the Django settings are reading from this secrets file during the migration?