0

I'm trying to migrate from LocalExecutor to CeleryExecutor in Airflow 2.1.3 using Docker with Redis. I made separate containers for webserver, scheduler, worker, redis and database. Problem: tasks are queued but not executed.

docker-compose.yml:

version: "3.3"

services:
  redis:
    image: redis:6.0.9-alpine
    healthcheck:
    test: ["CMD", "redis-cli", "ping"]
    interval: 5s
    timeout: 30s
    retries: 50
    restart: always
    ports:
      - "6793:6793"

  database:
    image: postgres:12-alpine
    restart: always
    environment:
      POSTGRES_DB: airflow
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
    volumes:
      - airflow_database:/var/lib/postgresql/data/

  webserver:
    image: airflow:latest
    restart: always
    depends_on:
      - database
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      GUNICORN_CMD_ARGS: --log-level WARNING
      EXECUTOR: Celery
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    ports:
      - 8080:8080
      - 8888:8888
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  flower:
    image: airflow:latest
    restart: always
    depends_on:
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    ports:
      - "5555:5555"
    command: celery flower -b "redis://redis:6379/1"

  scheduler:
    image: airflow:latest
    restart: always
    depends_on:
      - webserver
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    command: scheduler

  worker:
    image: airflow:latest
    restart: always
    depends_on:
      - scheduler
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    healthcheck:
    test:
      - "CMD-SHELL"
      - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
    interval: 10s
    timeout: 10s
    retries: 5
    command: celery worker -b "redis://redis:6379/1" --result-backend "db+postgresql://airflow:airflow@database/airflow"

volumes:
  airflow_database:
  airflow_data:
  airflow_logs:
  staging_database:

Dockerfile, airflow.cfg, entrypoint.sh

All containers loaded normally. I tried to do celery_result_backend == broker_url == 'redis://redis:6379/1' but to no avail. The flower shows what the worker itself created, but the worker container doesn't show a single line of logs. I also tried to use the worker container separately - it did not help.

Flower

1
  • Please trim your code to make it easier to find your problem. Follow these guidelines to create a minimal reproducible example. Commented Mar 14, 2022 at 18:14

1 Answer 1

0

As I can see, there's an obvious port number inconsistency for Redis. '6793' above and '6379' down below

Sign up to request clarification or add additional context in comments.

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.