Django Celery issues within Docker

Having a major issue I cannot track down and I'm spinning my wheels. Any help would be greatly appreciated. I have django, celerybeat and celeryworker containers that all use the same base image of django.

local.yml

version: "3"

volumes:
  local_postgres_data: {}
  local_postgres_data_backups: {}

services:
  django: &django
    build:
      context: .
      dockerfile: ./compose/local/django/Dockerfile
    image: local_django
    container_name: django
    depends_on:
      - postgres
      - mailhog
      - redis
    volumes:
      - .:/app
    env_file:
      - ./.envs/.local/.django
      - ./.envs/.local/.postgres
    ports:
      - "8000:8000"
    command: /start
    restart: always

  postgres:
    build:
      context: .
      dockerfile: ./compose/production/postgres/Dockerfile
    image: production_postgres
    container_name: postgres
    volumes:
      - local_postgres_data:/var/lib/postgresql/data
      - local_postgres_data_backups:/backups
    env_file:
      - ./.envs/.local/.postgres
    ports:
      - "5432:5432"

  mailhog:
    image: mailhog/mailhog:v1.0.0
    container_name: mailhog
    ports:
      - "8025:8025"
    logging:
      driver: "none" # disable saving logs

  redis:
    image: redis:6
    container_name: redis

  celeryworker:
    <<: *django
    image: local_celeryworker
    container_name: celeryworker
    depends_on:
      - redis
      - postgres
      - mailhog
    ports: []
    command: /start-celeryworker
    restart: always

  celerybeat:
    <<: *django
    image: local_celerybeat
    container_name: celerybeat
    depends_on:
      - redis
      - postgres
      - mailhog
    ports: []
    command: /start-celerybeat
    restart: always

  node:
    build:
      context: .
      dockerfile: ./compose/local/node/Dockerfile
    image: local_node
    container_name: node
    depends_on:
      - django
    volumes:
      - .:/app
      # http://jdlm.info/articles/2016/03/06/lessons-building-node-app-docker.html
      - /app/node_modules
    command: npm run dev
    ports:
      - "3000:3000"
      # Expose browsersync UI: https://www.browsersync.io/docs/options/#option-ui
      - "3001:3001"

compose/local/django/Dockerfile

ARG PYTHON_VERSION=3.9-slim-bullseye

# define an alias for the specfic python version used in this file.
FROM python:${PYTHON_VERSION} as python

# Python build stage
FROM python as python-build-stage

ARG BUILD_ENVIRONMENT=local

# Install apt packages
RUN apt-get update && apt-get install --no-install-recommends -y \
  # dependencies for building Python packages
  build-essential \
  # psycopg2 dependencies
  libpq-dev

# Requirements are installed here to ensure they will be cached.
COPY ./requirements .

# Create Python Dependency and Sub-Dependency Wheels.
RUN pip wheel --wheel-dir /usr/src/app/wheels  \
  -r ${BUILD_ENVIRONMENT}.txt


# Python 'run' stage
FROM python as python-run-stage

ARG BUILD_ENVIRONMENT=local
ARG APP_HOME=/app

ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
ENV BUILD_ENV ${BUILD_ENVIRONMENT}

WORKDIR ${APP_HOME}

# Install required system dependencies
RUN apt-get update && apt-get install --no-install-recommends -y \
  # psycopg2 dependencies
  libpq-dev \
  # Translations dependencies
  gettext \
  # cleaning up unused files
  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
  && rm -rf /var/lib/apt/lists/*

# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
# copy python dependency wheels from python-build-stage
COPY --from=python-build-stage /usr/src/app/wheels  /wheels/

# use wheels to install python dependencies
RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
  && rm -rf /wheels/

COPY ./compose/production/django/entrypoint /entrypoint
RUN sed -i 's/\r$//g' /entrypoint
RUN chmod +x /entrypoint

COPY ./compose/local/django/start /start
RUN sed -i 's/\r$//g' /start
RUN chmod +x /start


COPY ./compose/local/django/celery/worker/start /start-celeryworker
RUN sed -i 's/\r$//g' /start-celeryworker
RUN chmod +x /start-celeryworker

COPY ./compose/local/django/celery/beat/start /start-celerybeat
RUN sed -i 's/\r$//g' /start-celerybeat
RUN chmod +x /start-celerybeat

COPY ./compose/local/django/celery/flower/start /start-flower
RUN sed -i 's/\r$//g' /start-flower
RUN chmod +x /start-flower


# copy application code to WORKDIR
COPY . ${APP_HOME}

ENTRYPOINT ["/entrypoint"]

config/celery_app.py

import os

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")

app = Celery("myapp")

# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object("django.conf:settings", namespace="CELERY")

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()

config/__init__.py

# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery_app import app as celery_app

__all__ = ("celery_app",)

compose/local/django/celery/beat/start

#!/bin/bash

set -o errexit
set -o nounset


rm -f './celerybeat.pid'
celery -A config.celery_app beat -l INFO

compose/local/django/celery/worker/start

#!/bin/bash

set -o errexit
set -o nounset


watchgod celery.__main__.main --args -A config.celery_app worker -l INFO

When running docker-compose up without celeryworker or celerybeat the modules are found fine in the django container. But as soon as I add the celeryworker or celeryworker I get the following about not being able to find modules.

celery beat v5.2.7 (dawn-chorus) is starting.
2022-08-09 14:07:41,879 [celery.utils.dispatch.signal:280] ERROR - Signal handler <bound method DjangoFixup.on_import_modules of <celery.fixups.django.DjangoFixup object at 0x7f6cff68e670>> raised: ModuleNotFoundError("No module named 'customadmin'")
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 82, in on_import_modules
    self.worker_fixup.validate_models()
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 120, in validate_models
    self.django_setup()
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 116, in django_setup
    django.setup()
  File "/usr/local/lib/python3.9/site-packages/django/__init__.py", line 24, in setup
    apps.populate(settings.INSTALLED_APPS)
  File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 91, in populate
    app_config = AppConfig.create(entry)
  File "/usr/local/lib/python3.9/site-packages/django/apps/config.py", line 224, in create
    import_module(entry)
  File "/usr/local/lib/python3.9/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
  File "<frozen importlib._bootstrap>", line 984, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'customadmin'
2022-08-09 14:07:41,880 [celery.utils.dispatch.signal:280] ERROR - Signal handler <promise@0x7f6cff6491f0 --> <bound method Celery._autodiscover_tasks of <Celery bahamar at 0x7f6cff68eee0>>> raised: AppRegistryNotReady("Apps aren't loaded yet.")
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/usr/local/lib/python3.9/site-packages/vine/promises.py", line 160, in __call__
    return self.throw()
  File "/usr/local/lib/python3.9/site-packages/vine/promises.py", line 157, in __call__
    retval = fun(*final_args, **final_kwargs)
  File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 689, in _autodiscover_tasks
    return self._autodiscover_tasks_from_fixups(related_name)
  File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 698, in _autodiscover_tasks_from_fixups
    return self._autodiscover_tasks_from_names([
  File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 701, in <listcomp>
    for pkg in fixup.autodiscover_tasks()
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 92, in autodiscover_tasks
    return [config.name for config in apps.get_app_configs()]
  File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 145, in get_app_configs
    self.check_apps_ready()
  File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 136, in check_apps_ready
    raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
Traceback (most recent call last):
  File "/usr/local/bin/celery", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.9/site-packages/celery/__main__.py", line 15, in main
    sys.exit(_main())
  File "/usr/local/lib/python3.9/site-packages/celery/bin/celery.py", line 217, in main
    return celery(auto_envvar_prefix="CELERY")
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/click/decorators.py", line 26, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/celery/bin/base.py", line 134, in caller
    return f(ctx, *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/celery/bin/beat.py", line 72, in beat
    return beat().run()
  File "/usr/local/lib/python3.9/site-packages/celery/apps/beat.py", line 75, in run
    self.init_loader()
  File "/usr/local/lib/python3.9/site-packages/celery/apps/beat.py", line 124, in init_loader
    self.app.loader.init_worker()
  File "/usr/local/lib/python3.9/site-packages/celery/loaders/base.py", line 111, in init_worker
    self.import_default_modules()
  File "/usr/local/lib/python3.9/site-packages/celery/loaders/base.py", line 105, in import_default_modules
    raise response
  File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 82, in on_import_modules
    self.worker_fixup.validate_models()
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 120, in validate_models
    self.django_setup()
  File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 116, in django_setup
    django.setup()
  File "/usr/local/lib/python3.9/site-packages/django/__init__.py", line 24, in setup
    apps.populate(settings.INSTALLED_APPS)
  File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 91, in populate
    app_config = AppConfig.create(entry)
  File "/usr/local/lib/python3.9/site-packages/django/apps/config.py", line 224, in create
    import_module(entry)
  File "/usr/local/lib/python3.9/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
  File "<frozen importlib._bootstrap>", line 984, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'customadmin'

customadmin just so happens to be the first app of many apps in my LOCAL_APPS.

• Django 3.2.14
• celery 5.2.7
• django-celery-beat 2.3.0
• django-celery-results 2.2.0