django - Handle Periodic Tasks Django and Celery

Question

I am trying to integrate Celery with an existing Django app using Docker. I have been trying to follow this tutorial (1). However, I keep bumping with the same error no matter what I try. I am using Django==2.2, Celery==4.4.1 Docker==19.03.8 Docker-compose==1.25.4 on Windows 10.

My project is structured in the following way (I skip some files for simplicity):

├── .gitignore
├── docker-compose.yml
├── Dockerfile
├── src
     ├── core
     │   ├── __init__.py
     │   ├── urls.py
     │   └── wsgi.py
     │   ├── celery.py
     │   ├── tasks.py
     │   ├── settings
     │   │   ├── base.py
     │   │   └── development.py
     ├── requirements.txt
     ├── .env
     └── manage.py

My docker-compose.yml:

version: '3'
services:
   python:
     build:
       context: .
       dockerfile: Dockerfile
     command: python /src/manage.py runserver 0.0.0.0:8000
     volumes:
       - ./src:/src
     depends_on: 
       - db
       - redis
     ports:
       - 8000:8000
     links:
       - db
       - redis
   db:
     image: postgres:11
     environment:
       - POSTGRES_PASSWORD=postgres
     ports:
       - "5432:5432"
   redis:
     image: redis
     ports:
       - "6379:6379"
   celery-beat:
      build:
        context: .
        dockerfile: Dockerfile
        command: celery worker -A core -l debug
     env_file:
        - "src/.env"
     environment:
       - SECRET_KEY=kobl@t=yw9d*0y%jt2gjnq78=u!z_rrxb&w8e47l!(jz@m79zy
     depends_on:
        - redis
 celery:
   build:
     context: .
     dockerfile: Dockerfile
   command: celery worker -A core-l debug
   env_file:
     - "src/.env"
   depends_on:
     - redis

Running 'docker-compose logs celery' after building gives the following:

celery_1       | Traceback (most recent call last):
celery_1       |   File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 42, in __get__
celery_1       |     return obj.__dict__[self.__name__]
celery_1       | KeyError: 'data'
celery_1       |
celery_1       | During handling of the above exception, another exception occurred:
celery_1       |
celery_1       | Traceback (most recent call last):
celery_1       |   File "/usr/local/bin/celery", line 8, in <module>
celery_1       |     sys.exit(main())
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/__main__.py", line 16, in main
celery_1       |     _main()
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 322, in main
celery_1       |     cmd.execute_from_commandline(argv)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 496, in execute_from_comm
andline
celery_1       |     super(CeleryCommand, self).execute_from_commandline(argv)))
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 298, in execute_from_comman
dline
celery_1       |     return self.handle_argv(self.prog_name, argv[1:])
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 488, in handle_argv
celery_1       |     return self.execute(command, argv)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 420, in execute
celery_1       |     ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 221, in run_from_argv
celery_1       |     *self.parse_options(prog_name, argv, command))
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 421, in parse_options
celery_1       |     self.parser = self.create_parser(prog_name, command)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 437, in create_parser
celery_1       |     self.add_arguments(parser)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 277, in add_arguments
celery_1       |     default=conf.worker_state_db,
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 134, in __getattr_
_
celery_1       |     return self[k]
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 437, in __getitem_
_
celery_1       |     return getitem(k)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 286, in __getitem_
_
celery_1       |     return mapping[_key]
celery_1       |   File "/usr/local/lib/python3.6/collections/__init__.py", line 987, in __getitem__
celery_1       |     if key in self.data:
celery_1       |   File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 44, in __get__
celery_1       |     value = obj.__dict__[self.__name__] = self.__get(obj)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 141, in data
celery_1       |     return self.callback()
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 935, in _finalize_pending_c
onf
celery_1       |     conf = self._conf = self._load_config()
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 945, in _load_config
celery_1       |     self.loader.config_from_object(self._config_source)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/loaders/base.py", line 131, in config_from_obj
ect
celery_1       |     self._conf = force_mapping(obj)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 54, in force_mappi
ng
celery_1       |     if isinstance(m, (LazyObject, LazySettings)):
celery_1       |   File "/usr/local/lib/python3.6/site-packages/django/utils/functional.py", line 256, in inner
celery_1       |     self._setup()
celery_1       |   File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 66, in _setup
celery_1       |     self._wrapped = Settings(settings_module)
celery_1       |   File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 176, in __init__
celery_1       |     raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")
celery_1       | django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.

Running 'docker-compose celery-beat' after building:

 celery-beat_1  | Traceback (most recent call last):
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 42, in __get__
celery-beat_1  |     return obj.__dict__[self.__name__]
celery-beat_1  | KeyError: 'data'
celery-beat_1  |
celery-beat_1  | During handling of the above exception, another exception occurred:
celery-beat_1  |
celery-beat_1  | Traceback (most recent call last):
celery-beat_1  |   File "/usr/local/bin/celery", line 8, in <module>
celery-beat_1  |     sys.exit(main())
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/__main__.py", line 16, in main
celery-beat_1  |     _main()
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 322, in main
celery-beat_1  |     cmd.execute_from_commandline(argv)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 496, in execute_from_comm
andline
celery-beat_1  |     super(CeleryCommand, self).execute_from_commandline(argv)))
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 298, in execute_from_comman
dline
celery-beat_1  |     return self.handle_argv(self.prog_name, argv[1:])
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 488, in handle_argv
celery-beat_1  |     return self.execute(command, argv)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 420, in execute
celery-beat_1  |     ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 221, in run_from_argv
celery-beat_1  |     *self.parse_options(prog_name, argv, command))
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 421, in parse_options
celery-beat_1  |     self.parser = self.create_parser(prog_name, command)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 437, in create_parser
celery-beat_1  |     self.add_arguments(parser)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 277, in add_arguments
celery-beat_1  |     default=conf.worker_state_db,
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 134, in __getattr_
_
celery-beat_1  |     return self[k]
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 437, in __getitem_
_
celery-beat_1  |     return getitem(k)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 286, in __getitem_
_
celery-beat_1  |     return mapping[_key]
celery-beat_1  |   File "/usr/local/lib/python3.6/collections/__init__.py", line 987, in __getitem__
celery-beat_1  |     if key in self.data:
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 44, in __get__
celery-beat_1  |     value = obj.__dict__[self.__name__] = self.__get(obj)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 141, in data
celery-beat_1  |     return self.callback()
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 935, in _finalize_pending_c
onf
celery-beat_1  |     conf = self._conf = self._load_config()
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 945, in _load_config
celery-beat_1  |     self.loader.config_from_object(self._config_source)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/loaders/base.py", line 131, in config_from_obj
ect
celery-beat_1  |     self._conf = force_mapping(obj)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 54, in force_mappi
ng
celery-beat_1  |     if isinstance(m, (LazyObject, LazySettings)):
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/django/utils/functional.py", line 256, in inner
celery-beat_1  |     self._setup()
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 66, in _setup
celery-beat_1  |     self._wrapped = Settings(settings_module)
celery-beat_1  |   File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 176, in __init__
celery-beat_1  |     raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")
celery-beat_1  | django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.

However, 'docker-compose config' shows that the SECRET_KEY is there:

services:
  celery:
    build:
      context: C:\Users\project
      dockerfile: Dockerfile
    command: celery worker -A core-l debug
    depends_on:
    - redis
    environment:
      DEBUG: "False"
      SECRET_KEY: kobl@t=yw9d*0y%jt2gjnq78=u!z_rrxb&w8e47l!(jz@m79zy
  celery-beat:
    build:
      context: C:\Users\project
      dockerfile: Dockerfile
    command: celery worker -A core-l debug
    depends_on:
    - redis
    environment:
      DEBUG: "False"
      SECRET_KEY: kobl@t=yw9d*0y%jt2gjnq78=u!z_rrxb&w8e47l!(jz@m79zy
  db:
    environment:
      POSTGRES_PASSWORD: postgres
    image: postgres:11
    ports:
    - 5432:5432/tcp
  python:
    build:
      context: C:\Users\project
      dockerfile: Dockerfile
    command: python /src/manage.py runserver 0.0.0.0:8000
    depends_on:
    - db
    - redis
    links:
    - db
    - redis
    ports:
    - 8000:8000/tcp
    volumes:
    - C:\Users\project\src:/src:rw
  redis:
    image: redis
    ports:
    - 6379:6379/tcp
version: '3.0'

标签： djangodockercelery