feat: do not run celery workers in development

In development, edx-platform runs with CELERY_ALWAYS_EAGER=True, which means that lms-worker and cms-worker never catch celery tasks! This change was heavily inspired by: #1041
overhangio · Oct 1, 2024 · 2d37e6c · 2d37e6c
1 parent 0119667
commit 2d37e6c
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 47 deletions.
diff --git a/changelog.d/20241001_123236_regis_celery.md b/changelog.d/20241001_123236_regis_celery.md
@@ -0,0 +1 @@
+- [Improvement] Do not run useless celery workers (lms-worker, cms-worker) in development. This should save us ~700MB memory. (by @arbrandes, @regisb).
diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py
@@ -73,6 +73,7 @@ def run_task(self, service: str, command: str) -> int:
 
 class BaseComposeContext(BaseTaskContext):
     NAME: t.Literal["local", "dev"]
+    OPENEDX_SERVICES: list[str] = ["lms", "cms"]
 
     def job_runner(self, config: Config) -> ComposeTaskRunner:
         raise NotImplementedError
@@ -292,7 +293,7 @@ def restart(context: BaseComposeContext, services: list[str]) -> None:
     else:
         for service in services:
             if service == "openedx":
-                command += ["lms", "lms-worker", "cms", "cms-worker"]
+                command += context.OPENEDX_SERVICES
             else:
                 command.append(service)
     context.job_runner(config).docker_compose(*command)

diff --git a/tutor/commands/local.py b/tutor/commands/local.py
@@ -29,6 +29,7 @@ def __init__(self, root: str, config: Config):
 # pylint: disable=too-few-public-methods
 class LocalContext(compose.BaseComposeContext):
     NAME = "local"
+    OPENEDX_SERVICES = ["lms", "cms", "lms-worker", "cms-worker"]
 
     def job_runner(self, config: Config) -> LocalTaskRunner:
         return LocalTaskRunner(self.root, config)

diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml
@@ -32,12 +32,6 @@ services:
     ports:
         - "8001:8000"
 
-  lms-worker:
-    <<: *openedx-service
-
-  cms-worker:
-    <<: *openedx-service
-
   # Additional service for watching theme changes
   watchthemes:
     <<: *openedx-service

diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml
@@ -25,4 +25,43 @@ services:
           {{ patch("local-docker-compose-caddy-aliases")|indent(10) }}
     {% endif %}
 
+  ############# LMS and CMS workers
+  lms-worker:
+    image: {{ DOCKER_IMAGE_OPENEDX }}
+    environment:
+      SERVICE_VARIANT: lms
+      DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
+    command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default
+    restart: unless-stopped
+    volumes:
+      - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
+      - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
+      - ../apps/openedx/config:/openedx/config:ro
+      - ../../data/lms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
+      {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %}
+      - {{ mount }}
+      {%- endfor %}
+    depends_on:
+      - lms
+
+  cms-worker:
+    image: {{ DOCKER_IMAGE_OPENEDX }}
+    environment:
+      SERVICE_VARIANT: cms
+      DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
+    command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default
+    restart: unless-stopped
+    volumes:
+      - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
+      - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
+      - ../apps/openedx/config:/openedx/config:ro
+      - ../../data/cms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
+      {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %}
+      - {{ mount }}
+      {%- endfor %}
+    depends_on:
+      - cms
+
   {{ patch("local-docker-compose-prod-services")|indent(2) }}
diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml
@@ -151,44 +151,4 @@ services:
       {% if RUN_SMTP %}- smtp{% endif %}
       {{ patch("local-docker-compose-cms-dependencies")|indent(6) }}
 
-  ############# LMS and CMS workers
-
-  lms-worker:
-    image: {{ DOCKER_IMAGE_OPENEDX }}
-    environment:
-      SERVICE_VARIANT: lms
-      DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
-    command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default
-    restart: unless-stopped
-    volumes:
-      - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
-      - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
-      - ../apps/openedx/config:/openedx/config:ro
-      - ../../data/lms:/openedx/data
-      - ../../data/openedx-media:/openedx/media
-      {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %}
-      - {{ mount }}
-      {%- endfor %}
-    depends_on:
-      - lms
-
-  cms-worker:
-    image: {{ DOCKER_IMAGE_OPENEDX }}
-    environment:
-      SERVICE_VARIANT: cms
-      DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
-    command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default
-    restart: unless-stopped
-    volumes:
-      - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
-      - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
-      - ../apps/openedx/config:/openedx/config:ro
-      - ../../data/cms:/openedx/data
-      - ../../data/openedx-media:/openedx/media
-      {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %}
-      - {{ mount }}
-      {%- endfor %}
-    depends_on:
-      - cms
-
   {{ patch("local-docker-compose-services")|indent(2) }}