diff --git a/changelog.d/20241001_123236_regis_celery.md b/changelog.d/20241001_123236_regis_celery.md new file mode 100644 index 0000000000..745d7a14e8 --- /dev/null +++ b/changelog.d/20241001_123236_regis_celery.md @@ -0,0 +1 @@ +- [Improvement] Do not run useless celery workers (lms-worker, cms-worker) in development. This should save us ~700MB memory. (by @arbrandes, @regisb). diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index cde530c0a2..bf0c2fd7af 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -73,6 +73,7 @@ def run_task(self, service: str, command: str) -> int: class BaseComposeContext(BaseTaskContext): NAME: t.Literal["local", "dev"] + OPENEDX_SERVICES: list[str] = ["lms", "cms"] def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError @@ -292,7 +293,7 @@ def restart(context: BaseComposeContext, services: list[str]) -> None: else: for service in services: if service == "openedx": - command += ["lms", "lms-worker", "cms", "cms-worker"] + command += context.OPENEDX_SERVICES else: command.append(service) context.job_runner(config).docker_compose(*command) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 320edaab79..a9a6141524 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -29,6 +29,7 @@ def __init__(self, root: str, config: Config): # pylint: disable=too-few-public-methods class LocalContext(compose.BaseComposeContext): NAME = "local" + OPENEDX_SERVICES = ["lms", "cms", "lms-worker", "cms-worker"] def job_runner(self, config: Config) -> LocalTaskRunner: return LocalTaskRunner(self.root, config) diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 6c435a7917..acbaa56925 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -32,12 +32,6 @@ services: ports: - "8001:8000" - lms-worker: - <<: *openedx-service - - cms-worker: - <<: *openedx-service - # Additional service for watching theme changes watchthemes: <<: *openedx-service diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index 9f24a4af11..aa08f24121 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -25,4 +25,43 @@ services: {{ patch("local-docker-compose-caddy-aliases")|indent(10) }} {% endif %} + ############# LMS and CMS workers + lms-worker: + image: {{ DOCKER_IMAGE_OPENEDX }} + environment: + SERVICE_VARIANT: lms + DJANGO_SETTINGS_MODULE: lms.envs.tutor.production + command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default + restart: unless-stopped + volumes: + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + - ../../data/lms:/openedx/data + - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %} + - {{ mount }} + {%- endfor %} + depends_on: + - lms + + cms-worker: + image: {{ DOCKER_IMAGE_OPENEDX }} + environment: + SERVICE_VARIANT: cms + DJANGO_SETTINGS_MODULE: cms.envs.tutor.production + command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default + restart: unless-stopped + volumes: + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + - ../../data/cms:/openedx/data + - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %} + - {{ mount }} + {%- endfor %} + depends_on: + - cms + {{ patch("local-docker-compose-prod-services")|indent(2) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 63c2d3c7d3..f4bfeaad76 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -151,44 +151,4 @@ services: {% if RUN_SMTP %}- smtp{% endif %} {{ patch("local-docker-compose-cms-dependencies")|indent(6) }} - ############# LMS and CMS workers - - lms-worker: - image: {{ DOCKER_IMAGE_OPENEDX }} - environment: - SERVICE_VARIANT: lms - DJANGO_SETTINGS_MODULE: lms.envs.tutor.production - command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default - restart: unless-stopped - volumes: - - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - - ../apps/openedx/config:/openedx/config:ro - - ../../data/lms:/openedx/data - - ../../data/openedx-media:/openedx/media - {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %} - - {{ mount }} - {%- endfor %} - depends_on: - - lms - - cms-worker: - image: {{ DOCKER_IMAGE_OPENEDX }} - environment: - SERVICE_VARIANT: cms - DJANGO_SETTINGS_MODULE: cms.envs.tutor.production - command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default - restart: unless-stopped - volumes: - - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - - ../apps/openedx/config:/openedx/config:ro - - ../../data/cms:/openedx/data - - ../../data/openedx-media:/openedx/media - {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %} - - {{ mount }} - {%- endfor %} - depends_on: - - cms - {{ patch("local-docker-compose-services")|indent(2) }}