PoC for Ragna meta package

.github/actions/setup-env/action.yml

@@ -7,7 +7,7 @@ inputs:
     description: "Python version to install."
   optional-dependencies:
     default: "true"
-    description: "Install ragna with [all], i.e. all optional dependencies"
+    description: "Whether to install the optional dependencies"
 
 runs:
   using: composite
@@ -64,14 +64,12 @@ runs:
 
     - name: Install ragna
       shell: bash -el {0}
-      run: |
-        if [[ ${{ inputs.optional-dependencies }} == true ]]
-        then
-          PROJECT_PATH='.[all]'
-        else
-          PROJECT_PATH='.'
-        fi
-        pip install --editable "${PROJECT_PATH}"
+      run: pip install --editable .
+
+    - name: Install optional dependencies
+      if: ${{ inputs.optional-dependencies }} == 'true'
+      shell: bash -el {0}
+      run: pip install --requirement ./requirements-optional.txt
 
     - name: Display development environment
       shell: bash -el {0}

.github/workflows/docker.yml

@@ -92,7 +92,7 @@ jobs:
           cache-to: type=local,dest=${{ steps.cache.outputs.cache-dir }}
           tags: ${{ steps.metadata.outputs.tagged-image-name }}
           build-args: |
-            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA=${{ steps.metadata.outputs.version }}
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE=${{ steps.metadata.outputs.version }}
           # Unfortunately, there currently seems to be no way to build a multiplatform
           # image and access a single one for the host platform afterwards. Thus, we
           # only build a multiplatform image when we also want to push to the registry.

.github/workflows/update-docker-requirements.yml

@@ -6,6 +6,7 @@ on:
       - ".github/workflows/update-docker-requirements.yml"
       - "pyproject.toml"
       - "requirements-docker.lock"
+      - "requirements-optional.txt"
   workflow_dispatch:
 
 permissions:
@@ -37,7 +38,7 @@ jobs:
           pip-compile \
             --quiet \
             --strip-extras --output-file=requirements-docker.lock \
-            --extra=all pyproject.toml
+            pyproject.toml requirements-optional.txt
 
           NEEDS_UPDATE=$(git diff --quiet && echo 'false' || echo 'true')
           echo "needs-update=${NEEDS_UPDATE}" | tee --append $GITHUB_OUTPUT

Dockerfile

@@ -17,10 +17,10 @@ COPY pyproject.toml .
 #    the ragna folder only includes files that we are tracking. Thus, we just include
 #    everything manually.
 # 2. We need to pass the version expliclitly as
-#    --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA=...,
+#    --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE=...,
 #    since setuptools-scm cannot infer the version
 RUN echo '[tool.setuptools.package-data]\n"*" = ["*"]' >> pyproject.toml
-ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA
+ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE
 RUN pip install --progress-bar=off --no-deps .
 
 WORKDIR /var/ragna

meta-package/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = [
+    "setuptools>=64",
+    "setuptools_scm>=8",
+    "tomlkit; python_version<'3.11'"
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "Ragna"
+description = "RAG orchestration framework"
+license = {file = "LICENSE"}
+authors = [
+    { name = "Ragna Development Team", email = "[email protected]" },
+]
+readme = "README.md"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+requires-python = ">=3.9"
+dynamic = ["version", "dependencies"]
+
+[project.urls]
+Homepage = "https://ragna.chat"
+Documentation = "https://ragna.chat"
+Changelog = "https://ragna.chat/en/stable/references/release-notes/"
+Repository = "https://github.com/Quansight/ragna"
+
+[tool.setuptools_scm]
+root = ".."
+version_scheme = "release-branch-semver"
+local_scheme = "node-and-date"

meta-package/setup.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from setuptools import setup
+from setuptools_scm import Configuration, get_version
+
+HERE = Path(__file__).parent
+PROJECT_ROOT = HERE.parent
+
+
+config = Configuration.from_file(PROJECT_ROOT / "pyproject.toml")
+version = get_version(
+    root=str(PROJECT_ROOT),
+    version_scheme=config.version_scheme,
+    local_scheme=config.local_scheme,
+)
+install_requires = [f"{config.dist_name}=={version}"]
+
+with open(PROJECT_ROOT / "requirements-optional.txt") as file:
+    install_requires.extend(file.read().splitlines())
+
+
+setup(install_requires=install_requires)

pyproject.toml

@@ -1,12 +1,13 @@
 [build-system]
 requires = [
-    "setuptools>=45",
-    "setuptools_scm[toml]>=6.2",
+    "setuptools>=64",
+    "setuptools_scm>=8",
+    "tomlkit; python_version<'3.11'",
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "Ragna"
+name = "ragna-base"
 description = "RAG orchestration framework"
 license = {file = "LICENSE"}
 authors = [
@@ -50,24 +51,10 @@ Documentation = "https://ragna.chat"
 Changelog = "https://ragna.chat/en/stable/references/release-notes/"
 Repository = "https://github.com/Quansight/ragna"
 
-[project.optional-dependencies]
-# to update the array below, run scripts/update_optional_dependencies.py
-all = [
-    "chromadb>=0.4.13",
-    "httpx_sse",
-    "ijson",
-    "lancedb>=0.2",
-    "pyarrow",
-    "pymupdf>=1.23.6",
-    "python-docx",
-    "python-pptx",
-    "tiktoken",
-]
-
 [tool.setuptools_scm]
 write_to = "ragna/_version.py"
 version_scheme = "release-branch-semver"
-local_scheme = "node-and-timestamp"
+local_scheme = "node-and-date"
 
 [project.scripts]
 ragna = "ragna.__main__:app"

requirements-docker.lock

@@ -2,10 +2,10 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --extra=all --output-file=requirements-docker.lock --strip-extras pyproject.toml
+#    pip-compile --output-file=requirements-docker.lock --strip-extras pyproject.toml requirements-optional.txt
 #
 aiofiles==23.2.1
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 annotated-types==0.6.0
     # via pydantic
 anyio==4.2.0
@@ -46,7 +46,7 @@ charset-normalizer==3.3.2
 chroma-hnswlib==0.7.3
     # via chromadb
 chromadb==0.4.22
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 click==8.1.7
     # via
     #   lancedb
@@ -65,11 +65,11 @@ deprecated==1.2.14
 deprecation==2.1.0
     # via lancedb
 emoji==2.9.0
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 fastapi==0.109.0
     # via
-    #   Ragna (pyproject.toml)
     #   chromadb
+    #   ragna-base (pyproject.toml)
 filelock==3.13.1
     # via huggingface-hub
 flatbuffers==23.5.26
@@ -95,9 +95,9 @@ httpcore==1.0.2
 httptools==0.6.1
     # via uvicorn
 httpx==0.26.0
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 httpx-sse==0.4.0
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 huggingface-hub==0.20.2
     # via tokenizers
 humanfriendly==10.0
@@ -108,7 +108,7 @@ idna==3.6
     #   httpx
     #   requests
 ijson==3.2.3
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 importlib-metadata==6.11.0
     # via opentelemetry-api
 importlib-resources==6.1.1
@@ -118,7 +118,7 @@ jinja2==3.1.3
 kubernetes==29.0.0
     # via chromadb
 lancedb==0.4.4
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 linkify-it-py==2.0.2
     # via panel
 lxml==5.1.0
@@ -203,18 +203,18 @@ overrides==7.4.0
     #   lancedb
 packaging==23.2
     # via
-    #   Ragna (pyproject.toml)
     #   bokeh
     #   build
     #   deprecation
     #   huggingface-hub
     #   onnxruntime
+    #   ragna-base (pyproject.toml)
 pandas==2.1.4
     # via
     #   bokeh
     #   panel
 panel==1.4.2
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 param==2.0.1
     # via
     #   panel
@@ -238,7 +238,7 @@ py==1.11.0
     # via retry
 pyarrow==14.0.2
     # via
-    #   Ragna (pyproject.toml)
+    #   -r requirements-optional.txt
     #   pylance
 pyasn1==0.5.1
     # via
@@ -248,25 +248,25 @@ pyasn1-modules==0.3.0
     # via google-auth
 pydantic==2.5.3
     # via
-    #   Ragna (pyproject.toml)
     #   chromadb
     #   fastapi
     #   lancedb
     #   pydantic-settings
+    #   ragna-base (pyproject.toml)
 pydantic-core==2.14.6
     # via
-    #   Ragna (pyproject.toml)
     #   pydantic
+    #   ragna-base (pyproject.toml)
 pydantic-settings==2.1.0
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 pygments==2.17.2
     # via rich
 pyjwt==2.8.0
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 pylance==0.9.6
     # via lancedb
 pymupdf==1.23.15
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 pymupdfb==1.23.9
     # via pymupdf
 pypika==0.48.9
@@ -279,15 +279,15 @@ python-dateutil==2.8.2
     #   pandas
     #   posthog
 python-docx==1.1.0
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 python-dotenv==1.0.0
     # via
     #   pydantic-settings
     #   uvicorn
 python-multipart==0.0.6
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 python-pptx==0.6.23
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 pytz==2023.3.post1
     # via pandas
 pyviz-comms==3.0.1
@@ -301,11 +301,11 @@ pyyaml==6.0.1
     #   lancedb
     #   uvicorn
 questionary==2.0.1
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 ratelimiter==1.2.0.post0
     # via lancedb
 redis==5.0.1
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 regex==2023.12.25
     # via tiktoken
 requests==2.31.0
@@ -323,7 +323,7 @@ requests-oauthlib==1.3.1
 retry==0.9.2
     # via lancedb
 rich==13.7.0
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 rsa==4.9
     # via google-auth
 semver==3.0.2
@@ -339,21 +339,21 @@ sniffio==1.3.0
     #   anyio
     #   httpx
 sqlalchemy==2.0.25
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 starlette==0.35.1
     # via
-    #   Ragna (pyproject.toml)
     #   fastapi
+    #   ragna-base (pyproject.toml)
 sympy==1.12
     # via onnxruntime
 tenacity==8.2.3
     # via chromadb
 tiktoken==0.5.2
-    # via Ragna (pyproject.toml)
+    # via -r requirements-optional.txt
 tokenizers==0.15.0
     # via chromadb
 tomlkit==0.12.3
-    # via Ragna (pyproject.toml)
+    # via ragna-base (pyproject.toml)
 tornado==6.4
     # via bokeh
 tqdm==4.66.1
@@ -364,8 +364,8 @@ tqdm==4.66.1
     #   panel
 typer==0.9.0
     # via
-    #   Ragna (pyproject.toml)
     #   chromadb
+    #   ragna-base (pyproject.toml)
 typing-extensions==4.9.0
     # via
     #   chromadb
@@ -388,8 +388,8 @@ urllib3==2.1.0
     #   requests
 uvicorn==0.26.0
     # via
-    #   Ragna (pyproject.toml)
     #   chromadb
+    #   ragna-base (pyproject.toml)
 uvloop==0.19.0
     # via uvicorn
 watchfiles==0.21.0

requirements-optional.txt

@@ -0,0 +1,9 @@
+chromadb>=0.4.13
+httpx_sse
+ijson
+lancedb>=0.2
+pyarrow
+pymupdf>=1.23.6
+python-docx
+python-pptx
+tiktoken