diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 60684969..31510fdf 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -7,7 +7,7 @@ inputs: description: "Python version to install." optional-dependencies: default: "true" - description: "Install ragna with [all], i.e. all optional dependencies" + description: "Whether to install the optional dependencies" runs: using: composite @@ -64,14 +64,12 @@ runs: - name: Install ragna shell: bash -el {0} - run: | - if [[ ${{ inputs.optional-dependencies }} == true ]] - then - PROJECT_PATH='.[all]' - else - PROJECT_PATH='.' - fi - pip install --editable "${PROJECT_PATH}" + run: pip install --editable . + + - name: Install optional dependencies + if: ${{ inputs.optional-dependencies }} == 'true' + shell: bash -el {0} + run: pip install --requirement ./requirements-optional.txt - name: Display development environment shell: bash -el {0} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5e26e6f9..95e88cd2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -92,7 +92,7 @@ jobs: cache-to: type=local,dest=${{ steps.cache.outputs.cache-dir }} tags: ${{ steps.metadata.outputs.tagged-image-name }} build-args: | - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA=${{ steps.metadata.outputs.version }} + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE=${{ steps.metadata.outputs.version }} # Unfortunately, there currently seems to be no way to build a multiplatform # image and access a single one for the host platform afterwards. Thus, we # only build a multiplatform image when we also want to push to the registry. diff --git a/.github/workflows/update-docker-requirements.yml b/.github/workflows/update-docker-requirements.yml index 1fcf0d89..937583ce 100644 --- a/.github/workflows/update-docker-requirements.yml +++ b/.github/workflows/update-docker-requirements.yml @@ -6,6 +6,7 @@ on: - ".github/workflows/update-docker-requirements.yml" - "pyproject.toml" - "requirements-docker.lock" + - "requirements-optional.txt" workflow_dispatch: permissions: @@ -37,7 +38,7 @@ jobs: pip-compile \ --quiet \ --strip-extras --output-file=requirements-docker.lock \ - --extra=all pyproject.toml + pyproject.toml requirements-optional.txt NEEDS_UPDATE=$(git diff --quiet && echo 'false' || echo 'true') echo "needs-update=${NEEDS_UPDATE}" | tee --append $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index de260634..a5f8c01c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,10 @@ COPY pyproject.toml . # the ragna folder only includes files that we are tracking. Thus, we just include # everything manually. # 2. We need to pass the version expliclitly as -# --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA=..., +# --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE=..., # since setuptools-scm cannot infer the version RUN echo '[tool.setuptools.package-data]\n"*" = ["*"]' >> pyproject.toml -ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA +ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAGNA_BASE RUN pip install --progress-bar=off --no-deps . WORKDIR /var/ragna diff --git a/meta-package/pyproject.toml b/meta-package/pyproject.toml new file mode 100644 index 00000000..8509564e --- /dev/null +++ b/meta-package/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = [ + "setuptools>=64", + "setuptools_scm>=8", + "tomlkit; python_version<'3.11'" +] +build-backend = "setuptools.build_meta" + +[project] +name = "Ragna" +description = "RAG orchestration framework" +license = {file = "LICENSE"} +authors = [ + { name = "Ragna Development Team", email = "connect@quansight.com" }, +] +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +requires-python = ">=3.9" +dynamic = ["version", "dependencies"] + +[project.urls] +Homepage = "https://ragna.chat" +Documentation = "https://ragna.chat" +Changelog = "https://ragna.chat/en/stable/references/release-notes/" +Repository = "https://github.com/Quansight/ragna" + +[tool.setuptools_scm] +root = ".." +version_scheme = "release-branch-semver" +local_scheme = "node-and-date" diff --git a/meta-package/setup.py b/meta-package/setup.py new file mode 100644 index 00000000..1d6ab725 --- /dev/null +++ b/meta-package/setup.py @@ -0,0 +1,22 @@ +from pathlib import Path + +from setuptools import setup +from setuptools_scm import Configuration, get_version + +HERE = Path(__file__).parent +PROJECT_ROOT = HERE.parent + + +config = Configuration.from_file(PROJECT_ROOT / "pyproject.toml") +version = get_version( + root=str(PROJECT_ROOT), + version_scheme=config.version_scheme, + local_scheme=config.local_scheme, +) +install_requires = [f"{config.dist_name}=={version}"] + +with open(PROJECT_ROOT / "requirements-optional.txt") as file: + install_requires.extend(file.read().splitlines()) + + +setup(install_requires=install_requires) diff --git a/pyproject.toml b/pyproject.toml index 9ceb1569..71a7e01a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ [build-system] requires = [ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", + "setuptools>=64", + "setuptools_scm>=8", + "tomlkit; python_version<'3.11'", ] build-backend = "setuptools.build_meta" [project] -name = "Ragna" +name = "ragna-base" description = "RAG orchestration framework" license = {file = "LICENSE"} authors = [ @@ -50,24 +51,10 @@ Documentation = "https://ragna.chat" Changelog = "https://ragna.chat/en/stable/references/release-notes/" Repository = "https://github.com/Quansight/ragna" -[project.optional-dependencies] -# to update the array below, run scripts/update_optional_dependencies.py -all = [ - "chromadb>=0.4.13", - "httpx_sse", - "ijson", - "lancedb>=0.2", - "pyarrow", - "pymupdf>=1.23.6", - "python-docx", - "python-pptx", - "tiktoken", -] - [tool.setuptools_scm] write_to = "ragna/_version.py" version_scheme = "release-branch-semver" -local_scheme = "node-and-timestamp" +local_scheme = "node-and-date" [project.scripts] ragna = "ragna.__main__:app" diff --git a/requirements-docker.lock b/requirements-docker.lock index 21d52e6c..18f21fb2 100644 --- a/requirements-docker.lock +++ b/requirements-docker.lock @@ -2,10 +2,10 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --extra=all --output-file=requirements-docker.lock --strip-extras pyproject.toml +# pip-compile --output-file=requirements-docker.lock --strip-extras pyproject.toml requirements-optional.txt # aiofiles==23.2.1 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) annotated-types==0.6.0 # via pydantic anyio==4.2.0 @@ -46,7 +46,7 @@ charset-normalizer==3.3.2 chroma-hnswlib==0.7.3 # via chromadb chromadb==0.4.22 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt click==8.1.7 # via # lancedb @@ -65,11 +65,11 @@ deprecated==1.2.14 deprecation==2.1.0 # via lancedb emoji==2.9.0 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) fastapi==0.109.0 # via - # Ragna (pyproject.toml) # chromadb + # ragna-base (pyproject.toml) filelock==3.13.1 # via huggingface-hub flatbuffers==23.5.26 @@ -95,9 +95,9 @@ httpcore==1.0.2 httptools==0.6.1 # via uvicorn httpx==0.26.0 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) httpx-sse==0.4.0 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt huggingface-hub==0.20.2 # via tokenizers humanfriendly==10.0 @@ -108,7 +108,7 @@ idna==3.6 # httpx # requests ijson==3.2.3 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt importlib-metadata==6.11.0 # via opentelemetry-api importlib-resources==6.1.1 @@ -118,7 +118,7 @@ jinja2==3.1.3 kubernetes==29.0.0 # via chromadb lancedb==0.4.4 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt linkify-it-py==2.0.2 # via panel lxml==5.1.0 @@ -203,18 +203,18 @@ overrides==7.4.0 # lancedb packaging==23.2 # via - # Ragna (pyproject.toml) # bokeh # build # deprecation # huggingface-hub # onnxruntime + # ragna-base (pyproject.toml) pandas==2.1.4 # via # bokeh # panel panel==1.4.2 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) param==2.0.1 # via # panel @@ -238,7 +238,7 @@ py==1.11.0 # via retry pyarrow==14.0.2 # via - # Ragna (pyproject.toml) + # -r requirements-optional.txt # pylance pyasn1==0.5.1 # via @@ -248,25 +248,25 @@ pyasn1-modules==0.3.0 # via google-auth pydantic==2.5.3 # via - # Ragna (pyproject.toml) # chromadb # fastapi # lancedb # pydantic-settings + # ragna-base (pyproject.toml) pydantic-core==2.14.6 # via - # Ragna (pyproject.toml) # pydantic + # ragna-base (pyproject.toml) pydantic-settings==2.1.0 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) pygments==2.17.2 # via rich pyjwt==2.8.0 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) pylance==0.9.6 # via lancedb pymupdf==1.23.15 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt pymupdfb==1.23.9 # via pymupdf pypika==0.48.9 @@ -279,15 +279,15 @@ python-dateutil==2.8.2 # pandas # posthog python-docx==1.1.0 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt python-dotenv==1.0.0 # via # pydantic-settings # uvicorn python-multipart==0.0.6 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) python-pptx==0.6.23 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt pytz==2023.3.post1 # via pandas pyviz-comms==3.0.1 @@ -301,11 +301,11 @@ pyyaml==6.0.1 # lancedb # uvicorn questionary==2.0.1 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) ratelimiter==1.2.0.post0 # via lancedb redis==5.0.1 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) regex==2023.12.25 # via tiktoken requests==2.31.0 @@ -323,7 +323,7 @@ requests-oauthlib==1.3.1 retry==0.9.2 # via lancedb rich==13.7.0 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) rsa==4.9 # via google-auth semver==3.0.2 @@ -339,21 +339,21 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy==2.0.25 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) starlette==0.35.1 # via - # Ragna (pyproject.toml) # fastapi + # ragna-base (pyproject.toml) sympy==1.12 # via onnxruntime tenacity==8.2.3 # via chromadb tiktoken==0.5.2 - # via Ragna (pyproject.toml) + # via -r requirements-optional.txt tokenizers==0.15.0 # via chromadb tomlkit==0.12.3 - # via Ragna (pyproject.toml) + # via ragna-base (pyproject.toml) tornado==6.4 # via bokeh tqdm==4.66.1 @@ -364,8 +364,8 @@ tqdm==4.66.1 # panel typer==0.9.0 # via - # Ragna (pyproject.toml) # chromadb + # ragna-base (pyproject.toml) typing-extensions==4.9.0 # via # chromadb @@ -388,8 +388,8 @@ urllib3==2.1.0 # requests uvicorn==0.26.0 # via - # Ragna (pyproject.toml) # chromadb + # ragna-base (pyproject.toml) uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 00000000..16ae5be6 --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1,9 @@ +chromadb>=0.4.13 +httpx_sse +ijson +lancedb>=0.2 +pyarrow +pymupdf>=1.23.6 +python-docx +python-pptx +tiktoken diff --git a/scripts/update_optional_dependencies.py b/scripts/update_optional_dependencies.py index a99f206c..f93f04e8 100644 --- a/scripts/update_optional_dependencies.py +++ b/scripts/update_optional_dependencies.py @@ -2,15 +2,13 @@ from functools import reduce from pathlib import Path -import tomlkit -import tomlkit.items from packaging.requirements import Requirement import ragna from ragna.core import Assistant, SourceStorage HERE = Path(__file__).parent -PYPROJECT_TOML = HERE / ".." / "pyproject.toml" +PROJECT_ROOT = HERE.parent def main(): @@ -18,7 +16,10 @@ def main(): extract_builtin_document_handler_requirements(), extract_builtin_component_requirements(), ) - update_pyproject_toml(optional_dependencies) + + with open(PROJECT_ROOT / "requirements-optional.txt", "w") as file: + for dependency in optional_dependencies: + file.write(f"{dependency}\n") def make_optional_dependencies(*optional_requirements): @@ -57,20 +58,6 @@ def extract_builtin_component_requirements(): return dict(requirements) -def update_pyproject_toml(optional_dependencies): - with open(PYPROJECT_TOML) as file: - document = tomlkit.load(file) - - document["project"]["optional-dependencies"]["all"] = tomlkit.items.Array( - list(map(tomlkit.items.String.from_raw, optional_dependencies)), - trivia=tomlkit.items.Trivia(), - multiline=True, - ) - - with open(PYPROJECT_TOML, "w") as file: - tomlkit.dump(document, file) - - def append_version_specifiers(version_specifiers, obj): for requirement in obj.requirements(): if not isinstance(requirement, ragna.core.PackageRequirement):