From 5d9730de3f3cbf16685ec13eb4c6e4afd3d1b805 Mon Sep 17 00:00:00 2001 From: Miles Date: Sat, 27 Apr 2024 16:45:52 +0200 Subject: [PATCH] Add blosc2 to CLI, libcramjam features and own CI (#156) * Add each codec as a feature to libcramjam and remove experimental feature from python crates * Add libcramjam specific CI * Add blosc2 to cramjam-cli --- .cargo/config | 12 + .github/workflows/CI-libcramjam.yml | 72 ++ .github/workflows/{CI.yml => CI-python.yml} | 113 +- .gitignore | 2 + Cargo.lock | 1289 ------------------- cramjam-cli/Cargo.toml | 12 +- cramjam-cli/src/main.rs | 5 + cramjam-cli/tests/test_cli.py | 11 +- cramjam-python/Cargo.toml | 10 +- cramjam-python/README.md | 3 +- cramjam-python/src/lib.rs | 5 - libcramjam/Cargo.toml | 50 +- libcramjam/src/capi.rs | 192 ++- libcramjam/src/lib.rs | 36 +- 14 files changed, 382 insertions(+), 1430 deletions(-) create mode 100644 .github/workflows/CI-libcramjam.yml rename .github/workflows/{CI.yml => CI-python.yml} (83%) delete mode 100644 Cargo.lock diff --git a/.cargo/config b/.cargo/config index 146dc31d..57229ecf 100644 --- a/.cargo/config +++ b/.cargo/config @@ -4,5 +4,17 @@ rustflags = [ "-C", "link-arg=dynamic_lookup", ] +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "link-arg=-lgcc"] + +[target.i686-unknown-linux-musl] +rustflags = ["-C", "link-arg=-lgcc"] + +[target.armv7-unknown-linux-musleabihf] +rustflags = ["-C", "link-arg=-lgcc"] + +[target.aarch64-unknown-linux-musl] +rustflags = ["-C", "link-arg=-lgcc"] + [net] git-fetch-with-cli = true diff --git a/.github/workflows/CI-libcramjam.yml b/.github/workflows/CI-libcramjam.yml new file mode 100644 index 00000000..b1ab1da6 --- /dev/null +++ b/.github/workflows/CI-libcramjam.yml @@ -0,0 +1,72 @@ +name: CI-libcramjam + +on: + push: + branches: + - master + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + build-test: + name: capi=${{ matrix.capi }}-codec=${{ matrix.codec || 'all' }} + runs-on: ubuntu-latest + strategy: + fail-fast: ${{ !( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} + matrix: + capi: + - true + - false + codec: + - zstd + - blosc2 + - gzip + - brotli + - lz4 + - xz + - deflate + - bzip2 + - null # Use all codecs + + # TODO: codecs not implemented in capi feature + exclude: + - capi: true + codec: blosc2 + - capi: true + codec: xz + - capi: true + codec: deflate + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + name: Rust Cache + + - name: Audit + if: | + !matrix.codec && matrix.capi + run: cargo install cargo-audit && cargo audit + + - name: Test capi and single codec + if: matrix.capi && matrix.codec + run: cargo test -p libcramjam --no-default-features --features ${{ matrix.codec }} --features capi --lib + + - name: Test capi and all codecs + if: matrix.capi && !matrix.codec + run: cargo test -p libcramjam --features capi --lib + + - name: Test no capi and all codecs + if: | + !matrix.capi && !matrix.codec + run: cargo test -p libcramjam --lib + + - name: Test no capi and single codec + if: | + !matrix.capi && matrix.codec + run: cargo test -p libcramjam --lib --no-default-features --features ${{ matrix.codec }} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI-python.yml similarity index 83% rename from .github/workflows/CI.yml rename to .github/workflows/CI-python.yml index 48536619..dbf2bfce 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI-python.yml @@ -1,4 +1,4 @@ -name: CI +name: CI-python on: push: @@ -11,12 +11,16 @@ on: - released - prereleased +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + jobs: build-test: runs-on: ${{ matrix.conf.os }} name: ${{ matrix.conf.os }}-${{ matrix.conf.target }}-${{ matrix.package.name }}-${{ matrix.python-version }}-${{ matrix.conf.python-architecture }}-${{ matrix.conf.manylinux }} strategy: - fail-fast: ${{ !( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} + fail-fast: ${{ !( startsWith(github.ref, 'refs/heads/master') || !startsWith(github.ref, 'refs/tags/') ) }} matrix: python-version: - '3.8' @@ -136,7 +140,7 @@ jobs: package: { name: cramjam-cli, dir: cramjam-cli } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -147,11 +151,6 @@ jobs: with: target: ${{ matrix.conf.target-triple }} - - uses: Swatinem/rust-cache@v2 - name: Rust Cache - with: - prefix-key: ${{ matrix.conf.os }}-${{ matrix.conf.manylinux }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.python-architecture }}-${{ matrix.package.name }} - - name: Setup (Mac) if: runner.os == 'macOS' run: | @@ -161,14 +160,8 @@ jobs: rustup target add x86_64-apple-darwin echo "MACOSX_DEPLOYMENT_TARGET=10.12" >> $GITHUB_ENV - - # blosc2-rs has a difficult time compiling on x86_64 mac w/ missing symbol __cpu_model. - # and python less than 3.10 is not available on macos-14 M1. So we'll grab the already compiled one - # Download blosc2 2.14.3 pre-built lib - wget https://files.pythonhosted.org/packages/64/98/6f0e0e66f395c5123c74c7acc4626622e34fa1ddf2b45f0684d7a9c81bee/blosc2-2.6.2-cp312-cp312-macosx_10_9_x86_64.whl -O blosc2.whl - unzip blosc2.whl - echo "BLOSC2_INSTALL_PREFIX=$(pwd)/blosc2-2.6.2.data/data" >> $GITHUB_ENV - echo "DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/blosc2-2.6.2.data/data/lib64:$(pwd)/blosc2-2.6.2.data/data/lib" >> $GITHUB_ENV + echo "BLOSC2_INSTALL_PREFIX=$(pwd)/blosc2" >> $GITHUB_ENV + echo "DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/blosc2/lib64:$(pwd)/blosc2/lib" >> $GITHUB_ENV - name: Setup (Windows) if: runner.os == 'Windows' @@ -189,9 +182,6 @@ jobs: $Env:LIB += ";$Env:BLOSC2_INSTALL_PREFIX\lib;$Env:BLOSC2_INSTALL_PREFIX\lib64;$Env:BLOSC2_INSTALL_PREFIX\bin" echo "LIB=$Env:LIB" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - # Now just build blosc2-sys to ensure caching doesn't get in the way and sanity purposes - cargo build --release --target ${{ matrix.conf.target-triple }} --package blosc2-sys --target-dir build -vv - - name: Setup (Linux) if: runner.os == 'Linux' env: @@ -209,20 +199,14 @@ jobs: # so we'll just use 'cross' to build and pass it in for the action for all archs # nothing special for the revision pin, just deterministic install cargo install cross --git https://github.com/cross-rs/cross --rev 6d097fb - cross build --release --target $TARGET_TRIPLE --package blosc2-sys --target-dir build + cross build --release --target $TARGET_TRIPLE --package blosc2-sys --target-dir build --manifest-path ${{ matrix.package.dir }}/Cargo.toml blosc2_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep blosc2-sys) mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$blosc2_sys_dir/out $BLOSC2_INSTALL_PREFIX tree -L 2 $BLOSC2_INSTALL_PREFIX - # Redundant to do audit and tests for all python versions - - name: Audit - if: runner.os == 'Linux' && matrix.conf.target == 'x86_64' && matrix.python-version == '3.12' - run: cargo install cargo-audit && cargo audit - - # Redundant to do audit and tests for all python versions - name: Rust Tests if: matrix.conf.target == 'x86_64' && !startsWith(matrix.python-version, 'pypy') && matrix.python-version == '3.12' - run: cargo test --no-default-features --release --lib + run: cargo test -p ${{ matrix.package.dir }} - name: Build wheel (Linux) if: runner.os == 'Linux' @@ -230,61 +214,48 @@ jobs: with: target: ${{ matrix.conf.target }} manylinux: ${{ matrix.conf.manylinux }} - docker-options: -e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2 - args: -i ${{ matrix.python-version }} --release --out dist --manifest-path ${{ matrix.package.dir }}/Cargo.toml --features use-system-blosc2 --features blosc2-static + docker-options: -e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2 -e LD_LIBRARY_PATH=${{ github.workspace }}/blosc2/lib:${{ github.workspace }}/blosc2/lib64 + args: -i ${{ matrix.python-version }} --release --out dist --manifest-path ${{ matrix.package.dir }}/Cargo.toml --features use-system-blosc2 before-script-linux: ls -l $BLOSC2_INSTALL_PREFIX - name: Build wheel (Windows) if: runner.os == 'Windows' + shell: bash run: | python -m pip install maturin delvewheel - maturin build ` - -i python ` - --release ` - --out wheels ` - --target ${{ matrix.conf.target-triple }} ` - --features blosc2-static ` - --features blosc2-shared ` - --features use-system-blosc2 ` + maturin build \ + -i python \ + --release \ + --out wheels \ + --target ${{ matrix.conf.target-triple }} \ + --no-default-features \ + --features extension-module \ + --features blosc2-static \ --manifest-path ${{ matrix.package.dir }}/Cargo.toml - # Repair wheel - $file = Get-ChildItem -Path ".\wheels\" -Filter "*.whl" | Select-Object -First 1 - if ($file -ne $null) { - & delvewheel repair -v $file.FullName -w dist - } else { - Write-Host "No .whl file found in the wheels directory." - } - - # Remove location of blosc2 to ensure fixing wheel works - Remove-Item -Path $env:BLOSC2_INSTALL_PREFIX -Recurse -Force -ErrorAction SilentlyContinue + name=$(ls -1 wheels/ | head -n 1) + delvewheel repair -v wheels/$name -w dist - name: Build wheel (MacOS) if: runner.os == 'macOS' - run: | - python -m pip install maturin delocate - - # TODO: probably remove this when cramjam-cli has blosc2 support and cramjam is released w/ blosc2 - # so cramjam-cli can be tested against it. - if [ "${{ matrix.package.name }}" = "cramjam-cli" ]; then - export USE_BLOSC2="" - else - export USE_BLOSC2="--features use-system-blosc2" - fi - - maturin build \ - -i python \ - --release \ - --out dist \ - --target ${{ matrix.conf.target-triple }} \ - --manifest-path ${{ matrix.package.dir }}/Cargo.toml $USE_BLOSC2 + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.conf.target-triple }} + # OSX complains with __cpu_model from blosc2 statically compiled so only enabled shared + args: -i python --release --out dist --manifest-path ${{ matrix.package.dir }}/Cargo.toml --no-default-features --features extension-module --features blosc2-shared + - name: Fix wheel (MacOS) + if: runner.os == 'macOS' + run: | + python -m pip install delocate delocate-wheel -v dist/*.whl - # Ensure wheel isn't linking to something we've built outside of it + # Remove pointers back to built lib location to ensure tests later don't link back + unset DYLD_FALLBACK_LIBRARY_PATH + unset BLOSC2_INSTALL_PREFIX rm -rf $BLOSC2_INSTALL_PREFIX - + - name: Install built wheel and Test (Native) # TODO: I'm not sure but the actual collection of tests on windows using pypy3.10 takes forever and/or fails if: | @@ -349,16 +320,18 @@ jobs: venv/bin/python -c 'import cramjam' || venv/bin/cramjam-cli --help - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 + if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} with: name: dist path: dist + overwrite: true build-sdist: name: Build sdists runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.12 @@ -369,13 +342,14 @@ jobs: - name: Build sdist cramjam-cli run: python -m build --sdist cramjam-cli/ -o ./dist - name: Upload sdists - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist pypi-publish: name: Upload ${{ matrix.package }} release to PyPI + if: startsWith(github.ref, 'refs/tags/') strategy: fail-fast: false matrix: @@ -383,7 +357,6 @@ jobs: - cramjam - cramjam-cli runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') needs: [build-test, build-sdist] environment: name: pypi diff --git a/.gitignore b/.gitignore index 7067b3ed..f5e37467 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ benchenv/ cramjam-python/_build +Cargo.lock + # Distribution / packaging .Python diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index c017b476..00000000 --- a/Cargo.lock +++ /dev/null @@ -1,1289 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "anstream" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" - -[[package]] -name = "anstyle-parse" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" -dependencies = [ - "anstyle", - "windows-sys", -] - -[[package]] -name = "assert_cmd" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe" -dependencies = [ - "bstr", - "doc-comment", - "predicates", - "predicates-core", - "predicates-tree", - "wait-timeout", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - -[[package]] -name = "blosc2-rs" -version = "0.2.3+2.14.3" -source = "git+https://github.com/milesgranger/blosc2-rs.git#35ee5f6c77b137b473a1a9b46a0a782a359de016" -dependencies = [ - "blosc2-sys", - "parking_lot", -] - -[[package]] -name = "blosc2-sys" -version = "0.2.3+2.14.3" -source = "git+https://github.com/milesgranger/blosc2-rs.git#35ee5f6c77b137b473a1a9b46a0a782a359de016" -dependencies = [ - "cmake", - "libc", - "pkg-config", -] - -[[package]] -name = "brotli" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata 0.1.10", -] - -[[package]] -name = "bytesize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" - -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cbindgen" -version = "0.24.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b922faaf31122819ec80c4047cc684c6979a087366c069611e33649bf98e18d" -dependencies = [ - "clap 3.2.25", - "heck 0.4.1", - "indexmap", - "log", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn 1.0.109", - "tempfile", - "toml", -] - -[[package]] -name = "cc" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" -dependencies = [ - "jobserver", - "libc", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap", - "strsim 0.10.0", - "termcolor", - "textwrap", -] - -[[package]] -name = "clap" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" -dependencies = [ - "anstream", - "anstyle", - "clap_lex 0.7.0", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.59", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - -[[package]] -name = "cramjam-cli" -version = "0.1.1" -dependencies = [ - "bytesize", - "clap 4.5.4", - "libcramjam", -] - -[[package]] -name = "cramjam-python" -version = "2.8.4-rc1" -dependencies = [ - "libcramjam", - "pyo3", - "pyo3-build-config", -] - -[[package]] -name = "crc32fast" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - -[[package]] -name = "either" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" - -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fastrand" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" - -[[package]] -name = "flate2" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "indoc" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" - -[[package]] -name = "inline-c" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "340dd3d6102fa919bd20987024a6d84954c36ec691ac1efea37742ee983c8dd5" -dependencies = [ - "assert_cmd", - "cc", - "inline-c-macro", - "lazy_static", - "predicates", - "regex", - "rustc_version", - "target-lexicon 0.11.2", - "tempfile", -] - -[[package]] -name = "inline-c-macro" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f5621ec7adacda881d7c2826c064f5c29c72fd44333f97df61b458a583ae15" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "jobserver" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" -dependencies = [ - "libc", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.153" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "libcramjam" -version = "0.4.0" -dependencies = [ - "blosc2-rs", - "brotli", - "bzip2", - "cbindgen", - "flate2", - "inline-c", - "libc", - "libdeflate-sys", - "libdeflater", - "lz4", - "snap", - "xz2", - "zstd", -] - -[[package]] -name = "libdeflate-sys" -version = "1.19.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9caa76c8cc6ee8c4efcf8f4514a812ebcad3aa7d3b548efe4d26da1203f177" -dependencies = [ - "cc", -] - -[[package]] -name = "libdeflater" -version = "1.19.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265a985bd31e5f22e2b2ac107cbed44c6ccf40ae236e46963cd00dd213e4bd03" -dependencies = [ - "libdeflate-sys", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "lz4" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" -dependencies = [ - "libc", - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "memchr" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - -[[package]] -name = "num-traits" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.48.5", -] - -[[package]] -name = "pest" -version = "2.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" -dependencies = [ - "memchr", - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp", - "itertools", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" - -[[package]] -name = "predicates-tree" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" -dependencies = [ - "predicates-core", - "termtree", -] - -[[package]] -name = "proc-macro2" -version = "1.0.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" -dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "parking_lot", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" -dependencies = [ - "once_cell", - "python3-dll-a", - "target-lexicon 0.12.14", -] - -[[package]] -name = "pyo3-ffi" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn 2.0.59", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn 2.0.59", -] - -[[package]] -name = "python3-dll-a" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f07cd4412be8fa09a721d40007c483981bbe072cd6a21f2e83e04ec8f8343f" -dependencies = [ - "cc", -] - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "regex" -version = "1.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.6", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - -[[package]] -name = "regex-automata" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" - -[[package]] -name = "rustc_version" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "0.38.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" -dependencies = [ - "bitflags 2.5.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "ryu" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - -[[package]] -name = "serde" -version = "1.0.197" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.197" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.59", -] - -[[package]] -name = "serde_json" -version = "1.0.115" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "snap" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" - -[[package]] -name = "target-lexicon" -version = "0.12.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" - -[[package]] -name = "tempfile" -version = "3.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" -dependencies = [ - "cfg-if", - "fastrand", - "rustix", - "windows-sys", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - -[[package]] -name = "textwrap" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" - -[[package]] -name = "thiserror" -version = "1.0.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.59", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unindent" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" - -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - -[[package]] -name = "wait-timeout" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" -dependencies = [ - "libc", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" - -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - -[[package]] -name = "zstd" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/cramjam-cli/Cargo.toml b/cramjam-cli/Cargo.toml index e490900f..64ab9d09 100644 --- a/cramjam-cli/Cargo.toml +++ b/cramjam-cli/Cargo.toml @@ -1,15 +1,15 @@ [package] name = "cramjam-cli" -version = "0.1.1" +version = "0.2.0-rc1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = [] -experimental = ["libcramjam/blosc2"] -use-system-blosc2 = ["experimental", "libcramjam/use-system-blosc2"] -blosc2-static = ["experimental", "libcramjam/blosc2-static"] -blosc2-shared = ["experimental", "libcramjam/blosc2-shared"] +default = ["blosc2-shared", "blosc2-static"] +extension-module = [] # only here for uniformity w/ cramjam-python's feat set +use-system-blosc2 = ["libcramjam/use-system-blosc2"] +blosc2-static = ["libcramjam/blosc2-static"] +blosc2-shared = ["libcramjam/blosc2-shared"] [dependencies] clap = { version = "^4.2", features = ["derive"] } diff --git a/cramjam-cli/src/main.rs b/cramjam-cli/src/main.rs index dac00a11..c1141957 100644 --- a/cramjam-cli/src/main.rs +++ b/cramjam-cli/src/main.rs @@ -45,6 +45,7 @@ enum Codec { Gzip(Config), Deflate(Config), Bzip2(Config), + Blosc2(Config), } trait ReadableDowncast: Read + Any { @@ -121,6 +122,10 @@ pub fn main() -> io::Result<()> { let start = Instant::now(); let nbytes = match m.codec { + Codec::Blosc2(conf) => match conf.action { + Action::Compress => libcramjam::blosc2::compress(input, &mut output), + Action::Decompress => libcramjam::blosc2::decompress(input, &mut output), + }, Codec::Snappy(conf) => match conf.action { Action::Compress => libcramjam::snappy::compress(input, &mut output), Action::Decompress => libcramjam::snappy::decompress(input, &mut output), diff --git a/cramjam-cli/tests/test_cli.py b/cramjam-cli/tests/test_cli.py index 12107d3e..48a4d28f 100644 --- a/cramjam-cli/tests/test_cli.py +++ b/cramjam-cli/tests/test_cli.py @@ -11,6 +11,11 @@ VARIANTS = ("snappy", "brotli", "bzip2", "lz4", "gzip", "deflate", "zstd") +# TODO: after blosc2 is moved out of cramjam experimental +if not hasattr(cramjam, "blosc2") and hasattr(cramjam, "experimental") and hasattr(cramjam.experimental, "blosc2"): + cramjam.blosc2 = cramjam.experimental.blosc2 + VARIANTS = (*VARIANTS, "blosc2") + # Some OS can be slow or have higher variability in their runtimes on CI settings.register_profile("local", deadline=timedelta(milliseconds=1000)) settings.register_profile("CI", deadline=None, max_examples=10) @@ -38,7 +43,8 @@ def test_cli_file_to_file(data, variant): run_command(cmd) expected = bytes(getattr(cramjam, variant).compress(data)) - assert expected == compressed_file.read_bytes() + if variant != "blosc2": + assert expected == compressed_file.read_bytes() decompressed_file = pathlib.Path(tmpdir).joinpath("decompressed.txt") run_command( @@ -59,7 +65,8 @@ def test_cli_file_to_stdout(data, variant): out = run_command(cmd) expected = bytes(getattr(cramjam, variant).compress(data)) - assert expected == out + if variant != "blosc2": + assert expected == out compressed = pathlib.Path(tmpdir).joinpath(f"compressed.txt.{variant}") compressed.write_bytes(expected) diff --git a/cramjam-python/Cargo.toml b/cramjam-python/Cargo.toml index 1b2576e8..9692a045 100644 --- a/cramjam-python/Cargo.toml +++ b/cramjam-python/Cargo.toml @@ -13,15 +13,13 @@ name = "cramjam" crate-type = ["cdylib"] [features] -default = ["extension-module", "experimental"] +default = ["extension-module"] extension-module = ["pyo3/extension-module"] generate-import-lib = ["pyo3/generate-import-lib"] # needed for Windows PyPy builds -experimental = ["libcramjam/blosc2"] - -blosc2-static = ["experimental", "libcramjam/blosc2-static"] -blosc2-shared = ["experimental", "libcramjam/blosc2-shared"] -use-system-blosc2 = ["experimental", "libcramjam/use-system-blosc2"] +blosc2-static = ["libcramjam/blosc2-static"] +blosc2-shared = ["libcramjam/blosc2-shared"] +use-system-blosc2 = ["libcramjam/use-system-blosc2"] [dependencies] diff --git a/cramjam-python/README.md b/cramjam-python/README.md index 0285b117..4740f4b2 100644 --- a/cramjam-python/README.md +++ b/cramjam-python/README.md @@ -1,7 +1,7 @@ # cramjam-python [![Code Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) -[![CI](https://github.com/milesgranger/pyrus-cramjam/workflows/CI/badge.svg?branch=master)](https://github.com/milesgranger/pyrus-cramjam/actions?query=branch=master) +[![CI](https://github.com/milesgranger/cramjam/workflows/CI-python/badge.svg?branch=master)](https://github.com/milesgranger/cramjam/actions?query=branch=master) [![PyPI](https://img.shields.io/pypi/v/cramjam.svg)](https://pypi.org/project/cramjam) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/cramjam/badges/version.svg)](https://anaconda.org/conda-forge/cramjam) [![Downloads](https://pepy.tech/badge/cramjam/month)](https://pepy.tech/project/cramjam) @@ -43,6 +43,7 @@ Available algorithms: - [X] Deflate (cramjam.deflate) - [X] ZSTD (cramjam.zstd) - [X] XZ / LZMA (cramjam.xz) +- [X] Blosc2 (cramjam.experimental.blosc2) All available for use as: diff --git a/cramjam-python/src/lib.rs b/cramjam-python/src/lib.rs index c1ab1132..e9fb71a8 100644 --- a/cramjam-python/src/lib.rs +++ b/cramjam-python/src/lib.rs @@ -51,15 +51,12 @@ //! b'some bytes here' //! ``` -#[cfg(feature = "experimental")] pub mod blosc2; - pub mod brotli; pub mod bzip2; pub mod deflate; pub mod exceptions; -#[cfg(feature = "experimental")] pub mod experimental; pub mod gzip; @@ -395,8 +392,6 @@ fn cramjam(py: Python, m: &PyModule) -> PyResult<()> { make_submodule!(py -> m -> deflate); make_submodule!(py -> m -> xz); make_submodule!(py -> m -> zstd); - - #[cfg(feature = "experimental")] make_submodule!(py -> m -> experimental); Ok(()) diff --git a/libcramjam/Cargo.toml b/libcramjam/Cargo.toml index 4fb65991..f151c93b 100644 --- a/libcramjam/Cargo.toml +++ b/libcramjam/Cargo.toml @@ -1,32 +1,40 @@ [package] -name = "libcramjam" -version = "0.4.0" -edition = "2021" -license = "MIT" +name = "libcramjam" +version = "0.4.0" +edition = "2021" +license = "MIT" description = "Compression library combining a plethora of algorithms in a similar as possible API" -readme = "README.md" +readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = [] -capi = ["libc"] -blosc2 = ["blosc2-rs"] -blosc2-static = ["blosc2", "blosc2-rs/static"] -blosc2-shared = ["blosc2", "blosc2-rs/shared"] +default = ["snappy", "lz4", "bzip2", "brotli", "xz", "zstd", "gzip", "deflate", "blosc2"] +capi = ["dep:libc"] +snappy = ["dep:snap"] +lz4 = ["dep:lz4"] +bzip2 = ["dep:bzip2"] +brotli = ["dep:brotli"] +xz = ["dep:xz2"] +zstd = ["dep:zstd"] +gzip = ["dep:libdeflater", "dep:libdeflate-sys", "dep:flate2"] +deflate = ["dep:libdeflater", "dep:libdeflate-sys", "dep:flate2"] +blosc2 = ["dep:blosc2-rs"] +blosc2-static = ["blosc2", "blosc2-rs/static"] +blosc2-shared = ["blosc2", "blosc2-rs/shared"] use-system-blosc2 = ["blosc2", "blosc2-rs/use-system-blosc2"] [dependencies] -snap = "^1" -brotli = { version = "^3", default-features = false, features = ["std", "ffi-api"] } -bzip2 = "^0.4" -lz4 = "^1" -flate2 = "^1" -blosc2-rs = { git = "https://github.com/milesgranger/blosc2-rs.git", optional = true } -libdeflater = "^1" -libdeflate-sys = "<1.20.0"# TODO: requires gcc>=4.9 not available on Python's CI wheel builds -zstd = "0.13.0" -libc = { version = "0.2", optional = true } -xz2 = { version = "0.1.7", features = ["static"] } +libc = { version = "0.2", optional = true } +snap = { version = "^1", optional = true } +brotli = { version = "^3", default-features = false, features = ["std", "ffi-api"], optional = true } +bzip2 = { version = "^0.4", optional = true } +lz4 = { version = "^1", optional = true } +flate2 = { version = "^1", optional = true } +libdeflater = { version = "^1", optional = true } +libdeflate-sys = { version = "<1.20.0", optional = true } # TODO: requires gcc>=4.9 not available on Python's CI wheel builds +blosc2-rs = { git = "https://github.com/milesgranger/blosc2-rs.git", optional = true } +zstd = { version = "^0.13", optional = true } +xz2 = { version = "0.1.7", features = ["static"], optional = true } [build-dependencies] cbindgen = "^0.24" diff --git a/libcramjam/src/capi.rs b/libcramjam/src/capi.rs index 4a7233c2..8a345608 100644 --- a/libcramjam/src/capi.rs +++ b/libcramjam/src/capi.rs @@ -5,7 +5,20 @@ use std::io::Cursor; use std::io::Write; use std::slice; -use crate::{brotli, bzip2, deflate, gzip, lz4, snappy, zstd}; +#[cfg(feature = "brotli")] +use crate::brotli; +#[cfg(feature = "bzip2")] +use crate::bzip2; +#[cfg(feature = "deflate")] +use crate::deflate; +#[cfg(feature = "gzip")] +use crate::gzip; +#[cfg(feature = "lz4")] +use crate::lz4; +#[cfg(feature = "snappy")] +use crate::snappy; +#[cfg(feature = "zstd")] +use crate::zstd; #[repr(C)] pub struct Buffer { @@ -50,20 +63,35 @@ impl From> for Buffer { #[derive(Debug, Copy, Clone)] #[repr(C)] pub enum Codec { + #[cfg(feature = "snappy")] #[allow(dead_code)] Snappy, + + #[cfg(feature = "snappy")] #[allow(dead_code)] SnappyRaw, + + #[cfg(feature = "bzip2")] #[allow(dead_code)] Bzip2, + + #[cfg(feature = "lz4")] #[allow(dead_code)] Lz4, + + #[cfg(feature = "lz4")] #[allow(dead_code)] Lz4Block, + + #[cfg(feature = "zstd")] #[allow(dead_code)] Zstd, + + #[cfg(feature = "gzip")] #[allow(dead_code)] Gzip, + + #[cfg(feature = "brotli")] #[allow(dead_code)] Brotli, } @@ -72,25 +100,42 @@ pub enum Codec { #[derive(Debug)] #[repr(C)] pub enum StreamingCodec { + #[cfg(feature = "bzip2")] #[allow(dead_code)] StreamingBzip2, + + #[cfg(feature = "snappy")] #[allow(dead_code)] StreamingSnappy, + + #[cfg(feature = "lz4")] #[allow(dead_code)] StreamingLz4, + + #[cfg(feature = "zstd")] #[allow(dead_code)] StreamingZstd, + + #[cfg(feature = "gzip")] #[allow(dead_code)] StreamingGzip, + + #[cfg(feature = "brotli")] #[allow(dead_code)] StreamingBrotli, } +#[cfg(feature = "snappy")] type SnappyFrameCompressor = snappy::snap::write::FrameEncoder>; +#[cfg(feature = "bzip2")] type Bzip2Compressor = bzip2::bzip2::write::BzEncoder>; +#[cfg(feature = "lz4")] type Lz4Compressor = crate::lz4::lz4::Encoder>; +#[cfg(feature = "gzip")] type GzipCompressor = crate::gzip::flate2::write::GzEncoder>; +#[cfg(feature = "brotli")] type BrotliCompressor = brotli::brotli::CompressorWriter>; +#[cfg(feature = "zstd")] type ZstdCompressor<'a> = crate::zstd::zstd::Encoder<'a, Vec>; type Decompressor = Cursor>; @@ -128,8 +173,10 @@ pub extern "C" fn decompress( ) -> Buffer { let mut decompressed = Cursor::new(vec![]); let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); - let ret = match codec { + let ret: Result = match codec { + #[cfg(feature = "snappy")] Codec::Snappy => snappy::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::decompress_vec(compressed.get_ref()).map(|v| { let len = v.len(); *decompressed.get_mut() = v; @@ -137,11 +184,17 @@ pub extern "C" fn decompress( compressed.set_position(input_len as _); // todo, assuming it read the whole thing len }), + #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "brotli")] Codec::Brotli => brotli::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "gzip")] Codec::Gzip => gzip::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "zstd")] Codec::Zstd => zstd::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "lz4")] Codec::Lz4 => lz4::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "lz4")] Codec::Lz4Block => lz4::block::decompress_vec(compressed.get_ref()).map(|v| { let len = v.len(); *decompressed.get_mut() = v; @@ -183,11 +236,13 @@ pub extern "C" fn compress( error_to_ptr("Requires compression >= 0", error); return Buffer::empty(); } - let level = Some(level as _); + let level = Some(level); let mut compressed = Cursor::new(vec![]); let mut decompressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); - let ret = match codec { + let ret: Result = match codec { + #[cfg(feature = "snappy")] Codec::Snappy => snappy::compress(&mut decompressed, &mut compressed), + #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::compress_vec(decompressed.get_ref()).map(|v| { let len = v.len(); *compressed.get_mut() = v; @@ -195,19 +250,26 @@ pub extern "C" fn compress( decompressed.set_position(input_len as _); len }), - Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level), - Codec::Brotli => brotli::compress(&mut decompressed, &mut compressed, level), - Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level), - Codec::Zstd => zstd::compress(&mut decompressed, &mut compressed, level.map(|v| v as i32)), - Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level), + #[cfg(feature = "bzip2")] + Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "brotli")] + Codec::Brotli => brotli::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "gzip")] + Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "zstd")] + Codec::Zstd => zstd::compress(&mut decompressed, &mut compressed, level.map(|v: i32| v as i32)), + #[cfg(feature = "lz4")] + Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), // TODO: Support passing acceleration - Codec::Lz4Block => lz4::block::compress_vec(decompressed.get_ref(), level, None, Some(true)).map(|v| { - let len = v.len(); - *compressed.get_mut() = v; - compressed.set_position(len as _); - decompressed.set_position(input_len as _); - len - }), // TODO + #[cfg(feature = "lz4")] + Codec::Lz4Block => lz4::block::compress_vec(decompressed.get_ref(), level.map(|v| v as _), None, Some(true)) + .map(|v| { + let len = v.len(); + *compressed.get_mut() = v; + compressed.set_position(len as _); + decompressed.set_position(input_len as _); + len + }), // TODO }; match ret { Ok(n) => { @@ -242,14 +304,22 @@ pub extern "C" fn decompress_into( let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); let mut decompressed = Cursor::new(unsafe { std::slice::from_raw_parts_mut(output, output_len) }); - let ret = match codec { + let ret: Result = match codec { + #[cfg(feature = "snappy")] Codec::Snappy => snappy::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::decompress(compressed.get_ref(), decompressed.get_mut()), + #[cfg(feature = "bzip2")] Codec::Bzip2 => bzip2::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "brotli")] Codec::Brotli => brotli::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "gzip")] Codec::Gzip => gzip::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "zstd")] Codec::Zstd => zstd::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "lz4")] Codec::Lz4 => lz4::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "lz4")] Codec::Lz4Block => lz4::block::decompress_into(&compressed.get_ref(), decompressed.get_mut(), None), }; match ret { @@ -284,18 +354,26 @@ pub extern "C" fn compress_into( error_to_ptr("Requires compression >= 0", error); return; } - let level = Some(level as _); + let level = Some(level); - let ret = match codec { + let ret: Result = match codec { + #[cfg(feature = "snappy")] Codec::Snappy => snappy::compress(&mut decompressed, &mut compressed), + #[cfg(feature = "snappy")] Codec::SnappyRaw => snappy::raw::compress(decompressed, &mut compressed), - Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level), - Codec::Brotli => brotli::compress(&mut decompressed, &mut compressed, level), - Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level), - Codec::Zstd => zstd::compress(&mut decompressed, &mut compressed, level.map(|v| v as i32)), - Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level), + #[cfg(feature = "bzip2")] + Codec::Bzip2 => bzip2::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "brotli")] + Codec::Brotli => brotli::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "gzip")] + Codec::Gzip => gzip::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), + #[cfg(feature = "zstd")] + Codec::Zstd => zstd::compress(&mut decompressed, &mut compressed, level.map(|v: i32| v as i32)), + #[cfg(feature = "lz4")] + Codec::Lz4 => lz4::compress(&mut decompressed, &mut compressed, level.map(|v| v as _)), // TODO: Support passing acceleration - Codec::Lz4Block => lz4::block::compress_into(decompressed, compressed, level, None, Some(true)), + #[cfg(feature = "lz4")] + Codec::Lz4Block => lz4::block::compress_into(decompressed, compressed, level.map(|v| v as _), None, Some(true)), }; match ret { Ok(n) => { @@ -315,6 +393,7 @@ pub extern "C" fn compress_into( #[allow(unused_variables)] pub extern "C" fn compressor_init(codec: StreamingCodec, level: i32, error: &mut *mut c_char) -> *mut c_void { match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { if level < 0 { error_to_ptr("Bzip2 requires compression level >= 0", error); @@ -323,6 +402,7 @@ pub extern "C" fn compressor_init(codec: StreamingCodec, level: i32, error: &mut let compressor = bzip2::bzip2::write::BzEncoder::new(vec![], bzip2::bzip2::Compression::new(level as _)); Box::into_raw(Box::new(compressor)) as _ } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { if level < 0 { error_to_ptr("Brotli requires compression level >= 0", error); @@ -331,6 +411,7 @@ pub extern "C" fn compressor_init(codec: StreamingCodec, level: i32, error: &mut let compressor = brotli::make_write_compressor(vec![], Some(level as _)); Box::into_raw(Box::new(compressor)) as _ } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { if level < 1 { error_to_ptr("Gzip requires compression level >= 1", error); @@ -339,14 +420,17 @@ pub extern "C" fn compressor_init(codec: StreamingCodec, level: i32, error: &mut let compressor = gzip::flate2::write::GzEncoder::new(vec![], gzip::flate2::Compression::new(level as _)); Box::into_raw(Box::new(compressor)) as _ } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = zstd::zstd::Encoder::new(vec![], level); Box::into_raw(Box::new(compressor)) as _ } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = snappy::snap::write::FrameEncoder::new(vec![]); Box::into_raw(Box::new(compressor)) as _ } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { if level < 0 { error_to_ptr("Lz4 requires compression level >= 0", error); @@ -363,21 +447,27 @@ pub extern "C" fn free_compressor(codec: StreamingCodec, compressor_ptr: &mut *m if !(*compressor_ptr).is_null() { { match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let _ = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; } @@ -390,36 +480,42 @@ pub extern "C" fn free_compressor(codec: StreamingCodec, compressor_ptr: &mut *m #[no_mangle] pub extern "C" fn compressor_inner(codec: StreamingCodec, compressor_ptr: &mut *mut c_void) -> Buffer { match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; let buffer = Buffer::from(compressor.get_ref()); *compressor_ptr = Box::into_raw(compressor) as _; buffer } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; let buffer = Buffer::from(compressor.writer()); @@ -437,6 +533,7 @@ pub extern "C" fn compressor_finish( error: &mut *mut c_char, ) -> Buffer { let buf = match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; match compressor.finish() { @@ -447,6 +544,7 @@ pub extern "C" fn compressor_finish( } } } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; if let Err(err) = compressor.flush() { @@ -455,6 +553,7 @@ pub extern "C" fn compressor_finish( } Buffer::from(compressor.into_inner()) } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; match compressor.finish() { @@ -465,6 +564,7 @@ pub extern "C" fn compressor_finish( } } } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; match compressor.finish() { @@ -475,6 +575,7 @@ pub extern "C" fn compressor_finish( } } } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; match compressor.into_inner() { @@ -485,6 +586,7 @@ pub extern "C" fn compressor_finish( } } } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; let (w, ret) = compressor.finish(); @@ -504,6 +606,7 @@ pub extern "C" fn compressor_finish( #[no_mangle] pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut *mut c_void, error: &mut *mut c_char) { match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; if let Err(err) = compressor.flush() { @@ -511,6 +614,7 @@ pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut * } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; if let Err(err) = compressor.flush() { @@ -518,6 +622,7 @@ pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut * } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; if let Err(err) = compressor.flush() { @@ -525,6 +630,7 @@ pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut * } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; if let Err(err) = compressor.flush() { @@ -532,6 +638,7 @@ pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut * } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; if let Err(err) = compressor.flush() { @@ -539,6 +646,7 @@ pub extern "C" fn compressor_flush(codec: StreamingCodec, compressor_ptr: &mut * } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; if let Err(err) = compressor.flush() { @@ -561,6 +669,7 @@ pub extern "C" fn compressor_compress( ) { let mut decompressed = Cursor::new(unsafe { slice::from_raw_parts(input, input_len) }); match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Bzip2Compressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -574,6 +683,7 @@ pub extern "C" fn compressor_compress( } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut BrotliCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -587,6 +697,7 @@ pub extern "C" fn compressor_compress( } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut GzipCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -600,6 +711,7 @@ pub extern "C" fn compressor_compress( } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut ZstdCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -613,6 +725,7 @@ pub extern "C" fn compressor_compress( } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut SnappyFrameCompressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -626,6 +739,7 @@ pub extern "C" fn compressor_compress( } *compressor_ptr = Box::into_raw(compressor) as _; } + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => { let mut compressor = unsafe { Box::from_raw(*compressor_ptr as *mut Lz4Compressor) }; match std::io::copy(&mut decompressed, &mut compressor) { @@ -715,12 +829,18 @@ pub extern "C" fn decompressor_decompress( let mut decompressed = unsafe { Box::from_raw(*decompressor_ptr as *mut Decompressor) }; let start_pos = decompressed.position(); let mut compressed = Cursor::new(unsafe { std::slice::from_raw_parts(input, input_len) }); - let ret = match codec { + let ret: Result = match codec { + #[cfg(feature = "bzip2")] StreamingCodec::StreamingBzip2 => bzip2::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "gzip")] StreamingCodec::StreamingGzip => gzip::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "brotli")] StreamingCodec::StreamingBrotli => brotli::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "zstd")] StreamingCodec::StreamingZstd => zstd::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "snappy")] StreamingCodec::StreamingSnappy => snappy::decompress(&mut compressed, &mut decompressed), + #[cfg(feature = "lz4")] StreamingCodec::StreamingLz4 => lz4::decompress(&mut compressed, &mut decompressed), }; match ret { @@ -736,48 +856,57 @@ pub extern "C" fn decompressor_decompress( } /* -------- Codec specific functions ----------*/ +#[cfg(feature = "lz4")] #[no_mangle] pub extern "C" fn lz4_frame_max_compression_level() -> usize { lz4::LZ4_ACCELERATION_MAX as _ } +#[cfg(feature = "lz4")] #[no_mangle] pub extern "C" fn lz4_frame_max_compressed_len(input_len: usize, compression_level: i32) -> usize { lz4::compress_bound(input_len, Some(compression_level as _)) } +#[cfg(feature = "lz4")] #[no_mangle] #[allow(unused_variables)] pub extern "C" fn lz4_block_max_compressed_len(input_len: usize, error: &mut *mut c_char) -> usize { lz4::block::compress_bound(input_len, Some(true)) } +#[cfg(feature = "deflate")] #[no_mangle] pub extern "C" fn deflate_max_compressed_len(input_len: usize, level: i32) -> usize { deflate::compress_bound(input_len, Some(level)) } +#[cfg(feature = "gzip")] #[no_mangle] pub extern "C" fn gzip_max_compressed_len(input_len: usize, level: i32) -> usize { let level = if level < 0 { 0 } else { level }; gzip::compress_bound(input_len, Some(level)).unwrap() } +#[cfg(feature = "zstd")] #[no_mangle] pub extern "C" fn zstd_max_compressed_len(input_len: usize) -> usize { zstd::compress_bound(input_len) } +#[cfg(feature = "snappy")] #[no_mangle] pub extern "C" fn snappy_raw_max_compressed_len(input_len: usize) -> usize { snap::raw::max_compress_len(input_len) } +#[cfg(feature = "brotli")] #[no_mangle] pub extern "C" fn brotli_max_compressed_len(input_len: usize) -> usize { brotli::compress_bound(input_len) } +#[cfg(feature = "snappy")] #[no_mangle] pub extern "C" fn snappy_raw_decompressed_len(input: *const u8, input_len: usize, error: &mut *mut c_char) -> isize { let input = unsafe { slice::from_raw_parts(input, input_len) }; @@ -796,6 +925,7 @@ mod tests { const DATA: &[u8; 5] = b"bytes"; + #[cfg(feature = "lz4")] #[test] fn test_lz4_frame_max_compressed_len() { // A known simple test case, expected len taken from lz4/lz4 repo @@ -803,6 +933,7 @@ mod tests { assert_eq!(len, 65544); } + #[cfg(feature = "lz4")] #[test] fn test_lz4_block_max_compressed_len() { let mut error: *mut c_char = std::ptr::null_mut(); @@ -811,12 +942,14 @@ mod tests { assert_eq!(len, 30); } + #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_max_compressed_len() { let len = snappy_raw_max_compressed_len(10); assert_eq!(len, 43); } + #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_decompressed_len() { let uncompressed = b"bytes"; @@ -830,17 +963,20 @@ mod tests { assert_eq!(len as usize, uncompressed.len()); } + #[cfg(feature = "snappy")] #[test] fn test_snappy_roundtrip() { let mut expected = vec![]; snappy::compress(Cursor::new(DATA), &mut expected).unwrap(); roundtrip(Codec::Snappy, &expected, 0); } + #[cfg(feature = "snappy")] #[test] fn test_snappy_raw_roundtrip() { let expected = snappy::raw::compress_vec(DATA).unwrap(); roundtrip(Codec::SnappyRaw, &expected, 0); } + #[cfg(feature = "lz4")] #[test] fn test_lz4_roundtrip() { let mut expected = Cursor::new(vec![]); @@ -848,11 +984,13 @@ mod tests { let expected = expected.into_inner(); roundtrip(Codec::Lz4, &expected, 6); } + #[cfg(feature = "lz4")] #[test] fn test_lz4_block_roundtrip() { let expected = lz4::block::compress_vec(DATA, Some(6), Some(1), Some(true)).unwrap(); roundtrip(Codec::Lz4Block, &expected, 6); } + #[cfg(feature = "bzip2")] #[test] fn test_bzip2_roundtrip() { let mut expected = Cursor::new(vec![]); @@ -860,6 +998,7 @@ mod tests { let expected = expected.into_inner(); roundtrip(Codec::Bzip2, &expected, 6); } + #[cfg(feature = "brotli")] #[test] fn test_brotli_roundtrip() { let mut expected = Cursor::new(vec![]); @@ -867,6 +1006,7 @@ mod tests { let expected = expected.into_inner(); roundtrip(Codec::Brotli, &expected, 6); } + #[cfg(feature = "zstd")] #[test] fn test_zstd_roundtrip() { let mut expected = Cursor::new(vec![]); diff --git a/libcramjam/src/lib.rs b/libcramjam/src/lib.rs index 9d4613c5..8581669e 100644 --- a/libcramjam/src/lib.rs +++ b/libcramjam/src/lib.rs @@ -1,18 +1,24 @@ #[cfg(feature = "blosc2")] pub mod blosc2; - +#[cfg(feature = "brotli")] pub mod brotli; +#[cfg(feature = "bzip2")] pub mod bzip2; +#[cfg(feature = "capi")] +mod capi; +#[cfg(feature = "deflate")] pub mod deflate; +#[cfg(feature = "gzip")] pub mod gzip; +#[cfg(feature = "lz4")] pub mod lz4; +#[cfg(feature = "snappy")] pub mod snappy; +#[cfg(feature = "xz")] pub mod xz; +#[cfg(feature = "zstd")] pub mod zstd; -#[cfg(feature = "capi")] -mod capi; - #[cfg(test)] mod tests { @@ -78,24 +84,46 @@ mod tests { } // Expected compressed_len, subsequent args are supplied to the variant's `compress` call. + #[cfg(feature = "snappy")] test_variant!(snappy, compressed_len = 2_572_398); + + #[cfg(feature = "gzip")] test_variant!(gzip, compressed_len = 157_192, None); + + #[cfg(feature = "brotli")] test_variant!(brotli, compressed_len = 128, None); + + #[cfg(feature = "bzip2")] test_variant!(bzip2, compressed_len = 14_207, None); + + #[cfg(feature = "deflate")] test_variant!(deflate, compressed_len = 157_174, None); + + #[cfg(feature = "zstd")] test_variant!(zstd, compressed_len = 4990, None); + + #[cfg(feature = "lz4")] test_variant!(lz4, compressed_len = 303_278, None); #[cfg(feature = "blosc2")] test_variant!(blosc2, compressed_len = 791_923); + #[cfg(feature = "xz")] #[allow(non_upper_case_globals)] const format: Option = None; + #[allow(non_upper_case_globals)] + #[cfg(feature = "xz")] const check: Option = None; + #[allow(non_upper_case_globals)] + #[cfg(feature = "xz")] const filters: Option = None; + #[allow(non_upper_case_globals)] + #[cfg(feature = "xz")] const opts: Option = None; + + #[cfg(feature = "xz")] test_variant!(xz, compressed_len = 8_020, None, format, check, filters, opts); }