diff --git a/Cargo.lock b/Cargo.lock index 27a4e3bd..d4160755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -799,7 +799,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.9.0", "scopeguard", ] @@ -878,6 +878,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "cryo_python" +version = "0.1.0" +dependencies = [ + "cryo_cli", + "pyo3", + "pyo3-asyncio", + "tokio", +] + [[package]] name = "crypto-bigint" version = "0.5.2" @@ -2046,6 +2056,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + [[package]] name = "inout" version = "0.1.3" @@ -2348,6 +2364,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -3183,6 +3208,79 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pyo3" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset 0.8.0", + "parking_lot 0.12.1", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-asyncio" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3564762e37035cfc486228e10b0528460fa026d681b5763873c693aa0d5c260" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + +[[package]] +name = "pyo3-build-config" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quanta" version = "0.9.3" @@ -4028,6 +4126,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f6b473c37f9add4cf1df5b4d66a8ef58ab6c895f1a3b3f949cf3e21230140e" +[[package]] +name = "target-lexicon" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8e77cb757a61f51b947ec4a7e3646efd825b73561db1c232a8ccb639e611a0" + [[package]] name = "tempfile" version = "3.6.0" @@ -4358,6 +4462,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 808da1d3..0a7f87df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/cli", "crates/freeze", + "crates/python", ] [workspace.package] diff --git a/crates/python/.github/workflows/CI.yml b/crates/python/.github/workflows/CI.yml new file mode 100644 index 00000000..cc8262ea --- /dev/null +++ b/crates/python/.github/workflows/CI.yml @@ -0,0 +1,124 @@ +# TODO: test and implement + +## This file is autogenerated by maturin v1.1.0 +## To update, run +## +## maturin generate-ci github +## +#name: CI + +#on: +# push: +# branches: +# - main +# - master +# tags: +# - '*' +# pull_request: +# workflow_dispatch: + +#permissions: +# contents: read + +#jobs: +# linux: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: ['3.7', '3.8', '3.9', '3.10'] +# target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Build wheels +# uses: PyO3/maturin-action@v1 +# with: +# target: ${{ matrix.target }} +# args: --release --out dist --find-interpreter +# sccache: 'true' +# manylinux: auto +# - name: Upload wheels +# uses: actions/upload-artifact@v3 +# with: +# name: wheels +# path: dist + +# windows: +# runs-on: windows-latest +# strategy: +# matrix: +# target: [x64, x86] +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# architecture: ${{ matrix.target }} +# - name: Build wheels +# uses: PyO3/maturin-action@v1 +# with: +# target: ${{ matrix.target }} +# args: --release --out dist --find-interpreter +# sccache: 'true' +# - name: Upload wheels +# uses: actions/upload-artifact@v3 +# with: +# name: wheels +# path: dist + +# macos: +# runs-on: macos-latest +# strategy: +# matrix: +# python-version: ['3.7', '3.8', '3.9', '3.10'] +# target: [x86_64, aarch64] +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Build wheels +# uses: PyO3/maturin-action@v1 +# with: +# target: ${{ matrix.target }} +# args: --release --out dist --find-interpreter +# sccache: 'true' +# - name: Upload wheels +# uses: actions/upload-artifact@v3 +# with: +# name: wheels +# path: dist + +# sdist: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - name: Build sdist +# uses: PyO3/maturin-action@v1 +# with: +# command: sdist +# args: --out dist +# - name: Upload sdist +# uses: actions/upload-artifact@v3 +# with: +# name: wheels +# path: dist + +# release: +# name: Release +# runs-on: ubuntu-latest +# if: "startsWith(github.ref, 'refs/tags/')" +# needs: [linux, windows, macos, sdist] +# steps: +# - uses: actions/download-artifact@v3 +# with: +# name: wheels +# - name: Publish to PyPI +# uses: PyO3/maturin-action@v1 +# env: +# MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} +# with: +# command: upload +# args: --skip-existing * diff --git a/crates/python/.gitignore b/crates/python/.gitignore new file mode 100644 index 00000000..af3ca5ef --- /dev/null +++ b/crates/python/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version \ No newline at end of file diff --git a/crates/python/Cargo.toml b/crates/python/Cargo.toml new file mode 100644 index 00000000..1c990bc7 --- /dev/null +++ b/crates/python/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cryo_python" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "cryo" +crate-type = ["cdylib"] + +[dependencies] +cryo_cli = { version = "0.1.0", path = "../cli" } +pyo3 = { version = "0.18.0", features = ["extension-module"] } +pyo3-asyncio = { version = "0.18.0", features = ["tokio-runtime"] } +tokio = "1.29.0" diff --git a/crates/python/pyproject.toml b/crates/python/pyproject.toml new file mode 100644 index 00000000..bf112f53 --- /dev/null +++ b/crates/python/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["maturin>=1.1,<2.0"] +build-backend = "maturin" + +[project] +name = "cryo_python" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + +[tool.maturin] +python-source = "python" +features = ["pyo3/extension-module"] diff --git a/crates/python/python/cryo/__init__.py b/crates/python/python/cryo/__init__.py new file mode 100644 index 00000000..1995f8e9 --- /dev/null +++ b/crates/python/python/cryo/__init__.py @@ -0,0 +1,10 @@ +from .cryo import * + +__doc__ = cryo.__doc__ +if hasattr(cryo, "__all__"): + __all__ = cryo.__all__ + + +async def freeze(*args, **kwargs): + return await _freeze(*args, **kwargs) + diff --git a/crates/python/python/cryo/collect.py b/crates/python/python/cryo/collect.py new file mode 100644 index 00000000..58489d00 --- /dev/null +++ b/crates/python/python/cryo/collect.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +def collect_blocks(): + pass + diff --git a/crates/python/src/freeze_adapter.rs b/crates/python/src/freeze_adapter.rs new file mode 100644 index 00000000..3ae3e054 --- /dev/null +++ b/crates/python/src/freeze_adapter.rs @@ -0,0 +1,124 @@ +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::*; + +use cryo_cli::run; +use cryo_cli::Args; + +#[pyfunction( + signature = ( + datatype, + blocks, + *, + align = false, + reorg_buffer = 0, + include_columns = None, + exclude_columns = None, + columns = None, + hex = false, + sort = None, + rpc = None, + network_name = None, + requests_per_second = None, + max_concurrent_requests = None, + max_concurrent_chunks = None, + max_concurrent_blocks = None, + dry = false, + chunk_size = 1000, + n_chunks = None, + output_dir = ".".to_string(), + file_suffix = None, + overwrite = false, + csv = false, + json = false, + row_group_size = None, + n_row_groups = None, + no_stats = false, + compression = vec!["lz4".to_string()], + contract = None, + topic0 = None, + topic1 = None, + topic2 = None, + topic3 = None, + log_request_size = 1, + ) +)] +#[allow(clippy::too_many_arguments)] +pub fn _freeze( + py: Python<'_>, + datatype: Vec, + blocks: Vec, + align: bool, + reorg_buffer: u64, + include_columns: Option>, + exclude_columns: Option>, + columns: Option>, + hex: bool, + sort: Option>, + rpc: Option, + network_name: Option, + requests_per_second: Option, + max_concurrent_requests: Option, + max_concurrent_chunks: Option, + max_concurrent_blocks: Option, + dry: bool, + chunk_size: u64, + n_chunks: Option, + output_dir: String, + file_suffix: Option, + overwrite: bool, + csv: bool, + json: bool, + row_group_size: Option, + n_row_groups: Option, + no_stats: bool, + compression: Vec, + contract: Option, + topic0: Option, + topic1: Option, + topic2: Option, + topic3: Option, + log_request_size: u64, +) -> PyResult<&PyAny> { + let args = Args { + datatype, + blocks, + align, + reorg_buffer, + include_columns, + exclude_columns, + columns, + hex, + sort, + rpc, + network_name, + requests_per_second, + max_concurrent_requests, + max_concurrent_chunks, + max_concurrent_blocks, + dry, + chunk_size, + n_chunks, + output_dir, + file_suffix, + overwrite, + csv, + json, + row_group_size, + n_row_groups, + no_stats, + compression, + contract, + topic0, + topic1, + topic2, + topic3, + log_request_size, + }; + + pyo3_asyncio::tokio::future_into_py(py, async move { + match run(args).await { + Ok(()) => Ok(Python::with_gil(|py| py.None())), + Err(_e) => Err(PyErr::new::("failed")), + } + }) +} diff --git a/crates/python/src/lib.rs b/crates/python/src/lib.rs new file mode 100644 index 00000000..214dc4d1 --- /dev/null +++ b/crates/python/src/lib.rs @@ -0,0 +1,18 @@ +mod freeze_adapter; + +use pyo3::prelude::*; +// use crate::freeze_adapter; + +/// Formats the sum of two numbers as string. +#[pyfunction] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// A Python module implemented in Rust. +#[pymodule] +fn cryo(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + m.add_function(wrap_pyfunction!(freeze_adapter::_freeze, m)?)?; + Ok(()) +}