From 5456da04586f339c04b6a3b2b9949498e61db442 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 11 Sep 2024 13:23:43 +0200 Subject: [PATCH 01/10] Nightly Distribution --- .github/workflows/distribution.yml | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/distribution.yml diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml new file mode 100644 index 0000000..dcd0d01 --- /dev/null +++ b/.github/workflows/distribution.yml @@ -0,0 +1,39 @@ +# +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Stable Extension Distribution Pipeline +on: + pull_request: + branches: + - main + paths-ignore: + - '**/README.md' + - 'doc/**' + push: + branches: + - main + paths-ignore: + - '**/README.md' + - 'doc/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-stable-build: + name: Build extension binaries + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0 + with: + duckdb_version: v1.1.0 + extension_name: substrait + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable-build + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.0 + secrets: inherit + with: + duckdb_version: v1.1.0 + extension_name: substrait \ No newline at end of file From 2293c8008a4dd62962240f704ce96a5d61cf8f8a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 11 Sep 2024 14:03:44 +0200 Subject: [PATCH 02/10] excldue wasm and windows --- .github/workflows/distribution.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index dcd0d01..c502ccf 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -27,6 +27,7 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0 with: duckdb_version: v1.1.0 + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" extension_name: substrait duckdb-stable-deploy: @@ -36,4 +37,5 @@ jobs: secrets: inherit with: duckdb_version: v1.1.0 + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" extension_name: substrait \ No newline at end of file From 33924e016e62dfc39919486343b07d4b1a57a0b7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 14:18:30 +0200 Subject: [PATCH 03/10] Add extension-ci-tools submodule --- .gitmodules | 3 +++ extension-ci-tools | 1 + 2 files changed, 4 insertions(+) create mode 160000 extension-ci-tools diff --git a/.gitmodules b/.gitmodules index 81fceeb..82a23a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "substrait"] path = substrait url = https://github.com/substrait-io/substrait +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = https://github.com/duckdb/extension-ci-tools.git diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..69ec500 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit 69ec500f2330913633481af5602728a3cd4b7468 From 455a5bf3d0b85b46d2d1546b7e95e0242c4ef8e5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 14:27:14 +0200 Subject: [PATCH 04/10] Rely on extension-ci-tools for Makefile --- Makefile | 54 +++++++++++++----------------------------- extension_config.cmake | 7 ++++++ 2 files changed, 23 insertions(+), 38 deletions(-) create mode 100644 extension_config.cmake diff --git a/Makefile b/Makefile index c211163..901db44 100644 --- a/Makefile +++ b/Makefile @@ -1,47 +1,25 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) all: release -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -EXTRA_CMAKE_VARIABLES := - -# These flags will make DuckDB build the extension -EXTRA_CMAKE_VARIABLES += -DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="tpch;json" ${OSX_ARCH_FLAG} -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_NAMES="substrait" -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_SHOULD_LINK=1 -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_LOAD_TESTS=1 -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_PATH=$(PROJ_DIR) -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_TEST_PATH=$(PROJ_DIR)test -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_INCLUDE_PATH="$(PROJ_DIR)src/include" -export - -DUCKDB_DIRECTORY= -ifndef DUCKDB_DIR - DUCKDB_DIRECTORY=./duckdb -else - DUCKDB_DIRECTORY=${DUCKDB_DIR} +# Configuration of extension +EXT_NAME=substrait +EXT_CONFIG=${PROJ_DIR}extension_config.cmake + +CORE_EXTENSIONS='tpch;json' + +# Set this flag during building to enable the benchmark runner +ifeq (${BUILD_BENCHMARK}, 1) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DBUILD_BENCHMARKS=1 endif +# Include the Makefile from extension-ci-tools +include extension-ci-tools/makefiles/duckdb_extension.Makefile + pull: git submodule init git submodule update --recursive --remote -clean: - rm -rf ${DUCKDB_DIRECTORY}/build - rm -rf testext - cd ${DUCKDB_DIRECTORY} && make clean - -# Main builds -debug: -# Have to actually cd here because the makefile assumes it's called from within duckdb - cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . debug - -release: -# Have to actually cd here because the makefile assumes it's called from within duckdb - cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . release - # Client builds %_js: export BUILD_NODE=1 debug_js: debug @@ -62,10 +40,10 @@ release_python: release test: test_release test_release: release - ${DUCKDB_DIRECTORY}/build/release/test/unittest "$(PROJ_DIR)test/*" + build/release/test/unittest "$(PROJ_DIR)test/*" test_debug: debug - ${DUCKDB_DIRECTORY}/build/debug/test/unittest "$(PROJ_DIR)test/*" + build/debug/test/unittest "$(PROJ_DIR)test/*" # Client tests test_python: test_debug_python @@ -88,4 +66,4 @@ format: rm .clang-format update: - git submodule update --remote --merge \ No newline at end of file + git submodule update --remote --merge diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..9338c96 --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,7 @@ +# This file is included by DuckDB's build system. It specifies which extension to load + +# Extension from this repo +duckdb_extension_load(substrait + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/include +) From 0909f8f62a9e0663ffed460e0352d4e6a85521b9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 14:27:25 +0200 Subject: [PATCH 05/10] Bump duckdb to v1.1.0 --- duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duckdb b/duckdb index a6e32b1..fa5c2fe 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit a6e32b115826ba543e32a733cb92f68fd0549186 +Subproject commit fa5c2fe15f3da5f32397b009196c0895fce60820 From 921da878cc29992a1cf804a82ea0a74eca0299ad Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 15:06:55 +0200 Subject: [PATCH 06/10] Fixup sql.yml, using extension-ci-tools --- .github/workflows/distribution.yml | 2 +- .github/workflows/main_distribution.yml | 27 +++++++++++++++++++++ .github/workflows/sql.yml | 31 ------------------------- 3 files changed, 28 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/main_distribution.yml delete mode 100644 .github/workflows/sql.yml diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index c502ccf..74e831e 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -38,4 +38,4 @@ jobs: with: duckdb_version: v1.1.0 exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" - extension_name: substrait \ No newline at end of file + extension_name: substrait diff --git a/.github/workflows/main_distribution.yml b/.github/workflows/main_distribution.yml new file mode 100644 index 0000000..b974a08 --- /dev/null +++ b/.github/workflows/main_distribution.yml @@ -0,0 +1,27 @@ +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Main Extension Distribution Pipeline +on: + pull_request: + paths-ignore: + - '**/README.md' + - 'doc/**' + push: + paths-ignore: + - '**/README.md' + - 'doc/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-main-build: + name: Build extension binaries + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main + with: + duckdb_version: main + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" + extension_name: substrait + diff --git a/.github/workflows/sql.yml b/.github/workflows/sql.yml deleted file mode 100644 index a849445..0000000 --- a/.github/workflows/sql.yml +++ /dev/null @@ -1,31 +0,0 @@ -on: [push, pull_request,repository_dispatch] - -defaults: - run: - shell: bash - -jobs: - sql: - name: SQL - runs-on: macos-latest - env: - GEN: ninja - - steps: - - name: Install Ninja - run: brew install ninja - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Update DuckDB submodule - run: | - git config --global --add safe.directory '*' - make pull - - - name: Build - run: make release - - - name: Test - run: make test_release \ No newline at end of file From 86808652ec691b0e8efde1b9e7d88bbd272bc82e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 15:08:44 +0200 Subject: [PATCH 07/10] Fixup python.yml --- .github/workflows/python.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 274a102..5597e7d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,16 +18,12 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - uses: actions/setup-python@v2 with: python-version: '3.11' - - name: Update DuckDB submodule - run: | - git config --global --add safe.directory '*' - make pull - - name: Build DuckDB (Python) run: | cd duckdb/tools/pythonpkg @@ -45,4 +41,4 @@ jobs: - name: Test Python run: | - (cd test/python && python -m pytest) \ No newline at end of file + (cd test/python && python -m pytest) From 7a2bc057c24bea3eb86ee70f5aef751c84a77333 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 15:09:58 +0200 Subject: [PATCH 08/10] Add empty vcpkg.json --- vcpkg.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 vcpkg.json diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..9757fe1 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,4 @@ +{ + "dependencies": [ + ] +} From 50a43170dc94900971b1cf1fd02cb9c505394a99 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 15:33:47 +0200 Subject: [PATCH 09/10] Fixup paths (tests and README) --- README.md | 2 +- test/python/conftest.py | 2 +- test/python/test_adbc.py | 2 +- test/r/test_substrait.R | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bbe9c86..3f86a18 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ make To run, run the bundled `duckdb` shell: ``` - ./duckdb/build/release/duckdb + ./build/release/duckdb ``` Then, load the Substrait - DuckDB extension like so: diff --git a/test/python/conftest.py b/test/python/conftest.py index d83ce39..2a19f0c 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -23,7 +23,7 @@ def add_extension(extension_name, conn: Union[str, DuckDBPyConnection] = '') -> 'allow_unsigned_extensions' : 'true' } conn = duckdb.connect(conn or '', config=config) - file_path = f"'{dir}/../../duckdb/build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'" + file_path = f"'{dir}/../../build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'" conn.execute(f"LOAD {file_path}") return conn diff --git a/test/python/test_adbc.py b/test/python/test_adbc.py index 1d577ae..e80198b 100644 --- a/test/python/test_adbc.py +++ b/test/python/test_adbc.py @@ -19,7 +19,7 @@ def find_substrait(): # Paths to search for extensions - build = normpath(join(dirname(__file__), "../../duckdb/build/")) + build = normpath(join(dirname(__file__), "../../build/")) extension = "extension/*/*.duckdb_extension" extension_search_patterns = [ diff --git a/test/r/test_substrait.R b/test/r/test_substrait.R index 8a6610f..0caa43a 100644 --- a/test/r/test_substrait.R +++ b/test/r/test_substrait.R @@ -7,7 +7,7 @@ load_extension <- function() { file_directory <- getwd() con <- dbConnect(duckdb::duckdb(config=list("allow_unsigned_extensions"="true"))) - dbExecute(con, sprintf("LOAD '%s/../../duckdb/build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type)) + dbExecute(con, sprintf("LOAD '%s/../../build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type)) return (con) } From ccd7929a6e5c3b632482d706dd09573503e0e676 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 11 Sep 2024 16:34:38 +0200 Subject: [PATCH 10/10] Do LOAD_TESTS --- Makefile | 6 ------ extension_config.cmake | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 901db44..36723f9 100644 --- a/Makefile +++ b/Makefile @@ -36,12 +36,6 @@ release_r: release debug_python: debug release_python: release -# Main tests -test: test_release - -test_release: release - build/release/test/unittest "$(PROJ_DIR)test/*" - test_debug: debug build/debug/test/unittest "$(PROJ_DIR)test/*" diff --git a/extension_config.cmake b/extension_config.cmake index 9338c96..7665aeb 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -4,4 +4,5 @@ duckdb_extension_load(substrait SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/include + LOAD_TESTS )