From 051dcde8fb95acd27f5a3ef4b9f9d11356b75f8e Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 25 Oct 2023 15:13:55 +0530 Subject: [PATCH 1/3] feat: Store branch and dataset for benchmarks --- .github/workflows/manual-benchmark.yaml | 9 ++++--- tools/upload_results_postgres.sh | 33 ++++++++++++++----------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/manual-benchmark.yaml b/.github/workflows/manual-benchmark.yaml index 26af06fb..6e8df1e9 100644 --- a/.github/workflows/manual-benchmark.yaml +++ b/.github/workflows/manual-benchmark.yaml @@ -7,9 +7,9 @@ on: qdrant_branch: description: "Branch of qdrant to benchmark" default: dev - benchmark_table: - description: "Postgres table to store benchmark results" - default: benchmark_manual + dataset: + description: "Dataset to benchmark" + default: laion-small-clip jobs: runManualBenchmark: @@ -28,5 +28,6 @@ jobs: export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} export QDRANT_VERSION=${{ inputs.qdrant_branch }} - export POSTGRES_TABLE=${{ inputs.benchmark_table }} + export DATASETS=${{ inputs.dataset }} + export POSTGRES_TABLE=benchmark_manual bash -x tools/run_ci.sh diff --git a/tools/upload_results_postgres.sh b/tools/upload_results_postgres.sh index e07e0d95..d278453c 100644 --- a/tools/upload_results_postgres.sh +++ b/tools/upload_results_postgres.sh @@ -4,24 +4,29 @@ # Read search results from json file and upload it to postgres # # Assume table: -#create table benchmark ( -# id SERIAL PRIMARY key, -# engine VARCHAR(255), -# measure_timestamp TIMESTAMP, -# upload_time real, -# indexing_time real, -# rps real, -# mean_precisions real, -# p95_time real, -# p99_time real, -# memory_usage real -#); +# create table benchmark ( +# id SERIAL PRIMARY key, +# engine VARCHAR(255), +# branch VARCCHAR(255), +# dataset VARCHAR(255), +# measure_timestamp TIMESTAMP, +# upload_time real, +# indexing_time real, +# rps real, +# mean_precisions real, +# p95_time real, +# p99_time real, +# memory_usage real +# ); SEARCH_RESULTS_FILE=${SEARCH_RESULTS_FILE:-""} UPLOAD_RESULTS_FILE=${UPLOAD_RESULTS_FILE:-""} MEMORY_USAGE_FILE=${MEMORY_USAGE_FILE:-""} POSTGRES_TABLE=${POSTGRES_TABLE:-"benchmark"} +QDRANT_VERSION=${QDRANT_VERSION:-"dev"} +DATASETS=${DATASETS:-"laion-small-clip"} + if [[ -z "$SEARCH_RESULTS_FILE" ]]; then echo "SEARCH_RESULTS_FILE is not set" exit 1 @@ -53,7 +58,7 @@ MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")} docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c " -INSERT INTO ${POSTGRES_TABLE} (engine, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage) -VALUES ('qdrant-ci', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE}); +INSERT INTO ${POSTGRES_TABLE} (engine, branch, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage) +VALUES ('qdrant-ci', ${QDRANT_VERSION}, ${DATASETS}, '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE}); " From 99b7b1d74372d33571b985cf81c4c0efec2ef347 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 25 Oct 2023 15:19:32 +0530 Subject: [PATCH 2/3] fix: Fix spacing --- tools/upload_results_postgres.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/upload_results_postgres.sh b/tools/upload_results_postgres.sh index d278453c..92c016e1 100644 --- a/tools/upload_results_postgres.sh +++ b/tools/upload_results_postgres.sh @@ -7,8 +7,8 @@ # create table benchmark ( # id SERIAL PRIMARY key, # engine VARCHAR(255), -# branch VARCCHAR(255), -# dataset VARCHAR(255), +# branch VARCCHAR(255), +# dataset VARCHAR(255), # measure_timestamp TIMESTAMP, # upload_time real, # indexing_time real, From 98fa9861de179e6e0aee3ca20c3913b84997500d Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 25 Oct 2023 15:32:25 +0530 Subject: [PATCH 3/3] fix: Use quotes and mention dataset in job name --- .github/workflows/manual-benchmark.yaml | 2 +- tools/upload_results_postgres.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-benchmark.yaml b/.github/workflows/manual-benchmark.yaml index 6e8df1e9..c9dbebee 100644 --- a/.github/workflows/manual-benchmark.yaml +++ b/.github/workflows/manual-benchmark.yaml @@ -13,7 +13,7 @@ on: jobs: runManualBenchmark: - name: manual benchmark - ${{ inputs.qdrant_branch }} + name: manual benchmark - ${{ inputs.qdrant_branch }} - ${{ inputs.dataset }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/tools/upload_results_postgres.sh b/tools/upload_results_postgres.sh index 92c016e1..d299f3c3 100644 --- a/tools/upload_results_postgres.sh +++ b/tools/upload_results_postgres.sh @@ -59,6 +59,6 @@ MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")} docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c " INSERT INTO ${POSTGRES_TABLE} (engine, branch, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage) -VALUES ('qdrant-ci', ${QDRANT_VERSION}, ${DATASETS}, '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE}); +VALUES ('qdrant-ci', '${QDRANT_VERSION}', '${DATASETS}', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE}); "