Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Store branch and dataset for benchmarks #68

Merged
merged 3 commits into from
Oct 25, 2023
Merged

feat: Store branch and dataset for benchmarks #68

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
051dcde
feat: Store branch and dataset for benchmarks
KShivendu Oct 25, 2023
99b7b1d
fix: Fix spacing
KShivendu Oct 25, 2023
98fa986
fix: Use quotes and mention dataset in job name
KShivendu Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .github/workflows/manual-benchmark.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ on:
qdrant_branch:
description: "Branch of qdrant to benchmark"
default: dev
benchmark_table:
description: "Postgres table to store benchmark results"
default: benchmark_manual
dataset:
description: "Dataset to benchmark"
default: laion-small-clip

jobs:
runManualBenchmark:
Expand All @@ -28,5 +28,6 @@ jobs:
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
export QDRANT_VERSION=${{ inputs.qdrant_branch }}
export POSTGRES_TABLE=${{ inputs.benchmark_table }}
export DATASETS=${{ inputs.dataset }}
export POSTGRES_TABLE=benchmark_manual
bash -x tools/run_ci.sh
33 changes: 19 additions & 14 deletions tools/upload_results_postgres.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,29 @@
# Read search results from json file and upload it to postgres
#
# Assume table:
#create table benchmark (
# id SERIAL PRIMARY key,
# engine VARCHAR(255),
# measure_timestamp TIMESTAMP,
# upload_time real,
# indexing_time real,
# rps real,
# mean_precisions real,
# p95_time real,
# p99_time real,
# memory_usage real
#);
# create table benchmark (
# id SERIAL PRIMARY key,
# engine VARCHAR(255),
# branch VARCCHAR(255),
# dataset VARCHAR(255),
# measure_timestamp TIMESTAMP,
# upload_time real,
# indexing_time real,
# rps real,
# mean_precisions real,
# p95_time real,
# p99_time real,
# memory_usage real
# );

SEARCH_RESULTS_FILE=${SEARCH_RESULTS_FILE:-""}
UPLOAD_RESULTS_FILE=${UPLOAD_RESULTS_FILE:-""}
MEMORY_USAGE_FILE=${MEMORY_USAGE_FILE:-""}
POSTGRES_TABLE=${POSTGRES_TABLE:-"benchmark"}

QDRANT_VERSION=${QDRANT_VERSION:-"dev"}
DATASETS=${DATASETS:-"laion-small-clip"}

if [[ -z "$SEARCH_RESULTS_FILE" ]]; then
echo "SEARCH_RESULTS_FILE is not set"
exit 1
Expand Down Expand Up @@ -53,7 +58,7 @@ MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}


docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c "
INSERT INTO ${POSTGRES_TABLE} (engine, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
VALUES ('qdrant-ci', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
INSERT INTO ${POSTGRES_TABLE} (engine, branch, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
VALUES ('qdrant-ci', ${QDRANT_VERSION}, ${DATASETS}, '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
"

Loading