From 0b71bb7c8eb217c3d8646e7d952db46f825a8190 Mon Sep 17 00:00:00 2001 From: j23414 Date: Thu, 29 Dec 2022 18:11:02 -0800 Subject: [PATCH] edit: Use a permalink for each script This would allow us to version the software we use in this workflow without being affected by upstream changes until we want to bump the version. This design adds more maintenance to this workflow, but it also protects users against unexpected issues that are outside of their control. Discussed in https://github.com/nextstrain/ebola/pull/6#discussion_r1048835183 --- Snakefile | 3 ++- .../snakemake_rules/fetch_sequences.smk | 6 +++--- .../snakemake_rules/slack_notifications.smk | 4 ++-- ingest/workflow/snakemake_rules/transform.smk | 18 +++++++++--------- .../snakemake_rules/trigger_rebuild.smk | 2 +- ingest/workflow/snakemake_rules/upload.smk | 6 +++--- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Snakefile b/Snakefile index 944953e..f37d87d 100644 --- a/Snakefile +++ b/Snakefile @@ -37,6 +37,7 @@ rule wrangle_metadata: metadata="results/wrangled_metadata.tsv", params: strain_id=lambda w: config.get("strain_id_field", "strain"), + wrangle_metadata_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/scripts/wrangle_metadata.py" shell: """ if [[ ! -d bin ]]; then @@ -253,7 +254,7 @@ rule final_strain_name: root_sequence="auspice/zika_root-sequence.json", params: display_strain_field=lambda w: config.get("display_strain_field", "strain"), - set_final_strain_name_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/scripts/set_final_strain_name.py", + set_final_strain_name_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/scripts/set_final_strain_name.py", shell: """ if [[ ! -d bin ]]; then diff --git a/ingest/workflow/snakemake_rules/fetch_sequences.smk b/ingest/workflow/snakemake_rules/fetch_sequences.smk index 158869b..2844949 100644 --- a/ingest/workflow/snakemake_rules/fetch_sequences.smk +++ b/ingest/workflow/snakemake_rules/fetch_sequences.smk @@ -24,9 +24,9 @@ rule fetch_from_genbank: genbank_ndjson="data/genbank_{serotype}.ndjson", params: serotype_tax_id=download_serotype, - csv_to_ndjson_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/csv-to-ndjson", - fetch_from_genbank_url="https://raw.githubusercontent.com/nextstrain/dengue/new_ingest/ingest/bin/fetch-from-genbank", - genbank_url_url="https://raw.githubusercontent.com/nextstrain/dengue/new_ingest/ingest/bin/genbank-url", # Update if dengue merged + csv_to_ndjson_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/csv-to-ndjson", + fetch_from_genbank_url="https://raw.githubusercontent.com/nextstrain/dengue/ca659008bfbe4b3f799e11ecd106a0b95977fe93/ingest/bin/fetch-from-genbank", + genbank_url_url="https://raw.githubusercontent.com/nextstrain/dengue/ca659008bfbe4b3f799e11ecd106a0b95977fe93/ingest/bin/genbank-url", # Update if dengue merged shell: """ if [[ ! -d bin ]]; then diff --git a/ingest/workflow/snakemake_rules/slack_notifications.smk b/ingest/workflow/snakemake_rules/slack_notifications.smk index 638f84a..91d5ffc 100644 --- a/ingest/workflow/snakemake_rules/slack_notifications.smk +++ b/ingest/workflow/snakemake_rules/slack_notifications.smk @@ -28,7 +28,7 @@ rule notify_on_genbank_record_change: touch("data/notify/genbank-record-change.done"), params: s3_src=S3_SRC, - notify_on_record_change_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/notify-on-record-change", + notify_on_record_change_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/notify-on-record-change", shell: """ if [[ ! -d bin ]]; then @@ -52,7 +52,7 @@ rule notify_on_metadata_diff: touch("data/notify/metadata-diff.done"), params: s3_src=S3_SRC, - notify_on_diff_url = "https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/notify-on-diff", + notify_on_diff_url = "https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/notify-on-diff", shell: """ if [[ ! -d bin ]]; then diff --git a/ingest/workflow/snakemake_rules/transform.smk b/ingest/workflow/snakemake_rules/transform.smk index c6b29a1..e48b15a 100644 --- a/ingest/workflow/snakemake_rules/transform.smk +++ b/ingest/workflow/snakemake_rules/transform.smk @@ -62,15 +62,15 @@ rule transform: metadata_columns=config["transform"]["metadata_columns"], id_field=config["transform"]["id_field"], sequence_field=config["transform"]["sequence_field"], - transform_field_names_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-field-names", - transform_string_fields_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-string-fields", - transform_strain_names_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-strain-names", - transform_date_fields_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-date-fields", - transform_genbank_location_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-genbank-location", - transform_authors_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/transform-authors", - apply_geolocation_rules_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/apply-geolocation-rules", - merge_user_metadata_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/merge-user-metadata", - ndjson_to_tsv_and_fasta_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/ndjson-to-tsv-and-fasta", + transform_field_names_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-field-names", + transform_string_fields_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-string-fields", + transform_strain_names_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-strain-names", + transform_date_fields_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-date-fields", + transform_genbank_location_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-genbank-location", + transform_authors_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/transform-authors", + apply_geolocation_rules_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/apply-geolocation-rules", + merge_user_metadata_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/merge-user-metadata", + ndjson_to_tsv_and_fasta_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/ndjson-to-tsv-and-fasta", shell: """ if [[ ! -d bin ]]; then diff --git a/ingest/workflow/snakemake_rules/trigger_rebuild.smk b/ingest/workflow/snakemake_rules/trigger_rebuild.smk index 6048dd8..aa5e52a 100644 --- a/ingest/workflow/snakemake_rules/trigger_rebuild.smk +++ b/ingest/workflow/snakemake_rules/trigger_rebuild.smk @@ -14,7 +14,7 @@ rule trigger_build: output: touch("data/trigger/rebuild.done") params: - trigger_on_new_data_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/trigger-on-new-data" + trigger_on_new_data_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/trigger-on-new-data" shell: """ if [[ ! -d bin ]]; then diff --git a/ingest/workflow/snakemake_rules/upload.smk b/ingest/workflow/snakemake_rules/upload.smk index 95ad81e..db2de41 100644 --- a/ingest/workflow/snakemake_rules/upload.smk +++ b/ingest/workflow/snakemake_rules/upload.smk @@ -54,9 +54,9 @@ rule upload_to_s3: quiet="" if send_notifications else "--quiet", s3_dst=config["upload"].get("s3", {}).get("dst", ""), cloudfront_domain=config["upload"].get("s3", {}).get("cloudfront_domain", ""), - upload_to_s3_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/upload-to-s3", - sha256sum_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/sha256sum", - cloudfront_invalidate_url="https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/cloudfront-invalidate" + upload_to_s3_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/upload-to-s3", + sha256sum_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/sha256sum", + cloudfront_invalidate_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/ingest/bin/cloudfront-invalidate" shell: """ if [[ ! -d bin ]]; then