From 2498befcd5c70fea69e20156e694dec9f47aa2d6 Mon Sep 17 00:00:00 2001 From: "C. Titus Brown" Date: Mon, 29 Apr 2024 11:10:28 -0700 Subject: [PATCH] add actual config scripts for jetstream instances --- extra/jetstream/minto_install.ctb.sh | 71 +++++++++++++++++++++++++++ extra/jetstream/titus-configure.sh | 47 ++++++++++++++++++ extra/jetstream/titus-get-ordinate.sh | 5 ++ extra/jetstream/titus-get-plot-amr.sh | 5 ++ extra/jetstream/titus-init.sh | 7 +++ 5 files changed, 135 insertions(+) create mode 100755 extra/jetstream/minto_install.ctb.sh create mode 100755 extra/jetstream/titus-configure.sh create mode 100755 extra/jetstream/titus-get-ordinate.sh create mode 100755 extra/jetstream/titus-get-plot-amr.sh create mode 100755 extra/jetstream/titus-init.sh diff --git a/extra/jetstream/minto_install.ctb.sh b/extra/jetstream/minto_install.ctb.sh new file mode 100755 index 0000000..bd3eeb2 --- /dev/null +++ b/extra/jetstream/minto_install.ctb.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -e +set -x + +cd $HOME + +. /opt/miniconda3/etc/profile.d/conda.sh + +# Get MIntO + +git clone --depth 1 https://github.com/microbiomix/MIntO.git + +# Create MIntO conda env + +conda config --set channel_priority strict +conda create --yes -n MIntO mamba snakemake=7 -c conda-forge -c bioconda +conda activate MIntO +export MINTO_DIR="$HOME/MIntO" +cd $MINTO_DIR + +# Create conda envs + +cat >> $MINTO_DIR/smk/precreate_envs.smk <<__EOM__ +import os +my_envs = ['MIntO_base.yml', 'avamb.yml', 'mags.yml', 'checkm2.yml', 'gene_annotation.yml', 'r_pkgs.yml'] +rule make_all_envs: + input: + expand("created-{name}", name=my_envs) +for env_file in my_envs: + rule: + output: + temp("created-%s" % env_file) + conda: + config["minto_dir"]+"/envs/%s" % env_file + shell: + "touch {output}" +__EOM__ + +snakemake --use-conda --conda-prefix $MINTO_DIR/conda_env --config minto_dir=$MINTO_DIR --cores 4 -s $MINTO_DIR/smk/precreate_envs.smk +mamba clean --tarballs --yes + +# Patch assembly to avoid indexing contig files + +sed -i "s/\.len//" $MINTO_DIR/smk/assembly.smk + +# Set up minimal database downloads + +sed -i -e "s|minto_dir: /mypath/MIntO|minto_dir: $MINTO_DIR|" $MINTO_DIR/configuration/dependencies.yaml +cat >> $MINTO_DIR/configuration/dependencies.yaml <<__EOM__ +enable_GTDB: no +enable_phylophlan: no +enable_metaphlan: no +enable_motus: no +__EOM__ + +# Download databases + +snakemake --use-conda --restart-times 1 --keep-going --latency-wait 60 --cores 14 --resources mem=50 --jobs 4 --shadow-prefix="./.snakemake" --conda-prefix $MINTO_DIR/conda_env --snakefile $MINTO_DIR/smk/dependencies.smk --configfile $MINTO_DIR/configuration/dependencies.yaml -U checkm2_db Kofam_db functional_db_descriptions KEGG_maps download_fetchMGs + + +# Download Tutorial data + +mkdir -p $HOME/tutorial/metaG +cd $HOME/tutorial/metaG + +# Get tarball, extract and delete tarball + +#wget --no-check-certificate https://arumugamlab.sund.ku.dk/tutorials/202404_MIntO/tutorial_data.tar.bz2 +#tar xfj tutorial_data.tar.bz2 +#rm tutorial_data.tar.bz2 +tar xfj /opt/shared/tutorial_data.tar.bz2 diff --git a/extra/jetstream/titus-configure.sh b/extra/jetstream/titus-configure.sh new file mode 100755 index 0000000..031aa1e --- /dev/null +++ b/extra/jetstream/titus-configure.sh @@ -0,0 +1,47 @@ +#! /bin/bash +set -x +set -e + +. /opt/miniconda3/etc/profile.d/conda.sh + +# add my ssh key +#echo ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHFz3WLVqV+0md4OkZi/S0a79cOO7Ax8S4Dledp832JhMQ0GJ0ZlmEnWZrIv83KnRexpAEi5w6H1aSackGjucgQ= t@TitusMatsalmoth.attlocal.net >> ~/.ssh/authorized_keys + +mkdir ~/data/ + +cd ~/data/ +#curl -JLO http://farm.cse.ucdavis.edu/~ctbrown/transfer/IBD_tutorial_subset.tar.gz +tar xzf /opt/shared/IBD_tutorial_subset.tar.gz +#curl -JLO http://farm.cse.ucdavis.edu/~ctbrown/transfer/tutorial_other.tar.gz +tar xzf /opt/shared/tutorial_other.tar.gz + +mkdir ~/databases/ +cd ~/databases/ +ln -fs /opt/shared/gtdb-rs214-k31.zip . +ln -fs /opt/shared/gtdb-rs214.lineages.csv.gz . +#curl -JLO https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs214/gtdb-rs214-k31.zip +#curl -JLO https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs214/gtdb-rs214.lineages.csv.gz + +cd ~/ + +# fix up conda channels +conda config --add channels defaults +conda config --add channels bioconda +conda config --add channels conda-forge +conda config --set channel_priority strict + +## pre-download packages + +# single metagenome tutorial +mamba create -n test.tax -y sourmash sourmash_plugin_branchwater +conda activate test.tax +pip install sourmash_plugin_abundhist sourmash_plugin_venn + +# comparison tutorial +mamba create -n test.smash -y sourmash scikit-learn + +# AMRfinder tutorial +mamba create -n test.amrfinder -y ncbi-amrfinderplus megahit prodigal csvtk + +## minto install +bash /opt/shared/minto_install.ctb.sh diff --git a/extra/jetstream/titus-get-ordinate.sh b/extra/jetstream/titus-get-ordinate.sh new file mode 100755 index 0000000..d561c34 --- /dev/null +++ b/extra/jetstream/titus-get-ordinate.sh @@ -0,0 +1,5 @@ +#! /bin/bash +mkdir ~/scripts +cd ~/scripts +curl -JL -o ordinate.py https://raw.githubusercontent.com/ngs-docs/2024-pig-paradigm-workshop/main/scripts/ordinate.py +chmod +x ordinate.py diff --git a/extra/jetstream/titus-get-plot-amr.sh b/extra/jetstream/titus-get-plot-amr.sh new file mode 100755 index 0000000..682cb90 --- /dev/null +++ b/extra/jetstream/titus-get-plot-amr.sh @@ -0,0 +1,5 @@ +#! /bin/bash +mkdir ~/scripts +cd ~/scripts +curl -JL -o plot_amr.py https://raw.githubusercontent.com/ngs-docs/2024-pig-paradigm-workshop/main/scripts/plot_amr.py +chmod +x plot_amr.py diff --git a/extra/jetstream/titus-init.sh b/extra/jetstream/titus-init.sh new file mode 100755 index 0000000..0d6a642 --- /dev/null +++ b/extra/jetstream/titus-init.sh @@ -0,0 +1,7 @@ +#! /bin/bash +set -x +set -e + +# add my ssh key +echo ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHFz3WLVqV+0md4OkZi/S0a79cOO7Ax8S4Dledp832JhMQ0GJ0ZlmEnWZrIv83KnRexpAEi5w6H1aSackGjucgQ= t@TitusMatsalmoth.attlocal.net >> ~/.ssh/authorized_keys +