PacBio IsoSeq analysis loading changes #522
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Unit tests" | |
| on: [push, pull_request] | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail | |
| defaults: | |
| run: | |
| shell: bash -l -e -o pipefail {0} | |
| env: | |
| SINGULARITY_VERSION: "3.11.1" | |
| PERL_CACHE: ~/perl5 # Perlbrew and CPAN modules installed here, cached | |
| NPG_LIB: ~/perl5npg # NPG modules installed here, not cached | |
| WTSI_NPG_BUILD_BRANCH: ${{ github.base_ref || github.ref }} | |
| strategy: | |
| matrix: | |
| include: | |
| - perl: "5.22.4" | |
| irods: "4.2.7" | |
| client_image: "ghcr.io/wtsi-npg/ub-16.04-irods-clients-4.2.7:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-16.04-irods-4.2.7:latest" | |
| experimental: false | |
| - perl: "5.22.4" | |
| irods: "4.2.11" | |
| client_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.11:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest" | |
| experimental: false | |
| - perl: "5.34.1" | |
| irods: "4.2.11" | |
| client_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.11:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest" | |
| experimental: false | |
| - perl: "5.22.4" | |
| irods: "4.2.12" | |
| client_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.12:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.12:latest" | |
| experimental: false | |
| - perl: "5.22.4" | |
| irods: "4.3.1" | |
| client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3.1:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3.1:latest" | |
| experimental: true | |
| - perl: "5.22.4" | |
| irods: "4.3.2" | |
| client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3.2:latest" | |
| server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3.2:latest" | |
| experimental: true | |
| services: | |
| irods-server: | |
| image: ${{ matrix.server_image }} | |
| ports: | |
| - "1247:1247" | |
| - "20000-20199:20000-20199" | |
| options: >- | |
| --health-cmd "nc -z -v localhost 1247" | |
| --health-start-period 30s | |
| --health-interval 10s | |
| --health-timeout 20s | |
| --health-retries 6 | |
| steps: | |
| - name: "Set environmental variables based on other environmental variables" | |
| run: | | |
| echo "SINGULARITY_CACHEDIR=$HOME/.singularity-cache" >> $GITHUB_ENV | |
| # '~' in SINGULARITY_CACHEDIR value (from say a env: section in this YAML) is not expanded by | |
| # singularity so that paths used are misleading/unclear | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| architecture: 'x64' | |
| - name: "Install OS dependencies" | |
| run: | | |
| sudo apt-get update | |
| # https://github.com/actions/runner-images/issues/2139 | |
| sudo apt-get remove -y nginx libgd3 | |
| sudo apt-get install -qq uuid-dev # For Perl UUID module | |
| sudo apt-get install -qq libgd-dev libgd-text-perl # For npg_tracking | |
| sudo apt-get install -qq liblzma-dev # For npg_qc | |
| sudo apt-get install -qq pigz # for BioNano run publication in npg_irods | |
| - name: "Install Singularity" | |
| run: | | |
| pkg="singularity-ce_${SINGULARITY_VERSION}-$(lsb_release -sc)_amd64.deb" | |
| curl -sSL -O "https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/$pkg" | |
| curl -sSL -O "https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/sha256sums" | |
| grep "$os_release" sha256sums | sha256sum | |
| sudo apt-get install -q -y "./$pkg" | |
| # Avoid the runner's loop devices being exhausted | |
| # sudo singularity config global --set "shared loop devices" yes | |
| sudo singularity config global --set "max loop devices" 128 | |
| # Enable DBus for this user, so that rootless cgroups works and we can create | |
| # Singularity service instances. See https://github.com/containers/podman/issues/5443 | |
| systemctl --user enable --now dbus.socket | |
| echo DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$(id -u)/bus" >> $GITHUB_ENV | |
| - name: "Cache Singularity images" | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.SINGULARITY_CACHEDIR }} | |
| key: ${{ runner.os }}-singularity | |
| - name: "Install iRODS client wrappers" | |
| env: | |
| DOCKER_IMAGE: ${{ matrix.client_image }} | |
| run: | | |
| # Use -s option for the wrappers to enable re-usable service instances | |
| # Install is to HOME rather than workspace to avoid clashes with repo e.g. in bin/ | |
| singularity exec docker://$DOCKER_IMAGE singularity-wrapper -s -p $HOME/.local install | |
| echo $HOME/.local/bin >> $GITHUB_PATH | |
| - name: "Configure iRODS clients" | |
| env: | |
| DOCKER_IMAGE: ${{ matrix.client_image }} | |
| run: | | |
| mkdir -p "$HOME/.irods" | |
| cat <<'EOF' > "$HOME/.irods/irods_environment.json" | |
| { | |
| "irods_host": "localhost", | |
| "irods_port": 1247, | |
| "irods_user_name": "irods", | |
| "irods_zone_name": "testZone", | |
| "irods_home": "/testZone/home/irods", | |
| "irods_default_resource": "replResc", | |
| "irods_default_hash_scheme": "MD5" | |
| } | |
| EOF | |
| echo "irods" | script -q -c "iinit" /dev/null | |
| ienv | |
| ils | |
| ilsresc | |
| baton-do --version | |
| - name: "Cache Perl" | |
| id: cache-perl | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.PERL_CACHE }} | |
| key: ${{ runner.os }}-${{ matrix.perl }}-perl | |
| - name: "Install Perlbrew" | |
| if: steps.cache-perl.outputs.cache-hit != 'true' | |
| run: | | |
| curl -sSL https://install.perlbrew.pl -o perlbrew.sh | |
| sha256sum -c .github/workflows/perlbrew.sha256 | |
| export PERLBREW_ROOT=${{ env.PERL_CACHE }} | |
| sh perlbrew.sh | |
| source ${{ env.PERL_CACHE }}/etc/bashrc | |
| perlbrew available | |
| perlbrew install --notest perl-${{ matrix.perl }} | |
| perlbrew use perl-${{ matrix.perl }} | |
| perlbrew install-cpanm | |
| - name: "Initialize Perlbrew" | |
| run: | | |
| echo "source ${{ env.PERL_CACHE }}/etc/bashrc" >> "$HOME/.bash_profile" | |
| - name: "Install Perl dependencies" | |
| run: | | |
| cpanm --local-lib=${{ env.PERL_CACHE }} local::lib | |
| eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB") | |
| eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib) | |
| # Required for npg_qc | |
| cpanm --quiet --notest https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz | |
| # Work around the circular dependency between npg_irods and | |
| # npg_ml_warehouse. The latter dependency requires the former (which | |
| # is this repository itself). | |
| export PERL5LIB="$GITHUB_WORKSPACE/lib:$PERL5LIB" | |
| ./scripts/install_wsi_dependencies.sh "$NPG_LIB" \ | |
| perl-dnap-utilities \ | |
| perl-irods-wrap \ | |
| ml_warehouse \ | |
| npg_ml_warehouse \ | |
| npg_tracking \ | |
| npg_seq_common \ | |
| npg_qc | |
| cpanm --installdeps --notest . | |
| - name: "Install Python dependencies" | |
| run: | | |
| python3 -m pip install git+https://github.com/wtsi-npg/npg_id_generation@$WTSI_NPG_BUILD_BRANCH || | |
| python3 -m pip install git+https://github.com/wtsi-npg/npg_id_generation@devel | |
| - name: "Log install failure" | |
| if: ${{ failure() }} | |
| run: | | |
| find ~/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20 {} \; | |
| - name: "Run tests" | |
| run: | | |
| eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib) | |
| eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB") | |
| export PERL5LIB="$PWD:$PERL5LIB" | |
| perl Build.PL | |
| export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE="$HOME/.irods/irods_environment.json" | |
| ./Build test --verbose |