diff --git a/update-pytorch.sh b/update-pytorch.sh index 141890a..b2d936b 100755 --- a/update-pytorch.sh +++ b/update-pytorch.sh @@ -1,26 +1,26 @@ #!/bin/bash -# old non-updated compute platforms: -# cu75 cu80 cu90 cu91 cu92 cu100 cu101 cu102 cu110 cu111 cu113 cu115 cu116 cu117 cu117_pypi_cudnn -# rocm3.10 rocm3.7 rocm3.8 rocm4.0.1 rocm4.1 rocm4.2 rocm4.3.1 rocm4.5.2 rocm5.0 rocm5.1.1 rocm5.2 rocm5.3 rocm5.4.2 rocm5.5 rocm5.6 rocm5.7 -for d in . cpu cpu-cxx11-abi cpu_pypi_pkg cu118 cu121 cu124 rocm6.0 rocm6.1 -do - d="whl/$d" - [ "$d" == "whl/." ] && d="whl" +# update a PyPI index from PyTorch = https://download.pytorch.org/$d with d=whl or whl/ +# this copies the content (main url + follows links to projects) and does 2 updates: +# 1. copies the content in a "simple/" sub-directory to match the convention from PEP 503 simple (that allows other APIs in parallel) +# 2. updates the links to binary packages by adding "https://download.pytorch.org" prefix to PyTorch-provided "/whl/*" path to link back to PyTorch binaries in their home location +function updateIndex() { + local d=$1 + + # will copy source $d to $d/simple mkdir -p $d/simple - dir="$(pwd)" + local dir="$(pwd)" cd $d/simple + + # projects list curl -s https://download.pytorch.org/$d/ | grep -v 'TIMESTAMP 1' > index.html - count="$(cat index.html | cut -d '>' -f 2 | cut -d '<' -f 1 | grep -cve '^\s*$')" + local count="$(cat index.html | cut -d '>' -f 2 | cut -d '<' -f 1 | grep -cve '^\s*$')" - echo "https://download.pytorch.org/$d/ $count => $d/simple/" - if [ $count -lt 40 ] - then - echo "failing because low packages count for $d: $count (intermittent download failure?)" - exit 1 - fi + echo "https://download.pytorch.org/$d/ => $d/simple/" + checkCount $d $count 40 - i=0 + # copy also content of each project + local i=0 for p in `cat index.html | cut -d '>' -f 2 | cut -d '<' -f 1` do mkdir -p $p @@ -31,20 +31,51 @@ do | grep -v 'TIMESTAMP 1' \ > index.html - count="$(cat index.html | grep -c 'https://download.pytorch.org/whl/')" - echo "$i $d/$p/ => $d/simple/$p/ $count" - if [ $count -lt 1 ] - then - echo "failing because low packages count for $d/$p: $count (intermittent download failure?)" - exit 1 - fi + local pcount="$(cat index.html | grep -c 'https://download.pytorch.org/whl/')" + printf "%5d / $count $d/$p/ => $d/simple/$p/ $pcount\n" $i + checkCount $d/$p $count 1 cd .. done echo cd "$dir" +} + +function checkCount() { + local content="$1" + local count="$2" + local minimum="$3" + + if [ $count -lt $minimum ] + then + echo "!!! failing because low packages count for $content: $count (probably intermittent download failure)" + exit 1 + fi +} + +# update main PyTorch index, that contains everything, whatever the compute platform +updateIndex "whl" + +# see resulting updates +git update-index -q --refresh +git diff-index --name-status HEAD + +if `git diff-index --quiet HEAD` +then + echo "no update found in PyTorch main index." + exit 0 +fi +echo "updates found in PyTorch main index: updating also compute platform specific ones..." + +# update compute-platform specific indexes +# ignore old non-updated ones: +# - cu75 cu80 cu90 cu91 cu92 cu100 cu101 cu102 cu110 cu111 cu113 cu115 cu116 cu117 cu117_pypi_cudnn +# - rocm3.10 rocm3.7 rocm3.8 rocm4.0.1 rocm4.1 rocm4.2 rocm4.3.1 rocm4.5.2 rocm5.0 rocm5.1.1 rocm5.2 rocm5.3 rocm5.4.2 rocm5.5 rocm5.6 rocm5.7 +for d in cpu cpu-cxx11-abi cpu_pypi_pkg cu118 cu121 cu124 rocm6.0 rocm6.1 +do + updateIndex "whl/$d" done du -sh whl/* -for d in whl/simple whl/*/simple ; do echo "$(ls $d | wc -l) $d" ; done > summary.txt +for d in whl/simple whl/*/simple ; do echo "$(ls $d | wc -l | xargs) $d" ; done > summary.txt cat summary.txt