From: bones_was_here Date: Tue, 18 Mar 2025 05:35:44 +0000 (+1000) Subject: rsync: implement mirror support with push updates and dual-stack autoselect X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=70d66e495fe19c251653e7b1dec3af09871fa421;p=xonotic%2Fxonotic.git rsync: implement mirror support with push updates and dual-stack autoselect --- diff --git a/misc/infrastructure/update-rsync.sh b/misc/infrastructure/update-rsync.sh deleted file mode 100755 index dbad1945..00000000 --- a/misc/infrastructure/update-rsync.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -set -ex - -d=$1 - -cd /var/rsync/autobuild - -for BUILD in '' -high -mappingsupport; do - rm -rf .new - mkdir .new - cd .new - if ! unzip ~/Xonotic-"$d$BUILD".zip; then - good=false - #for f in ~/Xonotic-"$d$BUILD".zip; do - # unzip "$f" && good=true && break - #done - $good - fi - cd Xonotic/data - for X in *"$d"*; do - pre=${X%$d*} - post=${X##*$d} - mv "$X" "$pre"rsync"$post" - done - cd ../../.. - rm -rf Xonotic.old - mv Xonotic"$BUILD" Xonotic.old || true - mv .new/Xonotic Xonotic"$BUILD" - rmdir .new - rm -rf Xonotic.old -done diff --git a/misc/infrastructure/xonotic-release-build.cron b/misc/infrastructure/xonotic-release-build.cron index 9491caba..d1e08f34 100755 --- a/misc/infrastructure/xonotic-release-build.cron +++ b/misc/infrastructure/xonotic-release-build.cron @@ -99,13 +99,40 @@ if [ -z "$RELEASETYPEDIR" ]; then fi rsync -vaSHP Xonotic*.zip Xonotic*.sha512 autobuild-release-uploader@beta.xonotic.org:"$RELEASETYPEDIR" if [ -z "$RELEASETYPEDIR" ]; then - ssh autobuild-release-uploader@beta.xonotic.org sh ../../update-rsync.sh "$d" + # Update one package at a time (to reduce temp space requirement), on all mirrors simultaneously + rm -rf rsynctmp && mkdir rsynctmp && cd rsynctmp + for suffix in '' -high -mappingsupport; do + package="Xonotic$suffix" + unzip ../Xonotic-"$d$suffix".zip + [ "$package" != Xonotic ] && mv Xonotic "$package" + while read firstword restoflineignored; do + mirror=${firstword%%//*} + [ -z $mirror ] && continue + # if desired an extra word could be parsed from mirrors.txt to change /var/rsync + # ports and usernames should be specified in .ssh/config + rpath=/var/rsync/autobuild + rsync -rtzlyhhS --delete-delay --executability --stats --link-dest="$rpath/$package" \ + "$package/" "$mirror:$rpath/.$package.new/" \ + && ssh "$mirror" " + set -e + cd $rpath + rm -rf .$package.old + [ -e $package ] && mv $package .$package.old + mv .$package.new $package + rm -rf .$package.old + " || echo "Failed to update $package on rsync mirror $mirror" | $IRCSPAM \ + & + done < ../misc/tools/rsync-updater/mirrors.txt + wait + rm -rf "$package" + done + cd .. && rm -rf rsynctmp # also upload pipeline files for .gitlab-ci.yml in xonotic-data.pk3dir cp data/stormkeep-????????????????????????????????????????-????????????????????????????????????????.pk3 stormkeep.pk3 cd data/xonotic-maps.pk3dir # need correct path inside pk3 - 7z a -tzip -mx=9 ../../stormkeep.pk3 maps/stormkeep.mapinfo maps/stormkeep.waypoints* - cd ~/xonotic-release-build + 7z a -tzip -mx=9 ../../stormkeep.pk3 maps/stormkeep.mapinfo maps/stormkeep.waypoints* + cd ../.. PIPELINE_FILES="gmqcc/gmqcc Xonotic/xonotic-linux64-dedicated stormkeep.pk3" rsync -vSP $PIPELINE_FILES autobuild-release-uploader@beta.xonotic.org:../pipeline-bin/ rm -f stormkeep.pk3 diff --git a/misc/tools/all/release.subr b/misc/tools/all/release.subr index fee5d75e..62334a31 100644 --- a/misc/tools/all/release.subr +++ b/misc/tools/all/release.subr @@ -681,19 +681,21 @@ case "$cmd" in release-rsync) release_common # make sure everything we need is installed and updated - verbose "$d0"/misc/tools/msys2-linux.sh --schroot=sid rsync dash openssl + verbose "$d0"/misc/tools/msys2-linux.sh --schroot=sid rsync dash openssl sed targetroot="$PWD/Xonotic" verbose cd "$HOME/msys64" # see msys2-linux.sh verbose cp --parents \ usr/bin/msys-2.0.dll \ usr/bin/msys-crypto-3.dll \ usr/bin/msys-iconv-2.dll \ + usr/bin/msys-intl-8.dll \ usr/bin/msys-lz4-1.dll \ usr/bin/msys-ssl-3.dll \ usr/bin/msys-xxhash-0.dll \ usr/bin/msys-zstd-1.dll \ usr/bin/openssl.exe \ usr/bin/rsync.exe \ + usr/bin/sed.exe \ usr/ssl/cert.pem \ "$targetroot/misc/tools/rsync-updater/" # msys2 sh.exe is currently bash diff --git a/misc/tools/rsync-updater/mirrors.txt b/misc/tools/rsync-updater/mirrors.txt new file mode 100644 index 00000000..07ec02be --- /dev/null +++ b/misc/tools/rsync-updater/mirrors.txt @@ -0,0 +1,3 @@ +// One mirror per line, the first word is the FQDN, the second is the location +rsync.xonotic.org DE // admins: Xonotic Core Team +rsync.xonotic.au AU // admins: rly and bones_was_here diff --git a/misc/tools/rsync-updater/update-to-autobuild.sh b/misc/tools/rsync-updater/update-to-autobuild.sh index 17cb618c..a78b5eb5 100755 --- a/misc/tools/rsync-updater/update-to-autobuild.sh +++ b/misc/tools/rsync-updater/update-to-autobuild.sh @@ -9,12 +9,11 @@ if ! command -v rsync > /dev/null; then exit 1 fi -if ! command -v rsync-ssl >/dev/null; then - export PATH="$PWD/usr/bin:$PATH" -fi +# always prefer our own rsync-ssl script, we need its --ipv4 and --ipv6 option support +export PATH="$PWD/usr/bin:$PATH" # openssl is the only option, as gnutls-cli is broken in rsync-ssl and stunnel doesn't verify the cert. -rsynccmd=rsync-ssl +rsynccmd="rsync-ssl --timeout=3" if ! command -v openssl > /dev/null; then if [ $interactive = false ]; then printf >&2 "\033[1;31mFATAL: openssl not found, please install the openssl package!\033[m\n" @@ -27,7 +26,7 @@ if ! command -v openssl > /dev/null; then read -r secchoice [ "$secchoice" = n ] || [ "$secchoice" = N ] && exit 1 done - rsynccmd=rsync + rsynccmd="rsync --contimeout=3" fi case "${0##*/}" in @@ -79,7 +78,7 @@ elif [ -e "Xonotic-high" ]; then printf "\033[1;35mFound manually created 'Xonotic-high' package override\033[m\n" package="Xonotic-high" fi -url="rsync.xonotic.org/$buildtype/$package" +source="$buildtype/$package" excludes= if [ -n "$XONOTIC_INCLUDE_ALL" ]; then @@ -131,8 +130,56 @@ else esac fi +bestmirror=rsync.xonotic.org +if [ -z "$(rsync --help | sed -En 's/(--write-devices)/\1/p')" ]; then + printf "\033[1;33mNOTE: your rsync version is too old for mirror autoselect and modern compression, expect bad performance. Please update to rsync 3.2.0 or later!\033[m\n" +else + printf "Updating mirror list ... " + out=$($rsynccmd -t "rsync://$bestmirror/autobuild/Xonotic/misc/tools/rsync-updater/mirrors.txt" mirrors.txt 2>&1) \ + && printf "\033[0;32mOK\033[m\n" \ + || printf "\033[1;31mFAILED\n\033[0;31m$out\033[m\n" | sed '2,${s/^/ /}' + + bestspeed=-1 + while read firstword secondword restoflineignored; do + mirror=${firstword%%//*} + [ -z $mirror ] && continue + location=${secondword%%//*} + # Sometimes perf differs greatly between v6 and v4 + for ipv in ipv6 ipv4; do + printf "Testing mirror \033[36m$mirror\033[m [$location] $ipv ... \033[m" + # not the most rigorous benchmark, but fit for purpose and unaffected by local filesystem perf + # NB: /dev/null as the DEST arg is intended + if out=$(LC_ALL=C.UTF-8 $rsynccmd --compress-choice=none --write-devices --stats --$ipv "rsync://$mirror/$source/GPL-3" /dev/null 2>&1); then + # parse the speed from the --stats output (integer part only), and strip any commas (thousands separators) + speed=$(printf "$out" | sed -En 's/.* ([0-9,]+)\.?[0-9]* bytes\/sec$/\1/p' | sed 's/,//g') + if [ -n "$speed" ]; then + printf "\033[0;32mOK, speed $speed\033[m\n" + else + printf "\033[1;33mfailed to parse speed value!\n\033[0;33m$out\033[m\n" | sed '2,${s/^/ /}' + speed=0 + fi + if [ $speed -gt $bestspeed ]; then + bestspeed=$speed + bestmirror=$mirror + bestipv=$ipv + fi + elif [ $ipv = ipv6 ]; then + # omit error text to reduce spam as the ISP may not support v6 + printf "\033[0;31mFAILED\033[m\n" + else + printf "\033[0;31mFAILED\n\033[0;31m$out\033[m\n" | sed '2,${s/^/ /}' + fi + done + done < mirrors.txt + if [ $bestspeed -eq -1 ]; then + printf "\033[1;31mFATAL: all mirror tests failed, no internet?\033[m\n" + exit 1 + fi + options="$options --$bestipv" +fi + resolvedtarget=$(cd $target && [ "${PWD#$HOME}" != "$PWD" ] && printf "~${PWD#$HOME}" || printf "$PWD") -printf "Updating \033[1;34m$resolvedtarget\033[m from \033[0;36m$url \033[m...\n" +printf "Updating \033[1;34m$resolvedtarget\033[m from \033[0;36m$bestmirror/$source \033[m$bestipv ...\n" targetname=$(cd "$target" && printf "${PWD##*/}") if [ "$1" = "-y" ] || [ "$1" = "--yes" ]; then @@ -148,4 +195,4 @@ until [ "$choice" = y ] || [ "$choice" = Y ]; do done # exec ensures this script stops before it's updated to prevent potential glitches -exec $rsynccmd $options $excludes "rsync://$url/" "$target" +exec $rsynccmd $options $excludes "rsync://$bestmirror/$source/" "$target" diff --git a/misc/tools/rsync-updater/usr/bin/rsync-ssl b/misc/tools/rsync-updater/usr/bin/rsync-ssl index 54447b52..d8c79650 100755 --- a/misc/tools/rsync-updater/usr/bin/rsync-ssl +++ b/misc/tools/rsync-updater/usr/bin/rsync-ssl @@ -4,6 +4,7 @@ # because gnutls-cli is broken and stunnel doesn't verify the server's certificate. # Other differences: +# - supports -4 -6 and --ipv4 --ipv6 options # - supports spaces in the path to this script # By default this script takes rsync args and hands them off to the actual @@ -20,6 +21,8 @@ if [ "$1" = --HELPER ]; then shift + [ "$1" = "-4" ] || [ "$1" = "-6" ] && ipv=$1 && shift + port="${RSYNC_PORT:-0}" if [ "$port" -eq 0 ]; then port="${RSYNC_SSL_PORT:-874}" @@ -39,9 +42,14 @@ if [ "$1" = --HELPER ]; then exit 1 fi - exec openssl s_client -verify_return_error -verify 4 -quiet -verify_quiet -servername $hostname -verify_hostname $hostname -connect $hostname:$port + exec openssl s_client -verify_return_error -verify 4 -quiet -verify_quiet -servername $hostname -verify_hostname $hostname -connect $hostname:$port $ipv fi +for arg in "$@"; do + [ "$arg" = "--ipv6" ] || [ "$arg" = "-6" ] && ipv=-6 && break + [ "$arg" = "--ipv4" ] || [ "$arg" = "-4" ] && ipv=-4 && break +done + # The --rsh parameter string will need to be word split between $0 and --HELPER when executed # but $0 may contain space(s) and must not be split, hence its extra quotes. -exec rsync --rsh="'$0' --HELPER" "${@}" +exec rsync --rsh="'$0' --HELPER $ipv" "${@}"