From: bones_was_here <bones_was_here@xonotic.au>
Date: Tue, 11 Mar 2025 21:00:09 +0000 (+1000)
Subject: rsync: automate Windows binary maintenance
X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=14f49831b55bee588653d45274b2aaf9ba8f11cf;p=xonotic%2Fxonotic.git

rsync: automate Windows binary maintenance

Sources from msys2 rather than cygwin because msys2 uses ~standard
pacman which allows a linux build server to fetch the current bins
cleanly, also msys2 has newer rsync and openssl.

Increases minimum Windows rsync OS requirement to Windows 10 x64,
see https://www.msys2.org/docs/windows_support/

Removes redundant copy and chmod batch steps.
The chmod was a workaround only needed due to running rsync with
--executability (needed on linux but it was probably a mistake to use it
on windows where it causes problems due to windows' lack of a direct
equivalent of the unix execute permission).

Reduces remaining batch spam.
---

diff --git a/misc/tools/all/release.subr b/misc/tools/all/release.subr
index b64e0484..428b3bff 100644
--- a/misc/tools/all/release.subr
+++ b/misc/tools/all/release.subr
@@ -678,9 +678,26 @@ case "$cmd" in
 			Xonotic/gmqcc \
 			Xonotic/source/gmqcc
 		;;
+	release-rsync)
+		release_common
+		# make sure everything we need is installed and updated
+		verbose "$d0"/misc/tools/msys2-linux.sh --schroot=sid rsync
+		targetroot="$PWD/Xonotic"
+		verbose cd "$HOME/msys64" # see msys2-linux.sh
+		verbose cp --parents \
+			usr/bin/msys-2.0.dll \
+			usr/bin/msys-crypto-3.dll \
+			usr/bin/msys-iconv-2.dll \
+			usr/bin/msys-lz4-1.dll \
+			usr/bin/msys-xxhash-0.dll \
+			usr/bin/msys-zstd-1.dll \
+			usr/bin/rsync.exe \
+			"$targetroot/misc/tools/rsync-updater/"
+		;;
 	release)
 		release_common
 		verbose "$SELF" release-prepare
+		verbose "$SELF" release-rsync
 		verbose "$SELF" release-maps
 		verbose "$SELF" release-libs
 		verbose "$SELF" release-engine
diff --git a/misc/tools/msys2-linux.sh b/misc/tools/msys2-linux.sh
new file mode 100755
index 00000000..cd823aff
--- /dev/null
+++ b/misc/tools/msys2-linux.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# (re)initialises an msys2 environment for obtaining current binaries cleanly on linux
+# requires packages (in schroot if used): pacman-package-manager 7.0.0 and makepkg 7.0.0
+
+# https://www.msys2.org/docs/windows_support/
+TBURL=https://repo.msys2.org/distrib/msys2-x86_64-latest.tar.zst
+MSYSROOT=msys64
+cd "$HOME"
+
+case "$1" in
+	--schroot=*)
+		SCHROOT="schroot -c ${1#--schroot=} --"
+		shift
+		;;
+esac
+
+# there's a bunch of first-time setup in etc/profile which calls more in etc/post-install/
+# most isn't relevant or safe for our purposes but we do need pacman keys
+# see: etc/post-install/07-pacman-key.post
+pacman_init() {
+	export PACMAN_KEYRING_DIR=etc/pacman.d/gnupg        # equivalent to pacman-key --gpgdir
+	export KEYRING_IMPORT_DIR=usr/share/pacman/keyrings # equivalent to pacman-key --populate-from
+	export CONFIG=etc/pacman.conf                       # equivalent to pacman-key --config
+	export GNUPGHOME=etc/pacman.d/gnupg                 # tell gpg to use this instead of ~/.gnupg
+	set -ex
+	cd $MSYSROOT
+	pacman-key --init
+	pacman-key --populate msys2 || true
+	# msys2 gpg has this server as compiled default, debian's default server doesn't have the keys
+	# and we need this persistent
+	echo "keyserver hkps://keyserver.ubuntu.com" >> etc/pacman.d/gnupg/gpg.conf
+	pacman-key --refresh-keys || true
+	gpgconf --kill all
+}
+
+case "$1" in
+	pacman_init)
+		pacman_init
+		;;
+	*)
+		[ $(id -u) -eq 0 ] && (printf "\n\033[1;31mDo not run as root!\033[m\n"; exit 1)
+		[ -n "$ABORT" ] && (printf "\n\033[1;31mToo much fail, giving up.\033[m\n"; exit 1)
+		set -ex
+		if [ ! -d $MSYSROOT ]; then
+			rm -f "${TBURL##*/}"
+			curl -O "$TBURL"
+			tar --zstd -xf "${TBURL##*/}"
+			rm -f "${TBURL##*/}"
+			$SCHROOT fakeroot "$0" pacman_init
+			export ABORT=true
+		fi
+		# update msys2 base and all packages, or delete and redownload if that fails
+		# NOTE: sometimes this can print an error (for a specific package install) without failing,
+		# eg "could not change the root directory (Operation not permitted)" (fakechroot doesn't fix, just changes error)
+		# but it doesn't matter for our purposes because the files we need still get updated.
+		$SCHROOT fakeroot pacman --sysroot $MSYSROOT --noconfirm -Syu || (rm -rf $MSYSROOT && exec "$0" "$@")
+		# install specified packages if they're not already installed
+		if ! $SCHROOT pacman --sysroot $MSYSROOT -Q "$@" ; then
+			$SCHROOT fakeroot pacman --sysroot $MSYSROOT --noconfirm -S "$@"
+		fi
+		# some cleanup
+		$SCHROOT fakeroot pacman --sysroot $MSYSROOT --noconfirm -Sc
+		;;
+esac
diff --git a/misc/tools/rsync-updater/chmod.exe b/misc/tools/rsync-updater/chmod.exe
deleted file mode 100644
index b4955404..00000000
Binary files a/misc/tools/rsync-updater/chmod.exe and /dev/null differ
diff --git a/misc/tools/rsync-updater/cyggcc_s-1.dll b/misc/tools/rsync-updater/cyggcc_s-1.dll
deleted file mode 100644
index f6e242b4..00000000
Binary files a/misc/tools/rsync-updater/cyggcc_s-1.dll and /dev/null differ
diff --git a/misc/tools/rsync-updater/cygiconv-2.dll b/misc/tools/rsync-updater/cygiconv-2.dll
deleted file mode 100644
index 9a00513c..00000000
Binary files a/misc/tools/rsync-updater/cygiconv-2.dll and /dev/null differ
diff --git a/misc/tools/rsync-updater/cygintl-8.dll b/misc/tools/rsync-updater/cygintl-8.dll
deleted file mode 100644
index a98eb67b..00000000
Binary files a/misc/tools/rsync-updater/cygintl-8.dll and /dev/null differ
diff --git a/misc/tools/rsync-updater/cygpopt-0.dll b/misc/tools/rsync-updater/cygpopt-0.dll
deleted file mode 100644
index d44aeb9b..00000000
Binary files a/misc/tools/rsync-updater/cygpopt-0.dll and /dev/null differ
diff --git a/misc/tools/rsync-updater/cygwin1.dll b/misc/tools/rsync-updater/cygwin1.dll
deleted file mode 100644
index 8152109e..00000000
Binary files a/misc/tools/rsync-updater/cygwin1.dll and /dev/null differ
diff --git a/misc/tools/rsync-updater/rsync.exe b/misc/tools/rsync-updater/rsync.exe
deleted file mode 100644
index cafd87ad..00000000
Binary files a/misc/tools/rsync-updater/rsync.exe and /dev/null differ
diff --git a/misc/tools/rsync-updater/update-to-autobuild.bat b/misc/tools/rsync-updater/update-to-autobuild.bat
index 5c02c30c..0c8491ae 100644
--- a/misc/tools/rsync-updater/update-to-autobuild.bat
+++ b/misc/tools/rsync-updater/update-to-autobuild.bat
@@ -3,9 +3,11 @@ setlocal
 
 if "%1" == "did-copy" goto copied
 cd %~dp0
-rmdir /s /q %TEMP%\xonotic-rsync-updater
+rmdir /s /q %TEMP%\xonotic-rsync-updater  2>NUL
 mkdir %TEMP%\xonotic-rsync-updater
-for %%f in (*.exe *.dll *.bat) do copy /b %%f %TEMP%\xonotic-rsync-updater\
+copy /b %~nx0 %TEMP%\xonotic-rsync-updater\  >NUL
+:: windows has no cp -r equivalent, this seems least-bad
+robocopy usr %TEMP%\xonotic-rsync-updater\usr /e   >NUL
 %TEMP%\xonotic-rsync-updater\%~n0 did-copy
 :: can only get here if above batch file couldn't be created
 pause
@@ -19,7 +21,7 @@ if /i not "%choice%" == "Y" goto end
 set buildtype=release
 if "%~n0" == "update-to-autobuild" set buildtype=autobuild
 
-set options=-Prtzil --executability --delete-after --delete-excluded --stats
+set options=-Prtzil --delete-after --delete-excluded --stats
 
 if exist ..\..\..\.git goto xonoticdatagit
 if exist Xonotic goto xonoticswitchtonormal
@@ -104,12 +106,11 @@ if "%ProgramFiles(x86)%" == "" goto bit32
 	goto endbit
 :endbit
 
-for %%f in (*.exe *.dll) do copy /b %%f %TEMP%\xonotic-rsync-updater\
 cd %target%
 echo Updating %CD% from %url% ...
-%TEMP%\xonotic-rsync-updater\rsync %options% %excludes% rsync://%url%/ %target%
-%TEMP%\xonotic-rsync-updater\chmod -R a+x %target%
+%TEMP%\xonotic-rsync-updater\usr\bin\rsync %options% %excludes% rsync://%url%/ %target%
 
 :end
 pause
-rmdir /s /q %TEMP%\xonotic-rsync-updater
+:: hack: delete running batch file without error by deleting after batch exit
+(goto) 2>NUL & rmdir /s /q %TEMP%\xonotic-rsync-updater
diff --git a/misc/tools/rsync-updater/update-to-release.bat b/misc/tools/rsync-updater/update-to-release.bat
index 14735bd1..115a32db 100644
--- a/misc/tools/rsync-updater/update-to-release.bat
+++ b/misc/tools/rsync-updater/update-to-release.bat
@@ -3,9 +3,11 @@ setlocal
 
 if "%1" == "did-copy" goto copied
 cd %~dp0
-rmdir /s /q %TEMP%\xonotic-rsync-updater
+rmdir /s /q %TEMP%\xonotic-rsync-updater  2>NUL
 mkdir %TEMP%\xonotic-rsync-updater
-for %%f in (*.exe *.dll *.bat) do copy /b %%f %TEMP%\xonotic-rsync-updater\
+copy /b %~nx0 %TEMP%\xonotic-rsync-updater\  >NUL
+:: windows has no cp -r equivalent, this seems least-bad
+robocopy usr %TEMP%\xonotic-rsync-updater\usr /e   >NUL
 %TEMP%\xonotic-rsync-updater\%~n0 did-copy
 :: can only get here if above batch file couldn't be created
 pause
@@ -19,7 +21,7 @@ if /i not "%choice%" == "Y" goto end
 set buildtype=release
 if "%~n0" == "update-to-autobuild" set buildtype=autobuild
 
-set options=-Prtzil --executability --delete-after --delete-excluded --stats
+set options=-Prtzil --delete-after --delete-excluded --stats
 
 if exist ..\..\..\.git goto xonoticdatagit
 if exist ..\..\..\data goto xonoticdata
@@ -104,12 +106,11 @@ if "%ProgramFiles(x86)%" == "" goto bit32
 	goto endbit
 :endbit
 
-for %%f in (*.exe *.dll) do copy /b %%f %TEMP%\xonotic-rsync-updater\
 cd %target%
 echo Updating %CD% from %url% ...
-%TEMP%\xonotic-rsync-updater\rsync %options% %excludes% rsync://%url%/ %target%
-%TEMP%\xonotic-rsync-updater\chmod -R a+x %target%
+%TEMP%\xonotic-rsync-updater\usr\bin\rsync %options% %excludes% rsync://%url%/ %target%
 
 :end
 pause
-rmdir /s /q %TEMP%\xonotic-rsync-updater
+:: hack: delete running batch file without error by deleting after batch exit
+(goto) 2>NUL & rmdir /s /q %TEMP%\xonotic-rsync-updater