From: Dr. Jaska <drjaska83@gmail.com>
Date: Mon, 8 Jan 2024 15:13:52 +0000 (+0000)
Subject: Create an usable tool for local hash testing
X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=dc939c02c3330feef03b6f28054677fdac2cc493;p=xonotic%2Fxonotic-data.pk3dir.git

Create an usable tool for local hash testing
---

diff --git a/.gitignore b/.gitignore
index cfb3f6fcb..86069d63d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,11 @@ cscope*
 .DS_Store
 .idea/
 Thumbs.db
+
+# sv_game hashtest local test re-use
+data/maps/
+data/maps/_init.bsp
+data/maps/stormkeep.mapinfo
+data/maps/stormkeep.waypoints
+data/maps/stormkeep.waypoints.cache
+data/stormkeep.pk3
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 273fc045a..aa2c10557 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,117 +1,71 @@
-workflow:
-  rules:
-    - if: $CI_COMMIT_MESSAGE =~ /Transifex autosync/
-      when: never
-    - when: always
-
-before_script:
-  - ln -s $PWD data/xonotic-data.pk3dir
-
-  - export MAKEFLAGS=-j$(nproc); echo MAKEFLAGS=$MAKEFLAGS
-# - export CC="gcc -pipe -march=native -mtune=native"
-  - export CC="gcc -pipe"
-
-  - >
-    if wget -nv https://beta.xonotic.org/pipeline-bin/gmqcc ; then
-      export QCC="$PWD/gmqcc"
-      chmod +x "$QCC"
-    else
-      git clone --depth=1 --branch=main https://gitlab.com/xonotic/gmqcc.git gmqcc
-      make -C gmqcc || exit 1
-      export QCC="$PWD/gmqcc/gmqcc"
-    fi
-
-  # Makefile: don't complain about lack of tags (fetching them is slow)
-  - export QCCFLAGS_WATERMARK=gitlab_pipeline
-  # Makefile: don't compress anything or complain about lack of zip program
-  - export ZIP=/bin/true
-
-test_compilation_units:
-  rules:
-    - changes:
-      - qcsrc/**/*
-  stage: test
-  script:
-    - make test
-
-test_sv_game:
-  stage: test
-  script:
-    - >
-      if wget -nv https://beta.xonotic.org/pipeline-bin/xonotic-linux64-dedicated ; then
-        export ENGINE="$PWD/xonotic-linux64-dedicated"
-        chmod +x "$ENGINE"
-      else
-        git clone --depth=1 --branch=div0-stable https://gitlab.com/xonotic/darkplaces.git darkplaces
-        make -C darkplaces sv-release || exit 1
-        export ENGINE="$PWD/darkplaces/darkplaces-dedicated -xonotic"
-      fi
-    - export ENGINE="$ENGINE -noconfig -nohome"
-
-    - make qc || exit 1
-
-    - mkdir -p data/maps
-    - wget -nv -O data/maps/_init.bsp https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/_init/_init.bsp
-
-    - while read LINE; do
-        echo $LINE;
-        [ "$LINE" = "All tests OK" ] && PASS=1;
-      done < <(${ENGINE} +developer 1 +map _init +sv_cmd runtest +wait +quit)
-    - test "$PASS" = "1" || { echo 'sv_cmd runtest failed!'; exit 1; }
-
-    - ${ENGINE} +map _init +sv_cmd dumpnotifs +wait +quit
-    - diff notifications.cfg data/data/notifications_dump.cfg ||
-        { echo 'Please update notifications.cfg using `dumpnotifs`!'; exit 1; }
-
-#    - wget -nv -O data/stormkeep.pk3 http://beta.xonotic.org/autobuild-bsp/latest/stormkeep.pk3
-# ^^ INCORRECT: /latest/stormkeep.pk3 is the most recently built, not necessarily the one built from master!
-# we can't get the one from master directly as there's no /stable/stormkeep.pk3 or /master/stormkeep.pk3
-# and we can't run misc/tools/xonotic-map-compiler-autobuild as it uses commit hashes from xonotic-maps.pk3dir to generate filenames
-# but the autobuild server can run it and provide us the resulting pk3:
-    - wget -nv -O data/stormkeep.pk3 https://beta.xonotic.org/pipeline-bin/stormkeep.pk3
-# see also: misc/infrastructure/xonotic-release-build.cron
-    - wget -nv -O data/maps/stormkeep.mapinfo https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.mapinfo
-    - wget -nv -O data/maps/stormkeep.waypoints https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.waypoints
-    - wget -nv -O data/maps/stormkeep.waypoints.cache https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.waypoints.cache
-
-    - EXPECT=210a5126bffa3cd2acdb8d62bcec9e11
-    - HASH=$(${ENGINE} +exec serverbench.cfg
-      | tee /dev/stderr
-      | grep '^:'
-      | grep -v '^:gamestart:'
-      | grep -v '^:anticheat:'
-      | md5sum | awk '{ print $1 }')
-    - echo 'expected:' $EXPECT
-    - echo '  actual:' $HASH
-    - test "$HASH" == "$EXPECT"
-    - exit $?
-
-
-# NOTE: The generated docs are incomplete - they don't contain code behind SVQC CSQC MENUQC GAMEQC ifdefs.
-# With them added to PREDEFINED, it would take over half an hour to generate the docs and even then
-# they might not be complete. Doxygen doesn't handle #elif and might not understand some QC definitions.
-#doxygen:  # rename to 'pages' when gitlab.com allows pages to exceed 100MiB
-#  before_script:
-#    - ln -s $PWD data/xonotic-data.pk3dir # is this needed?
-#    - apt-get update
-#    - apt-get -y install doxygen graphviz
-#  stage: deploy
-#  script:
-#    - cd qcsrc && doxygen
-#    - mv html ../public
-#    - mkdir -p ~/.ssh
-#    - for i in {0..0}; do eval $(printf "echo \$id_rsa_%02d\n" $i) >> ~/.ssh/id_rsa_base64; done
-#    - base64 --decode ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-#    - chmod 600 ~/.ssh/id_rsa
-#    - echo -e "Host *\n\tStrictHostKeyChecking no\n\tLogLevel ERROR\n" >> ~/.ssh/config
-#    - git config --global user.name "Gitlab CI"
-#    - git config --global user.email "<>"
-#    - git clone --single-branch --depth 1 ${DEPLOY_HOST}:${DEPLOY_REPO} ~/deploy_
-#    - mkdir ~/deploy && mv ~/deploy_/.git ~/deploy && rm -r ~/deploy_
-#    - cp -r ../public/* ~/deploy
-#    - cd ~/deploy && git add -A . && git commit -m "Deploy ${CI_BUILD_REF}" && git push origin gh-pages
-#  artifacts:
-#    paths:
-#      - public
-#  only:
-#    - master
+workflow:
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /Transifex autosync/
+      when: never
+    - when: always
+
+before_script:
+  - ln -s $PWD data/xonotic-data.pk3dir
+
+  - export MAKEFLAGS=-j$(nproc); echo MAKEFLAGS=$MAKEFLAGS
+  - export CC="gcc -pipe"
+
+  - >
+    if wget -nv https://beta.xonotic.org/pipeline-bin/gmqcc ; then
+      export QCC="$PWD/gmqcc"
+      chmod +x "$QCC"
+    else
+      git clone --depth=1 --branch=main https://gitlab.com/xonotic/gmqcc.git gmqcc
+      make -C gmqcc || exit 1
+      export QCC="$PWD/gmqcc/gmqcc"
+    fi
+
+  # Makefile: don't complain about lack of tags (fetching them is slow)
+  - export QCCFLAGS_WATERMARK=gitlab_pipeline
+  # Makefile: don't compress anything or complain about lack of zip program
+  - export ZIP=/bin/true
+
+test_compilation_units:
+  rules:
+    - changes:
+      - qcsrc/**/*
+  stage: test
+  script:
+    - make test
+
+test_sv_game:
+  stage: test
+  script:
+    - export EXPECT=210a5126bffa3cd2acdb8d62bcec9e11
+    - qcsrc/tools/sv_game-hashtest.sh
+    - exit $?
+
+
+# NOTE: The generated docs are incomplete - they don't contain code behind SVQC CSQC MENUQC GAMEQC ifdefs.
+# With them added to PREDEFINED, it would take over half an hour to generate the docs and even then
+# they might not be complete. Doxygen doesn't handle #elif and might not understand some QC definitions.
+#doxygen:  # rename to 'pages' when gitlab.com allows pages to exceed 100MiB
+#  before_script:
+#    - ln -s $PWD data/xonotic-data.pk3dir # is this needed?
+#    - apt-get update
+#    - apt-get -y install doxygen graphviz
+#  stage: deploy
+#  script:
+#    - cd qcsrc && doxygen
+#    - mv html ../public
+#    - mkdir -p ~/.ssh
+#    - for i in {0..0}; do eval $(printf "echo \$id_rsa_%02d\n" $i) >> ~/.ssh/id_rsa_base64; done
+#    - base64 --decode ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+#    - chmod 600 ~/.ssh/id_rsa
+#    - echo -e "Host *\n\tStrictHostKeyChecking no\n\tLogLevel ERROR\n" >> ~/.ssh/config
+#    - git config --global user.name "Gitlab CI"
+#    - git config --global user.email "<>"
+#    - git clone --single-branch --depth 1 ${DEPLOY_HOST}:${DEPLOY_REPO} ~/deploy_
+#    - mkdir ~/deploy && mv ~/deploy_/.git ~/deploy && rm -r ~/deploy_
+#    - cp -r ../public/* ~/deploy
+#    - cd ~/deploy && git add -A . && git commit -m "Deploy ${CI_BUILD_REF}" && git push origin gh-pages
+#  artifacts:
+#    paths:
+#      - public
+#  only:
+#    - master
diff --git a/qcsrc/tools/sv_game-hashtest.sh b/qcsrc/tools/sv_game-hashtest.sh
new file mode 100755
index 000000000..f3f0eb233
--- /dev/null
+++ b/qcsrc/tools/sv_game-hashtest.sh
@@ -0,0 +1,235 @@
+#!/bin/bash
+
+# abort on error(non-0 return code) and abort on unset variable
+set -eu
+
+# refuse to run on Windows' MINGW due to symlinking issues
+# `ln -s "$PWD" data/xonotic-data.pk3dir` seems to not work in any form
+# even with fully relative paths or `ln -s ../ xonotic-data.pk3dir`
+# FIXME?
+case "$(uname)" in
+	MINGW*) printf "%s\n%s\n" "This file is not suitable for Windows' MINGW" \
+		"without modifications due to the lack of proper symlink support"; exit 1 ;;
+esac
+
+printf "Testing for dependencies\n"
+# command -q / --quiet one day? :(
+command -V awk       > /dev/null
+command -V chmod     > /dev/null
+command -V git       > /dev/null
+command -V grep      > /dev/null
+command -V make      > /dev/null
+command -V md5sum    > /dev/null
+command -V mkdir     > /dev/null
+command -V mktemp    > /dev/null
+command -V rm        > /dev/null
+command -V rmdir     > /dev/null
+command -V sed       > /dev/null
+command -V tee       > /dev/null
+command -V test      > /dev/null
+command -V true      > /dev/null
+command -V printf    > /dev/null
+command -V wget      > /dev/null
+printf "All dependencies found\n"
+
+createdtoday() {
+	# check if a file's creation date is today
+	if [ "$(stat -c '%y' "$1" | cut -d ' ' -f 1)" = "$(date -I)" ]
+	then
+		#echo "$1 was created today"
+		return 0
+	else
+		#echo "$1 was not created today"
+		return 1
+	fi
+}
+
+hashtestcleanup() {
+	# allow for error return codes in this function as this may be ran on interrupt
+	# right after starting where all files to clean up don't exist yet
+	set +e
+
+	# unset trap
+	trap - EXIT INT QUIT TERM
+
+	# Few files' removal has been disabled for file reuse
+	# It's possible to get rate limited with enough testing
+
+	rm lock
+	rm data/darkplaces_history.txt
+	rm data/xonotic-data.pk3dir
+	#rm data/stormkeep.pk3
+	rm data/data/defaultSVQC.cfg
+	rm data/data/hits---1.plot
+	rm data/data/hits---2.plot
+	rm data/data/hits---3.plot
+	rm data/data/notifications_dump.cfg
+	rm data/data/server.db
+	rmdir data/data/
+	#rm data/maps/_init.bsp
+	#rm data/maps/stormkeep.mapinfo
+	#rm data/maps/stormkeep.waypoints
+	#rm data/maps/stormkeep.waypoints.cache
+	#rmdir data/maps/
+
+	set -e
+}
+trap "hashtestcleanup" EXIT INT QUIT TERM
+
+# cd xonotic-data.pk3dir
+cd "$(dirname "$0")/../../"
+
+WORKINGDIR="$PWD"
+
+TMPDIR="$PWD/.tmp"
+
+# if xonotic-data.pk3dir/data/xonotic-data.pk3dir isn't a symlink then link it
+if [ -e data/xonotic-data.pk3dir ]
+then # file exists
+	if ! [ -L data/xonotic-data.pk3dir ]
+	then # file exists but it's not a symlink, replace it
+		if [ -d data/xonotic-data.pk3dir ]
+		then # it's a dir
+			rmdir data/xonotic-data.pk3dir
+			ln -s "$PWD" data/xonotic-data.pk3dir
+		else # it's not a dir
+			rm data/xonotic-data.pk3dir
+			ln -s "$PWD" data/xonotic-data.pk3dir
+		fi
+	else # it is a symlink, verify where it points
+		if [ "$(realpath data/xonotic-data.pk3dir)" != "$PWD" ]
+		then # wrong place, recreate it
+			rm data/xonotic-data.pk3dir
+			ln -s "$PWD" data/xonotic-data.pk3dir
+		fi
+	fi
+else # no file exists there
+	ln -s "$PWD" data/xonotic-data.pk3dir
+fi
+
+MAKEFLAGS=-j$(nproc)
+export MAKEFLAGS
+printf "%s\n" "MAKEFLAGS=$MAKEFLAGS"
+export CC="gcc -pipe"
+
+# precompiled binary is executable and its creation date is today
+if [ -x "$TMPDIR/gmqcc-bin" ] && createdtoday "$TMPDIR/gmqcc-bin"
+then
+	export QCC="$TMPDIR/gmqcc-bin"
+else # previously compiled is executable and its creation date is today
+	if [ -x "$TMPDIR/gmqcc/gmqcc" ] && createdtoday "$TMPDIR/gmqcc/gmqcc"
+	then
+		export QCC="$TMPDIR/gmqcc/gmqcc"
+	else # nothing reusable exists
+		# prefer a precompiled binary
+		if wget -nv https://beta.xonotic.org/pipeline-bin/gmqcc -O "$TMPDIR/gmqcc-bin"
+		then
+			export QCC="$TMPDIR/gmqcc-bin"
+			chmod +x "$QCC"
+		else
+			if [ -d "$TMPDIR/gmqcc" ]
+			then
+				cd "$TMPDIR/gmqcc"
+				git checkout main
+				git pull
+				cd "$WORKINGDIR"
+			else
+				git clone --depth=1 --branch=main https://gitlab.com/xonotic/gmqcc.git "$TMPDIR/gmqcc"
+			fi
+			make -C "$TMPDIR/gmqcc" || exit 1
+			export QCC="$TMPDIR/gmqcc/gmqcc"
+		fi
+	fi
+fi
+
+# Makefile: don't complain about lack of tags (fetching them is slow)
+export QCCFLAGS_WATERMARK=gitlab_pipeline
+# Makefile: don't compress anything or complain about lack of zip program
+export ZIP=true
+
+
+if [ "$(uname):$(uname -m)" = "Linux:x86_64" ] && [ -x "$TMPDIR/xonotic-linux64-dedicated" ] && createdtoday "$TMPDIR/xonotic-linux64-dedicated"
+then # precompiled binary is executable and its creation date is today
+	export ENGINE="$TMPDIR/xonotic-linux64-dedicated"
+else
+	if [ -x "$TMPDIR/darkplaces/darkplaces-dedicated" ] && createdtoday "$TMPDIR/darkplaces/darkplaces-dedicated"
+	then # previously compiled is executable and its creation date is today
+		export ENGINE="$TMPDIR/darkplaces/darkplaces-dedicated -xonotic"
+	else # nothing reusable exists
+		# prefer a precompiled binary
+		if [ "$(uname):$(uname -m)" = "Linux:x86_64" ] && wget -nv https://beta.xonotic.org/pipeline-bin/xonotic-linux64-dedicated -O "$TMPDIR/xonotic-linux64-dedicated"
+		then
+			export ENGINE="$TMPDIR/xonotic-linux64-dedicated"
+			chmod +x "$ENGINE"
+		else
+			if [ -d "$TMPDIR/darkplaces" ]
+			then
+				cd "$TMPDIR/darkplaces"
+				#git checkout master
+				git pull --autostash
+				cd "$WORKINGDIR"
+			else
+				git clone --depth=1 https://gitlab.com/xonotic/darkplaces.git "$TMPDIR/darkplaces"
+			fi
+			make -C "$TMPDIR/darkplaces" sv-release || exit 1
+			export ENGINE="$TMPDIR/darkplaces/darkplaces-dedicated -xonotic"
+		fi
+	fi
+fi
+export ENGINE="$ENGINE -noconfig -nohome"
+
+make qc || exit 1
+
+mkdir -p data/maps
+
+createdtoday "data/maps/_init.bsp" \
+	|| wget -nv -O data/maps/_init.bsp https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/_init/_init.bsp
+
+while read -r LINE
+do
+	printf "%s\n" "$LINE"
+	[ "$LINE" = "All tests OK" ] && PASS=1
+done < <(${ENGINE} +developer 1 +map _init +sv_cmd runtest +wait +quit)
+test "$PASS" = "1" || { printf 'sv_cmd runtest failed!'; exit 1; }
+
+${ENGINE} +map _init +sv_cmd dumpnotifs +wait +quit
+diff notifications.cfg data/data/notifications_dump.cfg ||
+	{ printf "Please update notifications.cfg using \`dumpnotifs\`!"; exit 1; }
+
+createdtoday "data/stormkeep.pk3" \
+	|| wget -nv -O data/stormkeep.pk3 https://beta.xonotic.org/pipeline-bin/stormkeep.pk3
+createdtoday "data/maps/stormkeep.mapinfo" \
+	|| wget -nv -O data/maps/stormkeep.mapinfo https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.mapinfo
+createdtoday "data/maps/stormkeep.waypoints" \
+	|| wget -nv -O data/maps/stormkeep.waypoints https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.waypoints
+createdtoday "data/maps/stormkeep.waypoints.cache" \
+	|| wget -nv -O data/maps/stormkeep.waypoints.cache https://gitlab.com/xonotic/xonotic-maps.pk3dir/raw/master/maps/stormkeep.waypoints.cache
+
+set +u
+if [ -z "$EXPECT" ]
+then
+	# find the line with expected hash from .gitlab-ci.yml, extract the hash and remove carriage return
+	EXPECT="$(grep 'EXPECT=' './.gitlab-ci.yml' | cut -d '=' -f 2 | tr -d $'\r')"
+fi
+set -u
+HASH=$(${ENGINE} +exec serverbench.cfg \
+      | tee /dev/stderr \
+      | grep '^:' \
+      | grep -v '^:gamestart:' \
+      | grep -v '^:anticheat:' \
+      | md5sum | awk '{ print $1 }')
+
+hashtestcleanup
+
+if [ "$HASH" = "$EXPECT" ]
+then # green ok print
+	printf "\033[32m%s\033[0m\n" "expected: $EXPECT"
+	printf "\033[32m%s\033[0m\n" "  actual: $HASH"
+	printf "\033[32m%s\033[0m\n" "hashes match"
+	exit 0
+else # red error print
+	printf "\033[32m%s\033[0m\n" "expected: $EXPECT"
+	printf "\033[32m%s\033[0m\n" "  actual: $HASH"
+	printf "\033[31m%s\033[0m\n" "!!! ERROR: HASHES DO NOT MATCH !!!"
+	exit 1
+fi