From: Rudolf Polzer Date: Sat, 3 Dec 2011 12:59:09 +0000 (+0100) Subject: add a symlink based deduplication script X-Git-Tag: xonotic-v0.6.0~54 X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=daf9b2977b88ca376ad894068b999c379a466c0f;p=xonotic%2Fxonotic.git add a symlink based deduplication script --- diff --git a/misc/tools/symlink-deduplicate.sh b/misc/tools/symlink-deduplicate.sh new file mode 100644 index 00000000..d367635c --- /dev/null +++ b/misc/tools/symlink-deduplicate.sh @@ -0,0 +1,117 @@ +#!/bin/sh + +use_magnet_to_acquire_checksum_faster() +# ___________________ +# ,--'' ~~~~~~~^^^~._ '. +# ,.-' ~~~~~~~~~~^^^^^~~~._._ \ +# | /^^^^^| /^^^^^^^^\\ \ \ +# ,/___ < o> < (OO) > _ \ +# /'/, |- . ----.\ +# |(|-'^^;,- ,| __ ^~~^^^^^^^; |\ +# \\` | <;_ __ |`--- ..-^^/- | || +# \`-|Oq-.____`________~='^^|__,/ ' // +# \ || | | | | \ ..-;| / '/ +# | ||#|#|the|==|game!|'^` |/' /' +# | \\\\^\***|***| \ ,,;' / +# | `-=\_\__\___\__..-' ,.- - ,/ +# | . `-_ ------ _,-'^-'^,-' +# | `-._________..--''^,-''^ +# \ ,...-'^ +# `----------'^ PROBLEM? +{ + magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"` + if [ -n "$magnet" ]; then + magnet=${magnet#* } + magnet=${magnet%% *} + echo "$magnet" + else + git hash-object "$1" + fi +} + +lastinfiles= +lastinfileshash= +acquire_checksum() +{ + if [ x"$1/../$2" = x"$lastinfiles" ]; then + _a_s=$lastinfileshash + else + _a_e=false + for _a_f in "$1" "$2"; do + case "$_a_f" in + */background_l2.tga|*/background_ingame_l2.tga) + _a_e=true + ;; + esac + done + if [ -n "$git_src_repo" ] && ! $_a_e; then + _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"` + if [ -n "$2" ]; then + _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"` + fi + else + _a_s=`git hash-object "$1"` + if [ -n "$2" ]; then + _a_s=$_a_s`git hash-object "$2"` + fi + fi + lastinfileshash=$_a_s + lastinfiles="$1/../$2" + fi + echo "$_a_s" +} + +make_relative_path() +{ + from=$1 + to=$2 + pre= + post= + while :; do + case "$from" in + */*) + case "$to" in + */*) + fromfirst=${from%%/*} + fromrest=${from#*/} + tofirst=${to%%/*} + torest=${to#*/} + if [ x"$fromfirst" = x"$tofirst" ]; then + from=$fromrest + to=$torest + else + to=$tofirst + post=/$torest + pre=../$pre + from=$fromrest # now we can only hit the ../ path or the bottom one + fi + ;; + *) + # from has path, to does not + # we need a ../ component then try again + pre=../$pre + from=${from#*/} + ;; + esac + ;; + *) + echo "$pre$to$post" + break + ;; + esac + done +} + +killed=0 +while IFS= read -r L; do + s=`acquire_checksum "$L"` + eval first=\$first_$s + if [ -n "$first" ]; then + first_r=`make_relative_path "$L" "$first"` + echo ln -snf "$first_r" "$L" + killed=$((`stat -c %s "$L"` + $killed)) + else + eval first_$s=\$L + fi +done +echo "Killed $(($killed / 1048576)) MiB"