From 3127fbe7af7436be0b4b8f76f749115a1410a69c Mon Sep 17 00:00:00 2001 From: Robert Pelnar Date: Tue, 19 Jun 2018 23:18:26 +0200 Subject: [PATCH] New ML support - optimized script lang-build.sh --- lang/lang-build.sh | 216 +++++++++++---------------------------------- 1 file changed, 51 insertions(+), 165 deletions(-) diff --git a/lang/lang-build.sh b/lang/lang-build.sh index bbd8f053..c98ee7b6 100644 --- a/lang/lang-build.sh +++ b/lang/lang-build.sh @@ -4,39 +4,35 @@ # generate lang_xx.bin (language binary file) # # Input files: -# lang_en.txt -# lang_en_xx.txt -# +# lang_en.txt or lang_en_xx.txt +# # Output files: +# lang_xx.bin # -# -# Selected language: -LNG=$1 -if [ -z "$LNG" ]; then LNG='cz'; fi +# Temporary files: +# lang_xx.tmp +# lang_xx.dat # #awk code to format ui16 variables for dd awk_ui16='{ h=int($1/256); printf("\\x%02x\\x%02x\n", int($1-256*h), h); }' +#startup message +echo "lang-build.sh started" >&2 + #exiting function finish() { if [ $1 -eq 0 ]; then - if [ -e lang_en.tmp ]; then rm lang_en.tmp; fi - if [ -e lang_en_$LNG.tmp ]; then rm lang_en_$LNG.tmp; fi - if [ -e lang_en_$LNG.dif ]; then rm lang_en_$LNG.dif; fi - fi -# echo >&2 - if [ $1 -eq 0 ]; then - echo "make_lang.sh finished with success" >&2 + echo "lang-build.sh finished with success" >&2 else - echo "make_lang.sh finished with errors!" >&2 + echo "lang-build.sh finished with errors!" >&2 fi exit $1 } #returns hexadecial data for lang code -lang_code() +lang_code_hex_data() # $1 - language code ('en', 'cz'...) { case "$1" in @@ -51,7 +47,6 @@ lang_code() echo '??' } -# write_header() # $1 - lang # $2 - size @@ -67,7 +62,7 @@ write_header() dd of=lang_$1.bin bs=1 count=2 seek=6 conv=notrunc 2>/dev/null /bin/echo -n -e $(echo -n "$(($4))" | awk "$awk_ui16") |\ dd of=lang_$1.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null - /bin/echo -n -e "$(lang_code $1)" |\ + /bin/echo -n -e "$(lang_code_hex_data $1)" |\ dd of=lang_$1.bin bs=1 count=2 seek=10 conv=notrunc 2>/dev/null sig_h=$(($5 / 65536)) /bin/echo -n -e $(echo -n "$sig_h" | awk "$awk_ui16") |\ @@ -77,176 +72,67 @@ write_header() dd of=lang_$1.bin bs=1 count=2 seek=12 conv=notrunc 2>/dev/null } -make_lang2() -# $1 - lang ('en', 'cz'...) +generate_binary() +# $1 - language code ('en', 'cz'...) { - rm -f lang_$LNG.tmp - rm -f lang_$LNG.dat - rm -f lang_$LNG.bin + echo "lang="$1 >&2 + #remove output and temporary files + rm -f lang_$1.bin + rm -f lang_$1.tmp + rm -f lang_$1.dat LNG=$1 #create lang_xx.tmp - different processing for 'en' language - if [ "$LNG" = "en" ]; then + if [ "$1" = "en" ]; then #remove comments and empty lines cat lang_en.txt | sed '/^$/d;/^#/d' else #remove comments and empty lines, print lines with translated text only - cat lang_en_$LNG.txt | sed '/^$/d;/^#/d' | sed -n 'n;p' - fi | sed 's/^\"\\x00\"$/\"\"/' > lang_$LNG.tmp + cat lang_en_$1.txt | sed '/^$/d;/^#/d' | sed -n 'n;p' + fi | sed 's/^\"\\x00\"$/\"\"/' > lang_$1.tmp #create lang_xx.dat (binary text data file) - cat lang_$LNG.tmp | sed 's/^\"/printf \"/;s/"$/\\x00\"/' | sh >lang_$LNG.dat + cat lang_$1.tmp | sed 's/^\"/printf \"/;s/"$/\\x00\"/' | sh >lang_$1.dat #calculate number of strings - count=$(grep -c '^"' lang_$LNG.tmp) - echo "count="$count + count=$(grep -c '^"' lang_$1.tmp) + echo "count="$count >&2 #calculate text data offset offs=$((16 + 2 * $count)) - echo "offs="$offs + echo "offs="$offs >&2 #calculate text data size - size=$(($offs + $(wc -c lang_$LNG.dat | cut -f1 -d' '))) - echo "size="$size + size=$(($offs + $(wc -c lang_$1.dat | cut -f1 -d' '))) + echo "size="$size >&2 #write header with empty signature and checksum - write_header $LNG $size $count 0x0000 0x00000000 + write_header $1 $size $count 0x0000 0x00000000 #write offset table - cat lang_$LNG.tmp | sed 's/^\"//;s/\"$//' |\ + offs_hex=$(cat lang_$1.tmp | sed 's/^\"//;s/\"$//' |\ sed 's/\\x[0-9a-f][0-9a-f]/\./g;s/\\[0-7][0-7][0-7]/\./g;s/\ /\./g' |\ - awk 'BEGIN { o='$offs';} { h=int(o/256); printf("%c%c",int(o-256*h), h); o+=(length($0)+1); }' |\ - dd of=./lang_$LNG.bin bs=1 seek=16 conv=notrunc 2>/dev/null + awk 'BEGIN { o='$offs';} { h=int(o/256); printf("\\x%02x\\x%02x",int(o-256*h), h); o+=(length($0)+1); }') + /bin/echo -n -e "$offs_hex" | dd of=./lang_$1.bin bs=1 seek=16 conv=notrunc 2>/dev/null #write binary text data - dd if=./lang_$LNG.dat of=./lang_$LNG.bin bs=1 seek=$offs conv=notrunc 2>/dev/null + dd if=./lang_$1.dat of=./lang_$1.bin bs=1 seek=$offs conv=notrunc 2>/dev/null + #write signature + if [ "$1" != "en" ]; then + dd if=lang_en.bin of=lang_$1.bin bs=1 count=4 skip=6 seek=12 conv=notrunc 2>/dev/null + fi #calculate and update checksum - chsum=$(cat lang_$LNG.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }') + chsum=$(cat lang_$1.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }') /bin/echo -n -e $(echo -n $((0x$chsum)) | awk "$awk_ui16") |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null + dd of=lang_$1.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null #remove temporary files - rm -f lang_$LNG.tmp - rm -f lang_$LNG.dat + rm -f lang_$1.tmp + rm -f lang_$1.dat } -make_lang2 $1 -exit +if [ -z "$1" ]; then set 'all'; fi -make_lang() -{ -LNG=$1 - -echo "make_lang.sh started" >&2 -echo "selected language=$LNG" >&2 - -#check if input files exists -echo -n " checking input files..." >&2 -if [ ! -e lang_en.txt ]; then echo "NG! file lang_en.txt not found!" >&2; exit 1; fi - -if [ ! -e lang_en_$LNG.txt ]; then echo "NG! file lang_en_$LNG.txt not found!" >&2; exit 1; fi -echo "OK" >&2 - -#filter comment and empty lines from key and dictionary files, create temporary files -echo -n " creating tmp files..." >&2 -cat lang_en.txt | sed "/^$/d;/^#/d" > lang_en.tmp -cat lang_en_$LNG.txt | sed "/^$/d;/^#/d" > lang_en_$LNG.tmp -echo "OK" >&2 -#cat lang_en_$LNG.tmp | sed 'n;d' >test1.txt - -#compare files using diff and check for differences -echo -n " comparing tmp files..." >&2 -if ! cat lang_en_$LNG.tmp | sed 'n;d' | diff lang_en.tmp - > lang_en_$LNG.dif; then - echo "NG!" >&2 - echo "Entries in lang_en_$LNG.txt are different from lang_en.txt!" >&2 - echo "please check lang_en_$LNG.dif" >&2 - finish 1 -fi -echo "OK" >&2 - -#generate lang_xx.txt (secondary language text data sorted by ids) -echo -n " generating lang_$LNG.txt..." >&2 -cat lang_en_$LNG.tmp | sed '1~2d' | sed "s/^\"\\\\x00/\"/" > lang_$LNG.txt -echo "OK" >&2 - -#generate lang_xx.dat (secondary language text data in binary form) -echo -n " generating lang_$LNG.dat..." >&2 -cat lang_$LNG.txt | sed "s/\\\\/\\\\\\\\/g" | while read s; do - s=${s#\"} - s=${s%\"} - /bin/echo -e -n "$s\x00" -done >lang_$LNG.dat -echo "OK" >&2 - -#calculate variables -lt_magic='\xa5\x5a\xb4\x4b' -lt_count=$(grep -c '^' lang_$LNG.txt) -lt_data_size=$(wc -c lang_$LNG.dat | cut -f1 -d' ') -lt_offs_size=$((2 * $lt_count)) -lt_size=$((16 + $lt_offs_size + $lt_data_size)) -lt_chsum=0 -lt_code='\xff\xff' -lt_resv1='\xff\xff\xff\xff' - -case "$LNG" in - *en*) lt_code='\x6e\x65' ;; - *cz*) lt_code='\x73\x63' ;; - *de*) lt_code='\x65\x64' ;; - *es*) lt_code='\x73\x65' ;; - *it*) lt_code='\x74\x69' ;; - *pl*) lt_code='\x6c\x70' ;; -esac - -#generate lang_xx.ofs (secondary language text data offset table) -echo -n " generating lang_$LNG.ofs..." >&2 -cat lang_$LNG.txt | sed "s/\\\\x[0-9a-f][0-9a-f]/\./g;s/\\\\[0-7][0-7][0-7]/\./g" |\ - awk 'BEGIN { o='$((16 + $lt_offs_size))';} { printf("%d\n",o); o+=(length($0)-1); }' > lang_$LNG.ofs -echo "OK" >&2 - -#generate lang_xx.bin (secondary language result binary file) -echo " generating lang_$LNG.bin:" >&2 -#create empty file -dd if=/dev/zero of=lang_$LNG.bin bs=1 count=$lt_size 2>/dev/null - -#write data to binary file with dd - -echo -n " writing header (16 bytes)..." >&2 -/bin/echo -n -e "$lt_magic" |\ - dd of=lang_$LNG.bin bs=1 count=4 seek=0 conv=notrunc 2>/dev/null -/bin/echo -n -e $(echo -n "$lt_size" | awk "$awk_ui16") |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=4 conv=notrunc 2>/dev/null -/bin/echo -n -e $(echo -n "$lt_count" | awk "$awk_ui16") |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=6 conv=notrunc 2>/dev/null -/bin/echo -n -e $(echo -n "$lt_chsum" | awk "$awk_ui16") |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null -/bin/echo -n -e "$lt_code" |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=10 conv=notrunc 2>/dev/null -/bin/echo -n -e "$lt_resv1" |\ - dd of=lang_$LNG.bin bs=1 count=4 seek=12 conv=notrunc 2>/dev/null -echo "OK" >&2 - -echo -n " writing offset table ($lt_offs_size bytes)..." >&2 -/bin/echo -n -e $(cat lang_$LNG.ofs | awk "$awk_ui16" | tr -d '\n'; echo) |\ - dd of=./lang_$LNG.bin bs=1 count=$lt_offs_size seek=16 conv=notrunc 2>/dev/null -echo "OK" >&2 - -echo -n " writing text data ($lt_data_size bytes)..." >&2 -dd if=./lang_$LNG.dat of=./lang_$LNG.bin bs=1 count=$lt_data_size seek=$((16 + $lt_offs_size)) conv=notrunc 2>/dev/null -echo "OK" >&2 - -#calculate and update checksum -lt_chsum=$(cat lang_$LNG.bin | xxd | cut -c11-49 | tr ' ' "\n" | sed '/^$/d' | awk 'BEGIN { sum = 0; } { sum += strtonum("0x"$1); if (sum > 0xffff) sum -= 0x10000; } END { printf("%x\n", sum); }') -/bin/echo -n -e $(echo -n $((0x$lt_chsum)) | awk "$awk_ui16") |\ - dd of=lang_$LNG.bin bs=1 count=2 seek=8 conv=notrunc 2>/dev/null - -echo " lang_table details:" >&2 -echo " lt_count = $lt_count" >&2 -echo " lt_size = $lt_size" >&2 -echo " lt_chsum = $lt_chsum" >&2 -} - -echo $LNG - -if [ "$LNG" = "all" ]; then - make_lang cz - make_lang de - make_lang es - make_lang it - make_lang pl - exit 0 +if [ "$1" = "all" ]; then + generate_binary 'en' + generate_binary 'cz' + generate_binary 'de' + generate_binary 'es' + generate_binary 'it' + generate_binary 'pl' else - make_lang $LNG + generate_binary $1 fi finish 0