From 476a71289324e6e8545976bff5bb8f8038ae2514 Mon Sep 17 00:00:00 2001
From: Robert Pelnar <robert.pelnar@seznam.cz>
Date: Mon, 21 May 2018 18:40:32 +0200
Subject: [PATCH] New multilanguage support - scripts for migration

---
 lang_upgrade/clean.sh        |   7 ++
 lang_upgrade/find_msgs.sh    |  63 +++++++++++++
 lang_upgrade/make_en_cz.sh   |  23 +++++
 lang_upgrade/make_msgs.sh    |  93 +++++++++++++++++++
 lang_upgrade/readme.txt      |  28 ++++++
 lang_upgrade/replace_msgs.sh | 168 +++++++++++++++++++++++++++++++++++
 6 files changed, 382 insertions(+)
 create mode 100644 lang_upgrade/clean.sh
 create mode 100644 lang_upgrade/find_msgs.sh
 create mode 100644 lang_upgrade/make_en_cz.sh
 create mode 100644 lang_upgrade/make_msgs.sh
 create mode 100644 lang_upgrade/readme.txt
 create mode 100644 lang_upgrade/replace_msgs.sh

diff --git a/lang_upgrade/clean.sh b/lang_upgrade/clean.sh
new file mode 100644
index 00000000..7f767c42
--- /dev/null
+++ b/lang_upgrade/clean.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+#
+
+rm msgs*.txt
+rm make_msgs.out
+rm messages.h
+rm messages.c
diff --git a/lang_upgrade/find_msgs.sh b/lang_upgrade/find_msgs.sh
new file mode 100644
index 00000000..dd5a55ec
--- /dev/null
+++ b/lang_upgrade/find_msgs.sh
@@ -0,0 +1,63 @@
+#!/bin/sh
+#
+echo 'find_msgs.sh'
+
+files=$(ls ../Firmware/*.c* | grep -v 'language'; ls ../Firmware/*.h | grep -v 'language'; )
+
+echo -n ' processing msgs_en.txt ...'
+#msgs=$(cat msgs_en.txt | cut -f 1 -d' ')
+msgs=$(cat msgs_en.txt msgs_common.txt | cut -f 1 -d' ')
+#msgs=$(cat msgs_common.txt | cut -f 1 -d' ')
+echo "$msgs" | while read msg; do
+ echo -n "$msg "
+ found=$(grep -c "$msg" $files | sed "/:0$/d;s/.*:/+/g")
+ echo $(("0"$found))
+done  | tee msgs_usage.txt_0
+cat msgs_usage.txt_0 | sort -k2 -n >msgs_usage.txt
+rm msgs_usage.txt_0
+echo "finished.."
+read
+exit
+
+#replace #define(length=xx,lines=xx)
+cat msgs_en.txt_0 | sed -E "s/^#define\(length=([0-9]*),[ \t]*lines=([0-9]*)\)[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\3 c=\1 r=\2 \4/g" >msgs_en.txt_1
+#replace #define(length=xx)
+cat msgs_en.txt_1 | sed -E "s/^#define\(length=([0-9]*)\)[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\2 c=\1 r=0 \3/g" >msgs_en.txt_2
+#replace #define
+cat msgs_en.txt_2 | sed -E "s/^#define[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\1 c=0 r=0 \2/g" >msgs_en.txt
+#remove tmp files
+rm msgs_en.txt_*
+echo "ok"
+
+echo -n ' processing language_cz.h ...'
+#list all defines from language_cz.h
+cat ../Firmware/language_cz.h | grep "^#define" >msgs_cz.txt_0
+cat msgs_cz.txt_0 | sed -E "s/^#define[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\1 \2/g" >msgs_cz.txt
+#remove tmp files
+rm msgs_cz.txt_*
+echo "ok"
+
+echo "finished.."
+read
+exit
+
+
+
+
+
+
+#cat msgs_en.txt_3 | grep "^XXX" >msgs_en.txt_4
+
+#cat msgs_en.txt_0 | sed -E "s/^#define\(length=([0-9]{2}), lines=([0-9]{2})\)[ ]*([^ ]*)/\3 \2 \1/g" >msgs_en.txt_1
+#cat msgs_en.txt_0 | sed -E "s/^#define\(length=([0-9]{2})\)[ ]*([^ ]*)/\2 \1/g" >msgs_en.txt_1
+#cat msgs_en.txt_1 | sed -E "s/^#define[ ]*([^ ]*)/\1/g" >msgs_en.txt_2
+#[0-9]{+2}
+#cat ../Firmware/language_en.h | sed "s/^#define(length=[0-9]*)//g" >msgs_en.txt_0
+#msgs=$(ls ../Firmware/*.c* | grep -v 'language'; ls ../Firmware/*.h | grep -v 'language'; )
+#echo "$files" | while read fn; do
+# if grep "MSG_SD_INSERTED" $fn >/dev/null; then
+#  echo $fn
+# fi
+#done
+echo "finished.."
+read
diff --git a/lang_upgrade/make_en_cz.sh b/lang_upgrade/make_en_cz.sh
new file mode 100644
index 00000000..48952ded
--- /dev/null
+++ b/lang_upgrade/make_en_cz.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+
+#rm lang_en_cz_0.txt
+#rm lang_en_cz_1.txt
+#(cat msgs_cz.txt  | sed -E "s/([^ ]*) (.*)/\1% \2/g"; cat msgs_en.txt | sed -E "s/([^ ]*) ([^ ]*) ([^ ]*) (.*)/\1 \4/g") | sort > lang_en_cz_0.txt
+#cat lang_en_cz_0.txt | sed -E "s/([^ %]*)[%]+ (.*)/\1 \2/g" > lang_en_cz_1.txt
+
+name0=""
+cat lang_en_cz_1.txt | while read name text; do
+ if [ -z "$name0" ]; then
+  name0=$name
+ else
+  if [ $name==$name0 ]; then
+   echo $name0" OK"
+  else
+   echo $name0" NG! "$name
+  fi
+  name0=''
+ fi
+done
+
+read
\ No newline at end of file
diff --git a/lang_upgrade/make_msgs.sh b/lang_upgrade/make_msgs.sh
new file mode 100644
index 00000000..83ff13d1
--- /dev/null
+++ b/lang_upgrade/make_msgs.sh
@@ -0,0 +1,93 @@
+#!/bin/sh
+#
+echo 'make_msgs.sh'
+if [ -e make_msgs.out ]; then rm make_msgs.out; fi
+CUSTOM_MENDEL_NAME='Prusa i3 MK3'
+
+calc_charcount()
+{
+ #extract texts for charcount calculation
+ cat $1 | cut -f$2- -d' ' | sed -E "s/\" \"//g" >_txt.txt_0
+ #replace printer name
+ cat _txt.txt_0 | sed -E "s/CUSTOM_MENDEL_NAME/\"$CUSTOM_MENDEL_NAME\"/g" >_txt.txt_1
+ #replace oct chars with space
+ cat _txt.txt_1 | sed -E "s/\\\\[0-7]{3}/ /g" >_txt.txt_2
+ #replace hex chars with space
+ cat _txt.txt_2 | sed -E "s/\\\\x[0-9A-Fa-f]{2}/ /g" >_txt.txt_3
+ #replace \" with '
+ cat _txt.txt_3 | sed -E "s/\\\\\"/'/g" >_txt.txt_4
+ #replace "_white_space_" with empty sequence - strigs
+ cat _txt.txt_4 | sed -E "s/\"[ \t]*\"//g" >_txt.txt_5
+ #replace " and white space at end of line
+ cat _txt.txt_5 | sed -E "s/\"[ \t]*$/\"/g" >_txt.txt_6
+ #replace all " with empty sequence
+ cat _txt.txt_6 | sed -E "s/\"//g" >_txt.txt_7
+ #calculate charcount
+ stat -c'%s' _txt.txt_7
+ rm _txt.txt_*
+}
+
+process_language_common()
+{
+ echo -n ' processing language_common.h ...' | tee -a make_msgs.out
+ #list all defines without '+' prefix from language_common.h
+ #cat ../Firmware/language_common.h | grep -E "^[+]*define" | sed "s/lenght/length/g" >msgs_common.txt_0
+ cat ../Firmware/language_common.h | grep -E "^define" | sed "s/lenght/length/g" >msgs_common.txt_0
+  #replace define and +define
+ cat msgs_common.txt_0 | sed -E "s/^[+]*define[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\1 \2/g" | sort >msgs_common.txt
+ #calculate msgcount
+ msgcount=$(grep -c '' msgs_common.txt)
+ #calculate charcount
+ charcount=$(calc_charcount msgs_common.txt 4)
+ #remove tmp files
+ rm msgs_common.txt_*
+ echo "ok ($msgcount messages, $charcount characters)" | tee -a make_msgs.out
+}
+
+
+process_language_en()
+{
+ echo -n ' processing language_en.h ...' | tee -a make_msgs.out
+ #list all defines from language_en.h
+ cat ../Firmware/language_en.h | grep "^#define" | sed "s/lenght/length/g" >msgs_en.txt_0
+ #replace #define(length=xx,lines=xx)
+ cat msgs_en.txt_0 | sed -E "s/^#define\(length=([0-9]*),[ \t]*lines=([0-9]*)\)[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\3 c=\1 r=\2 \4/g" >msgs_en.txt_1
+ #replace #define(length=xx)
+ cat msgs_en.txt_1 | sed -E "s/^#define\(length=([0-9]*)\)[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\2 c=\1 r=0 \3/g" >msgs_en.txt_2
+ #replace #define
+ cat msgs_en.txt_2 | sed -E "s/^#define[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\1 c=0 r=0 \2/g" | sort >msgs_en.txt
+ #calculate msgcount
+ msgcount=$(grep -c '' msgs_en.txt)
+ #calculate charcount
+ charcount=$(calc_charcount msgs_en.txt 4)
+ #remove tmp files
+ rm msgs_en.txt_*
+ echo "ok ($msgcount messages, $charcount characters)" | tee -a make_msgs.out
+}
+
+process_language_xx()
+{
+ echo -n " processing language_$1.h ..." | tee -a make_msgs.out
+ #list all defines from language_cz.h
+ cat "../Firmware/language_$1.h" | grep "^#define" >"msgs_$1.txt_0"
+ cat "msgs_$1.txt_0" | sed -E "s/^#define[ \t]*([^ \t]*)[ \t]*([^ \t]*[ \t]*\"[^\"]*\"*)/\1 \2/g" | sort >"msgs_$1.txt"
+ #calculate msgcount
+ msgcount=$(grep -c '' "msgs_$1.txt")
+ #calculate charcount
+ charcount=$(calc_charcount "msgs_$1.txt" 2)
+ #remove tmp files
+ rm "msgs_$1.txt_0"
+ echo "ok ($msgcount messages, $charcount characters)" | tee -a make_msgs.out
+}
+
+process_language_common
+process_language_en
+process_language_xx cz
+process_language_xx de
+process_language_xx it
+process_language_xx pl
+process_language_xx es
+
+
+echo "finished.."
+read
diff --git a/lang_upgrade/readme.txt b/lang_upgrade/readme.txt
new file mode 100644
index 00000000..7f92ed21
--- /dev/null
+++ b/lang_upgrade/readme.txt
@@ -0,0 +1,28 @@
+lang_upgrade - scripts for migration to new multilanguage support design
+
+
+0. clean.sh
+delete all output files
+
+1. make_msgs.sh
+Process all language_xx.h files and extract informations to msgs_xx.txt files in simple format.
+Every line in msgs_en.txt has following format: MSG_xx c=cc r=rr "text". 
+Every line in other msgs_xx.txt has simpler format: MSG_xx "text".
+MSG_xx is original message identifier, cc is column count (originaly length) and rr is row count (originaly lines).
+Output files msgs_xx.txt are sorted by message identifier (ascending order).
+make_msgs.sh also reports number of messages in each language_xx.h file and total number of characters in program memory.
+
+2. find_msgs.sh
+Find usage of each message and output listing in to file msgs_usage.txt in format: MSG_xx nn.
+MSG_xx is identifier, nn is number of occurrences. Output is sorted by number of occurrences (ascending order).
+
+3. replace_msgs.sh
+List all unused messages to file msgs_unused.txt.
+Copy all source files to folder ./source
+Replace all single-used messages in all ./source/*.c* files directly with the english version string constant and comment at end of line.
+Generate messages.h and messages.c source files with messages used twice and more.
+
+4. upgrade.sh
+Backup (move) all language*.h and language*.cpp files from source to folder '../backup'.
+Copy folder ./source/*.* to ../Firmware, new files will be messages.h, messages.c, language.h and other source will be replaced.
+After this step should be source compilable in english version, LANG_MODE in config.h is set to LANG_MODE_SINGLE.
diff --git a/lang_upgrade/replace_msgs.sh b/lang_upgrade/replace_msgs.sh
new file mode 100644
index 00000000..b390e6d0
--- /dev/null
+++ b/lang_upgrade/replace_msgs.sh
@@ -0,0 +1,168 @@
+#!/bin/sh
+# replace_msgs.sh - step3 - replace source and generate messages.h and messages.c files
+
+#mkdir ./source
+#files=$(ls ../Firmware/*.c* | grep -v 'language'; ls ../Firmware/*.h | grep -v 'language'; )
+#cp $files ./source/
+#exit
+
+files=$(ls ./source/*.c* | grep -v 'language'; ls ./source/*.h | grep -v 'language'; )
+#echo "$files"
+
+#list messages used only once
+ #msgs=$(cat msgs_usage.txt | grep " 1$" | cut -f1 -d' ')
+#make regular expression from the list - replace spaces with '\b|\b'
+ #msgs=$(echo $msgs | sed "s/ /\\\b\\\|\\\b/g")
+#filter this messages from msgs_en.txt to msgs_en_1.txt
+ #cat msgs_en.txt | grep "$msgs" > msgs_en_1.txt
+
+
+
+cat msgs_en_X.txt | sed "s/\\\\/\\\\\\\\/g;s/\//\\\\\\\\\//g" | while read name cols rows text; do
+ comment="$name $cols $rows"
+ if sed -i -E "s/(.*)(\b$name\b)(.*)$/\1_i\($text\)\3\/\/\/\/$comment/g" $files; then
+  echo "$name OK"
+ else
+  echo "$name NG!"
+ fi
+done | tee replace2.out
+
+read
+
+#grep "$msgs" $files > msg_.txt
+
+exit
+
+
+cat msgs_en.txt | grep
+
+echo "$msgs" | while read name; do
+# sed -i "s/\b$name\b/_i\(\"$text\"\)/g" ./source/ultralcd.cpp
+done
+read
+exit
+
+#
+name=MSG_INFO_NOZZLE_FAN
+text="Nozzle FAN:"
+#grep  "\b$name\b" $files
+sed -i "s/\b$name\b/_i\(\"$text\"\)/g" ./source/ultralcd.cpp
+read
+exit
+
+
+#unused messages will be listed in msgs_unused.txt.
+cat msgs_usage.txt | grep " 0$" | cut -f1 -d' ' >msgs_unused.txt
+
+#remove generated source files
+if [ -e messages.h ]; then rm messages.h; fi
+if [ -e messages.c ]; then rm messages.c; fi
+
+#messages used twice or more will be listed in messages.h and messages.cpp.
+msgs=$(cat msgs_usage.txt | grep -v " 0$" | cut -f1 -d' ')
+echo '//messages.h' > messages.h
+echo '#include <avr/pgmspace.h>' >> messages.h
+echo '//messages.c' >> messages.c
+echo '#include "messages.h"' >> messages.c
+echo -n '#define bool int
+' >> messages.c
+echo -n '#define true 1
+' >> messages.c
+echo -n '#define false 0
+' >> messages.c
+echo '#include "Configuration_prusa.h"' >> messages.c
+
+sync -f messages.c
+
+
+echo '//internationalized messages' | tee -a messages.h >> messages.c
+
+cat msgs_en.txt | sed 's/\\/\\\\/g' | while read msg cols rows text; do
+ if echo "$msgs" | grep "^$msg" >/dev/null; then
+  echo "extern const char $msg[] PROGMEM;" >> messages.h
+  echo "const char $msg[] PROGMEM = "$text";" >> messages.c
+  echo "$msg"
+ fi
+done
+
+echo '//not internationalized messages' | tee -a messages.h >> messages.c
+
+cat msgs_common.txt | sed 's/\\/\\\\/g' | while read msg text; do
+ if echo "$msgs" | grep "^$msg" >/dev/null; then
+  echo "extern const char $msg[] PROGMEM;" >> messages.h
+  echo "const char $msg[] PROGMEM = "$text";" >> messages.c
+  echo "$msg"
+ fi
+done
+
+read
+exit
+
+
+
+
+
+
+
+
+
+
+
+
+#messages used twice or more will be listed in messages.h and messages.cpp.
+#msgs=$(cat msgs_usage.txt | grep -v " 0$" | grep -v " 1$" | cut -f1 -d' ')
+msgs=$(cat msgs_usage.txt | grep -v " 0$" | cut -f1 -d' ')
+echo '//messages.h' > messages.h
+echo '//messages.c' > messages.c
+echo '#include <avr/pgmspace.h>' >> messages.h
+echo '#include <avr/pgmspace.h>' >> messages.c
+#internationalized messages
+echo '//internationalized messages' >> messages.h
+echo '//internationalized messages' >> messages.c
+msgs2=$(cat msgs_en.txt | while read msg cols rows text; do
+ if echo "$msgs" | grep "^$msg" >/dev/null; then
+  echo "extern const char $msg[] PROGMEM;" >> messages.h
+  echo "const char $msg[] PROGMEM = $text;" >> messages.c
+  echo $msg
+ fi
+done)
+#not internationalized messages
+echo '//not internationalized messages' >> messages.h
+echo '//not internationalized messages' >> messages.c
+cat msgs_common.txt | while read msg text; do
+ if echo "$msgs" | grep "^$msg" >/dev/null; then
+  if echo "$msgs2" | grep "^$msg" >/dev/null; then
+  else
+   echo "extern const char $msg[] PROGMEM;" >> messages.h
+   echo "const char $msg[] PROGMEM = $text;" >> messages.c
+   echo $msg
+  fi
+ fi
+done
+
+echo "finished..."
+read
+exit
+
+cat msgs_en.txt | while read msg cols rows text; do
+ if echo "$msgs" | grep "^$msg" >/dev/null; then
+  echo "$msg  $text"
+ fi
+done
+
+#; then
+
+#  echo "" $msg
+# fi
+#done
+
+#echo "$msgs" | while read msg; do
+# echo "" $msg
+#done
+
+#msg=MSG_WIZARD_FILAMENT_LOADED
+#txt="PSTR(\"Is filament loaded?\")"
+#fn=../ultralcd.cpp
+#sed -i "s/$msg/$txt/g" $fn
+
+read