From 4580b8a78cd3e3cde543be1269348c8be1f825cc Mon Sep 17 00:00:00 2001
From: "D.R.racer" <drracer@drracer.eu>
Date: Fri, 16 Jul 2021 07:37:11 +0200
Subject: [PATCH 01/40] Version changed (3.10.1 build 4587)

---
 Firmware/Configuration.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Firmware/Configuration.h b/Firmware/Configuration.h
index efdc63ee..76daa96c 100644
--- a/Firmware/Configuration.h
+++ b/Firmware/Configuration.h
@@ -18,10 +18,10 @@ extern PGM_P sPrinterName;
 // Firmware version
 #define FW_MAJOR 3
 #define FW_MINOR 10
-#define FW_REVISION 0
-#define FW_VERSION STR(FW_MAJOR) "." STR(FW_MINOR) "." STR(FW_REVISION)
+#define FW_REVISION 1
+#define FW_VERSION STR(FW_MAJOR) "." STR(FW_MINOR) "." STR(FW_REVISION) "-RC1"
 
-#define FW_COMMIT_NR 4481
+#define FW_COMMIT_NR 4587
 
 // FW_VERSION_UNKNOWN means this is an unofficial build.
 // The firmware should only be checked into github with this symbol.

From 1095b265702684acdfd81518792f3401fc60f838 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 25 May 2021 00:26:32 +0200
Subject: [PATCH 02/40] Add several low-level debugging tools

---
 tools/README.md     |  37 +++++++++++
 tools/dump_eeprom   |  17 +++++
 tools/dump_sram     |  17 +++++
 tools/elf_mem_map   | 153 ++++++++++++++++++++++++++++++++++++++++++++
 tools/noreset       |  12 ++++
 tools/update_eeprom |  54 ++++++++++++++++
 6 files changed, 290 insertions(+)
 create mode 100644 tools/README.md
 create mode 100755 tools/dump_eeprom
 create mode 100755 tools/dump_sram
 create mode 100755 tools/elf_mem_map
 create mode 100755 tools/noreset
 create mode 100755 tools/update_eeprom

diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 00000000..2b4c7b82
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,37 @@
+# Host debugging tools for Prusa MK3 firmware
+
+## Tools
+
+### ``dump_eeprom``
+
+Dump the content of the entire EEPROM using the D3 command.
+Requires ``printcore`` from [Pronterface].
+
+### ``dump_sram``
+
+Dump the content of the entire SRAM using the D2 command.
+Requires ``printcore`` from [Pronterface].
+
+### ``elf_mem_map``
+
+Generate a symbol table map starting directly from an ELF firmware with DWARF2 debugging information (which is the default using the stock board definition).
+
+When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range.
+
+This assumes the running firmware generating the dump and the elf file are the same.
+Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module.
+
+### ``update_eeprom``
+
+Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer.
+
+Given two EEPROM dumps, produces only the required instructions needed to update the contents from the first to the second. This is currently quite crude and assumes dumps are aligned (starting from the same address or same stride).
+
+Optionally writes the instructions to the specified port (requires ``printcore`` from [Pronterface]).
+
+### ``noreset``
+
+Set the required TTY flags on the specified port to avoid reset-on-connect for *subsequent* requests (issuing this command might still cause the printer to reset).
+
+
+[Pronterface]: https://github.com/kliment/Printrun
diff --git a/tools/dump_eeprom b/tools/dump_eeprom
new file mode 100755
index 00000000..5e546416
--- /dev/null
+++ b/tools/dump_eeprom
@@ -0,0 +1,17 @@
+#!/bin/sh
+prg=$(basename "$0")
+port="$1"
+if [ -z "$port" -o "$port" = "-h" ]
+then
+  echo "usage: $0 <port>" >&2
+  echo "Connect to <port> and dump the content of the EEPROM using D3 to stdout" >&2
+  exit 1
+fi
+
+set -e
+tmp=$(mktemp)
+trap "rm -f \"$tmp\"" EXIT
+
+echo D3 > "$tmp"
+printcore -v "$port" "$tmp" 2>&1 | \
+    sed -ne '/^RECV: D3 /,/RECV: ok$/s/^RECV: //p'
diff --git a/tools/dump_sram b/tools/dump_sram
new file mode 100755
index 00000000..93ef4e92
--- /dev/null
+++ b/tools/dump_sram
@@ -0,0 +1,17 @@
+#!/bin/sh
+prg=$(basename "$0")
+port="$1"
+if [ -z "$port" -o "$port" = "-h" ]
+then
+  echo "usage: $0 <port>" >&2
+  echo "Connect to <port> and dump the content of the SRAM using D2 to stdout" >&2
+  exit 1
+fi
+
+set -e
+tmp=$(mktemp)
+trap "rm -f \"$tmp\"" EXIT
+
+echo D2 > "$tmp"
+printcore -v "$port" "$tmp" 2>&1 | \
+    sed -ne '/^RECV: D2 /,/RECV: ok$/s/^RECV: //p'
diff --git a/tools/elf_mem_map b/tools/elf_mem_map
new file mode 100755
index 00000000..46f1e1c1
--- /dev/null
+++ b/tools/elf_mem_map
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+import argparse
+import elftools.elf.elffile
+import elftools.dwarf.descriptions
+from struct import unpack
+
+SRAM_OFFSET = 0x800000
+EEPROM_OFFSET = 0x810000
+FILL_BYTE = b'\0'
+
+
+def get_elf_globals(path):
+    fd = open(path, "rb")
+    if fd is None:
+        return
+    elffile = elftools.elf.elffile.ELFFile(fd)
+    if elffile is None or not elffile.has_dwarf_info():
+        return
+
+    # probably not needed, since we're decoding expressions manually
+    elftools.dwarf.descriptions.set_global_machine_arch(elffile.get_machine_arch())
+    dwarfinfo = elffile.get_dwarf_info()
+
+    grefs = []
+    for CU in dwarfinfo.iter_CUs():
+        for DIE in CU.iter_DIEs():
+            # handle only variable types
+            if DIE.tag != 'DW_TAG_variable':
+                continue
+            if 'DW_AT_name' not in DIE.attributes:
+                continue
+            if 'DW_AT_location' not in DIE.attributes:
+                continue
+            if 'DW_AT_type' not in DIE.attributes:
+                continue
+
+            # handle locations encoded directly as DW_OP_addr (leaf globals)
+            at_loc = DIE.attributes['DW_AT_location']
+            if at_loc.form != 'DW_FORM_block1' or at_loc.value[0] != 3:
+                continue
+            loc = (at_loc.value[1]) + (at_loc.value[2] << 8) \
+                + (at_loc.value[3] << 16) + (at_loc.value[4] << 24)
+            if loc < SRAM_OFFSET or loc >= EEPROM_OFFSET:
+                continue
+            loc -= SRAM_OFFSET
+
+            # variable name
+            name = DIE.attributes['DW_AT_name'].value.decode('ascii')
+
+            # recurse on type to find the leaf definition
+            type_DIE = DIE
+            while 'DW_AT_type' in type_DIE.attributes:
+                type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
+            byte_size = type_DIE.attributes.get('DW_AT_byte_size')
+            if byte_size is None:
+                continue
+            size = byte_size.value
+
+            grefs.append([name, loc, size])
+
+    return grefs
+
+
+def decode_dump(path):
+    fd = open(path, 'r')
+    if fd is None:
+        return None
+
+    buf_addr = None # starting address
+    buf_data = None # data
+
+    for line in fd:
+        tokens = line.split(maxsplit=1)
+        if len(tokens) == 0 or tokens[0] == 'ok':
+            break
+        elif len(tokens) < 2 or tokens[0] == 'D2':
+            continue
+
+        addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
+        data = bytes.fromhex(tokens[1])
+
+        if buf_addr is None:
+            buf_addr = addr
+            buf_data = data
+        else:
+            # grow buffer as needed
+            if addr < buf_addr:
+                buf_data = FILL_BYTE * (buf_addr - addr)
+                buf_addr = addr
+            addr_end = addr + len(data)
+            buf_end = buf_addr + len(buf_data)
+            if addr_end > buf_end:
+                buf_data += FILL_BYTE * (addr_end - buf_end)
+
+            # replace new part
+            rep_start = addr - buf_addr
+            rep_end = rep_start + len(data)
+            buf_data = buf_data[:rep_start] + data + buf_data[rep_end:]
+
+    return (buf_addr, buf_data)
+
+
+def annotate_refs(grefs, addr, data, width=45):
+    for name, loc, size in grefs:
+        if loc < addr:
+            continue
+        if loc + size > addr + len(data):
+            continue
+
+        pos = loc-addr
+        buf = data[pos:pos+size]
+
+        buf_repr = ''
+        if len(buf) in [1, 2, 4]:
+            # attempt to decode as integers
+            buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10)
+        if len(buf) in [4, 8]:
+            # attempt to decode as floats
+            buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0])
+
+        print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size,
+                                             buf_repr, buf.hex()))
+
+
+def print_map(grefs):
+    print('OFFSET\tSIZE\tNAME')
+    for name, loc, size in grefs:
+        print('{:x}\t{}\t{}'.format(loc, size, name))
+
+
+def main():
+    ap = argparse.ArgumentParser(description="""
+        Generate a symbol table map starting directly from an ELF
+        firmware with DWARF2 debugging information.
+        When used along with a memory dump obtained from the D2 g-code,
+        show the value of each symbol which is within the address range.
+    """)
+    ap.add_argument('elf', help='ELF file containing DWARF2 debugging information')
+    g = ap.add_mutually_exclusive_group(required=True)
+    g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
+    g.add_argument('--map', action='store_true', help='dump global memory map')
+    args = ap.parse_args()
+
+    grefs = get_elf_globals(args.elf)
+    grefs = list(sorted(grefs, key=lambda x: x[1]))
+    if args.dump is None:
+        print_map(grefs)
+    else:
+        addr, data = decode_dump(args.dump)
+        annotate_refs(grefs, addr, data)
+
+if __name__ == '__main__':
+    exit(main())
diff --git a/tools/noreset b/tools/noreset
new file mode 100755
index 00000000..3fd32416
--- /dev/null
+++ b/tools/noreset
@@ -0,0 +1,12 @@
+#!/bin/sh
+prg=$(basename "$0")
+port="$1"
+if [ -z "$port" -o "$port" = "-h" ]
+then
+  echo "usage: $0 <port>" >&2
+  echo "Set TTY flags on <port> to avoid reset-on-connect" >&2
+  exit 1
+fi
+
+set -e
+stty -F "$port" -hup
diff --git a/tools/update_eeprom b/tools/update_eeprom
new file mode 100755
index 00000000..a72f9d71
--- /dev/null
+++ b/tools/update_eeprom
@@ -0,0 +1,54 @@
+#!/bin/sh
+prg=$(basename "$0")
+
+# parse arguments
+while getopts f:h optname
+do
+  case $optname in
+  f)    port="$OPTARG" ;;
+  *)    help=1 ;;
+  esac
+done
+shift `expr $OPTIND - 1`
+
+old="$1"
+new="$2"
+
+if [ -z "$old" -o "$help" = "-h" -o "$#" -gt 2 ]
+then
+  echo "usage: $0 [-f <port>] <old dump> [<new dump>]" >&2
+  echo "Convert <old dump> to instructions to update instructions." >&2
+  echo "With <new dump>, generate instructions to update EEPROM changes only." >&2
+  echo "Optionally write such changes directly if <port> if given." >&2
+  exit 1
+fi
+
+set -e
+instr=$(mktemp)
+trap "rm -f \"$instr\"" EXIT
+
+convert()
+{
+    sed -ne 's/^\([0-9a-f]\{4\}\) \([0-9a-f ]*\)$/D3 Ax\1 C16 X\2/p' "$@"
+}
+
+if [ -z "$new" ]; then
+    # convert the instructions to updates
+    convert "$old" > "$instr"
+else
+    tmp1=$(mktemp)
+    tmp2=$(mktemp)
+    trap "rm -f \"$tmp1\" \"$tmp2\"" EXIT
+
+    convert "$old" > "$tmp1"
+    convert "$new" > "$tmp2"
+
+    comm -13 "$tmp1" "$tmp2" > "$instr"
+fi
+
+# write the instructions if requested
+if [ -z "$port" ]; then
+    cat "$instr"
+else
+    printcore -v "$port" "$instr"
+fi

From 4c6339ac4697e99e6190a8603a746759e6fe4c81 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 17:01:29 +0200
Subject: [PATCH 03/40] elf_mem_map: decode correctly void pointers

---
 tools/elf_mem_map | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 46f1e1c1..a4301f97 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -47,11 +47,15 @@ def get_elf_globals(path):
             # variable name
             name = DIE.attributes['DW_AT_name'].value.decode('ascii')
 
-            # recurse on type to find the leaf definition
+            # recurse on type to find the final storage definition
             type_DIE = DIE
-            while 'DW_AT_type' in type_DIE.attributes:
+            byte_size = None
+            while True:
+                if 'DW_AT_byte_size' in type_DIE.attributes:
+                    byte_size = type_DIE.attributes.get('DW_AT_byte_size')
+                if 'DW_AT_type' not in type_DIE.attributes:
+                    break
                 type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
-            byte_size = type_DIE.attributes.get('DW_AT_byte_size')
             if byte_size is None:
                 continue
             size = byte_size.value

From c321ba482137810ed6832fdb7a2ceb93a03b3590 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 17:15:21 +0200
Subject: [PATCH 04/40] elf_mem_map: also dump gaps between known regions

---
 tools/elf_mem_map | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index a4301f97..69f08428 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -104,7 +104,9 @@ def decode_dump(path):
     return (buf_addr, buf_data)
 
 
-def annotate_refs(grefs, addr, data, width=45):
+def annotate_refs(grefs, addr, data, width=45, gaps=True):
+    last_pos = None
+
     for name, loc, size in grefs:
         if loc < addr:
             continue
@@ -112,7 +114,8 @@ def annotate_refs(grefs, addr, data, width=45):
             continue
 
         pos = loc-addr
-        buf = data[pos:pos+size]
+        end_pos = pos + size
+        buf = data[pos:end_pos]
 
         buf_repr = ''
         if len(buf) in [1, 2, 4]:
@@ -122,8 +125,16 @@ def annotate_refs(grefs, addr, data, width=45):
             # attempt to decode as floats
             buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0])
 
+        if gaps and last_pos is not None and last_pos < pos:
+            # decode gaps
+            gap_size = pos - last_pos
+            gap_buf = data[last_pos:pos]
+            print('{:04x} {} {:4} R:{}'.format(last_pos, "*UNKNOWN*".ljust(width),
+                                               gap_size, gap_buf.hex()))
+
         print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size,
                                              buf_repr, buf.hex()))
+        last_pos = end_pos
 
 
 def print_map(grefs):

From f2192dc5e6a97b04415b1d8f6592d3ce9317a601 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 17:24:11 +0200
Subject: [PATCH 05/40] elf_mem_dump: fix unknown address

---
 tools/elf_mem_map | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 69f08428..ddfd5706 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -105,8 +105,7 @@ def decode_dump(path):
 
 
 def annotate_refs(grefs, addr, data, width=45, gaps=True):
-    last_pos = None
-
+    last_end = None
     for name, loc, size in grefs:
         if loc < addr:
             continue
@@ -125,16 +124,16 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True):
             # attempt to decode as floats
             buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0])
 
-        if gaps and last_pos is not None and last_pos < pos:
+        if gaps and last_end is not None and last_end < pos:
             # decode gaps
-            gap_size = pos - last_pos
-            gap_buf = data[last_pos:pos]
-            print('{:04x} {} {:4} R:{}'.format(last_pos, "*UNKNOWN*".ljust(width),
+            gap_size = pos - last_end
+            gap_buf = data[last_end:pos]
+            print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width),
                                                gap_size, gap_buf.hex()))
 
         print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size,
                                              buf_repr, buf.hex()))
-        last_pos = end_pos
+        last_end = end_pos
 
 
 def print_map(grefs):

From 40b737e33d98f0b56c2dfee94dcd53563a39fd19 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 18:13:43 +0200
Subject: [PATCH 06/40] elf_mem_map: switch to a named tuple for extensibility

---
 tools/elf_mem_map | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index ddfd5706..aad957b9 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -2,6 +2,7 @@
 import argparse
 import elftools.elf.elffile
 import elftools.dwarf.descriptions
+from collections import namedtuple
 from struct import unpack
 
 SRAM_OFFSET = 0x800000
@@ -9,6 +10,9 @@ EEPROM_OFFSET = 0x810000
 FILL_BYTE = b'\0'
 
 
+Entry = namedtuple('Entry', ['name', 'loc', 'size'])
+
+
 def get_elf_globals(path):
     fd = open(path, "rb")
     if fd is None:
@@ -60,7 +64,7 @@ def get_elf_globals(path):
                 continue
             size = byte_size.value
 
-            grefs.append([name, loc, size])
+            grefs.append(Entry(name, loc, size))
 
     return grefs
 
@@ -106,14 +110,14 @@ def decode_dump(path):
 
 def annotate_refs(grefs, addr, data, width=45, gaps=True):
     last_end = None
-    for name, loc, size in grefs:
-        if loc < addr:
+    for entry in grefs:
+        if entry.loc < addr:
             continue
-        if loc + size > addr + len(data):
+        if entry.loc + entry.size > addr + len(data):
             continue
 
-        pos = loc-addr
-        end_pos = pos + size
+        pos = entry.loc-addr
+        end_pos = pos + entry.size
         buf = data[pos:end_pos]
 
         buf_repr = ''
@@ -131,15 +135,15 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True):
             print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width),
                                                gap_size, gap_buf.hex()))
 
-        print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size,
-                                             buf_repr, buf.hex()))
+        print('{:04x} {} {:4}{} R:{}'.format(entry.loc, entry.name.ljust(width),
+                                             entry.size, buf_repr, buf.hex()))
         last_end = end_pos
 
 
 def print_map(grefs):
     print('OFFSET\tSIZE\tNAME')
-    for name, loc, size in grefs:
-        print('{:x}\t{}\t{}'.format(loc, size, name))
+    for entry in grefs:
+        print('{:x}\t{}\t{}'.format(entry.loc, entry.size, entry.name))
 
 
 def main():
@@ -156,7 +160,7 @@ def main():
     args = ap.parse_args()
 
     grefs = get_elf_globals(args.elf)
-    grefs = list(sorted(grefs, key=lambda x: x[1]))
+    grefs = list(sorted(grefs, key=lambda x: x.loc))
     if args.dump is None:
         print_map(grefs)
     else:

From 1de3fa51c9662e2b6f79aedb959c62f229bec3f3 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 18:26:33 +0200
Subject: [PATCH 07/40] elf_mem_map: decode doubles correctly

---
 tools/elf_mem_map | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index aad957b9..32f178dc 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -126,7 +126,8 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True):
             buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10)
         if len(buf) in [4, 8]:
             # attempt to decode as floats
-            buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0])
+            typ = 'f' if len(buf) == 4 else 'd'
+            buf_repr += ' F:' + '{:10.3f}'.format(unpack(typ, buf)[0])
 
         if gaps and last_end is not None and last_end < pos:
             # decode gaps

From 29b8c89ec2a7df5cf9dfcef693702890fe1c65e0 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 18:30:25 +0200
Subject: [PATCH 08/40] elf_mem_map: decode arrays (first dimension)

---
 tools/elf_mem_map | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 32f178dc..e518083d 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -54,17 +54,33 @@ def get_elf_globals(path):
             # recurse on type to find the final storage definition
             type_DIE = DIE
             byte_size = None
+            array_len = None
             while True:
                 if 'DW_AT_byte_size' in type_DIE.attributes:
                     byte_size = type_DIE.attributes.get('DW_AT_byte_size')
                 if 'DW_AT_type' not in type_DIE.attributes:
                     break
                 type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
+                if type_DIE.tag == 'DW_TAG_array_type':
+                    # fetch the first dimension (if known)
+                    range_DIE = next(type_DIE.iter_children())
+                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
+                       'DW_AT_upper_bound' in range_DIE.attributes:
+                        array_len = range_DIE.attributes['DW_AT_upper_bound'].value + 1
             if byte_size is None:
                 continue
             size = byte_size.value
 
-            grefs.append(Entry(name, loc, size))
+            if array_len is None or array_len == 1:
+                # plain entry
+                grefs.append(Entry(name, loc, size))
+            elif size == 1:
+                # string
+                grefs.append(Entry('{}[]'.format(name), loc, array_len))
+            else:
+                # expand array entries
+                for i in range(array_len):
+                    grefs.append(Entry('{}[{}]'.format(name, i), loc+size*i, size))
 
     return grefs
 

From 2718dbb42cbd3671a43fd52fe23547b121c4a6c8 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 18:57:14 +0200
Subject: [PATCH 09/40] elf_mem_map: array n-dimensional expansion

---
 tools/elf_mem_map | 45 +++++++++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 12 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index e518083d..fdfc6dad 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -13,6 +13,15 @@ FILL_BYTE = b'\0'
 Entry = namedtuple('Entry', ['name', 'loc', 'size'])
 
 
+def array_inc(loc, dim, idx=0):
+    if idx == len(dim):
+        return True
+    loc[idx] += 1
+    if loc[idx] == dim[idx]:
+        loc[idx] = 0
+        return array_inc(loc, dim, idx+1)
+    return False
+
 def get_elf_globals(path):
     fd = open(path, "rb")
     if fd is None:
@@ -54,7 +63,7 @@ def get_elf_globals(path):
             # recurse on type to find the final storage definition
             type_DIE = DIE
             byte_size = None
-            array_len = None
+            array_dim = []
             while True:
                 if 'DW_AT_byte_size' in type_DIE.attributes:
                     byte_size = type_DIE.attributes.get('DW_AT_byte_size')
@@ -62,25 +71,37 @@ def get_elf_globals(path):
                     break
                 type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
                 if type_DIE.tag == 'DW_TAG_array_type':
-                    # fetch the first dimension (if known)
-                    range_DIE = next(type_DIE.iter_children())
-                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
-                       'DW_AT_upper_bound' in range_DIE.attributes:
-                        array_len = range_DIE.attributes['DW_AT_upper_bound'].value + 1
+                    # fetch array dimensions (if known)
+                    for range_DIE in type_DIE.iter_children():
+                        if range_DIE.tag == 'DW_TAG_subrange_type' and \
+                           'DW_AT_upper_bound' in range_DIE.attributes:
+                            array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
             if byte_size is None:
                 continue
             size = byte_size.value
 
-            if array_len is None or array_len == 1:
+            if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
                 grefs.append(Entry(name, loc, size))
-            elif size == 1:
-                # string
-                grefs.append(Entry('{}[]'.format(name), loc, array_len))
+            elif len(array_dim) == 1 and size == 1:
+                # likely string, avoid expansion
+                grefs.append(Entry('{}[]'.format(name), loc, array_dim[0]))
             else:
                 # expand array entries
-                for i in range(array_len):
-                    grefs.append(Entry('{}[{}]'.format(name, i), loc+size*i, size))
+                array_pos = loc
+                array_loc = [0] * len(array_dim)
+                while True:
+                    # location index
+                    sfx = ''
+                    for d in range(len(array_dim)):
+                        sfx += '[{}]'.format(array_loc[d])
+
+                    grefs.append(Entry(name + sfx, array_pos, size))
+
+                    # advance
+                    array_pos += size
+                    if array_inc(array_loc, array_dim):
+                        break
 
     return grefs
 

From c311266a83e9f289c6ae3710cc1758d4bc8731b4 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 19:26:29 +0200
Subject: [PATCH 10/40] elf_mem_map: handle abstract locations

---
 tools/elf_mem_map | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index fdfc6dad..6d08abb9 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -40,11 +40,10 @@ def get_elf_globals(path):
             # handle only variable types
             if DIE.tag != 'DW_TAG_variable':
                 continue
-            if 'DW_AT_name' not in DIE.attributes:
-                continue
             if 'DW_AT_location' not in DIE.attributes:
                 continue
-            if 'DW_AT_type' not in DIE.attributes:
+            if 'DW_AT_name' not in DIE.attributes and \
+               'DW_AT_abstract_origin' not in DIE.attributes:
                 continue
 
             # handle locations encoded directly as DW_OP_addr (leaf globals)
@@ -57,7 +56,18 @@ def get_elf_globals(path):
                 continue
             loc -= SRAM_OFFSET
 
-            # variable name
+            # variable name/type
+            if 'DW_AT_name' not in DIE.attributes and \
+               'DW_AT_abstract_origin' in DIE.attributes:
+                DIE = DIE.get_DIE_from_attribute('DW_AT_abstract_origin')
+                if 'DW_AT_location' in DIE.attributes:
+                    # duplicate reference (handled directly), skip
+                    continue
+            if 'DW_AT_name' not in DIE.attributes:
+                continue
+            if 'DW_AT_type' not in DIE.attributes:
+                continue
+
             name = DIE.attributes['DW_AT_name'].value.decode('ascii')
 
             # recurse on type to find the final storage definition

From 1181e784849f9dd594e73fa05aed24927e7ad542 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 20:46:21 +0200
Subject: [PATCH 11/40] elf_mem_map: handle all pointer types correctly

---
 tools/elf_mem_map | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 6d08abb9..dc985ab0 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -75,12 +75,12 @@ def get_elf_globals(path):
             byte_size = None
             array_dim = []
             while True:
-                if 'DW_AT_byte_size' in type_DIE.attributes:
+                if byte_size is None and 'DW_AT_byte_size' in type_DIE.attributes:
                     byte_size = type_DIE.attributes.get('DW_AT_byte_size')
                 if 'DW_AT_type' not in type_DIE.attributes:
                     break
                 type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
-                if type_DIE.tag == 'DW_TAG_array_type':
+                if len(array_dim) == 0 and type_DIE.tag == 'DW_TAG_array_type':
                     # fetch array dimensions (if known)
                     for range_DIE in type_DIE.iter_children():
                         if range_DIE.tag == 'DW_TAG_subrange_type' and \

From bb8d171f34aef97e7f8cdaa245e61bcdd57d34bc Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 21:07:22 +0200
Subject: [PATCH 12/40] elf_mem_map: decode integers with correct endianness

---
 tools/elf_mem_map | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index dc985ab0..f4d187d1 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -170,7 +170,7 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True):
         buf_repr = ''
         if len(buf) in [1, 2, 4]:
             # attempt to decode as integers
-            buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10)
+            buf_repr += ' I:' + str(int.from_bytes(buf, 'little')).rjust(10)
         if len(buf) in [4, 8]:
             # attempt to decode as floats
             typ = 'f' if len(buf) == 4 else 'd'

From 9ddb5991f2ecc6f950bcb5e52a8999dc0a6df56e Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Wed, 2 Jun 2021 21:58:49 +0200
Subject: [PATCH 13/40] elf_mem_map: allow to disable gap dumps

---
 tools/elf_mem_map | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index f4d187d1..281566e8 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -202,6 +202,8 @@ def main():
         show the value of each symbol which is within the address range.
     """)
     ap.add_argument('elf', help='ELF file containing DWARF2 debugging information')
+    ap.add_argument('--no-gaps', action='store_true',
+                    help='do not dump memory inbetween known symbols')
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
     g.add_argument('--map', action='store_true', help='dump global memory map')
@@ -213,7 +215,7 @@ def main():
         print_map(grefs)
     else:
         addr, data = decode_dump(args.dump)
-        annotate_refs(grefs, addr, data)
+        annotate_refs(grefs, addr, data, gaps=not args.no_gaps)
 
 if __name__ == '__main__':
     exit(main())

From 615e8575bb1883518cb0d7280d8205748c3f1ab0 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 16:29:22 +0200
Subject: [PATCH 14/40] elf_mem_map: decode structs

---
 tools/elf_mem_map | 141 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 110 insertions(+), 31 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 281566e8..2d96221c 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -11,6 +11,7 @@ FILL_BYTE = b'\0'
 
 
 Entry = namedtuple('Entry', ['name', 'loc', 'size'])
+Member = namedtuple('Member', ['name', 'off', 'size'])
 
 
 def array_inc(loc, dim, idx=0):
@@ -22,7 +23,54 @@ def array_inc(loc, dim, idx=0):
         return array_inc(loc, dim, idx+1)
     return False
 
-def get_elf_globals(path):
+def get_type_size(type_DIE):
+    while True:
+        if 'DW_AT_byte_size' in type_DIE.attributes:
+            return type_DIE, type_DIE.attributes.get('DW_AT_byte_size').value
+        if 'DW_AT_type' not in type_DIE.attributes:
+            return None
+        type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
+
+def get_type_arrsize(type_DIE):
+    size = get_type_size(type_DIE)
+    if size is None:
+        return None
+    byte_size = size[1]
+    if size[0].tag != 'DW_TAG_pointer_type':
+        array_DIE = get_type_def(type_DIE, 'DW_TAG_array_type')
+        if array_DIE is not None:
+            for range_DIE in array_DIE.iter_children():
+                if range_DIE.tag == 'DW_TAG_subrange_type' and \
+                   'DW_AT_upper_bound' in range_DIE.attributes:
+                    dim = range_DIE.attributes['DW_AT_upper_bound'].value + 1
+                    byte_size *= dim
+    return byte_size
+
+def get_type_def(type_DIE, type_tag):
+    while True:
+        if type_DIE.tag == type_tag:
+            return type_DIE
+        if 'DW_AT_type' not in type_DIE.attributes:
+            return None
+        type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
+
+def get_FORM_block1(attr):
+    if attr.form != 'DW_FORM_block1':
+        return None
+    if attr.value[0] == 3: # OP_addr
+        return int.from_bytes(attr.value[1:], 'little')
+    if attr.value[0] == 35: # OP_plus_uconst (ULEB128)
+        v = 0
+        s = 0
+        for b in attr.value[1:]:
+            v |= b
+            s += 7
+            if not b & 0x100:
+                break
+        return v
+    return None
+
+def get_elf_globals(path, expand_structs, struct_gaps=True):
     fd = open(path, "rb")
     if fd is None:
         return
@@ -47,12 +95,8 @@ def get_elf_globals(path):
                 continue
 
             # handle locations encoded directly as DW_OP_addr (leaf globals)
-            at_loc = DIE.attributes['DW_AT_location']
-            if at_loc.form != 'DW_FORM_block1' or at_loc.value[0] != 3:
-                continue
-            loc = (at_loc.value[1]) + (at_loc.value[2] << 8) \
-                + (at_loc.value[3] << 16) + (at_loc.value[4] << 24)
-            if loc < SRAM_OFFSET or loc >= EEPROM_OFFSET:
+            loc = get_FORM_block1(DIE.attributes['DW_AT_location'])
+            if loc is None or loc < SRAM_OFFSET or loc >= EEPROM_OFFSET:
                 continue
             loc -= SRAM_OFFSET
 
@@ -70,32 +114,65 @@ def get_elf_globals(path):
 
             name = DIE.attributes['DW_AT_name'].value.decode('ascii')
 
-            # recurse on type to find the final storage definition
-            type_DIE = DIE
-            byte_size = None
-            array_dim = []
-            while True:
-                if byte_size is None and 'DW_AT_byte_size' in type_DIE.attributes:
-                    byte_size = type_DIE.attributes.get('DW_AT_byte_size')
-                if 'DW_AT_type' not in type_DIE.attributes:
-                    break
-                type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type')
-                if len(array_dim) == 0 and type_DIE.tag == 'DW_TAG_array_type':
-                    # fetch array dimensions (if known)
-                    for range_DIE in type_DIE.iter_children():
-                        if range_DIE.tag == 'DW_TAG_subrange_type' and \
-                           'DW_AT_upper_bound' in range_DIE.attributes:
-                            array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
-            if byte_size is None:
+            # get final storage size
+            size = get_type_size(DIE)
+            if size is None:
                 continue
-            size = byte_size.value
+            byte_size = size[1]
+
+            # fetch array dimensions (if known)
+            array_dim = []
+            array_DIE = get_type_def(DIE, 'DW_TAG_array_type')
+            if array_DIE is not None:
+                for range_DIE in array_DIE.iter_children():
+                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
+                       'DW_AT_upper_bound' in range_DIE.attributes:
+                        array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
+
+            # fetch structure members (one level only)
+            members = []
+            if expand_structs and size[0].tag != 'DW_TAG_pointer_type':
+                struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type')
+                if struct_DIE is not None:
+                    for member_DIE in struct_DIE.iter_children():
+                        if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes:
+                            m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii')
+                            m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location'])
+                            m_size = get_type_arrsize(member_DIE)
+                            members.append(Member(m_name, m_off, m_size))
+
+                if struct_gaps and len(members):
+                    # fill gaps in the middle
+                    members = list(sorted(members, key=lambda x: x.off))
+                    last_end = 0
+                    for member in members:
+                        if member.off > last_end:
+                            members.append(Member('*UNKNOWN*', last_end, member.off - last_end))
+                        last_end = member.off + member.size
+
+                if struct_gaps and len(members):
+                    # fill gap at the end
+                    members = list(sorted(members, key=lambda x: x.off))
+                    last = members[-1]
+                    last_end = last.off + last.size
+                    if byte_size > last_end:
+                        members.append(Member('*UNKNOWN*', last_end, byte_size - last_end))
+
+
+            def expand_members(entry, members):
+                if len(members) == 0:
+                    grefs.append(entry)
+                else:
+                    for member in members:
+                        grefs.append(Entry(entry.name + '.' + member.name,
+                                           entry.loc + member.off, member.size))
 
             if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
-                grefs.append(Entry(name, loc, size))
-            elif len(array_dim) == 1 and size == 1:
+                expand_members(Entry(name, loc, byte_size), members)
+            elif len(array_dim) == 1 and byte_size == 1:
                 # likely string, avoid expansion
-                grefs.append(Entry('{}[]'.format(name), loc, array_dim[0]))
+                grefs.append(Entry(name + '[]', loc, array_dim[0]))
             else:
                 # expand array entries
                 array_pos = loc
@@ -106,10 +183,10 @@ def get_elf_globals(path):
                     for d in range(len(array_dim)):
                         sfx += '[{}]'.format(array_loc[d])
 
-                    grefs.append(Entry(name + sfx, array_pos, size))
+                    expand_members(Entry(name + sfx, array_pos, byte_size), members)
 
                     # advance
-                    array_pos += size
+                    array_pos += byte_size
                     if array_inc(array_loc, array_dim):
                         break
 
@@ -204,12 +281,14 @@ def main():
     ap.add_argument('elf', help='ELF file containing DWARF2 debugging information')
     ap.add_argument('--no-gaps', action='store_true',
                     help='do not dump memory inbetween known symbols')
+    ap.add_argument('--no-expand-structs', action='store_true',
+                    help='do not decode structure data')
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
     g.add_argument('--map', action='store_true', help='dump global memory map')
     args = ap.parse_args()
 
-    grefs = get_elf_globals(args.elf)
+    grefs = get_elf_globals(args.elf, expand_structs=not args.no_expand_structs)
     grefs = list(sorted(grefs, key=lambda x: x.loc))
     if args.dump is None:
         print_map(grefs)

From 1d82d2da641aea497d19f483ce5de7c386cbc4c8 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 16:49:45 +0200
Subject: [PATCH 15/40] get_elf_map: do not reprocess members twice

---
 tools/elf_mem_map | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 2d96221c..cde11daf 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -145,7 +145,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                     # fill gaps in the middle
                     members = list(sorted(members, key=lambda x: x.off))
                     last_end = 0
-                    for member in members:
+                    for n in range(len(members)):
+                        member = members[n]
                         if member.off > last_end:
                             members.append(Member('*UNKNOWN*', last_end, member.off - last_end))
                         last_end = member.off + member.size

From a5635997b281e24f789dee211a8c13101943feb7 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 17:06:36 +0200
Subject: [PATCH 16/40] elf_mem_map: allow to annotate overlapping regions for
 clarity

---
 tools/elf_mem_map | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index cde11daf..b3b59978 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -233,7 +233,7 @@ def decode_dump(path):
     return (buf_addr, buf_data)
 
 
-def annotate_refs(grefs, addr, data, width=45, gaps=True):
+def annotate_refs(grefs, addr, data, width=45, gaps=True, overlaps=True):
     last_end = None
     for entry in grefs:
         if entry.loc < addr:
@@ -254,12 +254,16 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True):
             typ = 'f' if len(buf) == 4 else 'd'
             buf_repr += ' F:' + '{:10.3f}'.format(unpack(typ, buf)[0])
 
-        if gaps and last_end is not None and last_end < pos:
-            # decode gaps
-            gap_size = pos - last_end
-            gap_buf = data[last_end:pos]
-            print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width),
-                                               gap_size, gap_buf.hex()))
+        if last_end is not None:
+            if gaps and last_end < pos:
+                # decode gaps
+                gap_size = pos - last_end
+                gap_buf = data[last_end:pos]
+                print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width),
+                                                   gap_size, gap_buf.hex()))
+            if overlaps and last_end > pos + 1:
+                gap_size = pos - last_end
+                print('{:04x} {} {:4}'.format(addr+last_end, "*OVERLAP*".ljust(width), gap_size))
 
         print('{:04x} {} {:4}{} R:{}'.format(entry.loc, entry.name.ljust(width),
                                              entry.size, buf_repr, buf.hex()))
@@ -284,6 +288,8 @@ def main():
                     help='do not dump memory inbetween known symbols')
     ap.add_argument('--no-expand-structs', action='store_true',
                     help='do not decode structure data')
+    ap.add_argument('--overlaps', action='store_true',
+                    help='annotate overlaps greater than 1 byte')
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
     g.add_argument('--map', action='store_true', help='dump global memory map')
@@ -295,7 +301,9 @@ def main():
         print_map(grefs)
     else:
         addr, data = decode_dump(args.dump)
-        annotate_refs(grefs, addr, data, gaps=not args.no_gaps)
+        annotate_refs(grefs, addr, data,
+                      gaps=not args.no_gaps,
+                      overlaps=args.overlaps)
 
 if __name__ == '__main__':
     exit(main())

From 7f76f62af91f31acdf51cb5956afff1c76eb1ad8 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 17:17:00 +0200
Subject: [PATCH 17/40] elf_mem_map: fix uleb128 decoding (fixes incorrect
 member offsets)

---
 tools/elf_mem_map | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index b3b59978..28681167 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -63,10 +63,10 @@ def get_FORM_block1(attr):
         v = 0
         s = 0
         for b in attr.value[1:]:
-            v |= b
-            s += 7
-            if not b & 0x100:
+            v |= (b & 0x7f) << s
+            if b & 0x80 == 0:
                 break
+            s += 7
         return v
     return None
 

From 29513a369d69418d2d4a297a3b18b142bb4c86cf Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 17:22:25 +0200
Subject: [PATCH 18/40] elf_mem_map: allow to customize the name column's width

---
 tools/elf_mem_map | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 28681167..7a14c200 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -233,7 +233,7 @@ def decode_dump(path):
     return (buf_addr, buf_data)
 
 
-def annotate_refs(grefs, addr, data, width=45, gaps=True, overlaps=True):
+def annotate_refs(grefs, addr, data, width=46, gaps=True, overlaps=True):
     last_end = None
     for entry in grefs:
         if entry.loc < addr:
@@ -290,6 +290,8 @@ def main():
                     help='do not decode structure data')
     ap.add_argument('--overlaps', action='store_true',
                     help='annotate overlaps greater than 1 byte')
+    ap.add_argument('--name-width', type=int, default=46,
+                    help='set name column width')
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
     g.add_argument('--map', action='store_true', help='dump global memory map')
@@ -302,6 +304,7 @@ def main():
     else:
         addr, data = decode_dump(args.dump)
         annotate_refs(grefs, addr, data,
+                      width=args.name_width,
                       gaps=not args.no_gaps,
                       overlaps=args.overlaps)
 

From 776b82a6db7ad7110432b16e413841f4c0f206de Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 17:51:02 +0200
Subject: [PATCH 19/40] elf_mem_map: expand member arrays

---
 tools/elf_mem_map | 37 ++++++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 7a14c200..cb08ca5e 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -138,8 +138,39 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                         if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes:
                             m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii')
                             m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location'])
-                            m_size = get_type_arrsize(member_DIE)
-                            members.append(Member(m_name, m_off, m_size))
+                            m_byte_size = get_type_size(member_DIE)[1]
+
+                            # still expand member arrays
+                            m_array_dim = []
+                            m_array_DIE = get_type_def(member_DIE, 'DW_TAG_array_type')
+                            if m_array_DIE is not None:
+                                for range_DIE in m_array_DIE.iter_children():
+                                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
+                                       'DW_AT_upper_bound' in range_DIE.attributes:
+                                        m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
+
+                            if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1):
+                                # plain entry
+                                members.append(Member(m_name, m_off, m_byte_size))
+                            elif len(m_array_dim) == 1 and m_byte_size == 1:
+                                # likely string, avoid expansion
+                                members.append(Member(m_name + '[]', m_off, m_array_dim[0]))
+                            else:
+                                # expand array entries
+                                m_array_pos = m_off
+                                m_array_loc = [0] * len(m_array_dim)
+                                while True:
+                                    # location index
+                                    sfx = ''
+                                    for d in range(len(m_array_dim)):
+                                        sfx += '[{}]'.format(m_array_loc[d])
+
+                                    members.append(Member(m_name + sfx, m_array_pos, m_byte_size))
+
+                                    # advance
+                                    if array_inc(m_array_loc, m_array_dim):
+                                        break
+                                    m_array_pos += m_byte_size
 
                 if struct_gaps and len(members):
                     # fill gaps in the middle
@@ -187,9 +218,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                     expand_members(Entry(name + sfx, array_pos, byte_size), members)
 
                     # advance
-                    array_pos += byte_size
                     if array_inc(array_loc, array_dim):
                         break
+                    array_pos += byte_size
 
     return grefs
 

From cb4f5cff9f8c451eabc169857ef47d98deb16b9b Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 17:54:11 +0200
Subject: [PATCH 20/40] elf_mem_map: improve display of array-of-strings

---
 tools/elf_mem_map | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index cb08ca5e..86058618 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -149,6 +149,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                                        'DW_AT_upper_bound' in range_DIE.attributes:
                                         m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
 
+                            # likely string, remove one dimension
+                            if m_byte_size == 1 and len(m_array_dim) > 1:
+                                m_byte_size *= m_array_dim.pop()
                             if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1):
                                 # plain entry
                                 members.append(Member(m_name, m_off, m_byte_size))
@@ -199,6 +202,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                         grefs.append(Entry(entry.name + '.' + member.name,
                                            entry.loc + member.off, member.size))
 
+            # likely string, remove one dimension
+            if byte_size == 1 and len(array_dim) > 1:
+                byte_size *= array_dim.pop()
             if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
                 expand_members(Entry(name, loc, byte_size), members)

From 71ef94da2e3a6810a4711541d6b17dca8462abe9 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 18:03:36 +0200
Subject: [PATCH 21/40] elf_mem_map: improve alignment of arrays

---
 tools/elf_mem_map | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 86058618..b6aec2c6 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -149,8 +149,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                                        'DW_AT_upper_bound' in range_DIE.attributes:
                                         m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
 
-                            # likely string, remove one dimension
                             if m_byte_size == 1 and len(m_array_dim) > 1:
+                                # likely string, remove one dimension
                                 m_byte_size *= m_array_dim.pop()
                             if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1):
                                 # plain entry
@@ -166,7 +166,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                                     # location index
                                     sfx = ''
                                     for d in range(len(m_array_dim)):
-                                        sfx += '[{}]'.format(m_array_loc[d])
+                                        sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0'))
 
                                     members.append(Member(m_name + sfx, m_array_pos, m_byte_size))
 
@@ -202,8 +202,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                         grefs.append(Entry(entry.name + '.' + member.name,
                                            entry.loc + member.off, member.size))
 
-            # likely string, remove one dimension
             if byte_size == 1 and len(array_dim) > 1:
+                # likely string, remove one dimension
                 byte_size *= array_dim.pop()
             if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
@@ -219,7 +219,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                     # location index
                     sfx = ''
                     for d in range(len(array_dim)):
-                        sfx += '[{}]'.format(array_loc[d])
+                        sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0'))
 
                     expand_members(Entry(name + sfx, array_pos, byte_size), members)
 

From c875aef49ccdec2edda4a908c31bfbde1bf47c56 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 18:03:50 +0200
Subject: [PATCH 22/40] elf_mem_map: increase width again to fit new output

---
 tools/elf_mem_map | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index b6aec2c6..3bd0f23e 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -270,7 +270,7 @@ def decode_dump(path):
     return (buf_addr, buf_data)
 
 
-def annotate_refs(grefs, addr, data, width=46, gaps=True, overlaps=True):
+def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True):
     last_end = None
     for entry in grefs:
         if entry.loc < addr:
@@ -327,7 +327,7 @@ def main():
                     help='do not decode structure data')
     ap.add_argument('--overlaps', action='store_true',
                     help='annotate overlaps greater than 1 byte')
-    ap.add_argument('--name-width', type=int, default=46,
+    ap.add_argument('--name-width', type=int, default=50,
                     help='set name column width')
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')

From d1720cba512c60940a615f3826cb6477f46ea9dc Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 19:10:52 +0200
Subject: [PATCH 23/40] elf_mem_map: reduce some duplication

---
 tools/elf_mem_map | 148 ++++++++++++++++++++++++----------------------
 1 file changed, 78 insertions(+), 70 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 3bd0f23e..26694c80 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -70,6 +70,82 @@ def get_FORM_block1(attr):
         return v
     return None
 
+
+def get_array_dims(DIE):
+    array_DIE = get_type_def(DIE, 'DW_TAG_array_type')
+    if array_DIE is None:
+        return []
+
+    array_dim = []
+    for range_DIE in array_DIE.iter_children():
+        if range_DIE.tag == 'DW_TAG_subrange_type' and \
+           'DW_AT_upper_bound' in range_DIE.attributes:
+            array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
+    return array_dim
+
+
+def get_struct_members(DIE, size, expand_structs, struct_gaps):
+    if not expand_structs or size[0].tag == 'DW_TAG_pointer_type':
+        return []
+    struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type')
+    if struct_DIE is None:
+        return []
+
+    members = []
+    for member_DIE in struct_DIE.iter_children():
+        if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes:
+            m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii')
+            m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location'])
+            m_byte_size = get_type_size(member_DIE)[1]
+
+            # still expand member arrays
+            m_array_dim = get_array_dims(member_DIE)
+
+            if m_byte_size == 1 and len(m_array_dim) > 1:
+                # likely string, remove one dimension
+                m_byte_size *= m_array_dim.pop()
+            if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1):
+                # plain entry
+                members.append(Member(m_name, m_off, m_byte_size))
+            elif len(m_array_dim) == 1 and m_byte_size == 1:
+                # likely string, avoid expansion
+                members.append(Member(m_name + '[]', m_off, m_array_dim[0]))
+            else:
+                # expand array entries
+                m_array_pos = m_off
+                m_array_loc = [0] * len(m_array_dim)
+                while True:
+                    # location index
+                    sfx = ''
+                    for d in range(len(m_array_dim)):
+                        sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0'))
+                    members.append(Member(m_name + sfx, m_array_pos, m_byte_size))
+                    # advance
+                    if array_inc(m_array_loc, m_array_dim):
+                        break
+                    m_array_pos += m_byte_size
+
+    if struct_gaps and len(members):
+        # fill gaps in the middle
+        members = list(sorted(members, key=lambda x: x.off))
+        last_end = 0
+        for n in range(len(members)):
+            member = members[n]
+            if member.off > last_end:
+                members.append(Member('*UNKNOWN*', last_end, member.off - last_end))
+            last_end = member.off + member.size
+
+    if struct_gaps and len(members):
+        # fill gap at the end
+        members = list(sorted(members, key=lambda x: x.off))
+        last = members[-1]
+        last_end = last.off + last.size
+        if size[1] > last_end:
+            members.append(Member('*UNKNOWN*', last_end, byte_size - last_end))
+
+    return members
+
+
 def get_elf_globals(path, expand_structs, struct_gaps=True):
     fd = open(path, "rb")
     if fd is None:
@@ -121,78 +197,10 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
             byte_size = size[1]
 
             # fetch array dimensions (if known)
-            array_dim = []
-            array_DIE = get_type_def(DIE, 'DW_TAG_array_type')
-            if array_DIE is not None:
-                for range_DIE in array_DIE.iter_children():
-                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
-                       'DW_AT_upper_bound' in range_DIE.attributes:
-                        array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
+            array_dim = get_array_dims(DIE)
 
             # fetch structure members (one level only)
-            members = []
-            if expand_structs and size[0].tag != 'DW_TAG_pointer_type':
-                struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type')
-                if struct_DIE is not None:
-                    for member_DIE in struct_DIE.iter_children():
-                        if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes:
-                            m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii')
-                            m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location'])
-                            m_byte_size = get_type_size(member_DIE)[1]
-
-                            # still expand member arrays
-                            m_array_dim = []
-                            m_array_DIE = get_type_def(member_DIE, 'DW_TAG_array_type')
-                            if m_array_DIE is not None:
-                                for range_DIE in m_array_DIE.iter_children():
-                                    if range_DIE.tag == 'DW_TAG_subrange_type' and \
-                                       'DW_AT_upper_bound' in range_DIE.attributes:
-                                        m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1)
-
-                            if m_byte_size == 1 and len(m_array_dim) > 1:
-                                # likely string, remove one dimension
-                                m_byte_size *= m_array_dim.pop()
-                            if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1):
-                                # plain entry
-                                members.append(Member(m_name, m_off, m_byte_size))
-                            elif len(m_array_dim) == 1 and m_byte_size == 1:
-                                # likely string, avoid expansion
-                                members.append(Member(m_name + '[]', m_off, m_array_dim[0]))
-                            else:
-                                # expand array entries
-                                m_array_pos = m_off
-                                m_array_loc = [0] * len(m_array_dim)
-                                while True:
-                                    # location index
-                                    sfx = ''
-                                    for d in range(len(m_array_dim)):
-                                        sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0'))
-
-                                    members.append(Member(m_name + sfx, m_array_pos, m_byte_size))
-
-                                    # advance
-                                    if array_inc(m_array_loc, m_array_dim):
-                                        break
-                                    m_array_pos += m_byte_size
-
-                if struct_gaps and len(members):
-                    # fill gaps in the middle
-                    members = list(sorted(members, key=lambda x: x.off))
-                    last_end = 0
-                    for n in range(len(members)):
-                        member = members[n]
-                        if member.off > last_end:
-                            members.append(Member('*UNKNOWN*', last_end, member.off - last_end))
-                        last_end = member.off + member.size
-
-                if struct_gaps and len(members):
-                    # fill gap at the end
-                    members = list(sorted(members, key=lambda x: x.off))
-                    last = members[-1]
-                    last_end = last.off + last.size
-                    if byte_size > last_end:
-                        members.append(Member('*UNKNOWN*', last_end, byte_size - last_end))
-
+            members = get_struct_members(DIE, size, expand_structs, struct_gaps)
 
             def expand_members(entry, members):
                 if len(members) == 0:

From 7bdee552ce91bf133fb2fb7e00f87f668630bf01 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 20:36:19 +0200
Subject: [PATCH 24/40] elf_mem_map: add declaration position in --map

---
 tools/elf_mem_map | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 26694c80..f29a58cf 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -10,7 +10,7 @@ EEPROM_OFFSET = 0x810000
 FILL_BYTE = b'\0'
 
 
-Entry = namedtuple('Entry', ['name', 'loc', 'size'])
+Entry = namedtuple('Entry', ['name', 'loc', 'size', 'declpos'])
 Member = namedtuple('Member', ['name', 'off', 'size'])
 
 
@@ -160,6 +160,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
 
     grefs = []
     for CU in dwarfinfo.iter_CUs():
+        file_entries = dwarfinfo.line_program_for_CU(CU).header["file_entry"]
+
         for DIE in CU.iter_DIEs():
             # handle only variable types
             if DIE.tag != 'DW_TAG_variable':
@@ -196,6 +198,16 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                 continue
             byte_size = size[1]
 
+            # location of main definition
+            declpos = ''
+            if 'DW_AT_decl_file' in DIE.attributes and \
+               'DW_AT_decl_line' in DIE.attributes:
+                line = DIE.attributes['DW_AT_decl_line'].value
+                fname = DIE.attributes['DW_AT_decl_file'].value
+                if fname and fname - 1 < len(file_entries):
+                    fname = file_entries[fname-1].name.decode('ascii')
+                    declpos = '{}:{}'.format(fname, line)
+
             # fetch array dimensions (if known)
             array_dim = get_array_dims(DIE)
 
@@ -208,17 +220,18 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                 else:
                     for member in members:
                         grefs.append(Entry(entry.name + '.' + member.name,
-                                           entry.loc + member.off, member.size))
+                                           entry.loc + member.off, member.size,
+                                           entry.declpos))
 
             if byte_size == 1 and len(array_dim) > 1:
                 # likely string, remove one dimension
                 byte_size *= array_dim.pop()
             if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
-                expand_members(Entry(name, loc, byte_size), members)
+                expand_members(Entry(name, loc, byte_size, declpos), members)
             elif len(array_dim) == 1 and byte_size == 1:
                 # likely string, avoid expansion
-                grefs.append(Entry(name + '[]', loc, array_dim[0]))
+                grefs.append(Entry(name + '[]', loc, array_dim[0], declpos))
             else:
                 # expand array entries
                 array_pos = loc
@@ -229,7 +242,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                     for d in range(len(array_dim)):
                         sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0'))
 
-                    expand_members(Entry(name + sfx, array_pos, byte_size), members)
+                    expand_members(Entry(name + sfx, array_pos, byte_size, declpos), members)
 
                     # advance
                     if array_inc(array_loc, array_dim):
@@ -316,9 +329,9 @@ def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True):
 
 
 def print_map(grefs):
-    print('OFFSET\tSIZE\tNAME')
+    print('OFFSET\tSIZE\tNAME\tDECLPOS')
     for entry in grefs:
-        print('{:x}\t{}\t{}'.format(entry.loc, entry.size, entry.name))
+        print('{:x}\t{}\t{}\t{}'.format(entry.loc, entry.size, entry.name, entry.declpos))
 
 
 def main():

From 676b925c5f33a0a5f08bbcfb17941068965cb1b6 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 20:56:14 +0200
Subject: [PATCH 25/40] elf_mem_map: cleanup

---
 tools/elf_mem_map | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index f29a58cf..998366e9 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -84,9 +84,7 @@ def get_array_dims(DIE):
     return array_dim
 
 
-def get_struct_members(DIE, size, expand_structs, struct_gaps):
-    if not expand_structs or size[0].tag == 'DW_TAG_pointer_type':
-        return []
+def get_struct_members(DIE, entry, expand_structs, struct_gaps):
     struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type')
     if struct_DIE is None:
         return []
@@ -140,8 +138,8 @@ def get_struct_members(DIE, size, expand_structs, struct_gaps):
         members = list(sorted(members, key=lambda x: x.off))
         last = members[-1]
         last_end = last.off + last.size
-        if size[1] > last_end:
-            members.append(Member('*UNKNOWN*', last_end, byte_size - last_end))
+        if entry.size > last_end:
+            members.append(Member('*UNKNOWN*', last_end, entry.size - last_end))
 
     return members
 
@@ -212,7 +210,11 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
             array_dim = get_array_dims(DIE)
 
             # fetch structure members (one level only)
-            members = get_struct_members(DIE, size, expand_structs, struct_gaps)
+            entry = Entry(name, loc, byte_size, declpos)
+            if not expand_structs or size[0].tag == 'DW_TAG_pointer_type':
+                members = []
+            else:
+                members = get_struct_members(DIE, entry, expand_structs, struct_gaps)
 
             def expand_members(entry, members):
                 if len(members) == 0:
@@ -228,10 +230,11 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                 byte_size *= array_dim.pop()
             if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1):
                 # plain entry
-                expand_members(Entry(name, loc, byte_size, declpos), members)
+                expand_members(entry, members)
             elif len(array_dim) == 1 and byte_size == 1:
                 # likely string, avoid expansion
-                grefs.append(Entry(name + '[]', loc, array_dim[0], declpos))
+                grefs.append(Entry(entry.name + '[]', entry.loc,
+                                   array_dim[0], entry.declpos))
             else:
                 # expand array entries
                 array_pos = loc
@@ -241,9 +244,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
                     sfx = ''
                     for d in range(len(array_dim)):
                         sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0'))
-
-                    expand_members(Entry(name + sfx, array_pos, byte_size, declpos), members)
-
+                    expand_members(Entry(entry.name + sfx, array_pos,
+                                         byte_size, entry.declpos), members)
                     # advance
                     if array_inc(array_loc, array_dim):
                         break

From 54e24036a88c3d9ac758b1413361a8b8ff1232f3 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 21:54:42 +0200
Subject: [PATCH 26/40] elf_mem_map: add qdirstat output for space
 visualization

---
 tools/elf_mem_map | 44 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 998366e9..079adf35 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -4,6 +4,7 @@ import elftools.elf.elffile
 import elftools.dwarf.descriptions
 from collections import namedtuple
 from struct import unpack
+import re
 
 SRAM_OFFSET = 0x800000
 EEPROM_OFFSET = 0x810000
@@ -336,6 +337,43 @@ def print_map(grefs):
         print('{:x}\t{}\t{}\t{}'.format(entry.loc, entry.size, entry.name, entry.declpos))
 
 
+def print_qdirstat(grefs):
+    print('[qdirstat 1.0 cache file]')
+
+    entries = {}
+    for entry in grefs:
+        paths = list(filter(None, re.split(r'[\[\].]', entry.name)))
+        base = entries
+        for i in range(len(paths) - 1):
+            name = paths[i]
+            if name not in base:
+                base[name] = {}
+            base = base[name]
+        name = paths[-1]
+        base[name] = entry.size
+
+    def walker(root, prefix):
+        files = []
+        dirs = []
+
+        for name, entries in root.items():
+            if type(entries) == int:
+                files.append([name, entries])
+            else:
+                dirs.append([name, entries])
+
+        # print files
+        print('D\t{}\t{}\t0x0'.format(prefix, 0))
+        for name, size in files:
+            print('F\t{}\t{}\t0x0'.format(name, size))
+
+        # recurse directories
+        for name, entries in dirs:
+            walker(entries, prefix + '/' + name)
+
+    walker(entries, '/')
+
+
 def main():
     ap = argparse.ArgumentParser(description="""
         Generate a symbol table map starting directly from an ELF
@@ -355,12 +393,16 @@ def main():
     g = ap.add_mutually_exclusive_group(required=True)
     g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code')
     g.add_argument('--map', action='store_true', help='dump global memory map')
+    g.add_argument('--qdirstat', action='store_true',
+                   help='dump qdirstat-compatible size usage map')
     args = ap.parse_args()
 
     grefs = get_elf_globals(args.elf, expand_structs=not args.no_expand_structs)
     grefs = list(sorted(grefs, key=lambda x: x.loc))
-    if args.dump is None:
+    if args.map:
         print_map(grefs)
+    elif args.qdirstat:
+        print_qdirstat(grefs)
     else:
         addr, data = decode_dump(args.dump)
         annotate_refs(grefs, addr, data,

From 9958c449e347578f06c810d058539467b510f437 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 22:29:11 +0200
Subject: [PATCH 27/40] elf_mem_map: remove DWARF version except in help

Fix version to DWARF3, which is what we actually support.
---
 tools/README.md   | 2 +-
 tools/elf_mem_map | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/README.md b/tools/README.md
index 2b4c7b82..02f4813f 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -14,7 +14,7 @@ Requires ``printcore`` from [Pronterface].
 
 ### ``elf_mem_map``
 
-Generate a symbol table map starting directly from an ELF firmware with DWARF2 debugging information (which is the default using the stock board definition).
+Generate a symbol table map starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition).
 
 When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range.
 
diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 079adf35..65a95732 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -377,11 +377,11 @@ def print_qdirstat(grefs):
 def main():
     ap = argparse.ArgumentParser(description="""
         Generate a symbol table map starting directly from an ELF
-        firmware with DWARF2 debugging information.
+        firmware with DWARF3 debugging information.
         When used along with a memory dump obtained from the D2 g-code,
         show the value of each symbol which is within the address range.
     """)
-    ap.add_argument('elf', help='ELF file containing DWARF2 debugging information')
+    ap.add_argument('elf', help='ELF file containing DWARF debugging information')
     ap.add_argument('--no-gaps', action='store_true',
                     help='do not dump memory inbetween known symbols')
     ap.add_argument('--no-expand-structs', action='store_true',

From 9917689fdfe572e2462b3bcc3a69837575959f16 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 22:45:01 +0200
Subject: [PATCH 28/40] tools: update documentation for elf_mem_map

---
 tools/README.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tools/README.md b/tools/README.md
index 02f4813f..e7911501 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -14,9 +14,13 @@ Requires ``printcore`` from [Pronterface].
 
 ### ``elf_mem_map``
 
-Generate a symbol table map starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition).
+Generate a symbol table map with decoded information starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition).
 
-When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range.
+When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range of the dump.
+
+When used with ``--map`` and a single elf file, generate a map consisting of memory location and source location for each statically-addressed variable.
+
+With ``--qdirstat`` and a single elf file, generate a [qdirstat](https://github.com/shundhammer/qdirstat) compatible cache file which can be loaded to inspect memory utilization interactively in a treemap.
 
 This assumes the running firmware generating the dump and the elf file are the same.
 Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module.

From d98e1b1cd98faab573413e6e31e97f2f8198b92d Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sat, 5 Jun 2021 23:03:56 +0200
Subject: [PATCH 29/40] elf_mem_map: uniquify file names in qdirstat output

---
 tools/elf_mem_map | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 65a95732..f3ea03d0 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -350,6 +350,8 @@ def print_qdirstat(grefs):
                 base[name] = {}
             base = base[name]
         name = paths[-1]
+        if name in base:
+            name = '{}_{:x}'.format(entry.name, entry.loc)
         base[name] = entry.size
 
     def walker(root, prefix):

From 8ec4104840bb26f1c96a3268c91953ba6c92ed3b Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Sun, 6 Jun 2021 12:20:50 +0200
Subject: [PATCH 30/40] elf_mem_map: do not output registers in qdirstat output

---
 tools/elf_mem_map | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index f3ea03d0..2a7e8c30 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -6,6 +6,7 @@ from collections import namedtuple
 from struct import unpack
 import re
 
+SRAM_START = 0x200
 SRAM_OFFSET = 0x800000
 EEPROM_OFFSET = 0x810000
 FILL_BYTE = b'\0'
@@ -342,6 +343,10 @@ def print_qdirstat(grefs):
 
     entries = {}
     for entry in grefs:
+        # do not output registers when looking at space usage
+        if entry.loc < SRAM_START:
+            continue
+
         paths = list(filter(None, re.split(r'[\[\].]', entry.name)))
         base = entries
         for i in range(len(paths) - 1):

From 9f40fa68347ca6ace7669d7fe6a2ec08a2e41e3d Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Mon, 14 Jun 2021 16:03:02 +0200
Subject: [PATCH 31/40] elf_mem_map: parse D23 output directly

---
 tools/elf_mem_map | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index 2a7e8c30..b0a7db18 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -4,6 +4,7 @@ import elftools.elf.elffile
 import elftools.dwarf.descriptions
 from collections import namedtuple
 from struct import unpack
+import sys
 import re
 
 SRAM_START = 0x200
@@ -264,12 +265,26 @@ def decode_dump(path):
     buf_addr = None # starting address
     buf_data = None # data
 
-    for line in fd:
-        tokens = line.split(maxsplit=1)
-        if len(tokens) == 0 or tokens[0] == 'ok':
-            break
-        elif len(tokens) < 2 or tokens[0] == 'D2':
+    in_dump = False
+    for line in enumerate(fd):
+        line = (line[0], line[1].rstrip())
+        tokens = line[1].split(maxsplit=1)
+        if not in_dump:
+            if len(tokens) > 0 and tokens[0] in ['D2', 'D23']:
+                in_dump = True
             continue
+        else:
+            if len(tokens) < 1:
+                print('malformed line {}: {}'.format(*line), file=sys.stderr)
+                continue
+            elif tokens[0] == 'ok':
+                break
+            elif tokens[0] == 'reason:':
+                # ignored
+                continue
+            elif not re.match(r'[0-9a-fA-F]', tokens[0]):
+                print('malformed line {}: {}'.format(*line), file=sys.stderr)
+                continue
 
         addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
         data = bytes.fromhex(tokens[1])

From 06eab4ac11f3f62c1f3edf54f872696b949ec7e3 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 16:58:04 +0200
Subject: [PATCH 32/40] Handle XFLASH (D21) and serial (D23) dumps in
 elf_mem_map, add dump2bin

- Uniformly parse D2/D21/D23 dump types.
- Add dump2bin to parse/convert a dump into metadata and binary.
- Move the parsing into it's own module in order to be shared.
---
 tools/dump2bin    |  59 +++++++++++++++++
 tools/elf_mem_map |  75 ++++-----------------
 tools/lib/avr.py  |   4 ++
 tools/lib/dump.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 242 insertions(+), 61 deletions(-)
 create mode 100755 tools/dump2bin
 create mode 100644 tools/lib/avr.py
 create mode 100644 tools/lib/dump.py

diff --git a/tools/dump2bin b/tools/dump2bin
new file mode 100755
index 00000000..c7ab3603
--- /dev/null
+++ b/tools/dump2bin
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+import argparse
+import os, sys
+
+from lib.dump import decode_dump
+
+
+def main():
+    # parse the arguments
+    ap = argparse.ArgumentParser(description="""
+        Parse and decode a memory dump obtained from the D2/D21/D23 g-code
+        into readable metadata and binary. The output binary is padded and
+        extended to fit the original address range.
+    """)
+    ap.add_argument('-i', dest='info', action='store_true',
+                    help='display crash info only')
+    ap.add_argument('dump')
+    ap.add_argument('output', nargs='?')
+    args = ap.parse_args()
+
+    # decode the dump data
+    dump = decode_dump(args.dump)
+    if dump is None:
+        return os.EX_DATAERR
+
+    # output descriptors
+    if args.info:
+        o_fd = None
+        o_md = sys.stdout
+    elif args.output is None:
+        o_fd = sys.stdout.buffer
+        o_md = sys.stderr
+    else:
+        o_fd = open(args.output, 'wb')
+        o_md = sys.stdout
+
+    # output binary
+    if o_fd:
+        o_fd.write(dump.data)
+        o_fd.close()
+
+    # metadata
+    print('   dump type: {typ}\n'
+          'crash reason: {reason}\n'
+          '   registers: {regs}\n'
+          '          PC: {pc}\n'
+          '          SP: {sp}\n'
+          '      ranges: {ranges}'.format(
+              typ=dump.typ,
+              reason=dump.reason.name if dump.reason is not None else 'N/A',
+              regs=dump.regs,
+              pc='{:#x}'.format(dump.pc) if dump.pc is not None else 'N/A',
+              sp='{:#x}'.format(dump.sp) if dump.sp is not None else 'N/A',
+              ranges=str(dump.ranges)),
+          file=o_md)
+
+
+if __name__ == '__main__':
+    exit(main())
diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index b0a7db18..cb6e423c 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -4,13 +4,10 @@ import elftools.elf.elffile
 import elftools.dwarf.descriptions
 from collections import namedtuple
 from struct import unpack
-import sys
-import re
+import os
 
-SRAM_START = 0x200
-SRAM_OFFSET = 0x800000
-EEPROM_OFFSET = 0x810000
-FILL_BYTE = b'\0'
+from lib.dump import decode_dump
+from lib.avr import *
 
 
 Entry = namedtuple('Entry', ['name', 'loc', 'size', 'declpos'])
@@ -257,59 +254,6 @@ def get_elf_globals(path, expand_structs, struct_gaps=True):
     return grefs
 
 
-def decode_dump(path):
-    fd = open(path, 'r')
-    if fd is None:
-        return None
-
-    buf_addr = None # starting address
-    buf_data = None # data
-
-    in_dump = False
-    for line in enumerate(fd):
-        line = (line[0], line[1].rstrip())
-        tokens = line[1].split(maxsplit=1)
-        if not in_dump:
-            if len(tokens) > 0 and tokens[0] in ['D2', 'D23']:
-                in_dump = True
-            continue
-        else:
-            if len(tokens) < 1:
-                print('malformed line {}: {}'.format(*line), file=sys.stderr)
-                continue
-            elif tokens[0] == 'ok':
-                break
-            elif tokens[0] == 'reason:':
-                # ignored
-                continue
-            elif not re.match(r'[0-9a-fA-F]', tokens[0]):
-                print('malformed line {}: {}'.format(*line), file=sys.stderr)
-                continue
-
-        addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
-        data = bytes.fromhex(tokens[1])
-
-        if buf_addr is None:
-            buf_addr = addr
-            buf_data = data
-        else:
-            # grow buffer as needed
-            if addr < buf_addr:
-                buf_data = FILL_BYTE * (buf_addr - addr)
-                buf_addr = addr
-            addr_end = addr + len(data)
-            buf_end = buf_addr + len(buf_data)
-            if addr_end > buf_end:
-                buf_data += FILL_BYTE * (addr_end - buf_end)
-
-            # replace new part
-            rep_start = addr - buf_addr
-            rep_end = rep_start + len(data)
-            buf_data = buf_data[:rep_start] + data + buf_data[rep_end:]
-
-    return (buf_addr, buf_data)
-
-
 def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True):
     last_end = None
     for entry in grefs:
@@ -426,8 +370,17 @@ def main():
     elif args.qdirstat:
         print_qdirstat(grefs)
     else:
-        addr, data = decode_dump(args.dump)
-        annotate_refs(grefs, addr, data,
+        # fetch the memory data
+        dump = decode_dump(args.dump)
+        if dump is None:
+            return os.EX_DATAERR
+
+        # strip padding, if present
+        addr_start = dump.ranges[0][0]
+        addr_end = dump.ranges[-1][0]+dump.ranges[-1][1]
+        data = dump.data[addr_start:addr_end]
+
+        annotate_refs(grefs, addr_start, data,
                       width=args.name_width,
                       gaps=not args.no_gaps,
                       overlaps=args.overlaps)
diff --git a/tools/lib/avr.py b/tools/lib/avr.py
new file mode 100644
index 00000000..57f34479
--- /dev/null
+++ b/tools/lib/avr.py
@@ -0,0 +1,4 @@
+SRAM_START = 0x200
+SRAM_SIZE = 0x2000
+SRAM_OFFSET = 0x800000
+EEPROM_OFFSET = 0x810000
diff --git a/tools/lib/dump.py b/tools/lib/dump.py
new file mode 100644
index 00000000..1c541a2a
--- /dev/null
+++ b/tools/lib/dump.py
@@ -0,0 +1,165 @@
+import sys
+import re
+import enum
+import struct
+from . import avr
+
+
+FILL_BYTE = b'\0' # used to fill memory gaps in the dump
+
+class CrashReason(enum.IntEnum):
+    MANUAL = 0
+    STACK_ERROR = 1
+    WATCHDOG = 2
+    BAD_ISR = 3
+
+class Dump():
+    def __init__(self, typ, reason, regs, pc, sp, data, ranges):
+        self.typ = typ
+        self.reason = reason
+        self.regs = regs
+        self.pc = pc
+        self.sp = sp
+        self.data = data
+        self.ranges = ranges
+
+
+# expand the buffer identified by addr+data to fill the region start+size
+def region_expand(addr, data, start, size):
+    if start < addr:
+        data = FILL_BYTE * (addr - start) + data
+        addr = start
+    end = start + size
+    data_end = addr + len(data)
+    if end > data_end:
+        data += FILL_BYTE * (data_end - end)
+    return addr, data
+
+
+def merge_ranges(ranges):
+    ranges = list(sorted(ranges, key=lambda x: x[0]))
+    if len(ranges) < 2:
+        return ranges
+
+    ret = [ranges[0]]
+    for cur in ranges[1:]:
+        last = ret[-1]
+        last_end = last[0] + last[1]
+        if last_end < cur[0]:
+            ret.append(cur)
+        else:
+            cur_end = cur[0] + cur[1]
+            last = (last[0], max(last_end, cur_end) - last[0])
+            ret[-1] = last
+    return ret
+
+
+def decode_dump(path):
+    fd = open(path, 'r')
+    if fd is None:
+        return None
+
+    buf_addr = None # starting address
+    buf_data = None # data
+
+    typ = None      # dump type
+    reason = None   # crash reason
+    regs = None     # registers present
+    pc = None       # PC address
+    sp = None       # SP address
+    ranges = []     # dumped ranges
+
+    in_dump = False
+    for line in enumerate(fd):
+        line = (line[0], line[1].rstrip())
+        tokens = line[1].split(maxsplit=1)
+
+        def line_error():
+            print('malformed line {}: {}'.format(*line), file=sys.stderr)
+
+        # handle metadata
+        if not in_dump:
+            if len(tokens) > 0 and tokens[0] in ['D2', 'D21', 'D23']:
+                in_dump = True
+                typ = tokens[0]
+            continue
+        else:
+            if len(tokens) == 0:
+                line_error()
+                continue
+            elif tokens[0] == 'ok':
+                break
+            elif tokens[0] == 'error:' and len(tokens) == 2:
+                values = tokens[1].split(' ')
+                if typ == 'D23' and len(values) >= 3:
+                    reason = CrashReason(int(values[0], 0))
+                    pc = int(values[1], 0)
+                    sp = int(values[2], 0)
+                else:
+                    line_error()
+                continue
+            elif len(tokens) != 2 or not re.match(r'^[0-9a-fA-F]+$', tokens[0]):
+                line_error()
+                continue
+
+        # decode hex data
+        addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big')
+        data = bytes.fromhex(tokens[1])
+        ranges.append((addr, len(data)))
+
+        if buf_addr is None:
+            buf_addr = addr
+            buf_data = data
+        else:
+            # grow buffer as needed
+            buf_addr, buf_data = region_expand(buf_addr, buf_data,
+                                               addr, len(data))
+
+            # replace new part
+            rep_start = addr - buf_addr
+            rep_end = rep_start + len(data)
+            buf_data = buf_data[:rep_start] + data + buf_data[rep_end:]
+
+    # merge continuous ranges
+    ranges = merge_ranges(ranges)
+
+    if typ == 'D2':
+        # D2 doesn't guarantee registers to be present
+        regs = len(ranges) > 0 and \
+            ranges[0][0] == 0 and \
+            ranges[0][1] >= avr.SRAM_START
+
+        # fill to fit for easy loading
+        buf_addr, buf_data = region_expand(
+            buf_addr, buf_data, 0, avr.SRAM_START + avr.SRAM_SIZE)
+
+    elif typ == 'D23':
+        # check if the dump is complete
+        if len(ranges) != 1 or ranges[0][0] != 0 or \
+           ranges[0][1] != avr.SRAM_START + avr.SRAM_SIZE:
+            print('warning: incomplete D23 dump', file=sys.stderr)
+        else:
+            regs = True
+        if reason is None:
+            print('warning: no error line in D23', file=sys.stderr)
+
+    elif typ == 'D21':
+        if len(ranges) != 1 or len(buf_data) != (avr.SRAM_START + avr.SRAM_SIZE + 256):
+            print('error: incomplete D21 dump', file=sys.stderr)
+            return None
+
+        # decode the header structure
+        magic, regs_present, crash_reason, pc, sp = struct.unpack('<LBBLH', buf_data[0:12])
+        if magic != 0x55525547:
+            print('error: invalid dump header in D21', file=sys.stderr)
+            return None
+
+        regs = bool(regs_present)
+        reason = CrashReason(crash_reason)
+
+        # extract the data section
+        buf_addr = 0
+        buf_data = buf_data[256:]
+        ranges[0] = (0, len(buf_data))
+
+    return Dump(typ, reason, regs, pc, sp, buf_data, ranges)

From 500515eb0a72d66cecd5afd497c5da90ff207301 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 17:02:22 +0200
Subject: [PATCH 33/40] Add GDB utility functions to load/inspect binary dumps

---
 tools/utils.gdb | 82 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 tools/utils.gdb

diff --git a/tools/utils.gdb b/tools/utils.gdb
new file mode 100644
index 00000000..49548036
--- /dev/null
+++ b/tools/utils.gdb
@@ -0,0 +1,82 @@
+# -*- gdb-script -*-
+
+define load_dump
+  restore $arg0 binary 0x800000
+  set $pc = (((unsigned long)$arg1) - 2) << 1
+  set $sp = $arg2
+  where
+end
+
+document load_dump
+Load a crash dump, setup PC/SP and show the current backtrace
+Usage: load_dump <file> <pc-addr> <sp-addr>
+end
+
+
+define sp_skip
+  if $argc == 0
+    set $shift = 3
+  else
+    set $shift = $arg0
+  end
+  set $new_pc = ((((unsigned long)*(uint8_t*)($sp+$shift+1)) << 16) + \
+                 (((unsigned long)*(uint8_t*)($sp+$shift+2)) << 8) + \
+                 (((unsigned long)*(uint8_t*)($sp+$shift+3)) << 0)) << 1
+  set $new_sp = $sp+$shift+3
+  select-frame 0
+  set $saved_pc = $pc
+  set $saved_sp = $sp
+  set $pc = $new_pc
+  set $sp = $new_sp
+  where
+end
+
+document sp_skip
+TODO
+end
+
+
+define sp_restore
+  select-frame 0
+  set $pc = $saved_pc
+  set $sp = $saved_sp
+  where
+end
+
+document sp_restore
+TODO
+end
+
+
+define sp_test
+  sp_skip $arg0
+  set $pc = $saved_pc
+  set $sp = $saved_sp
+end
+
+document sp_test
+TODO
+end
+
+
+define sp_scan
+  dont-repeat
+
+  if $argc == 0
+    set $sp_end = 0x802200
+  else
+    set $sp_end = $arg0
+  end
+
+  set $sp_pos = $sp
+  while $sp_pos < ($sp_end-4)
+    set $sp_off = $sp_pos - $sp
+    printf "**** scanning %#x (+%u) ****\n", $sp_pos, $sp_off
+    sp_test $sp_off
+    set $sp_pos += 1
+  end
+end
+
+document sp_scan
+TODO
+end

From 11a6ac2f4fc41b1231efec63fdfe4746fb994710 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 17:08:32 +0200
Subject: [PATCH 34/40] dump parsing: refuse to continue on incomplete D23
 dumps

We should try harder to handle incomplete D21/D23 dumps in the future,
but until D21 handled too, bail in D23 as well.
---
 tools/lib/dump.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tools/lib/dump.py b/tools/lib/dump.py
index 1c541a2a..8d6b1059 100644
--- a/tools/lib/dump.py
+++ b/tools/lib/dump.py
@@ -137,9 +137,10 @@ def decode_dump(path):
         # check if the dump is complete
         if len(ranges) != 1 or ranges[0][0] != 0 or \
            ranges[0][1] != avr.SRAM_START + avr.SRAM_SIZE:
-            print('warning: incomplete D23 dump', file=sys.stderr)
-        else:
-            regs = True
+            print('error: incomplete D23 dump', file=sys.stderr)
+            return None
+
+        regs = True
         if reason is None:
             print('warning: no error line in D23', file=sys.stderr)
 

From 43b9a2d3df162930752c0b9d128ecfceac905a9c Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 17:12:24 +0200
Subject: [PATCH 35/40] dump: do not hard-code constants

---
 tools/lib/dump.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/lib/dump.py b/tools/lib/dump.py
index 8d6b1059..5d2ea724 100644
--- a/tools/lib/dump.py
+++ b/tools/lib/dump.py
@@ -5,7 +5,8 @@ import struct
 from . import avr
 
 
-FILL_BYTE = b'\0' # used to fill memory gaps in the dump
+FILL_BYTE  = b'\0'      # used to fill memory gaps in the dump
+DUMP_MAGIC = 0x55525547 # XFLASH dump magic
 
 class CrashReason(enum.IntEnum):
     MANUAL = 0
@@ -151,7 +152,7 @@ def decode_dump(path):
 
         # decode the header structure
         magic, regs_present, crash_reason, pc, sp = struct.unpack('<LBBLH', buf_data[0:12])
-        if magic != 0x55525547:
+        if magic != DUMP_MAGIC:
             print('error: invalid dump header in D21', file=sys.stderr)
             return None
 

From 8455c8e585cec53d80b0a9a25c1930a93f067a48 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 17:56:42 +0200
Subject: [PATCH 36/40] tools: Add/fix documentation

---
 tools/README.md   | 4 ++++
 tools/elf_mem_map | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tools/README.md b/tools/README.md
index e7911501..30648d0d 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -25,6 +25,10 @@ With ``--qdirstat`` and a single elf file, generate a [qdirstat](https://github.
 This assumes the running firmware generating the dump and the elf file are the same.
 Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module.
 
+### ``dump2bin``
+
+Parse and decode a memory dump obtained from the D2/D21/D23 g-code into readable metadata and binary. The output binary is padded and extended to fit the original address range.
+
 ### ``update_eeprom``
 
 Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer.
diff --git a/tools/elf_mem_map b/tools/elf_mem_map
index cb6e423c..df3fc0f5 100755
--- a/tools/elf_mem_map
+++ b/tools/elf_mem_map
@@ -344,7 +344,7 @@ def main():
     ap = argparse.ArgumentParser(description="""
         Generate a symbol table map starting directly from an ELF
         firmware with DWARF3 debugging information.
-        When used along with a memory dump obtained from the D2 g-code,
+        When used along with a memory dump obtained from the D2/D21/D23 g-code,
         show the value of each symbol which is within the address range.
     """)
     ap.add_argument('elf', help='ELF file containing DWARF debugging information')

From c79b1dcbfa6b112383543198681a7f827b0b61f0 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 17:59:41 +0200
Subject: [PATCH 37/40] tools: add dump_crash to recover XFLASH crash dumps

---
 tools/README.md  |  5 +++++
 tools/dump_crash | 17 +++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100755 tools/dump_crash

diff --git a/tools/README.md b/tools/README.md
index 30648d0d..63c1731d 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -12,6 +12,11 @@ Requires ``printcore`` from [Pronterface].
 Dump the content of the entire SRAM using the D2 command.
 Requires ``printcore`` from [Pronterface].
 
+### ``dump_crash``
+
+Dump the content of the last crash dump on MK3+ printers using D21.
+Requires ``printcore`` from [Pronterface].
+
 ### ``elf_mem_map``
 
 Generate a symbol table map with decoded information starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition).
diff --git a/tools/dump_crash b/tools/dump_crash
new file mode 100755
index 00000000..f74e8bf2
--- /dev/null
+++ b/tools/dump_crash
@@ -0,0 +1,17 @@
+#!/bin/sh
+prg=$(basename "$0")
+port="$1"
+if [ -z "$port" -o "$port" = "-h" ]
+then
+  echo "usage: $0 <port>" >&2
+  echo "Connect to <port> and dump the content of last crash using D21 to stdout" >&2
+  exit 1
+fi
+
+set -e
+tmp=$(mktemp)
+trap "rm -f \"$tmp\"" EXIT
+
+echo D21 > "$tmp"
+printcore -v "$port" "$tmp" 2>&1 | \
+    sed -ne '/^RECV: D21 /,/RECV: ok$/s/^RECV: //p'

From a697d00647d0ca85da8d0128850b4edef578869c Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Tue, 22 Jun 2021 18:01:21 +0200
Subject: [PATCH 38/40] tools: add __pycache__ to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index dae307af..26411502 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 .project
 .cproject
 Debug
+__pycache__
 Firmware/Configuration_prusa.h
 Firmware/Doc
 /Firmware/.vs/Firmware/v14

From 1b22aac9fcc68662cf22129bafc02c8197471426 Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Thu, 1 Jul 2021 13:16:02 +0200
Subject: [PATCH 39/40] tools: add xfimg2dump

---
 tools/README.md   |  4 ++++
 tools/lib/dump.py |  6 ++++--
 tools/xfimg2dump  | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100755 tools/xfimg2dump

diff --git a/tools/README.md b/tools/README.md
index 63c1731d..50516dac 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -34,6 +34,10 @@ Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) modu
 
 Parse and decode a memory dump obtained from the D2/D21/D23 g-code into readable metadata and binary. The output binary is padded and extended to fit the original address range.
 
+### ``xfimg2dump``
+
+Extract a crash dump from an external flash image and output the same format produced by the D21 g-code.
+
 ### ``update_eeprom``
 
 Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer.
diff --git a/tools/lib/dump.py b/tools/lib/dump.py
index 5d2ea724..7ce75606 100644
--- a/tools/lib/dump.py
+++ b/tools/lib/dump.py
@@ -5,8 +5,10 @@ import struct
 from . import avr
 
 
-FILL_BYTE  = b'\0'      # used to fill memory gaps in the dump
-DUMP_MAGIC = 0x55525547 # XFLASH dump magic
+FILL_BYTE   = b'\0'      # used to fill memory gaps in the dump
+DUMP_MAGIC  = 0x55525547 # XFLASH dump magic
+DUMP_OFFSET = 0x3d000    # XFLASH dump offset
+DUMP_SIZE   = 0x2300     # XFLASH dump size
 
 class CrashReason(enum.IntEnum):
     MANUAL = 0
diff --git a/tools/xfimg2dump b/tools/xfimg2dump
new file mode 100755
index 00000000..e1fe75ac
--- /dev/null
+++ b/tools/xfimg2dump
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import argparse
+import struct
+import os, sys
+
+from lib.dump import DUMP_MAGIC, DUMP_OFFSET, DUMP_SIZE
+
+
+def error(msg):
+    print(msg, file=sys.stderr)
+
+def main():
+    # parse the arguments
+    ap = argparse.ArgumentParser(description="""
+        Extract a crash dump from an external flash image and output
+        the same format produced by the D21 g-code.
+    """)
+    ap.add_argument('image')
+    args = ap.parse_args()
+
+    # read the image
+    off = DUMP_OFFSET
+    with open(args.image, 'rb') as fd:
+        fd.seek(off)
+        data = fd.read(DUMP_SIZE)
+    if len(data) != DUMP_SIZE:
+        error('incorrect image size')
+        return os.EX_DATAERR
+
+    # check for magic header
+    magic, = struct.unpack('<L', data[:4])
+    if magic != DUMP_MAGIC:
+        error('invalid dump magic or no dump')
+        return os.EX_DATAERR
+
+    # output D21 dump
+    print('D21 - read crash dump', end='')
+    for i in range(len(data)):
+        if i % 16 == 0:
+            print('\n{:06x} '.format(off + i), end='')
+        print(' {:02x}'.format(data[i]), end='')
+    print('\nok')
+
+
+if __name__ == '__main__':
+    exit(main())

From 4f403804745565a2e969753a0861cb7c374d43cd Mon Sep 17 00:00:00 2001
From: Yuri D'Elia <wavexx@thregr.org>
Date: Fri, 2 Jul 2021 20:57:12 +0200
Subject: [PATCH 40/40] tools: document functions in utils.gdb

---
 tools/utils.gdb | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tools/utils.gdb b/tools/utils.gdb
index 49548036..1d2922e1 100644
--- a/tools/utils.gdb
+++ b/tools/utils.gdb
@@ -9,7 +9,7 @@ end
 
 document load_dump
 Load a crash dump, setup PC/SP and show the current backtrace
-Usage: load_dump <file> <pc-addr> <sp-addr>
+Usage: load_dump <file> <PC-addr> <SP-addr>
 end
 
 
@@ -32,7 +32,9 @@ define sp_skip
 end
 
 document sp_skip
-TODO
+Decode the PC address at SP+offset, then show the resulting stack.
+The default (and minimum) offset is 3.
+Usage: sp_skip [off]
 end
 
 
@@ -44,7 +46,8 @@ define sp_restore
 end
 
 document sp_restore
-TODO
+Undo an sp_skip move (restore existing PC/SP positions)
+Usage: sp_restore
 end
 
 
@@ -55,7 +58,9 @@ define sp_test
 end
 
 document sp_test
-TODO
+Attempt to decode the PC address at SP+offset, then show the resulting stack.
+The default (and minimum) offset is 3.
+Usage: sp_test [off]
 end
 
 
@@ -78,5 +83,7 @@ define sp_scan
 end
 
 document sp_scan
-TODO
+Attempt to decode PC at any location starting from the SP+3 and up to SP-end
+(by default the end of the SRAM) and show the resulting stack at all locations.
+Usage: sp_scan [SP-end]
 end