From 4580b8a78cd3e3cde543be1269348c8be1f825cc Mon Sep 17 00:00:00 2001 From: "D.R.racer" Date: Fri, 16 Jul 2021 07:37:11 +0200 Subject: [PATCH 01/40] Version changed (3.10.1 build 4587) --- Firmware/Configuration.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Firmware/Configuration.h b/Firmware/Configuration.h index efdc63ee..76daa96c 100644 --- a/Firmware/Configuration.h +++ b/Firmware/Configuration.h @@ -18,10 +18,10 @@ extern PGM_P sPrinterName; // Firmware version #define FW_MAJOR 3 #define FW_MINOR 10 -#define FW_REVISION 0 -#define FW_VERSION STR(FW_MAJOR) "." STR(FW_MINOR) "." STR(FW_REVISION) +#define FW_REVISION 1 +#define FW_VERSION STR(FW_MAJOR) "." STR(FW_MINOR) "." STR(FW_REVISION) "-RC1" -#define FW_COMMIT_NR 4481 +#define FW_COMMIT_NR 4587 // FW_VERSION_UNKNOWN means this is an unofficial build. // The firmware should only be checked into github with this symbol. From 1095b265702684acdfd81518792f3401fc60f838 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 25 May 2021 00:26:32 +0200 Subject: [PATCH 02/40] Add several low-level debugging tools --- tools/README.md | 37 +++++++++++ tools/dump_eeprom | 17 +++++ tools/dump_sram | 17 +++++ tools/elf_mem_map | 153 ++++++++++++++++++++++++++++++++++++++++++++ tools/noreset | 12 ++++ tools/update_eeprom | 54 ++++++++++++++++ 6 files changed, 290 insertions(+) create mode 100644 tools/README.md create mode 100755 tools/dump_eeprom create mode 100755 tools/dump_sram create mode 100755 tools/elf_mem_map create mode 100755 tools/noreset create mode 100755 tools/update_eeprom diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..2b4c7b82 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,37 @@ +# Host debugging tools for Prusa MK3 firmware + +## Tools + +### ``dump_eeprom`` + +Dump the content of the entire EEPROM using the D3 command. +Requires ``printcore`` from [Pronterface]. + +### ``dump_sram`` + +Dump the content of the entire SRAM using the D2 command. +Requires ``printcore`` from [Pronterface]. + +### ``elf_mem_map`` + +Generate a symbol table map starting directly from an ELF firmware with DWARF2 debugging information (which is the default using the stock board definition). + +When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range. + +This assumes the running firmware generating the dump and the elf file are the same. +Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module. + +### ``update_eeprom`` + +Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer. + +Given two EEPROM dumps, produces only the required instructions needed to update the contents from the first to the second. This is currently quite crude and assumes dumps are aligned (starting from the same address or same stride). + +Optionally writes the instructions to the specified port (requires ``printcore`` from [Pronterface]). + +### ``noreset`` + +Set the required TTY flags on the specified port to avoid reset-on-connect for *subsequent* requests (issuing this command might still cause the printer to reset). + + +[Pronterface]: https://github.com/kliment/Printrun diff --git a/tools/dump_eeprom b/tools/dump_eeprom new file mode 100755 index 00000000..5e546416 --- /dev/null +++ b/tools/dump_eeprom @@ -0,0 +1,17 @@ +#!/bin/sh +prg=$(basename "$0") +port="$1" +if [ -z "$port" -o "$port" = "-h" ] +then + echo "usage: $0 " >&2 + echo "Connect to and dump the content of the EEPROM using D3 to stdout" >&2 + exit 1 +fi + +set -e +tmp=$(mktemp) +trap "rm -f \"$tmp\"" EXIT + +echo D3 > "$tmp" +printcore -v "$port" "$tmp" 2>&1 | \ + sed -ne '/^RECV: D3 /,/RECV: ok$/s/^RECV: //p' diff --git a/tools/dump_sram b/tools/dump_sram new file mode 100755 index 00000000..93ef4e92 --- /dev/null +++ b/tools/dump_sram @@ -0,0 +1,17 @@ +#!/bin/sh +prg=$(basename "$0") +port="$1" +if [ -z "$port" -o "$port" = "-h" ] +then + echo "usage: $0 " >&2 + echo "Connect to and dump the content of the SRAM using D2 to stdout" >&2 + exit 1 +fi + +set -e +tmp=$(mktemp) +trap "rm -f \"$tmp\"" EXIT + +echo D2 > "$tmp" +printcore -v "$port" "$tmp" 2>&1 | \ + sed -ne '/^RECV: D2 /,/RECV: ok$/s/^RECV: //p' diff --git a/tools/elf_mem_map b/tools/elf_mem_map new file mode 100755 index 00000000..46f1e1c1 --- /dev/null +++ b/tools/elf_mem_map @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +import argparse +import elftools.elf.elffile +import elftools.dwarf.descriptions +from struct import unpack + +SRAM_OFFSET = 0x800000 +EEPROM_OFFSET = 0x810000 +FILL_BYTE = b'\0' + + +def get_elf_globals(path): + fd = open(path, "rb") + if fd is None: + return + elffile = elftools.elf.elffile.ELFFile(fd) + if elffile is None or not elffile.has_dwarf_info(): + return + + # probably not needed, since we're decoding expressions manually + elftools.dwarf.descriptions.set_global_machine_arch(elffile.get_machine_arch()) + dwarfinfo = elffile.get_dwarf_info() + + grefs = [] + for CU in dwarfinfo.iter_CUs(): + for DIE in CU.iter_DIEs(): + # handle only variable types + if DIE.tag != 'DW_TAG_variable': + continue + if 'DW_AT_name' not in DIE.attributes: + continue + if 'DW_AT_location' not in DIE.attributes: + continue + if 'DW_AT_type' not in DIE.attributes: + continue + + # handle locations encoded directly as DW_OP_addr (leaf globals) + at_loc = DIE.attributes['DW_AT_location'] + if at_loc.form != 'DW_FORM_block1' or at_loc.value[0] != 3: + continue + loc = (at_loc.value[1]) + (at_loc.value[2] << 8) \ + + (at_loc.value[3] << 16) + (at_loc.value[4] << 24) + if loc < SRAM_OFFSET or loc >= EEPROM_OFFSET: + continue + loc -= SRAM_OFFSET + + # variable name + name = DIE.attributes['DW_AT_name'].value.decode('ascii') + + # recurse on type to find the leaf definition + type_DIE = DIE + while 'DW_AT_type' in type_DIE.attributes: + type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') + byte_size = type_DIE.attributes.get('DW_AT_byte_size') + if byte_size is None: + continue + size = byte_size.value + + grefs.append([name, loc, size]) + + return grefs + + +def decode_dump(path): + fd = open(path, 'r') + if fd is None: + return None + + buf_addr = None # starting address + buf_data = None # data + + for line in fd: + tokens = line.split(maxsplit=1) + if len(tokens) == 0 or tokens[0] == 'ok': + break + elif len(tokens) < 2 or tokens[0] == 'D2': + continue + + addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big') + data = bytes.fromhex(tokens[1]) + + if buf_addr is None: + buf_addr = addr + buf_data = data + else: + # grow buffer as needed + if addr < buf_addr: + buf_data = FILL_BYTE * (buf_addr - addr) + buf_addr = addr + addr_end = addr + len(data) + buf_end = buf_addr + len(buf_data) + if addr_end > buf_end: + buf_data += FILL_BYTE * (addr_end - buf_end) + + # replace new part + rep_start = addr - buf_addr + rep_end = rep_start + len(data) + buf_data = buf_data[:rep_start] + data + buf_data[rep_end:] + + return (buf_addr, buf_data) + + +def annotate_refs(grefs, addr, data, width=45): + for name, loc, size in grefs: + if loc < addr: + continue + if loc + size > addr + len(data): + continue + + pos = loc-addr + buf = data[pos:pos+size] + + buf_repr = '' + if len(buf) in [1, 2, 4]: + # attempt to decode as integers + buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10) + if len(buf) in [4, 8]: + # attempt to decode as floats + buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0]) + + print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size, + buf_repr, buf.hex())) + + +def print_map(grefs): + print('OFFSET\tSIZE\tNAME') + for name, loc, size in grefs: + print('{:x}\t{}\t{}'.format(loc, size, name)) + + +def main(): + ap = argparse.ArgumentParser(description=""" + Generate a symbol table map starting directly from an ELF + firmware with DWARF2 debugging information. + When used along with a memory dump obtained from the D2 g-code, + show the value of each symbol which is within the address range. + """) + ap.add_argument('elf', help='ELF file containing DWARF2 debugging information') + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') + g.add_argument('--map', action='store_true', help='dump global memory map') + args = ap.parse_args() + + grefs = get_elf_globals(args.elf) + grefs = list(sorted(grefs, key=lambda x: x[1])) + if args.dump is None: + print_map(grefs) + else: + addr, data = decode_dump(args.dump) + annotate_refs(grefs, addr, data) + +if __name__ == '__main__': + exit(main()) diff --git a/tools/noreset b/tools/noreset new file mode 100755 index 00000000..3fd32416 --- /dev/null +++ b/tools/noreset @@ -0,0 +1,12 @@ +#!/bin/sh +prg=$(basename "$0") +port="$1" +if [ -z "$port" -o "$port" = "-h" ] +then + echo "usage: $0 " >&2 + echo "Set TTY flags on to avoid reset-on-connect" >&2 + exit 1 +fi + +set -e +stty -F "$port" -hup diff --git a/tools/update_eeprom b/tools/update_eeprom new file mode 100755 index 00000000..a72f9d71 --- /dev/null +++ b/tools/update_eeprom @@ -0,0 +1,54 @@ +#!/bin/sh +prg=$(basename "$0") + +# parse arguments +while getopts f:h optname +do + case $optname in + f) port="$OPTARG" ;; + *) help=1 ;; + esac +done +shift `expr $OPTIND - 1` + +old="$1" +new="$2" + +if [ -z "$old" -o "$help" = "-h" -o "$#" -gt 2 ] +then + echo "usage: $0 [-f ] []" >&2 + echo "Convert to instructions to update instructions." >&2 + echo "With , generate instructions to update EEPROM changes only." >&2 + echo "Optionally write such changes directly if if given." >&2 + exit 1 +fi + +set -e +instr=$(mktemp) +trap "rm -f \"$instr\"" EXIT + +convert() +{ + sed -ne 's/^\([0-9a-f]\{4\}\) \([0-9a-f ]*\)$/D3 Ax\1 C16 X\2/p' "$@" +} + +if [ -z "$new" ]; then + # convert the instructions to updates + convert "$old" > "$instr" +else + tmp1=$(mktemp) + tmp2=$(mktemp) + trap "rm -f \"$tmp1\" \"$tmp2\"" EXIT + + convert "$old" > "$tmp1" + convert "$new" > "$tmp2" + + comm -13 "$tmp1" "$tmp2" > "$instr" +fi + +# write the instructions if requested +if [ -z "$port" ]; then + cat "$instr" +else + printcore -v "$port" "$instr" +fi From 4c6339ac4697e99e6190a8603a746759e6fe4c81 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 17:01:29 +0200 Subject: [PATCH 03/40] elf_mem_map: decode correctly void pointers --- tools/elf_mem_map | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 46f1e1c1..a4301f97 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -47,11 +47,15 @@ def get_elf_globals(path): # variable name name = DIE.attributes['DW_AT_name'].value.decode('ascii') - # recurse on type to find the leaf definition + # recurse on type to find the final storage definition type_DIE = DIE - while 'DW_AT_type' in type_DIE.attributes: + byte_size = None + while True: + if 'DW_AT_byte_size' in type_DIE.attributes: + byte_size = type_DIE.attributes.get('DW_AT_byte_size') + if 'DW_AT_type' not in type_DIE.attributes: + break type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') - byte_size = type_DIE.attributes.get('DW_AT_byte_size') if byte_size is None: continue size = byte_size.value From c321ba482137810ed6832fdb7a2ceb93a03b3590 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 17:15:21 +0200 Subject: [PATCH 04/40] elf_mem_map: also dump gaps between known regions --- tools/elf_mem_map | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index a4301f97..69f08428 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -104,7 +104,9 @@ def decode_dump(path): return (buf_addr, buf_data) -def annotate_refs(grefs, addr, data, width=45): +def annotate_refs(grefs, addr, data, width=45, gaps=True): + last_pos = None + for name, loc, size in grefs: if loc < addr: continue @@ -112,7 +114,8 @@ def annotate_refs(grefs, addr, data, width=45): continue pos = loc-addr - buf = data[pos:pos+size] + end_pos = pos + size + buf = data[pos:end_pos] buf_repr = '' if len(buf) in [1, 2, 4]: @@ -122,8 +125,16 @@ def annotate_refs(grefs, addr, data, width=45): # attempt to decode as floats buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0]) + if gaps and last_pos is not None and last_pos < pos: + # decode gaps + gap_size = pos - last_pos + gap_buf = data[last_pos:pos] + print('{:04x} {} {:4} R:{}'.format(last_pos, "*UNKNOWN*".ljust(width), + gap_size, gap_buf.hex())) + print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size, buf_repr, buf.hex())) + last_pos = end_pos def print_map(grefs): From f2192dc5e6a97b04415b1d8f6592d3ce9317a601 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 17:24:11 +0200 Subject: [PATCH 05/40] elf_mem_dump: fix unknown address --- tools/elf_mem_map | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 69f08428..ddfd5706 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -105,8 +105,7 @@ def decode_dump(path): def annotate_refs(grefs, addr, data, width=45, gaps=True): - last_pos = None - + last_end = None for name, loc, size in grefs: if loc < addr: continue @@ -125,16 +124,16 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True): # attempt to decode as floats buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0]) - if gaps and last_pos is not None and last_pos < pos: + if gaps and last_end is not None and last_end < pos: # decode gaps - gap_size = pos - last_pos - gap_buf = data[last_pos:pos] - print('{:04x} {} {:4} R:{}'.format(last_pos, "*UNKNOWN*".ljust(width), + gap_size = pos - last_end + gap_buf = data[last_end:pos] + print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width), gap_size, gap_buf.hex())) print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size, buf_repr, buf.hex())) - last_pos = end_pos + last_end = end_pos def print_map(grefs): From 40b737e33d98f0b56c2dfee94dcd53563a39fd19 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 18:13:43 +0200 Subject: [PATCH 06/40] elf_mem_map: switch to a named tuple for extensibility --- tools/elf_mem_map | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index ddfd5706..aad957b9 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -2,6 +2,7 @@ import argparse import elftools.elf.elffile import elftools.dwarf.descriptions +from collections import namedtuple from struct import unpack SRAM_OFFSET = 0x800000 @@ -9,6 +10,9 @@ EEPROM_OFFSET = 0x810000 FILL_BYTE = b'\0' +Entry = namedtuple('Entry', ['name', 'loc', 'size']) + + def get_elf_globals(path): fd = open(path, "rb") if fd is None: @@ -60,7 +64,7 @@ def get_elf_globals(path): continue size = byte_size.value - grefs.append([name, loc, size]) + grefs.append(Entry(name, loc, size)) return grefs @@ -106,14 +110,14 @@ def decode_dump(path): def annotate_refs(grefs, addr, data, width=45, gaps=True): last_end = None - for name, loc, size in grefs: - if loc < addr: + for entry in grefs: + if entry.loc < addr: continue - if loc + size > addr + len(data): + if entry.loc + entry.size > addr + len(data): continue - pos = loc-addr - end_pos = pos + size + pos = entry.loc-addr + end_pos = pos + entry.size buf = data[pos:end_pos] buf_repr = '' @@ -131,15 +135,15 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True): print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width), gap_size, gap_buf.hex())) - print('{:04x} {} {:4}{} R:{}'.format(loc, name.ljust(width), size, - buf_repr, buf.hex())) + print('{:04x} {} {:4}{} R:{}'.format(entry.loc, entry.name.ljust(width), + entry.size, buf_repr, buf.hex())) last_end = end_pos def print_map(grefs): print('OFFSET\tSIZE\tNAME') - for name, loc, size in grefs: - print('{:x}\t{}\t{}'.format(loc, size, name)) + for entry in grefs: + print('{:x}\t{}\t{}'.format(entry.loc, entry.size, entry.name)) def main(): @@ -156,7 +160,7 @@ def main(): args = ap.parse_args() grefs = get_elf_globals(args.elf) - grefs = list(sorted(grefs, key=lambda x: x[1])) + grefs = list(sorted(grefs, key=lambda x: x.loc)) if args.dump is None: print_map(grefs) else: From 1de3fa51c9662e2b6f79aedb959c62f229bec3f3 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 18:26:33 +0200 Subject: [PATCH 07/40] elf_mem_map: decode doubles correctly --- tools/elf_mem_map | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index aad957b9..32f178dc 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -126,7 +126,8 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True): buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10) if len(buf) in [4, 8]: # attempt to decode as floats - buf_repr += ' F:' + '{:10.3f}'.format(unpack('f', buf)[0]) + typ = 'f' if len(buf) == 4 else 'd' + buf_repr += ' F:' + '{:10.3f}'.format(unpack(typ, buf)[0]) if gaps and last_end is not None and last_end < pos: # decode gaps From 29b8c89ec2a7df5cf9dfcef693702890fe1c65e0 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 18:30:25 +0200 Subject: [PATCH 08/40] elf_mem_map: decode arrays (first dimension) --- tools/elf_mem_map | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 32f178dc..e518083d 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -54,17 +54,33 @@ def get_elf_globals(path): # recurse on type to find the final storage definition type_DIE = DIE byte_size = None + array_len = None while True: if 'DW_AT_byte_size' in type_DIE.attributes: byte_size = type_DIE.attributes.get('DW_AT_byte_size') if 'DW_AT_type' not in type_DIE.attributes: break type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') + if type_DIE.tag == 'DW_TAG_array_type': + # fetch the first dimension (if known) + range_DIE = next(type_DIE.iter_children()) + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + array_len = range_DIE.attributes['DW_AT_upper_bound'].value + 1 if byte_size is None: continue size = byte_size.value - grefs.append(Entry(name, loc, size)) + if array_len is None or array_len == 1: + # plain entry + grefs.append(Entry(name, loc, size)) + elif size == 1: + # string + grefs.append(Entry('{}[]'.format(name), loc, array_len)) + else: + # expand array entries + for i in range(array_len): + grefs.append(Entry('{}[{}]'.format(name, i), loc+size*i, size)) return grefs From 2718dbb42cbd3671a43fd52fe23547b121c4a6c8 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 18:57:14 +0200 Subject: [PATCH 09/40] elf_mem_map: array n-dimensional expansion --- tools/elf_mem_map | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index e518083d..fdfc6dad 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -13,6 +13,15 @@ FILL_BYTE = b'\0' Entry = namedtuple('Entry', ['name', 'loc', 'size']) +def array_inc(loc, dim, idx=0): + if idx == len(dim): + return True + loc[idx] += 1 + if loc[idx] == dim[idx]: + loc[idx] = 0 + return array_inc(loc, dim, idx+1) + return False + def get_elf_globals(path): fd = open(path, "rb") if fd is None: @@ -54,7 +63,7 @@ def get_elf_globals(path): # recurse on type to find the final storage definition type_DIE = DIE byte_size = None - array_len = None + array_dim = [] while True: if 'DW_AT_byte_size' in type_DIE.attributes: byte_size = type_DIE.attributes.get('DW_AT_byte_size') @@ -62,25 +71,37 @@ def get_elf_globals(path): break type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') if type_DIE.tag == 'DW_TAG_array_type': - # fetch the first dimension (if known) - range_DIE = next(type_DIE.iter_children()) - if range_DIE.tag == 'DW_TAG_subrange_type' and \ - 'DW_AT_upper_bound' in range_DIE.attributes: - array_len = range_DIE.attributes['DW_AT_upper_bound'].value + 1 + # fetch array dimensions (if known) + for range_DIE in type_DIE.iter_children(): + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) if byte_size is None: continue size = byte_size.value - if array_len is None or array_len == 1: + if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry grefs.append(Entry(name, loc, size)) - elif size == 1: - # string - grefs.append(Entry('{}[]'.format(name), loc, array_len)) + elif len(array_dim) == 1 and size == 1: + # likely string, avoid expansion + grefs.append(Entry('{}[]'.format(name), loc, array_dim[0])) else: # expand array entries - for i in range(array_len): - grefs.append(Entry('{}[{}]'.format(name, i), loc+size*i, size)) + array_pos = loc + array_loc = [0] * len(array_dim) + while True: + # location index + sfx = '' + for d in range(len(array_dim)): + sfx += '[{}]'.format(array_loc[d]) + + grefs.append(Entry(name + sfx, array_pos, size)) + + # advance + array_pos += size + if array_inc(array_loc, array_dim): + break return grefs From c311266a83e9f289c6ae3710cc1758d4bc8731b4 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 19:26:29 +0200 Subject: [PATCH 10/40] elf_mem_map: handle abstract locations --- tools/elf_mem_map | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index fdfc6dad..6d08abb9 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -40,11 +40,10 @@ def get_elf_globals(path): # handle only variable types if DIE.tag != 'DW_TAG_variable': continue - if 'DW_AT_name' not in DIE.attributes: - continue if 'DW_AT_location' not in DIE.attributes: continue - if 'DW_AT_type' not in DIE.attributes: + if 'DW_AT_name' not in DIE.attributes and \ + 'DW_AT_abstract_origin' not in DIE.attributes: continue # handle locations encoded directly as DW_OP_addr (leaf globals) @@ -57,7 +56,18 @@ def get_elf_globals(path): continue loc -= SRAM_OFFSET - # variable name + # variable name/type + if 'DW_AT_name' not in DIE.attributes and \ + 'DW_AT_abstract_origin' in DIE.attributes: + DIE = DIE.get_DIE_from_attribute('DW_AT_abstract_origin') + if 'DW_AT_location' in DIE.attributes: + # duplicate reference (handled directly), skip + continue + if 'DW_AT_name' not in DIE.attributes: + continue + if 'DW_AT_type' not in DIE.attributes: + continue + name = DIE.attributes['DW_AT_name'].value.decode('ascii') # recurse on type to find the final storage definition From 1181e784849f9dd594e73fa05aed24927e7ad542 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 20:46:21 +0200 Subject: [PATCH 11/40] elf_mem_map: handle all pointer types correctly --- tools/elf_mem_map | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 6d08abb9..dc985ab0 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -75,12 +75,12 @@ def get_elf_globals(path): byte_size = None array_dim = [] while True: - if 'DW_AT_byte_size' in type_DIE.attributes: + if byte_size is None and 'DW_AT_byte_size' in type_DIE.attributes: byte_size = type_DIE.attributes.get('DW_AT_byte_size') if 'DW_AT_type' not in type_DIE.attributes: break type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') - if type_DIE.tag == 'DW_TAG_array_type': + if len(array_dim) == 0 and type_DIE.tag == 'DW_TAG_array_type': # fetch array dimensions (if known) for range_DIE in type_DIE.iter_children(): if range_DIE.tag == 'DW_TAG_subrange_type' and \ From bb8d171f34aef97e7f8cdaa245e61bcdd57d34bc Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 21:07:22 +0200 Subject: [PATCH 12/40] elf_mem_map: decode integers with correct endianness --- tools/elf_mem_map | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index dc985ab0..f4d187d1 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -170,7 +170,7 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True): buf_repr = '' if len(buf) in [1, 2, 4]: # attempt to decode as integers - buf_repr += ' I:' + str(int.from_bytes(buf, 'big')).rjust(10) + buf_repr += ' I:' + str(int.from_bytes(buf, 'little')).rjust(10) if len(buf) in [4, 8]: # attempt to decode as floats typ = 'f' if len(buf) == 4 else 'd' From 9ddb5991f2ecc6f950bcb5e52a8999dc0a6df56e Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Wed, 2 Jun 2021 21:58:49 +0200 Subject: [PATCH 13/40] elf_mem_map: allow to disable gap dumps --- tools/elf_mem_map | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index f4d187d1..281566e8 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -202,6 +202,8 @@ def main(): show the value of each symbol which is within the address range. """) ap.add_argument('elf', help='ELF file containing DWARF2 debugging information') + ap.add_argument('--no-gaps', action='store_true', + help='do not dump memory inbetween known symbols') g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') g.add_argument('--map', action='store_true', help='dump global memory map') @@ -213,7 +215,7 @@ def main(): print_map(grefs) else: addr, data = decode_dump(args.dump) - annotate_refs(grefs, addr, data) + annotate_refs(grefs, addr, data, gaps=not args.no_gaps) if __name__ == '__main__': exit(main()) From 615e8575bb1883518cb0d7280d8205748c3f1ab0 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 16:29:22 +0200 Subject: [PATCH 14/40] elf_mem_map: decode structs --- tools/elf_mem_map | 141 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 110 insertions(+), 31 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 281566e8..2d96221c 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -11,6 +11,7 @@ FILL_BYTE = b'\0' Entry = namedtuple('Entry', ['name', 'loc', 'size']) +Member = namedtuple('Member', ['name', 'off', 'size']) def array_inc(loc, dim, idx=0): @@ -22,7 +23,54 @@ def array_inc(loc, dim, idx=0): return array_inc(loc, dim, idx+1) return False -def get_elf_globals(path): +def get_type_size(type_DIE): + while True: + if 'DW_AT_byte_size' in type_DIE.attributes: + return type_DIE, type_DIE.attributes.get('DW_AT_byte_size').value + if 'DW_AT_type' not in type_DIE.attributes: + return None + type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') + +def get_type_arrsize(type_DIE): + size = get_type_size(type_DIE) + if size is None: + return None + byte_size = size[1] + if size[0].tag != 'DW_TAG_pointer_type': + array_DIE = get_type_def(type_DIE, 'DW_TAG_array_type') + if array_DIE is not None: + for range_DIE in array_DIE.iter_children(): + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + dim = range_DIE.attributes['DW_AT_upper_bound'].value + 1 + byte_size *= dim + return byte_size + +def get_type_def(type_DIE, type_tag): + while True: + if type_DIE.tag == type_tag: + return type_DIE + if 'DW_AT_type' not in type_DIE.attributes: + return None + type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') + +def get_FORM_block1(attr): + if attr.form != 'DW_FORM_block1': + return None + if attr.value[0] == 3: # OP_addr + return int.from_bytes(attr.value[1:], 'little') + if attr.value[0] == 35: # OP_plus_uconst (ULEB128) + v = 0 + s = 0 + for b in attr.value[1:]: + v |= b + s += 7 + if not b & 0x100: + break + return v + return None + +def get_elf_globals(path, expand_structs, struct_gaps=True): fd = open(path, "rb") if fd is None: return @@ -47,12 +95,8 @@ def get_elf_globals(path): continue # handle locations encoded directly as DW_OP_addr (leaf globals) - at_loc = DIE.attributes['DW_AT_location'] - if at_loc.form != 'DW_FORM_block1' or at_loc.value[0] != 3: - continue - loc = (at_loc.value[1]) + (at_loc.value[2] << 8) \ - + (at_loc.value[3] << 16) + (at_loc.value[4] << 24) - if loc < SRAM_OFFSET or loc >= EEPROM_OFFSET: + loc = get_FORM_block1(DIE.attributes['DW_AT_location']) + if loc is None or loc < SRAM_OFFSET or loc >= EEPROM_OFFSET: continue loc -= SRAM_OFFSET @@ -70,32 +114,65 @@ def get_elf_globals(path): name = DIE.attributes['DW_AT_name'].value.decode('ascii') - # recurse on type to find the final storage definition - type_DIE = DIE - byte_size = None - array_dim = [] - while True: - if byte_size is None and 'DW_AT_byte_size' in type_DIE.attributes: - byte_size = type_DIE.attributes.get('DW_AT_byte_size') - if 'DW_AT_type' not in type_DIE.attributes: - break - type_DIE = type_DIE.get_DIE_from_attribute('DW_AT_type') - if len(array_dim) == 0 and type_DIE.tag == 'DW_TAG_array_type': - # fetch array dimensions (if known) - for range_DIE in type_DIE.iter_children(): - if range_DIE.tag == 'DW_TAG_subrange_type' and \ - 'DW_AT_upper_bound' in range_DIE.attributes: - array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) - if byte_size is None: + # get final storage size + size = get_type_size(DIE) + if size is None: continue - size = byte_size.value + byte_size = size[1] + + # fetch array dimensions (if known) + array_dim = [] + array_DIE = get_type_def(DIE, 'DW_TAG_array_type') + if array_DIE is not None: + for range_DIE in array_DIE.iter_children(): + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) + + # fetch structure members (one level only) + members = [] + if expand_structs and size[0].tag != 'DW_TAG_pointer_type': + struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type') + if struct_DIE is not None: + for member_DIE in struct_DIE.iter_children(): + if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes: + m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii') + m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location']) + m_size = get_type_arrsize(member_DIE) + members.append(Member(m_name, m_off, m_size)) + + if struct_gaps and len(members): + # fill gaps in the middle + members = list(sorted(members, key=lambda x: x.off)) + last_end = 0 + for member in members: + if member.off > last_end: + members.append(Member('*UNKNOWN*', last_end, member.off - last_end)) + last_end = member.off + member.size + + if struct_gaps and len(members): + # fill gap at the end + members = list(sorted(members, key=lambda x: x.off)) + last = members[-1] + last_end = last.off + last.size + if byte_size > last_end: + members.append(Member('*UNKNOWN*', last_end, byte_size - last_end)) + + + def expand_members(entry, members): + if len(members) == 0: + grefs.append(entry) + else: + for member in members: + grefs.append(Entry(entry.name + '.' + member.name, + entry.loc + member.off, member.size)) if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry - grefs.append(Entry(name, loc, size)) - elif len(array_dim) == 1 and size == 1: + expand_members(Entry(name, loc, byte_size), members) + elif len(array_dim) == 1 and byte_size == 1: # likely string, avoid expansion - grefs.append(Entry('{}[]'.format(name), loc, array_dim[0])) + grefs.append(Entry(name + '[]', loc, array_dim[0])) else: # expand array entries array_pos = loc @@ -106,10 +183,10 @@ def get_elf_globals(path): for d in range(len(array_dim)): sfx += '[{}]'.format(array_loc[d]) - grefs.append(Entry(name + sfx, array_pos, size)) + expand_members(Entry(name + sfx, array_pos, byte_size), members) # advance - array_pos += size + array_pos += byte_size if array_inc(array_loc, array_dim): break @@ -204,12 +281,14 @@ def main(): ap.add_argument('elf', help='ELF file containing DWARF2 debugging information') ap.add_argument('--no-gaps', action='store_true', help='do not dump memory inbetween known symbols') + ap.add_argument('--no-expand-structs', action='store_true', + help='do not decode structure data') g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') g.add_argument('--map', action='store_true', help='dump global memory map') args = ap.parse_args() - grefs = get_elf_globals(args.elf) + grefs = get_elf_globals(args.elf, expand_structs=not args.no_expand_structs) grefs = list(sorted(grefs, key=lambda x: x.loc)) if args.dump is None: print_map(grefs) From 1d82d2da641aea497d19f483ce5de7c386cbc4c8 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 16:49:45 +0200 Subject: [PATCH 15/40] get_elf_map: do not reprocess members twice --- tools/elf_mem_map | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 2d96221c..cde11daf 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -145,7 +145,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): # fill gaps in the middle members = list(sorted(members, key=lambda x: x.off)) last_end = 0 - for member in members: + for n in range(len(members)): + member = members[n] if member.off > last_end: members.append(Member('*UNKNOWN*', last_end, member.off - last_end)) last_end = member.off + member.size From a5635997b281e24f789dee211a8c13101943feb7 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 17:06:36 +0200 Subject: [PATCH 16/40] elf_mem_map: allow to annotate overlapping regions for clarity --- tools/elf_mem_map | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index cde11daf..b3b59978 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -233,7 +233,7 @@ def decode_dump(path): return (buf_addr, buf_data) -def annotate_refs(grefs, addr, data, width=45, gaps=True): +def annotate_refs(grefs, addr, data, width=45, gaps=True, overlaps=True): last_end = None for entry in grefs: if entry.loc < addr: @@ -254,12 +254,16 @@ def annotate_refs(grefs, addr, data, width=45, gaps=True): typ = 'f' if len(buf) == 4 else 'd' buf_repr += ' F:' + '{:10.3f}'.format(unpack(typ, buf)[0]) - if gaps and last_end is not None and last_end < pos: - # decode gaps - gap_size = pos - last_end - gap_buf = data[last_end:pos] - print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width), - gap_size, gap_buf.hex())) + if last_end is not None: + if gaps and last_end < pos: + # decode gaps + gap_size = pos - last_end + gap_buf = data[last_end:pos] + print('{:04x} {} {:4} R:{}'.format(addr+last_end, "*UNKNOWN*".ljust(width), + gap_size, gap_buf.hex())) + if overlaps and last_end > pos + 1: + gap_size = pos - last_end + print('{:04x} {} {:4}'.format(addr+last_end, "*OVERLAP*".ljust(width), gap_size)) print('{:04x} {} {:4}{} R:{}'.format(entry.loc, entry.name.ljust(width), entry.size, buf_repr, buf.hex())) @@ -284,6 +288,8 @@ def main(): help='do not dump memory inbetween known symbols') ap.add_argument('--no-expand-structs', action='store_true', help='do not decode structure data') + ap.add_argument('--overlaps', action='store_true', + help='annotate overlaps greater than 1 byte') g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') g.add_argument('--map', action='store_true', help='dump global memory map') @@ -295,7 +301,9 @@ def main(): print_map(grefs) else: addr, data = decode_dump(args.dump) - annotate_refs(grefs, addr, data, gaps=not args.no_gaps) + annotate_refs(grefs, addr, data, + gaps=not args.no_gaps, + overlaps=args.overlaps) if __name__ == '__main__': exit(main()) From 7f76f62af91f31acdf51cb5956afff1c76eb1ad8 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 17:17:00 +0200 Subject: [PATCH 17/40] elf_mem_map: fix uleb128 decoding (fixes incorrect member offsets) --- tools/elf_mem_map | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index b3b59978..28681167 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -63,10 +63,10 @@ def get_FORM_block1(attr): v = 0 s = 0 for b in attr.value[1:]: - v |= b - s += 7 - if not b & 0x100: + v |= (b & 0x7f) << s + if b & 0x80 == 0: break + s += 7 return v return None From 29513a369d69418d2d4a297a3b18b142bb4c86cf Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 17:22:25 +0200 Subject: [PATCH 18/40] elf_mem_map: allow to customize the name column's width --- tools/elf_mem_map | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 28681167..7a14c200 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -233,7 +233,7 @@ def decode_dump(path): return (buf_addr, buf_data) -def annotate_refs(grefs, addr, data, width=45, gaps=True, overlaps=True): +def annotate_refs(grefs, addr, data, width=46, gaps=True, overlaps=True): last_end = None for entry in grefs: if entry.loc < addr: @@ -290,6 +290,8 @@ def main(): help='do not decode structure data') ap.add_argument('--overlaps', action='store_true', help='annotate overlaps greater than 1 byte') + ap.add_argument('--name-width', type=int, default=46, + help='set name column width') g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') g.add_argument('--map', action='store_true', help='dump global memory map') @@ -302,6 +304,7 @@ def main(): else: addr, data = decode_dump(args.dump) annotate_refs(grefs, addr, data, + width=args.name_width, gaps=not args.no_gaps, overlaps=args.overlaps) From 776b82a6db7ad7110432b16e413841f4c0f206de Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 17:51:02 +0200 Subject: [PATCH 19/40] elf_mem_map: expand member arrays --- tools/elf_mem_map | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 7a14c200..cb08ca5e 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -138,8 +138,39 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes: m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii') m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location']) - m_size = get_type_arrsize(member_DIE) - members.append(Member(m_name, m_off, m_size)) + m_byte_size = get_type_size(member_DIE)[1] + + # still expand member arrays + m_array_dim = [] + m_array_DIE = get_type_def(member_DIE, 'DW_TAG_array_type') + if m_array_DIE is not None: + for range_DIE in m_array_DIE.iter_children(): + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) + + if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1): + # plain entry + members.append(Member(m_name, m_off, m_byte_size)) + elif len(m_array_dim) == 1 and m_byte_size == 1: + # likely string, avoid expansion + members.append(Member(m_name + '[]', m_off, m_array_dim[0])) + else: + # expand array entries + m_array_pos = m_off + m_array_loc = [0] * len(m_array_dim) + while True: + # location index + sfx = '' + for d in range(len(m_array_dim)): + sfx += '[{}]'.format(m_array_loc[d]) + + members.append(Member(m_name + sfx, m_array_pos, m_byte_size)) + + # advance + if array_inc(m_array_loc, m_array_dim): + break + m_array_pos += m_byte_size if struct_gaps and len(members): # fill gaps in the middle @@ -187,9 +218,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): expand_members(Entry(name + sfx, array_pos, byte_size), members) # advance - array_pos += byte_size if array_inc(array_loc, array_dim): break + array_pos += byte_size return grefs From cb4f5cff9f8c451eabc169857ef47d98deb16b9b Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 17:54:11 +0200 Subject: [PATCH 20/40] elf_mem_map: improve display of array-of-strings --- tools/elf_mem_map | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index cb08ca5e..86058618 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -149,6 +149,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): 'DW_AT_upper_bound' in range_DIE.attributes: m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) + # likely string, remove one dimension + if m_byte_size == 1 and len(m_array_dim) > 1: + m_byte_size *= m_array_dim.pop() if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1): # plain entry members.append(Member(m_name, m_off, m_byte_size)) @@ -199,6 +202,9 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): grefs.append(Entry(entry.name + '.' + member.name, entry.loc + member.off, member.size)) + # likely string, remove one dimension + if byte_size == 1 and len(array_dim) > 1: + byte_size *= array_dim.pop() if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry expand_members(Entry(name, loc, byte_size), members) From 71ef94da2e3a6810a4711541d6b17dca8462abe9 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 18:03:36 +0200 Subject: [PATCH 21/40] elf_mem_map: improve alignment of arrays --- tools/elf_mem_map | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 86058618..b6aec2c6 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -149,8 +149,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): 'DW_AT_upper_bound' in range_DIE.attributes: m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) - # likely string, remove one dimension if m_byte_size == 1 and len(m_array_dim) > 1: + # likely string, remove one dimension m_byte_size *= m_array_dim.pop() if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1): # plain entry @@ -166,7 +166,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): # location index sfx = '' for d in range(len(m_array_dim)): - sfx += '[{}]'.format(m_array_loc[d]) + sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0')) members.append(Member(m_name + sfx, m_array_pos, m_byte_size)) @@ -202,8 +202,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): grefs.append(Entry(entry.name + '.' + member.name, entry.loc + member.off, member.size)) - # likely string, remove one dimension if byte_size == 1 and len(array_dim) > 1: + # likely string, remove one dimension byte_size *= array_dim.pop() if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry @@ -219,7 +219,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): # location index sfx = '' for d in range(len(array_dim)): - sfx += '[{}]'.format(array_loc[d]) + sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0')) expand_members(Entry(name + sfx, array_pos, byte_size), members) From c875aef49ccdec2edda4a908c31bfbde1bf47c56 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 18:03:50 +0200 Subject: [PATCH 22/40] elf_mem_map: increase width again to fit new output --- tools/elf_mem_map | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index b6aec2c6..3bd0f23e 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -270,7 +270,7 @@ def decode_dump(path): return (buf_addr, buf_data) -def annotate_refs(grefs, addr, data, width=46, gaps=True, overlaps=True): +def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True): last_end = None for entry in grefs: if entry.loc < addr: @@ -327,7 +327,7 @@ def main(): help='do not decode structure data') ap.add_argument('--overlaps', action='store_true', help='annotate overlaps greater than 1 byte') - ap.add_argument('--name-width', type=int, default=46, + ap.add_argument('--name-width', type=int, default=50, help='set name column width') g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') From d1720cba512c60940a615f3826cb6477f46ea9dc Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 19:10:52 +0200 Subject: [PATCH 23/40] elf_mem_map: reduce some duplication --- tools/elf_mem_map | 148 ++++++++++++++++++++++++---------------------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 3bd0f23e..26694c80 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -70,6 +70,82 @@ def get_FORM_block1(attr): return v return None + +def get_array_dims(DIE): + array_DIE = get_type_def(DIE, 'DW_TAG_array_type') + if array_DIE is None: + return [] + + array_dim = [] + for range_DIE in array_DIE.iter_children(): + if range_DIE.tag == 'DW_TAG_subrange_type' and \ + 'DW_AT_upper_bound' in range_DIE.attributes: + array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) + return array_dim + + +def get_struct_members(DIE, size, expand_structs, struct_gaps): + if not expand_structs or size[0].tag == 'DW_TAG_pointer_type': + return [] + struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type') + if struct_DIE is None: + return [] + + members = [] + for member_DIE in struct_DIE.iter_children(): + if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes: + m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii') + m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location']) + m_byte_size = get_type_size(member_DIE)[1] + + # still expand member arrays + m_array_dim = get_array_dims(member_DIE) + + if m_byte_size == 1 and len(m_array_dim) > 1: + # likely string, remove one dimension + m_byte_size *= m_array_dim.pop() + if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1): + # plain entry + members.append(Member(m_name, m_off, m_byte_size)) + elif len(m_array_dim) == 1 and m_byte_size == 1: + # likely string, avoid expansion + members.append(Member(m_name + '[]', m_off, m_array_dim[0])) + else: + # expand array entries + m_array_pos = m_off + m_array_loc = [0] * len(m_array_dim) + while True: + # location index + sfx = '' + for d in range(len(m_array_dim)): + sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0')) + members.append(Member(m_name + sfx, m_array_pos, m_byte_size)) + # advance + if array_inc(m_array_loc, m_array_dim): + break + m_array_pos += m_byte_size + + if struct_gaps and len(members): + # fill gaps in the middle + members = list(sorted(members, key=lambda x: x.off)) + last_end = 0 + for n in range(len(members)): + member = members[n] + if member.off > last_end: + members.append(Member('*UNKNOWN*', last_end, member.off - last_end)) + last_end = member.off + member.size + + if struct_gaps and len(members): + # fill gap at the end + members = list(sorted(members, key=lambda x: x.off)) + last = members[-1] + last_end = last.off + last.size + if size[1] > last_end: + members.append(Member('*UNKNOWN*', last_end, byte_size - last_end)) + + return members + + def get_elf_globals(path, expand_structs, struct_gaps=True): fd = open(path, "rb") if fd is None: @@ -121,78 +197,10 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): byte_size = size[1] # fetch array dimensions (if known) - array_dim = [] - array_DIE = get_type_def(DIE, 'DW_TAG_array_type') - if array_DIE is not None: - for range_DIE in array_DIE.iter_children(): - if range_DIE.tag == 'DW_TAG_subrange_type' and \ - 'DW_AT_upper_bound' in range_DIE.attributes: - array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) + array_dim = get_array_dims(DIE) # fetch structure members (one level only) - members = [] - if expand_structs and size[0].tag != 'DW_TAG_pointer_type': - struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type') - if struct_DIE is not None: - for member_DIE in struct_DIE.iter_children(): - if member_DIE.tag == 'DW_TAG_member' and 'DW_AT_name' in member_DIE.attributes: - m_name = member_DIE.attributes['DW_AT_name'].value.decode('ascii') - m_off = get_FORM_block1(member_DIE.attributes['DW_AT_data_member_location']) - m_byte_size = get_type_size(member_DIE)[1] - - # still expand member arrays - m_array_dim = [] - m_array_DIE = get_type_def(member_DIE, 'DW_TAG_array_type') - if m_array_DIE is not None: - for range_DIE in m_array_DIE.iter_children(): - if range_DIE.tag == 'DW_TAG_subrange_type' and \ - 'DW_AT_upper_bound' in range_DIE.attributes: - m_array_dim.append(range_DIE.attributes['DW_AT_upper_bound'].value + 1) - - if m_byte_size == 1 and len(m_array_dim) > 1: - # likely string, remove one dimension - m_byte_size *= m_array_dim.pop() - if len(m_array_dim) == 0 or (len(m_array_dim) == 1 and m_array_dim[0] == 1): - # plain entry - members.append(Member(m_name, m_off, m_byte_size)) - elif len(m_array_dim) == 1 and m_byte_size == 1: - # likely string, avoid expansion - members.append(Member(m_name + '[]', m_off, m_array_dim[0])) - else: - # expand array entries - m_array_pos = m_off - m_array_loc = [0] * len(m_array_dim) - while True: - # location index - sfx = '' - for d in range(len(m_array_dim)): - sfx += '[{}]'.format(str(m_array_loc[d]).rjust(len(str(m_array_dim[d]-1)), '0')) - - members.append(Member(m_name + sfx, m_array_pos, m_byte_size)) - - # advance - if array_inc(m_array_loc, m_array_dim): - break - m_array_pos += m_byte_size - - if struct_gaps and len(members): - # fill gaps in the middle - members = list(sorted(members, key=lambda x: x.off)) - last_end = 0 - for n in range(len(members)): - member = members[n] - if member.off > last_end: - members.append(Member('*UNKNOWN*', last_end, member.off - last_end)) - last_end = member.off + member.size - - if struct_gaps and len(members): - # fill gap at the end - members = list(sorted(members, key=lambda x: x.off)) - last = members[-1] - last_end = last.off + last.size - if byte_size > last_end: - members.append(Member('*UNKNOWN*', last_end, byte_size - last_end)) - + members = get_struct_members(DIE, size, expand_structs, struct_gaps) def expand_members(entry, members): if len(members) == 0: From 7bdee552ce91bf133fb2fb7e00f87f668630bf01 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 20:36:19 +0200 Subject: [PATCH 24/40] elf_mem_map: add declaration position in --map --- tools/elf_mem_map | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 26694c80..f29a58cf 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -10,7 +10,7 @@ EEPROM_OFFSET = 0x810000 FILL_BYTE = b'\0' -Entry = namedtuple('Entry', ['name', 'loc', 'size']) +Entry = namedtuple('Entry', ['name', 'loc', 'size', 'declpos']) Member = namedtuple('Member', ['name', 'off', 'size']) @@ -160,6 +160,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): grefs = [] for CU in dwarfinfo.iter_CUs(): + file_entries = dwarfinfo.line_program_for_CU(CU).header["file_entry"] + for DIE in CU.iter_DIEs(): # handle only variable types if DIE.tag != 'DW_TAG_variable': @@ -196,6 +198,16 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): continue byte_size = size[1] + # location of main definition + declpos = '' + if 'DW_AT_decl_file' in DIE.attributes and \ + 'DW_AT_decl_line' in DIE.attributes: + line = DIE.attributes['DW_AT_decl_line'].value + fname = DIE.attributes['DW_AT_decl_file'].value + if fname and fname - 1 < len(file_entries): + fname = file_entries[fname-1].name.decode('ascii') + declpos = '{}:{}'.format(fname, line) + # fetch array dimensions (if known) array_dim = get_array_dims(DIE) @@ -208,17 +220,18 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): else: for member in members: grefs.append(Entry(entry.name + '.' + member.name, - entry.loc + member.off, member.size)) + entry.loc + member.off, member.size, + entry.declpos)) if byte_size == 1 and len(array_dim) > 1: # likely string, remove one dimension byte_size *= array_dim.pop() if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry - expand_members(Entry(name, loc, byte_size), members) + expand_members(Entry(name, loc, byte_size, declpos), members) elif len(array_dim) == 1 and byte_size == 1: # likely string, avoid expansion - grefs.append(Entry(name + '[]', loc, array_dim[0])) + grefs.append(Entry(name + '[]', loc, array_dim[0], declpos)) else: # expand array entries array_pos = loc @@ -229,7 +242,7 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): for d in range(len(array_dim)): sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0')) - expand_members(Entry(name + sfx, array_pos, byte_size), members) + expand_members(Entry(name + sfx, array_pos, byte_size, declpos), members) # advance if array_inc(array_loc, array_dim): @@ -316,9 +329,9 @@ def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True): def print_map(grefs): - print('OFFSET\tSIZE\tNAME') + print('OFFSET\tSIZE\tNAME\tDECLPOS') for entry in grefs: - print('{:x}\t{}\t{}'.format(entry.loc, entry.size, entry.name)) + print('{:x}\t{}\t{}\t{}'.format(entry.loc, entry.size, entry.name, entry.declpos)) def main(): From 676b925c5f33a0a5f08bbcfb17941068965cb1b6 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 20:56:14 +0200 Subject: [PATCH 25/40] elf_mem_map: cleanup --- tools/elf_mem_map | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index f29a58cf..998366e9 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -84,9 +84,7 @@ def get_array_dims(DIE): return array_dim -def get_struct_members(DIE, size, expand_structs, struct_gaps): - if not expand_structs or size[0].tag == 'DW_TAG_pointer_type': - return [] +def get_struct_members(DIE, entry, expand_structs, struct_gaps): struct_DIE = get_type_def(DIE, 'DW_TAG_structure_type') if struct_DIE is None: return [] @@ -140,8 +138,8 @@ def get_struct_members(DIE, size, expand_structs, struct_gaps): members = list(sorted(members, key=lambda x: x.off)) last = members[-1] last_end = last.off + last.size - if size[1] > last_end: - members.append(Member('*UNKNOWN*', last_end, byte_size - last_end)) + if entry.size > last_end: + members.append(Member('*UNKNOWN*', last_end, entry.size - last_end)) return members @@ -212,7 +210,11 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): array_dim = get_array_dims(DIE) # fetch structure members (one level only) - members = get_struct_members(DIE, size, expand_structs, struct_gaps) + entry = Entry(name, loc, byte_size, declpos) + if not expand_structs or size[0].tag == 'DW_TAG_pointer_type': + members = [] + else: + members = get_struct_members(DIE, entry, expand_structs, struct_gaps) def expand_members(entry, members): if len(members) == 0: @@ -228,10 +230,11 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): byte_size *= array_dim.pop() if len(array_dim) == 0 or (len(array_dim) == 1 and array_dim[0] == 1): # plain entry - expand_members(Entry(name, loc, byte_size, declpos), members) + expand_members(entry, members) elif len(array_dim) == 1 and byte_size == 1: # likely string, avoid expansion - grefs.append(Entry(name + '[]', loc, array_dim[0], declpos)) + grefs.append(Entry(entry.name + '[]', entry.loc, + array_dim[0], entry.declpos)) else: # expand array entries array_pos = loc @@ -241,9 +244,8 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): sfx = '' for d in range(len(array_dim)): sfx += '[{}]'.format(str(array_loc[d]).rjust(len(str(array_dim[d]-1)), '0')) - - expand_members(Entry(name + sfx, array_pos, byte_size, declpos), members) - + expand_members(Entry(entry.name + sfx, array_pos, + byte_size, entry.declpos), members) # advance if array_inc(array_loc, array_dim): break From 54e24036a88c3d9ac758b1413361a8b8ff1232f3 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 21:54:42 +0200 Subject: [PATCH 26/40] elf_mem_map: add qdirstat output for space visualization --- tools/elf_mem_map | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 998366e9..079adf35 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -4,6 +4,7 @@ import elftools.elf.elffile import elftools.dwarf.descriptions from collections import namedtuple from struct import unpack +import re SRAM_OFFSET = 0x800000 EEPROM_OFFSET = 0x810000 @@ -336,6 +337,43 @@ def print_map(grefs): print('{:x}\t{}\t{}\t{}'.format(entry.loc, entry.size, entry.name, entry.declpos)) +def print_qdirstat(grefs): + print('[qdirstat 1.0 cache file]') + + entries = {} + for entry in grefs: + paths = list(filter(None, re.split(r'[\[\].]', entry.name))) + base = entries + for i in range(len(paths) - 1): + name = paths[i] + if name not in base: + base[name] = {} + base = base[name] + name = paths[-1] + base[name] = entry.size + + def walker(root, prefix): + files = [] + dirs = [] + + for name, entries in root.items(): + if type(entries) == int: + files.append([name, entries]) + else: + dirs.append([name, entries]) + + # print files + print('D\t{}\t{}\t0x0'.format(prefix, 0)) + for name, size in files: + print('F\t{}\t{}\t0x0'.format(name, size)) + + # recurse directories + for name, entries in dirs: + walker(entries, prefix + '/' + name) + + walker(entries, '/') + + def main(): ap = argparse.ArgumentParser(description=""" Generate a symbol table map starting directly from an ELF @@ -355,12 +393,16 @@ def main(): g = ap.add_mutually_exclusive_group(required=True) g.add_argument('dump', nargs='?', help='RAM dump obtained from D2 g-code') g.add_argument('--map', action='store_true', help='dump global memory map') + g.add_argument('--qdirstat', action='store_true', + help='dump qdirstat-compatible size usage map') args = ap.parse_args() grefs = get_elf_globals(args.elf, expand_structs=not args.no_expand_structs) grefs = list(sorted(grefs, key=lambda x: x.loc)) - if args.dump is None: + if args.map: print_map(grefs) + elif args.qdirstat: + print_qdirstat(grefs) else: addr, data = decode_dump(args.dump) annotate_refs(grefs, addr, data, From 9958c449e347578f06c810d058539467b510f437 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 22:29:11 +0200 Subject: [PATCH 27/40] elf_mem_map: remove DWARF version except in help Fix version to DWARF3, which is what we actually support. --- tools/README.md | 2 +- tools/elf_mem_map | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/README.md b/tools/README.md index 2b4c7b82..02f4813f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -14,7 +14,7 @@ Requires ``printcore`` from [Pronterface]. ### ``elf_mem_map`` -Generate a symbol table map starting directly from an ELF firmware with DWARF2 debugging information (which is the default using the stock board definition). +Generate a symbol table map starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition). When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range. diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 079adf35..65a95732 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -377,11 +377,11 @@ def print_qdirstat(grefs): def main(): ap = argparse.ArgumentParser(description=""" Generate a symbol table map starting directly from an ELF - firmware with DWARF2 debugging information. + firmware with DWARF3 debugging information. When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range. """) - ap.add_argument('elf', help='ELF file containing DWARF2 debugging information') + ap.add_argument('elf', help='ELF file containing DWARF debugging information') ap.add_argument('--no-gaps', action='store_true', help='do not dump memory inbetween known symbols') ap.add_argument('--no-expand-structs', action='store_true', From 9917689fdfe572e2462b3bcc3a69837575959f16 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 22:45:01 +0200 Subject: [PATCH 28/40] tools: update documentation for elf_mem_map --- tools/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/README.md b/tools/README.md index 02f4813f..e7911501 100644 --- a/tools/README.md +++ b/tools/README.md @@ -14,9 +14,13 @@ Requires ``printcore`` from [Pronterface]. ### ``elf_mem_map`` -Generate a symbol table map starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition). +Generate a symbol table map with decoded information starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition). -When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range. +When used along with a memory dump obtained from the D2 g-code, show the value of each symbol which is within the address range of the dump. + +When used with ``--map`` and a single elf file, generate a map consisting of memory location and source location for each statically-addressed variable. + +With ``--qdirstat`` and a single elf file, generate a [qdirstat](https://github.com/shundhammer/qdirstat) compatible cache file which can be loaded to inspect memory utilization interactively in a treemap. This assumes the running firmware generating the dump and the elf file are the same. Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module. From d98e1b1cd98faab573413e6e31e97f2f8198b92d Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sat, 5 Jun 2021 23:03:56 +0200 Subject: [PATCH 29/40] elf_mem_map: uniquify file names in qdirstat output --- tools/elf_mem_map | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 65a95732..f3ea03d0 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -350,6 +350,8 @@ def print_qdirstat(grefs): base[name] = {} base = base[name] name = paths[-1] + if name in base: + name = '{}_{:x}'.format(entry.name, entry.loc) base[name] = entry.size def walker(root, prefix): From 8ec4104840bb26f1c96a3268c91953ba6c92ed3b Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Sun, 6 Jun 2021 12:20:50 +0200 Subject: [PATCH 30/40] elf_mem_map: do not output registers in qdirstat output --- tools/elf_mem_map | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index f3ea03d0..2a7e8c30 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -6,6 +6,7 @@ from collections import namedtuple from struct import unpack import re +SRAM_START = 0x200 SRAM_OFFSET = 0x800000 EEPROM_OFFSET = 0x810000 FILL_BYTE = b'\0' @@ -342,6 +343,10 @@ def print_qdirstat(grefs): entries = {} for entry in grefs: + # do not output registers when looking at space usage + if entry.loc < SRAM_START: + continue + paths = list(filter(None, re.split(r'[\[\].]', entry.name))) base = entries for i in range(len(paths) - 1): From 9f40fa68347ca6ace7669d7fe6a2ec08a2e41e3d Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Mon, 14 Jun 2021 16:03:02 +0200 Subject: [PATCH 31/40] elf_mem_map: parse D23 output directly --- tools/elf_mem_map | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index 2a7e8c30..b0a7db18 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -4,6 +4,7 @@ import elftools.elf.elffile import elftools.dwarf.descriptions from collections import namedtuple from struct import unpack +import sys import re SRAM_START = 0x200 @@ -264,12 +265,26 @@ def decode_dump(path): buf_addr = None # starting address buf_data = None # data - for line in fd: - tokens = line.split(maxsplit=1) - if len(tokens) == 0 or tokens[0] == 'ok': - break - elif len(tokens) < 2 or tokens[0] == 'D2': + in_dump = False + for line in enumerate(fd): + line = (line[0], line[1].rstrip()) + tokens = line[1].split(maxsplit=1) + if not in_dump: + if len(tokens) > 0 and tokens[0] in ['D2', 'D23']: + in_dump = True continue + else: + if len(tokens) < 1: + print('malformed line {}: {}'.format(*line), file=sys.stderr) + continue + elif tokens[0] == 'ok': + break + elif tokens[0] == 'reason:': + # ignored + continue + elif not re.match(r'[0-9a-fA-F]', tokens[0]): + print('malformed line {}: {}'.format(*line), file=sys.stderr) + continue addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big') data = bytes.fromhex(tokens[1]) From 06eab4ac11f3f62c1f3edf54f872696b949ec7e3 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 22 Jun 2021 16:58:04 +0200 Subject: [PATCH 32/40] Handle XFLASH (D21) and serial (D23) dumps in elf_mem_map, add dump2bin - Uniformly parse D2/D21/D23 dump types. - Add dump2bin to parse/convert a dump into metadata and binary. - Move the parsing into it's own module in order to be shared. --- tools/dump2bin | 59 +++++++++++++++++ tools/elf_mem_map | 75 ++++----------------- tools/lib/avr.py | 4 ++ tools/lib/dump.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+), 61 deletions(-) create mode 100755 tools/dump2bin create mode 100644 tools/lib/avr.py create mode 100644 tools/lib/dump.py diff --git a/tools/dump2bin b/tools/dump2bin new file mode 100755 index 00000000..c7ab3603 --- /dev/null +++ b/tools/dump2bin @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import argparse +import os, sys + +from lib.dump import decode_dump + + +def main(): + # parse the arguments + ap = argparse.ArgumentParser(description=""" + Parse and decode a memory dump obtained from the D2/D21/D23 g-code + into readable metadata and binary. The output binary is padded and + extended to fit the original address range. + """) + ap.add_argument('-i', dest='info', action='store_true', + help='display crash info only') + ap.add_argument('dump') + ap.add_argument('output', nargs='?') + args = ap.parse_args() + + # decode the dump data + dump = decode_dump(args.dump) + if dump is None: + return os.EX_DATAERR + + # output descriptors + if args.info: + o_fd = None + o_md = sys.stdout + elif args.output is None: + o_fd = sys.stdout.buffer + o_md = sys.stderr + else: + o_fd = open(args.output, 'wb') + o_md = sys.stdout + + # output binary + if o_fd: + o_fd.write(dump.data) + o_fd.close() + + # metadata + print(' dump type: {typ}\n' + 'crash reason: {reason}\n' + ' registers: {regs}\n' + ' PC: {pc}\n' + ' SP: {sp}\n' + ' ranges: {ranges}'.format( + typ=dump.typ, + reason=dump.reason.name if dump.reason is not None else 'N/A', + regs=dump.regs, + pc='{:#x}'.format(dump.pc) if dump.pc is not None else 'N/A', + sp='{:#x}'.format(dump.sp) if dump.sp is not None else 'N/A', + ranges=str(dump.ranges)), + file=o_md) + + +if __name__ == '__main__': + exit(main()) diff --git a/tools/elf_mem_map b/tools/elf_mem_map index b0a7db18..cb6e423c 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -4,13 +4,10 @@ import elftools.elf.elffile import elftools.dwarf.descriptions from collections import namedtuple from struct import unpack -import sys -import re +import os -SRAM_START = 0x200 -SRAM_OFFSET = 0x800000 -EEPROM_OFFSET = 0x810000 -FILL_BYTE = b'\0' +from lib.dump import decode_dump +from lib.avr import * Entry = namedtuple('Entry', ['name', 'loc', 'size', 'declpos']) @@ -257,59 +254,6 @@ def get_elf_globals(path, expand_structs, struct_gaps=True): return grefs -def decode_dump(path): - fd = open(path, 'r') - if fd is None: - return None - - buf_addr = None # starting address - buf_data = None # data - - in_dump = False - for line in enumerate(fd): - line = (line[0], line[1].rstrip()) - tokens = line[1].split(maxsplit=1) - if not in_dump: - if len(tokens) > 0 and tokens[0] in ['D2', 'D23']: - in_dump = True - continue - else: - if len(tokens) < 1: - print('malformed line {}: {}'.format(*line), file=sys.stderr) - continue - elif tokens[0] == 'ok': - break - elif tokens[0] == 'reason:': - # ignored - continue - elif not re.match(r'[0-9a-fA-F]', tokens[0]): - print('malformed line {}: {}'.format(*line), file=sys.stderr) - continue - - addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big') - data = bytes.fromhex(tokens[1]) - - if buf_addr is None: - buf_addr = addr - buf_data = data - else: - # grow buffer as needed - if addr < buf_addr: - buf_data = FILL_BYTE * (buf_addr - addr) - buf_addr = addr - addr_end = addr + len(data) - buf_end = buf_addr + len(buf_data) - if addr_end > buf_end: - buf_data += FILL_BYTE * (addr_end - buf_end) - - # replace new part - rep_start = addr - buf_addr - rep_end = rep_start + len(data) - buf_data = buf_data[:rep_start] + data + buf_data[rep_end:] - - return (buf_addr, buf_data) - - def annotate_refs(grefs, addr, data, width, gaps=True, overlaps=True): last_end = None for entry in grefs: @@ -426,8 +370,17 @@ def main(): elif args.qdirstat: print_qdirstat(grefs) else: - addr, data = decode_dump(args.dump) - annotate_refs(grefs, addr, data, + # fetch the memory data + dump = decode_dump(args.dump) + if dump is None: + return os.EX_DATAERR + + # strip padding, if present + addr_start = dump.ranges[0][0] + addr_end = dump.ranges[-1][0]+dump.ranges[-1][1] + data = dump.data[addr_start:addr_end] + + annotate_refs(grefs, addr_start, data, width=args.name_width, gaps=not args.no_gaps, overlaps=args.overlaps) diff --git a/tools/lib/avr.py b/tools/lib/avr.py new file mode 100644 index 00000000..57f34479 --- /dev/null +++ b/tools/lib/avr.py @@ -0,0 +1,4 @@ +SRAM_START = 0x200 +SRAM_SIZE = 0x2000 +SRAM_OFFSET = 0x800000 +EEPROM_OFFSET = 0x810000 diff --git a/tools/lib/dump.py b/tools/lib/dump.py new file mode 100644 index 00000000..1c541a2a --- /dev/null +++ b/tools/lib/dump.py @@ -0,0 +1,165 @@ +import sys +import re +import enum +import struct +from . import avr + + +FILL_BYTE = b'\0' # used to fill memory gaps in the dump + +class CrashReason(enum.IntEnum): + MANUAL = 0 + STACK_ERROR = 1 + WATCHDOG = 2 + BAD_ISR = 3 + +class Dump(): + def __init__(self, typ, reason, regs, pc, sp, data, ranges): + self.typ = typ + self.reason = reason + self.regs = regs + self.pc = pc + self.sp = sp + self.data = data + self.ranges = ranges + + +# expand the buffer identified by addr+data to fill the region start+size +def region_expand(addr, data, start, size): + if start < addr: + data = FILL_BYTE * (addr - start) + data + addr = start + end = start + size + data_end = addr + len(data) + if end > data_end: + data += FILL_BYTE * (data_end - end) + return addr, data + + +def merge_ranges(ranges): + ranges = list(sorted(ranges, key=lambda x: x[0])) + if len(ranges) < 2: + return ranges + + ret = [ranges[0]] + for cur in ranges[1:]: + last = ret[-1] + last_end = last[0] + last[1] + if last_end < cur[0]: + ret.append(cur) + else: + cur_end = cur[0] + cur[1] + last = (last[0], max(last_end, cur_end) - last[0]) + ret[-1] = last + return ret + + +def decode_dump(path): + fd = open(path, 'r') + if fd is None: + return None + + buf_addr = None # starting address + buf_data = None # data + + typ = None # dump type + reason = None # crash reason + regs = None # registers present + pc = None # PC address + sp = None # SP address + ranges = [] # dumped ranges + + in_dump = False + for line in enumerate(fd): + line = (line[0], line[1].rstrip()) + tokens = line[1].split(maxsplit=1) + + def line_error(): + print('malformed line {}: {}'.format(*line), file=sys.stderr) + + # handle metadata + if not in_dump: + if len(tokens) > 0 and tokens[0] in ['D2', 'D21', 'D23']: + in_dump = True + typ = tokens[0] + continue + else: + if len(tokens) == 0: + line_error() + continue + elif tokens[0] == 'ok': + break + elif tokens[0] == 'error:' and len(tokens) == 2: + values = tokens[1].split(' ') + if typ == 'D23' and len(values) >= 3: + reason = CrashReason(int(values[0], 0)) + pc = int(values[1], 0) + sp = int(values[2], 0) + else: + line_error() + continue + elif len(tokens) != 2 or not re.match(r'^[0-9a-fA-F]+$', tokens[0]): + line_error() + continue + + # decode hex data + addr = int.from_bytes(bytes.fromhex(tokens[0]), 'big') + data = bytes.fromhex(tokens[1]) + ranges.append((addr, len(data))) + + if buf_addr is None: + buf_addr = addr + buf_data = data + else: + # grow buffer as needed + buf_addr, buf_data = region_expand(buf_addr, buf_data, + addr, len(data)) + + # replace new part + rep_start = addr - buf_addr + rep_end = rep_start + len(data) + buf_data = buf_data[:rep_start] + data + buf_data[rep_end:] + + # merge continuous ranges + ranges = merge_ranges(ranges) + + if typ == 'D2': + # D2 doesn't guarantee registers to be present + regs = len(ranges) > 0 and \ + ranges[0][0] == 0 and \ + ranges[0][1] >= avr.SRAM_START + + # fill to fit for easy loading + buf_addr, buf_data = region_expand( + buf_addr, buf_data, 0, avr.SRAM_START + avr.SRAM_SIZE) + + elif typ == 'D23': + # check if the dump is complete + if len(ranges) != 1 or ranges[0][0] != 0 or \ + ranges[0][1] != avr.SRAM_START + avr.SRAM_SIZE: + print('warning: incomplete D23 dump', file=sys.stderr) + else: + regs = True + if reason is None: + print('warning: no error line in D23', file=sys.stderr) + + elif typ == 'D21': + if len(ranges) != 1 or len(buf_data) != (avr.SRAM_START + avr.SRAM_SIZE + 256): + print('error: incomplete D21 dump', file=sys.stderr) + return None + + # decode the header structure + magic, regs_present, crash_reason, pc, sp = struct.unpack(' Date: Tue, 22 Jun 2021 17:02:22 +0200 Subject: [PATCH 33/40] Add GDB utility functions to load/inspect binary dumps --- tools/utils.gdb | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tools/utils.gdb diff --git a/tools/utils.gdb b/tools/utils.gdb new file mode 100644 index 00000000..49548036 --- /dev/null +++ b/tools/utils.gdb @@ -0,0 +1,82 @@ +# -*- gdb-script -*- + +define load_dump + restore $arg0 binary 0x800000 + set $pc = (((unsigned long)$arg1) - 2) << 1 + set $sp = $arg2 + where +end + +document load_dump +Load a crash dump, setup PC/SP and show the current backtrace +Usage: load_dump +end + + +define sp_skip + if $argc == 0 + set $shift = 3 + else + set $shift = $arg0 + end + set $new_pc = ((((unsigned long)*(uint8_t*)($sp+$shift+1)) << 16) + \ + (((unsigned long)*(uint8_t*)($sp+$shift+2)) << 8) + \ + (((unsigned long)*(uint8_t*)($sp+$shift+3)) << 0)) << 1 + set $new_sp = $sp+$shift+3 + select-frame 0 + set $saved_pc = $pc + set $saved_sp = $sp + set $pc = $new_pc + set $sp = $new_sp + where +end + +document sp_skip +TODO +end + + +define sp_restore + select-frame 0 + set $pc = $saved_pc + set $sp = $saved_sp + where +end + +document sp_restore +TODO +end + + +define sp_test + sp_skip $arg0 + set $pc = $saved_pc + set $sp = $saved_sp +end + +document sp_test +TODO +end + + +define sp_scan + dont-repeat + + if $argc == 0 + set $sp_end = 0x802200 + else + set $sp_end = $arg0 + end + + set $sp_pos = $sp + while $sp_pos < ($sp_end-4) + set $sp_off = $sp_pos - $sp + printf "**** scanning %#x (+%u) ****\n", $sp_pos, $sp_off + sp_test $sp_off + set $sp_pos += 1 + end +end + +document sp_scan +TODO +end From 11a6ac2f4fc41b1231efec63fdfe4746fb994710 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 22 Jun 2021 17:08:32 +0200 Subject: [PATCH 34/40] dump parsing: refuse to continue on incomplete D23 dumps We should try harder to handle incomplete D21/D23 dumps in the future, but until D21 handled too, bail in D23 as well. --- tools/lib/dump.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/lib/dump.py b/tools/lib/dump.py index 1c541a2a..8d6b1059 100644 --- a/tools/lib/dump.py +++ b/tools/lib/dump.py @@ -137,9 +137,10 @@ def decode_dump(path): # check if the dump is complete if len(ranges) != 1 or ranges[0][0] != 0 or \ ranges[0][1] != avr.SRAM_START + avr.SRAM_SIZE: - print('warning: incomplete D23 dump', file=sys.stderr) - else: - regs = True + print('error: incomplete D23 dump', file=sys.stderr) + return None + + regs = True if reason is None: print('warning: no error line in D23', file=sys.stderr) From 43b9a2d3df162930752c0b9d128ecfceac905a9c Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 22 Jun 2021 17:12:24 +0200 Subject: [PATCH 35/40] dump: do not hard-code constants --- tools/lib/dump.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/lib/dump.py b/tools/lib/dump.py index 8d6b1059..5d2ea724 100644 --- a/tools/lib/dump.py +++ b/tools/lib/dump.py @@ -5,7 +5,8 @@ import struct from . import avr -FILL_BYTE = b'\0' # used to fill memory gaps in the dump +FILL_BYTE = b'\0' # used to fill memory gaps in the dump +DUMP_MAGIC = 0x55525547 # XFLASH dump magic class CrashReason(enum.IntEnum): MANUAL = 0 @@ -151,7 +152,7 @@ def decode_dump(path): # decode the header structure magic, regs_present, crash_reason, pc, sp = struct.unpack(' Date: Tue, 22 Jun 2021 17:56:42 +0200 Subject: [PATCH 36/40] tools: Add/fix documentation --- tools/README.md | 4 ++++ tools/elf_mem_map | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/README.md b/tools/README.md index e7911501..30648d0d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -25,6 +25,10 @@ With ``--qdirstat`` and a single elf file, generate a [qdirstat](https://github. This assumes the running firmware generating the dump and the elf file are the same. Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) module. +### ``dump2bin`` + +Parse and decode a memory dump obtained from the D2/D21/D23 g-code into readable metadata and binary. The output binary is padded and extended to fit the original address range. + ### ``update_eeprom`` Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer. diff --git a/tools/elf_mem_map b/tools/elf_mem_map index cb6e423c..df3fc0f5 100755 --- a/tools/elf_mem_map +++ b/tools/elf_mem_map @@ -344,7 +344,7 @@ def main(): ap = argparse.ArgumentParser(description=""" Generate a symbol table map starting directly from an ELF firmware with DWARF3 debugging information. - When used along with a memory dump obtained from the D2 g-code, + When used along with a memory dump obtained from the D2/D21/D23 g-code, show the value of each symbol which is within the address range. """) ap.add_argument('elf', help='ELF file containing DWARF debugging information') From c79b1dcbfa6b112383543198681a7f827b0b61f0 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 22 Jun 2021 17:59:41 +0200 Subject: [PATCH 37/40] tools: add dump_crash to recover XFLASH crash dumps --- tools/README.md | 5 +++++ tools/dump_crash | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100755 tools/dump_crash diff --git a/tools/README.md b/tools/README.md index 30648d0d..63c1731d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -12,6 +12,11 @@ Requires ``printcore`` from [Pronterface]. Dump the content of the entire SRAM using the D2 command. Requires ``printcore`` from [Pronterface]. +### ``dump_crash`` + +Dump the content of the last crash dump on MK3+ printers using D21. +Requires ``printcore`` from [Pronterface]. + ### ``elf_mem_map`` Generate a symbol table map with decoded information starting directly from an ELF firmware with DWARF debugging information (which is the default using the stock board definition). diff --git a/tools/dump_crash b/tools/dump_crash new file mode 100755 index 00000000..f74e8bf2 --- /dev/null +++ b/tools/dump_crash @@ -0,0 +1,17 @@ +#!/bin/sh +prg=$(basename "$0") +port="$1" +if [ -z "$port" -o "$port" = "-h" ] +then + echo "usage: $0 " >&2 + echo "Connect to and dump the content of last crash using D21 to stdout" >&2 + exit 1 +fi + +set -e +tmp=$(mktemp) +trap "rm -f \"$tmp\"" EXIT + +echo D21 > "$tmp" +printcore -v "$port" "$tmp" 2>&1 | \ + sed -ne '/^RECV: D21 /,/RECV: ok$/s/^RECV: //p' From a697d00647d0ca85da8d0128850b4edef578869c Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Tue, 22 Jun 2021 18:01:21 +0200 Subject: [PATCH 38/40] tools: add __pycache__ to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dae307af..26411502 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .project .cproject Debug +__pycache__ Firmware/Configuration_prusa.h Firmware/Doc /Firmware/.vs/Firmware/v14 From 1b22aac9fcc68662cf22129bafc02c8197471426 Mon Sep 17 00:00:00 2001 From: Yuri D'Elia Date: Thu, 1 Jul 2021 13:16:02 +0200 Subject: [PATCH 39/40] tools: add xfimg2dump --- tools/README.md | 4 ++++ tools/lib/dump.py | 6 ++++-- tools/xfimg2dump | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100755 tools/xfimg2dump diff --git a/tools/README.md b/tools/README.md index 63c1731d..50516dac 100644 --- a/tools/README.md +++ b/tools/README.md @@ -34,6 +34,10 @@ Requires Python3 and the [pyelftools](https://github.com/eliben/pyelftools) modu Parse and decode a memory dump obtained from the D2/D21/D23 g-code into readable metadata and binary. The output binary is padded and extended to fit the original address range. +### ``xfimg2dump`` + +Extract a crash dump from an external flash image and output the same format produced by the D21 g-code. + ### ``update_eeprom`` Given one EEPROM dump, convert the dump to update instructions that can be sent to a printer. diff --git a/tools/lib/dump.py b/tools/lib/dump.py index 5d2ea724..7ce75606 100644 --- a/tools/lib/dump.py +++ b/tools/lib/dump.py @@ -5,8 +5,10 @@ import struct from . import avr -FILL_BYTE = b'\0' # used to fill memory gaps in the dump -DUMP_MAGIC = 0x55525547 # XFLASH dump magic +FILL_BYTE = b'\0' # used to fill memory gaps in the dump +DUMP_MAGIC = 0x55525547 # XFLASH dump magic +DUMP_OFFSET = 0x3d000 # XFLASH dump offset +DUMP_SIZE = 0x2300 # XFLASH dump size class CrashReason(enum.IntEnum): MANUAL = 0 diff --git a/tools/xfimg2dump b/tools/xfimg2dump new file mode 100755 index 00000000..e1fe75ac --- /dev/null +++ b/tools/xfimg2dump @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import argparse +import struct +import os, sys + +from lib.dump import DUMP_MAGIC, DUMP_OFFSET, DUMP_SIZE + + +def error(msg): + print(msg, file=sys.stderr) + +def main(): + # parse the arguments + ap = argparse.ArgumentParser(description=""" + Extract a crash dump from an external flash image and output + the same format produced by the D21 g-code. + """) + ap.add_argument('image') + args = ap.parse_args() + + # read the image + off = DUMP_OFFSET + with open(args.image, 'rb') as fd: + fd.seek(off) + data = fd.read(DUMP_SIZE) + if len(data) != DUMP_SIZE: + error('incorrect image size') + return os.EX_DATAERR + + # check for magic header + magic, = struct.unpack(' Date: Fri, 2 Jul 2021 20:57:12 +0200 Subject: [PATCH 40/40] tools: document functions in utils.gdb --- tools/utils.gdb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/utils.gdb b/tools/utils.gdb index 49548036..1d2922e1 100644 --- a/tools/utils.gdb +++ b/tools/utils.gdb @@ -9,7 +9,7 @@ end document load_dump Load a crash dump, setup PC/SP and show the current backtrace -Usage: load_dump +Usage: load_dump end @@ -32,7 +32,9 @@ define sp_skip end document sp_skip -TODO +Decode the PC address at SP+offset, then show the resulting stack. +The default (and minimum) offset is 3. +Usage: sp_skip [off] end @@ -44,7 +46,8 @@ define sp_restore end document sp_restore -TODO +Undo an sp_skip move (restore existing PC/SP positions) +Usage: sp_restore end @@ -55,7 +58,9 @@ define sp_test end document sp_test -TODO +Attempt to decode the PC address at SP+offset, then show the resulting stack. +The default (and minimum) offset is 3. +Usage: sp_test [off] end @@ -78,5 +83,7 @@ define sp_scan end document sp_scan -TODO +Attempt to decode PC at any location starting from the SP+3 and up to SP-end +(by default the end of the SRAM) and show the resulting stack at all locations. +Usage: sp_scan [SP-end] end