tools/mpy_ld.py: Allow linking static libraries.

This commit introduces an additional symbol resolution mechanism to the
natmod linking process.  This allows the build scripts to look for required
symbols into selected libraries that are provided by the compiler
installation (libgcc and libm at the moment).

For example, using soft-float code in natmods, whilst technically possible,
was not an easy process and required some additional work to pull it off.
With this addition all the manual (and error-prone) operations have been
automated and folded into `tools/mpy_ld.py`.

Both newlib and picolibc toolchains are supported, albeit the latter may
require a bit of extra configuration depending on the environment the build
process runs on.  Picolibc's soft-float functions aren't in libm - in fact
the shipped libm is nothing but a stub - but they are inside libc.  This is
usually not a problem as these changes cater for that configuration quirk,
but on certain compilers the include paths used to find libraries in may
not be updated to take Picolibc's library directory into account.  The bare
metal RISC-V compiler shipped with the CI OS image (GCC 10.2.0 on Ubuntu
22.04LTS) happens to exhibit this very problem.

To work around that for CI builds, the Picolibc libraries' path is
hardcoded in the Makefile directives used by the linker, but this can be
changed by setting the PICOLIBC_ROOT environment library when building
natmods.

Signed-off-by: Volodymyr Shymanskyy <vshymanskyi@gmail.com>
Co-authored-by: Alessandro Gatti <a.gatti@frob.it>

tools/mpy_ld.py

@@ -30,6 +30,7 @@ Link .o files to .mpy
 
 import sys, os, struct, re
 from elftools.elf import elffile
+import ar_util
 
 sys.path.append(os.path.dirname(__file__) + "/../py")
 import makeqstrdata as qstrutil
@@ -664,7 +665,7 @@ def do_relocation_text(env, text_addr, r):
         R_XTENSA_PDIFF32,
         R_XTENSA_ASM_EXPAND,
     ):
-        if s.section.name.startswith(".text"):
+        if not hasattr(s, "section") or s.section.name.startswith(".text"):
             # it looks like R_XTENSA_[P]DIFF32 into .text is already correctly relocated,
             # and expand relaxations cannot occur in non-executable sections.
             return
@@ -1075,59 +1076,59 @@ def process_riscv32_relocation(env, text_addr, r):
     return addr, value
 
 
-def load_object_file(env, felf):
-    with open(felf, "rb") as f:
-        elf = elffile.ELFFile(f)
-        env.check_arch(elf["e_machine"])
+def load_object_file(env, f, felf):
+    elf = elffile.ELFFile(f)
+    env.check_arch(elf["e_machine"])
 
-        # Get symbol table
-        symtab = list(elf.get_section_by_name(".symtab").iter_symbols())
+    # Get symbol table
+    symtab = list(elf.get_section_by_name(".symtab").iter_symbols())
 
-        # Load needed sections from ELF file
-        sections_shndx = {}  # maps elf shndx to Section object
-        for idx, s in enumerate(elf.iter_sections()):
-            if s.header.sh_type in ("SHT_PROGBITS", "SHT_NOBITS"):
-                if s.data_size == 0:
-                    # Ignore empty sections
-                    pass
-                elif s.name.startswith((".literal", ".text", ".rodata", ".data.rel.ro", ".bss")):
-                    sec = Section.from_elfsec(s, felf)
-                    sections_shndx[idx] = sec
-                    if s.name.startswith(".literal"):
-                        env.literal_sections.append(sec)
-                    else:
-                        env.sections.append(sec)
-                elif s.name.startswith(".data"):
-                    raise LinkError("{}: {} non-empty".format(felf, s.name))
+    # Load needed sections from ELF file
+    sections_shndx = {}  # maps elf shndx to Section object
+    for idx, s in enumerate(elf.iter_sections()):
+        if s.header.sh_type in ("SHT_PROGBITS", "SHT_NOBITS"):
+            if s.data_size == 0:
+                # Ignore empty sections
+                pass
+            elif s.name.startswith((".literal", ".text", ".rodata", ".data.rel.ro", ".bss")):
+                sec = Section.from_elfsec(s, felf)
+                sections_shndx[idx] = sec
+                if s.name.startswith(".literal"):
+                    env.literal_sections.append(sec)
                 else:
-                    # Ignore section
-                    pass
-            elif s.header.sh_type in ("SHT_REL", "SHT_RELA"):
-                shndx = s.header.sh_info
-                if shndx in sections_shndx:
-                    sec = sections_shndx[shndx]
-                    sec.reloc_name = s.name
-                    sec.reloc = list(s.iter_relocations())
-                    for r in sec.reloc:
-                        r.sym = symtab[r["r_info_sym"]]
-
-        # Link symbols to their sections, and update known and unresolved symbols
-        for sym in symtab:
-            sym.filename = felf
-            shndx = sym.entry["st_shndx"]
+                    env.sections.append(sec)
+            elif s.name.startswith(".data"):
+                raise LinkError("{}: {} non-empty".format(felf, s.name))
+            else:
+                # Ignore section
+                pass
+        elif s.header.sh_type in ("SHT_REL", "SHT_RELA"):
+            shndx = s.header.sh_info
             if shndx in sections_shndx:
-                # Symbol with associated section
-                sym.section = sections_shndx[shndx]
-                if sym["st_info"]["bind"] in ("STB_GLOBAL", "STB_WEAK"):
-                    # Defined global symbol
-                    if sym.name in env.known_syms and not sym.name.startswith(
-                        "__x86.get_pc_thunk."
-                    ):
-                        raise LinkError("duplicate symbol: {}".format(sym.name))
-                    env.known_syms[sym.name] = sym
-            elif sym.entry["st_shndx"] == "SHN_UNDEF" and sym["st_info"]["bind"] == "STB_GLOBAL":
-                # Undefined global symbol, needs resolving
-                env.unresolved_syms.append(sym)
+                sec = sections_shndx[shndx]
+                sec.reloc_name = s.name
+                sec.reloc = list(s.iter_relocations())
+                for r in sec.reloc:
+                    r.sym = symtab[r["r_info_sym"]]
+
+    # Link symbols to their sections, and update known and unresolved symbols
+    dup_errors = []
+    for sym in symtab:
+        sym.filename = felf
+        shndx = sym.entry["st_shndx"]
+        if shndx in sections_shndx:
+            # Symbol with associated section
+            sym.section = sections_shndx[shndx]
+            if sym["st_info"]["bind"] in ("STB_GLOBAL", "STB_WEAK"):
+                # Defined global symbol
+                if sym.name in env.known_syms and not sym.name.startswith("__x86.get_pc_thunk."):
+                    dup_errors.append("duplicate symbol: {}".format(sym.name))
+                env.known_syms[sym.name] = sym
+        elif sym.entry["st_shndx"] == "SHN_UNDEF" and sym["st_info"]["bind"] == "STB_GLOBAL":
+            # Undefined global symbol, needs resolving
+            env.unresolved_syms.append(sym)
+    if len(dup_errors):
+        raise LinkError("\n".join(dup_errors))
 
 
 def link_objects(env, native_qstr_vals_len):
@@ -1188,6 +1189,8 @@ def link_objects(env, native_qstr_vals_len):
             ]
         )
     }
+
+    undef_errors = []
     for sym in env.unresolved_syms:
         assert sym["st_value"] == 0
         if sym.name == "_GLOBAL_OFFSET_TABLE_":
@@ -1205,7 +1208,10 @@ def link_objects(env, native_qstr_vals_len):
                 sym.section = mp_fun_table_sec
                 sym.mp_fun_table_offset = fun_table[sym.name]
             else:
-                raise LinkError("{}: undefined symbol: {}".format(sym.filename, sym.name))
+                undef_errors.append("{}: undefined symbol: {}".format(sym.filename, sym.name))
+
+    if len(undef_errors):
+        raise LinkError("\n".join(undef_errors))
 
     # Align sections, assign their addresses, and create full_text
     env.full_text = bytearray(env.arch.asm_jump(8))  # dummy, to be filled in later
@@ -1446,8 +1452,27 @@ def do_link(args):
     log(LOG_LEVEL_2, "qstr vals: " + ", ".join(native_qstr_vals))
     env = LinkEnv(args.arch)
     try:
-        for file in args.files:
-            load_object_file(env, file)
+        # Load object files
+        for fn in args.files:
+            with open(fn, "rb") as f:
+                load_object_file(env, f, fn)
+
+        if args.libs:
+            # Load archive info
+            archives = []
+            for item in args.libs:
+                archives.extend(ar_util.load_archive(item))
+            # List symbols to look for
+            syms = set(sym.name for sym in env.unresolved_syms)
+            # Resolve symbols from libs
+            lib_objs, _ = ar_util.resolve(archives, syms)
+            # Load extra object files from libs
+            for ar, obj in lib_objs:
+                obj_name = ar.fn + ":" + obj
+                log(LOG_LEVEL_2, "using " + obj_name)
+                with ar.open(obj) as f:
+                    load_object_file(env, f, obj_name)
+
         link_objects(env, len(native_qstr_vals))
         build_mpy(env, env.find_addr("mpy_init"), args.output, native_qstr_vals)
     except LinkError as er:
@@ -1458,13 +1483,16 @@ def do_link(args):
 def main():
     import argparse
 
-    cmd_parser = argparse.ArgumentParser(description="Run scripts on the pyboard.")
+    cmd_parser = argparse.ArgumentParser(description="Link native object files into a MPY bundle.")
     cmd_parser.add_argument(
         "--verbose", "-v", action="count", default=1, help="increase verbosity"
     )
     cmd_parser.add_argument("--arch", default="x64", help="architecture")
     cmd_parser.add_argument("--preprocess", action="store_true", help="preprocess source files")
     cmd_parser.add_argument("--qstrs", default=None, help="file defining additional qstrs")
+    cmd_parser.add_argument(
+        "--libs", "-l", dest="libs", action="append", help="static .a libraries to link"
+    )
     cmd_parser.add_argument(
         "--output", "-o", default=None, help="output .mpy file (default to input with .o->.mpy)"
     )