docs/develop/natmod: Add notes on Picolibc and natmods.

This commit adds some documentation on what are the limitations of using
Picolibc as a standard C library for native modules.

This also contains a reference to the "errno" issue when building
natmods on RV32 that the PR this commit is part of, as it is not obvious
how to approach this issue when encountered for the first time.

Signed-off-by: Alessandro Gatti <a.gatti@frob.it>

.github/workflows/ports_unix.yml

@@ -134,6 +134,20 @@ jobs:
       if: failure()
       run: tests/run-tests.py --print-failures
 
+  longlong:
+    runs-on: ubuntu-22.04 # use 22.04 to get python2, and libffi-dev:i386
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_32bit_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_longlong_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_longlong_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
   float:
     runs-on: ubuntu-latest
     steps:

docs/develop/natmod.rst

@@ -67,6 +67,9 @@ The known limitations are:
 
 * static BSS variables are not supported; workaround: use global BSS variables
 
+* thread-local storage variables are not supported on rv32imc; workaround: use
+  global BSS variables or allocate some space on the heap to store them
+
 So, if your C code has writable data, make sure the data is defined globally,
 without an initialiser, and only written to within functions.
 
@@ -225,6 +228,26 @@ other module, for example::
     print(factorial.factorial(10))
     # should display 3628800
 
+Using Picolibc when building modules
+------------------------------------
+
+Using `Picolibc <https://github.com/picolibc/picolibc>`_ as your C standard
+library is not only supported, but in fact it is the default for the rv32imc
+platform.  However, there are a couple of things worth mentioning to make sure
+you don't run into problems later when building code.
+
+Some pre-built Picolibc versions (for example, those provided by Ubuntu Linux
+as the ``picolibc-arm-none-eabi``, ``picolibc-riscv64-unknown-elf``, and
+``picolibc-xtensa-lx106-elf`` packages) assume thread-local storage (TLS) is
+available at runtime, but unfortunately MicroPython modules do not support that
+on some architectures (namely ``rv32imc``).  This means that some
+functionalities provided by Picolibc will default to use TLS, returning an
+error either during compilation or during linking.
+
+For an example on how this may affect you, the ``examples/natmod/btree``
+example module contains a workaround to make sure ``errno`` works (look for
+``__PICOLIBC_ERRNO_FUNCTION`` in the Makefile and follow the trail from there).
+
 Further examples
 ----------------
 

examples/natmod/btree/Makefile

@@ -36,6 +36,9 @@ ifeq ($(ARCH),xtensa)
 MPY_EXTERN_SYM_FILE=$(MPY_DIR)/ports/esp8266/boards/eagle.rom.addr.v6.ld
 endif
 
+# Use our own errno implementation if Picolibc is used
+CFLAGS += -D__PICOLIBC_ERRNO_FUNCTION=__errno
+
 include $(MPY_DIR)/py/dynruntime.mk
 
 # btree needs gnu99 defined

examples/rp2/pio_uart_rx.py

@@ -23,7 +23,7 @@ PIO_RX_PIN = Pin(3, Pin.IN, Pin.PULL_UP)
 @asm_pio(
     autopush=True,
     push_thresh=8,
-    in_shiftdir=rp2.PIO.SHIFT_RIGHT,
+    in_shiftdir=PIO.SHIFT_RIGHT,
     fifo_join=PIO.JOIN_RX,
 )
 def uart_rx_mini():
@@ -42,7 +42,7 @@ def uart_rx_mini():
 
 
 @asm_pio(
-    in_shiftdir=rp2.PIO.SHIFT_RIGHT,
+    in_shiftdir=PIO.SHIFT_RIGHT,
 )
 def uart_rx():
     # fmt: off

extmod/lwip-include/lwipopts_common.h

@@ -28,6 +28,9 @@
 
 #include "py/mpconfig.h"
 
+// This is needed to access `next_timeout` via `sys_timeouts_get_next_timeout()`.
+#define LWIP_TESTMODE                   1
+
 // This sys-arch protection is not needed.
 // Ports either protect lwIP code with flags, or run it at PendSV priority.
 #define SYS_ARCH_DECL_PROTECT(lev) do { } while (0)

extmod/modnetwork.h

@@ -82,6 +82,9 @@ extern const struct _mp_obj_type_t mp_network_ppp_lwip_type;
 #endif
 
 struct netif;
+
+void sys_untimeout_all_with_arg(void *arg);
+
 void mod_network_lwip_init(void);
 void mod_network_lwip_poll_wrapper(uint32_t ticks_ms);
 mp_obj_t mod_network_nic_ifconfig(struct netif *netif, size_t n_args, const mp_obj_t *args);

extmod/network_lwip.c

@@ -52,6 +52,19 @@ int mp_mod_network_prefer_dns_use_ip_version = 4;
 
 // Implementations of network methods that can be used by any interface.
 
+// This follows sys_untimeout but removes all timeouts with the given argument.
+void sys_untimeout_all_with_arg(void *arg) {
+    for (struct sys_timeo **t = sys_timeouts_get_next_timeout(); *t != NULL;) {
+        if ((*t)->arg == arg) {
+            struct sys_timeo *next = (*t)->next;
+            memp_free(MEMP_SYS_TIMEOUT, *t);
+            *t = next;
+        } else {
+            t = &(*t)->next;
+        }
+    }
+}
+
 // This function provides the implementation of nic.ifconfig, is deprecated and will be removed.
 // Use network.ipconfig and nic.ipconfig instead.
 mp_obj_t mod_network_nic_ifconfig(struct netif *netif, size_t n_args, const mp_obj_t *args) {

ports/alif/lwip_inc/lwipopts.h

@@ -1,49 +1,13 @@
 #ifndef MICROPY_INCLUDED_ALIF_LWIP_LWIPOPTS_H
 #define MICROPY_INCLUDED_ALIF_LWIP_LWIPOPTS_H
 
-#include <stdint.h>
-
-// This protection is not needed, instead we execute all lwIP code at PendSV priority
-#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0)
-#define SYS_ARCH_PROTECT(lev) do { } while (0)
-#define SYS_ARCH_UNPROTECT(lev) do { } while (0)
-
-#define NO_SYS                          1
-#define SYS_LIGHTWEIGHT_PROT            1
-#define MEM_ALIGNMENT                   4
-
-#define LWIP_CHKSUM_ALGORITHM           3
-#define LWIP_CHECKSUM_CTRL_PER_NETIF    1
-
-#define LWIP_ARP                        1
-#define LWIP_ETHERNET                   1
-#define LWIP_RAW                        1
-#define LWIP_NETCONN                    0
-#define LWIP_SOCKET                     0
-#define LWIP_STATS                      0
-#define LWIP_NETIF_HOSTNAME             1
 #define LWIP_NETIF_EXT_STATUS_CALLBACK  1
 
 #define LWIP_LOOPIF_MULTICAST           1
 #define LWIP_LOOPBACK_MAX_PBUFS         8
 
 #define LWIP_IPV6                       0
-#define LWIP_DHCP                       1
-#define LWIP_DHCP_CHECK_LINK_UP         1
-#define LWIP_DHCP_DOES_ACD_CHECK        0 // to speed DHCP up
-#define LWIP_DNS                        1
-#define LWIP_DNS_SUPPORT_MDNS_QUERIES   1
-#define LWIP_MDNS_RESPONDER             1
-#define LWIP_IGMP                       1
 
-#define LWIP_NUM_NETIF_CLIENT_DATA      LWIP_MDNS_RESPONDER
-#define MEMP_NUM_UDP_PCB                (4 + LWIP_MDNS_RESPONDER)
-#define MEMP_NUM_SYS_TIMEOUT            (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER)
-
-#define SO_REUSE                        1
-#define TCP_LISTEN_BACKLOG              1
-
-extern uint64_t se_services_rand64(void);
 #define LWIP_RAND() se_services_rand64()
 
 #define MEM_SIZE                        (16 * 1024)
@@ -55,6 +19,9 @@ extern uint64_t se_services_rand64(void);
 #define TCP_QUEUE_OOSEQ                 (1)
 #define MEMP_NUM_TCP_SEG                (2 * TCP_SND_QUEUELEN)
 
-typedef uint32_t sys_prot_t;
+// Include common lwIP configuration.
+#include "extmod/lwip-include/lwipopts_common.h"
+
+uint64_t se_services_rand64(void);
 
 #endif // MICROPY_INCLUDED_ALIF_LWIP_LWIPOPTS_H

ports/qemu/Makefile

@@ -191,12 +191,11 @@ test_full: $(BUILD)/firmware.elf
 	cd $(TOP)/tests && ./run-tests.py $(RUN_TESTS_FULL_ARGS) --via-mpy
 	cd $(TOP)/tests && ./run-tests.py $(RUN_TESTS_FULL_ARGS) --via-mpy --emit native
 
-# "btree" currently does not build for rv32imc (Picolibc TLS incompatibility).
 .PHONY: test_natmod
 test_natmod: $(BUILD)/firmware.elf
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
 	cd $(TOP)/tests && \
-		for natmod in deflate framebuf heapq random_basic re; do \
+		for natmod in btree deflate framebuf heapq random_basic re; do \
 		./run-natmodtests.py -p -d execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" extmod/$$natmod*.py; \
 		done
 

ports/rp2/mpnetworkport.c

@@ -30,6 +30,7 @@
 
 #if MICROPY_PY_LWIP
 
+#include "extmod/modnetwork.h"
 #include "shared/runtime/softtimer.h"
 #include "lwip/netif.h"
 #include "lwip/timeouts.h"
@@ -183,6 +184,10 @@ static void mp_network_netif_status_cb(struct netif *netif, netif_nsc_reason_t r
         mp_network_soft_timer.mode = SOFT_TIMER_MODE_PERIODIC;
         soft_timer_reinsert(&mp_network_soft_timer, LWIP_TICK_RATE_MS);
     }
+
+    if (reason == LWIP_NSC_NETIF_REMOVED) {
+        sys_untimeout_all_with_arg(netif);
+    }
 }
 
 #endif // MICROPY_PY_LWIP

ports/unix/Makefile

@@ -120,16 +120,6 @@ LDFLAGS += $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
 # Flags to link with pthread library
 LIBPTHREAD = -lpthread
 
-ifeq ($(MICROPY_FORCE_32BIT),1)
-# Note: you may need to install i386 versions of dependency packages,
-# starting with linux-libc-dev:i386
-ifeq ($(MICROPY_PY_FFI),1)
-ifeq ($(UNAME_S),Linux)
-CFLAGS += -I/usr/include/i686-linux-gnu
-endif
-endif
-endif
-
 ifeq ($(MICROPY_USE_READLINE),1)
 INC += -I$(TOP)/shared/readline
 CFLAGS += -DMICROPY_USE_READLINE=1

ports/unix/coverage.c

@@ -204,8 +204,20 @@ static mp_obj_t extra_coverage(void) {
         mp_printf(&mp_plat_print, "%d %+d % d\n", -123, 123, 123); // sign
         mp_printf(&mp_plat_print, "%05d\n", -123); // negative number with zero padding
         mp_printf(&mp_plat_print, "%ld\n", 123); // long
-        mp_printf(&mp_plat_print, "%lx\n", 0x123); // long hex
-        mp_printf(&mp_plat_print, "%X\n", 0x1abcdef); // capital hex
+        mp_printf(&mp_plat_print, "%lx\n", 0x123fl); // long hex
+        mp_printf(&mp_plat_print, "%lX\n", 0x123fl); // capital long hex
+        if (sizeof(mp_int_t) == 8) {
+            mp_printf(&mp_plat_print, "%llx\n", LLONG_MAX); // long long hex
+            mp_printf(&mp_plat_print, "%llX\n", LLONG_MAX); // capital long long hex
+            mp_printf(&mp_plat_print, "%llu\n", ULLONG_MAX); // unsigned long long
+        } else {
+            // fake for platforms without narrower mp_int_t
+            mp_printf(&mp_plat_print, "7fffffffffffffff\n", LLONG_MAX);
+            mp_printf(&mp_plat_print, "7FFFFFFFFFFFFFFF\n", LLONG_MAX);
+            mp_printf(&mp_plat_print, "18446744073709551615\n", ULLONG_MAX);
+        }
+        mp_printf(&mp_plat_print, "%p\n", (void *)0x789f); // pointer
+        mp_printf(&mp_plat_print, "%P\n", (void *)0x789f); // pointer uppercase
         mp_printf(&mp_plat_print, "%.2s %.3s '%4.4s' '%5.5q' '%.3q'\n", "abc", "abc", "abc", MP_QSTR_True, MP_QSTR_True); // fixed string precision
         mp_printf(&mp_plat_print, "%.*s\n", -1, "abc"); // negative string precision
         mp_printf(&mp_plat_print, "%b %b\n", 0, 1); // bools
@@ -216,11 +228,31 @@ static mp_obj_t extra_coverage(void) {
         #endif
         mp_printf(&mp_plat_print, "%d\n", 0x80000000); // should print signed
         mp_printf(&mp_plat_print, "%u\n", 0x80000000); // should print unsigned
-        mp_printf(&mp_plat_print, "%x\n", 0x80000000); // should print unsigned
-        mp_printf(&mp_plat_print, "%X\n", 0x80000000); // should print unsigned
+        mp_printf(&mp_plat_print, "%x\n", 0x8000000f); // should print unsigned
+        mp_printf(&mp_plat_print, "%X\n", 0x8000000f); // should print unsigned
         mp_printf(&mp_plat_print, "abc\n%"); // string ends in middle of format specifier
         mp_printf(&mp_plat_print, "%%\n"); // literal % character
         mp_printf(&mp_plat_print, ".%-3s.\n", "a"); // left adjust
+
+        // Check that all kinds of mp_printf arguments are parsed out
+        // correctly, by having a char argument before and after each main type
+        // of value that can be formatted.
+        mp_printf(&mp_plat_print, "%c%%%c\n", '<', '>');
+        mp_printf(&mp_plat_print, "%c%p%c\n", '<', (void *)0xaaaa, '>');
+        mp_printf(&mp_plat_print, "%c%b%c\n", '<', true, '>');
+        mp_printf(&mp_plat_print, "%c%d%c\n", '<', 0xaaaa, '>');
+        mp_printf(&mp_plat_print, "%c%ld%c\n", '<', 0xaaaal, '>');
+        mp_printf(&mp_plat_print, "%c" INT_FMT "%c\n", '<', (mp_int_t)0xaaaa, '>');
+        mp_printf(&mp_plat_print, "%c%s%c\n", '<', "test", '>');
+        mp_printf(&mp_plat_print, "%c%f%c\n", '<', MICROPY_FLOAT_CONST(1000.), '>');
+        mp_printf(&mp_plat_print, "%c%q%c\n", '<', (qstr)MP_QSTR_True, '>');
+        if (sizeof(mp_int_t) == 8) {
+            mp_printf(&mp_plat_print, "%c%lld%c\n", '<', LLONG_MAX, '>');
+        } else {
+            mp_printf(&mp_plat_print, "<9223372036854775807>\n");
+        }
+
+
     }
 
     // GC

ports/unix/variants/longlong/mpconfigvariant.h

@@ -0,0 +1,37 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+// This config exists to test that the MICROPY_LONGINT_IMPL_LONGLONG variant
+// (i.e. minimal form of "big integer" that's backed by 64-bit int only) builds
+// and passes tests.
+
+#define MICROPY_LONGINT_IMPL           (MICROPY_LONGINT_IMPL_LONGLONG)
+
+// Set base feature level.
+#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES)
+
+// Enable extra Unix features.
+#include "../mpconfigvariant_common.h"

ports/unix/variants/longlong/mpconfigvariant.mk

@@ -0,0 +1,8 @@
+# build interpreter with "bigints" implemented as "longlong"
+
+# otherwise, small int is essentially 64-bit
+MICROPY_FORCE_32BIT := 1
+
+MICROPY_PY_FFI := 0
+
+MPY_TOOL_FLAGS = -mlongint-impl longlong

ports/webassembly/objpyproxy.js

@@ -165,34 +165,35 @@ const py_proxy_handler = {
         if (prop === "_ref") {
             return target._ref;
         }
-        if (prop === "then") {
-            return null;
-        }
 
-        if (prop === Symbol.iterator) {
-            // Get the Python object iterator, and return a JavaScript generator.
-            const iter_ref = Module.ccall(
-                "proxy_c_to_js_get_iter",
-                "number",
-                ["number"],
-                [target._ref],
-            );
-            return function* () {
-                const value = Module._malloc(3 * 4);
-                while (true) {
-                    const valid = Module.ccall(
-                        "proxy_c_to_js_iternext",
-                        "number",
-                        ["number", "pointer"],
-                        [iter_ref, value],
-                    );
-                    if (!valid) {
-                        break;
+        // ignore both then and all symbols but Symbol.iterator
+        if (prop === "then" || typeof prop !== "string") {
+            if (prop === Symbol.iterator) {
+                // Get the Python object iterator, and return a JavaScript generator.
+                const iter_ref = Module.ccall(
+                    "proxy_c_to_js_get_iter",
+                    "number",
+                    ["number"],
+                    [target._ref],
+                );
+                return function* () {
+                    const value = Module._malloc(3 * 4);
+                    while (true) {
+                        const valid = Module.ccall(
+                            "proxy_c_to_js_iternext",
+                            "number",
+                            ["number", "pointer"],
+                            [iter_ref, value],
+                        );
+                        if (!valid) {
+                            break;
+                        }
+                        yield proxy_convert_mp_to_js_obj_jsside(value);
                     }
-                    yield proxy_convert_mp_to_js_obj_jsside(value);
-                }
-                Module._free(value);
-            };
+                    Module._free(value);
+                };
+            }
+            return undefined;
         }
 
         const value = Module._malloc(3 * 4);

py/misc.h

@@ -33,10 +33,15 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <stddef.h>
+#include <limits.h>
 
 typedef unsigned char byte;
 typedef unsigned int uint;
 
+#ifndef __has_builtin
+#define __has_builtin(x) (0)
+#endif
+
 /** generic ops *************************************************/
 
 #ifndef MIN
@@ -374,26 +379,23 @@ static inline bool mp_check(bool value) {
 static inline uint32_t mp_popcount(uint32_t x) {
     return __popcnt(x);
 }
-#else
+#else // _MSC_VER
 #define mp_clz(x) __builtin_clz(x)
 #define mp_clzl(x) __builtin_clzl(x)
 #define mp_clzll(x) __builtin_clzll(x)
 #define mp_ctz(x) __builtin_ctz(x)
 #define mp_check(x) (x)
-#if defined __has_builtin
 #if __has_builtin(__builtin_popcount)
 #define mp_popcount(x) __builtin_popcount(x)
-#endif
-#endif
-#if !defined(mp_popcount)
+#else
 static inline uint32_t mp_popcount(uint32_t x) {
     x = x - ((x >> 1) & 0x55555555);
     x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
     x = (x + (x >> 4)) & 0x0F0F0F0F;
     return (x * 0x01010101) >> 24;
 }
-#endif
-#endif
+#endif // __has_builtin(__builtin_popcount)
+#endif // _MSC_VER
 
 #define MP_FIT_UNSIGNED(bits, value) (((value) & (~0U << (bits))) == 0)
 #define MP_FIT_SIGNED(bits, value) \
@@ -426,4 +428,93 @@ static inline uint32_t mp_clz_mpi(mp_int_t x) {
     #endif
 }
 
+// Overflow-checked operations for long long
+
+// Integer overflow builtins were added to GCC 5, but __has_builtin only in GCC 10
+//
+// Note that the builtins has a defined result when overflow occurs, whereas the custom
+// functions below don't update the result if an overflow would occur (to avoid UB).
+#define MP_GCC_HAS_BUILTIN_OVERFLOW (__GNUC__ >= 5)
+
+#if __has_builtin(__builtin_umulll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW
+#define mp_mul_ull_overflow __builtin_umulll_overflow
+#else
+inline static bool mp_mul_ull_overflow(unsigned long long int x, unsigned long long int y, unsigned long long int *res) {
+    if (y > 0 && x > (ULLONG_MAX / y)) {
+        return true; // overflow
+    }
+    *res = x * y;
+    return false;
+}
+#endif
+
+#if __has_builtin(__builtin_smulll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW
+#define mp_mul_ll_overflow __builtin_smulll_overflow
+#else
+inline static bool mp_mul_ll_overflow(long long int x, long long int y, long long int *res) {
+    bool overflow;
+
+    // Check for multiply overflow; see CERT INT32-C
+    if (x > 0) { // x is positive
+        if (y > 0) { // x and y are positive
+            overflow = (x > (LLONG_MAX / y));
+        } else { // x positive, y nonpositive
+            overflow = (y < (LLONG_MIN / x));
+        } // x positive, y nonpositive
+    } else { // x is nonpositive
+        if (y > 0) { // x is nonpositive, y is positive
+            overflow = (x < (LLONG_MIN / y));
+        } else { // x and y are nonpositive
+            overflow = (x != 0 && y < (LLONG_MAX / x));
+        } // End if x and y are nonpositive
+    } // End if x is nonpositive
+
+    if (!overflow) {
+        *res = x * y;
+    }
+
+    return overflow;
+}
+#endif
+
+#if __has_builtin(__builtin_saddll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW
+#define mp_add_ll_overflow __builtin_saddll_overflow
+#else
+inline static bool mp_add_ll_overflow(long long int lhs, long long int rhs, long long int *res) {
+    bool overflow;
+
+    if (rhs > 0) {
+        overflow = (lhs > LLONG_MAX - rhs);
+    } else {
+        overflow = (lhs < LLONG_MIN - rhs);
+    }
+
+    if (!overflow) {
+        *res = lhs + rhs;
+    }
+
+    return overflow;
+}
+#endif
+
+#if __has_builtin(__builtin_ssubll_overflow) || MP_GCC_HAS_BUILTIN_OVERFLOW
+#define mp_sub_ll_overflow __builtin_ssubll_overflow
+#else
+inline static bool mp_sub_ll_overflow(long long int lhs, long long int rhs, long long int *res) {
+    bool overflow;
+
+    if (rhs > 0) {
+        overflow = (lhs < LLONG_MIN + rhs);
+    } else {
+        overflow = (lhs > LLONG_MAX + rhs);
+    }
+
+    if (!overflow) {
+        *res = lhs - rhs;
+    }
+
+    return overflow;
+}
+#endif
+
 #endif // MICROPY_INCLUDED_PY_MISC_H

py/mkrules.mk

@@ -268,7 +268,7 @@ submodules:
 	$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
 ifneq ($(GIT_SUBMODULES),)
 	$(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES)
-	$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \
+	$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) 2>/dev/null || \
 	  git submodule update --init $(GIT_SUBMODULES)
 endif
 .PHONY: submodules

py/mpprint.c

@@ -446,16 +446,36 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) {
             }
         }
 
-        // parse long specifiers (only for LP64 model where they make a difference)
-        #ifndef __LP64__
-        const
+        // parse long and long long specifiers (only where they make a difference)
+        #if defined(MICROPY_UNIX_COVERAGE) || (LONG_MAX > INT_MAX)
+        #define SUPPORT_L_FORMAT (1)
+        #else
+        #define SUPPORT_L_FORMAT (0)
         #endif
+        #if SUPPORT_L_FORMAT
         bool long_arg = false;
+        #endif
+
+        #if (MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D) || defined(_WIN64) || defined(MICROPY_UNIX_COVERAGE)
+        #define SUPPORT_LL_FORMAT (1)
+        #else
+        #define SUPPORT_LL_FORMAT (0)
+        #endif
+        #if SUPPORT_LL_FORMAT
+        bool long_long_arg = false;
+        #endif
+
         if (*fmt == 'l') {
             ++fmt;
-            #ifdef __LP64__
+            #if SUPPORT_L_FORMAT
             long_arg = true;
             #endif
+            #if SUPPORT_LL_FORMAT
+            if (*fmt == 'l') {
+                ++fmt;
+                long_long_arg = true;
+            }
+            #endif
         }
 
         if (*fmt == '\0') {
@@ -501,35 +521,50 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) {
                 chrs += mp_print_strn(print, str, len, flags, fill, width);
                 break;
             }
-            case 'd': {
-                mp_int_t val;
-                if (long_arg) {
-                    val = va_arg(args, long int);
-                } else {
-                    val = va_arg(args, int);
-                }
-                chrs += mp_print_int(print, val, 1, 10, 'a', flags, fill, width);
-                break;
-            }
+            case 'd':
+            case 'p':
+            case 'P':
             case 'u':
             case 'x':
             case 'X': {
-                int base = 16 - ((*fmt + 1) & 6); // maps char u/x/X to base 10/16/16
-                char fmt_c = (*fmt & 0xf0) - 'P' + 'A'; // maps char u/x/X to char a/a/A
+                char fmt_chr = *fmt;
                 mp_uint_t val;
-                if (long_arg) {
-                    val = va_arg(args, unsigned long int);
-                } else {
-                    val = va_arg(args, unsigned int);
+                if (fmt_chr == 'p' || fmt_chr == 'P') {
+                    val = va_arg(args, intptr_t);
                 }
-                chrs += mp_print_int(print, val, 0, base, fmt_c, flags, fill, width);
+                #if SUPPORT_LL_FORMAT
+                else if (long_long_arg) {
+                    val = va_arg(args, unsigned long long);
+                }
+                #endif
+                #if SUPPORT_L_FORMAT
+                else if (long_arg) {
+                    if (sizeof(long) != sizeof(mp_uint_t) && fmt_chr == 'd') {
+                        val = va_arg(args, long);
+                    } else {
+                        val = va_arg(args, unsigned long);
+                    }
+                }
+                #endif
+                else {
+                    if (sizeof(int) != sizeof(mp_uint_t) && fmt_chr == 'd') {
+                        val = va_arg(args, int);
+                    } else {
+                        val = va_arg(args, unsigned);
+                    }
+                }
+                int base;
+                // Map format char x/p/X/P to a/a/A/A for hex letters.
+                // It doesn't matter what d/u map to.
+                char fmt_c = (fmt_chr & 0xf0) - 'P' + 'A';
+                if (fmt_chr == 'd' || fmt_chr == 'u') {
+                    base = 10;
+                } else {
+                    base = 16;
+                }
+                chrs += mp_print_int(print, val, fmt_chr == 'd', base, fmt_c, flags, fill, width);
                 break;
             }
-            case 'p':
-            case 'P': // don't bother to handle upcase for 'P'
-                // Use unsigned long int to work on both ILP32 and LP64 systems
-                chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags, fill, width);
-                break;
             #if MICROPY_PY_BUILTINS_FLOAT
             case 'e':
             case 'E':
@@ -545,18 +580,6 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) {
                 #endif
                 break;
             }
-            #endif
-                // Because 'l' is eaten above, another 'l' means %ll.  We need to support
-                // this length specifier for OBJ_REPR_D (64-bit NaN boxing).
-                // TODO Either enable this unconditionally, or provide a specific config var.
-            #if (MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D) || defined(_WIN64)
-            case 'l': {
-                unsigned long long int arg_value = va_arg(args, unsigned long long int);
-                ++fmt;
-                assert(*fmt == 'u' || *fmt == 'd' || !"unsupported fmt char");
-                chrs += mp_print_int(print, arg_value, *fmt == 'd', 10, 'a', flags, fill, width);
-                break;
-            }
             #endif
             default:
                 // if it's not %% then it's an unsupported format character

py/objcode.c

@@ -69,6 +69,7 @@ static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_
     return consts;
 }
 
+#if !MICROPY_PREVIEW_VERSION_2
 static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) {
     // const mp_bytecode_prelude_t *prelude = &rc->prelude;
     uint start = 0;
@@ -106,6 +107,7 @@ static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) {
     m_del(byte, buffer, buffer_size);
     return o;
 }
+#endif
 
 static mp_obj_t code_colines_iter(mp_obj_t);
 static mp_obj_t code_colines_next(mp_obj_t);
@@ -198,12 +200,14 @@ static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
         case MP_QSTR_co_names:
             dest[0] = MP_OBJ_FROM_PTR(o->dict_locals);
             break;
+        #if !MICROPY_PREVIEW_VERSION_2
         case MP_QSTR_co_lnotab:
             if (!o->lnotab) {
                 o->lnotab = raw_code_lnotab(rc);
             }
             dest[0] = o->lnotab;
             break;
+        #endif
         case MP_QSTR_co_lines:
             dest[0] = MP_OBJ_FROM_PTR(&code_colines_obj);
             dest[1] = self_in;

py/objint_longlong.c

@@ -31,6 +31,7 @@
 #include "py/smallint.h"
 #include "py/objint.h"
 #include "py/runtime.h"
+#include "py/misc.h"
 
 #if MICROPY_PY_BUILTINS_FLOAT
 #include <math.h>
@@ -43,6 +44,10 @@
 const mp_obj_int_t mp_sys_maxsize_obj = {{&mp_type_int}, MP_SSIZE_MAX};
 #endif
 
+static void raise_long_long_overflow(void) {
+    mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("result overflows long long storage"));
+}
+
 mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf) {
     int delta = 1;
     if (!big_endian) {
@@ -120,7 +125,6 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) {
         // small int if the value fits without truncation
         case MP_UNARY_OP_HASH:
             return MP_OBJ_NEW_SMALL_INT((mp_int_t)o->val);
-
         case MP_UNARY_OP_POSITIVE:
             return o_in;
         case MP_UNARY_OP_NEGATIVE:
@@ -147,6 +151,8 @@ mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in) {
 mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
     long long lhs_val;
     long long rhs_val;
+    bool overflow = false;
+    long long result;
 
     if (mp_obj_is_small_int(lhs_in)) {
         lhs_val = MP_OBJ_SMALL_INT_VALUE(lhs_in);
@@ -167,13 +173,16 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i
     switch (op) {
         case MP_BINARY_OP_ADD:
         case MP_BINARY_OP_INPLACE_ADD:
-            return mp_obj_new_int_from_ll(lhs_val + rhs_val);
+            overflow = mp_add_ll_overflow(lhs_val, rhs_val, &result);
+            break;
         case MP_BINARY_OP_SUBTRACT:
         case MP_BINARY_OP_INPLACE_SUBTRACT:
-            return mp_obj_new_int_from_ll(lhs_val - rhs_val);
+            overflow = mp_sub_ll_overflow(lhs_val, rhs_val, &result);
+            break;
         case MP_BINARY_OP_MULTIPLY:
         case MP_BINARY_OP_INPLACE_MULTIPLY:
-            return mp_obj_new_int_from_ll(lhs_val * rhs_val);
+            overflow = mp_mul_ll_overflow(lhs_val, rhs_val, &result);
+            break;
         case MP_BINARY_OP_FLOOR_DIVIDE:
         case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE:
             if (rhs_val == 0) {
@@ -199,9 +208,21 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i
 
         case MP_BINARY_OP_LSHIFT:
         case MP_BINARY_OP_INPLACE_LSHIFT:
-            return mp_obj_new_int_from_ll(lhs_val << (int)rhs_val);
+            if ((int)rhs_val < 0) {
+                // negative shift not allowed
+                mp_raise_ValueError(MP_ERROR_TEXT("negative shift count"));
+            }
+            result = lhs_val << (int)rhs_val;
+            // Left-shifting of negative values is implementation defined in C, but assume compiler
+            // will give us typical 2s complement behaviour unless the value overflows
+            overflow = rhs_val > 0 && ((lhs_val >= 0 && result < lhs_val) || (lhs_val < 0 && result > lhs_val));
+            break;
         case MP_BINARY_OP_RSHIFT:
         case MP_BINARY_OP_INPLACE_RSHIFT:
+            if ((int)rhs_val < 0) {
+                // negative shift not allowed
+                mp_raise_ValueError(MP_ERROR_TEXT("negative shift count"));
+            }
             return mp_obj_new_int_from_ll(lhs_val >> (int)rhs_val);
 
         case MP_BINARY_OP_POWER:
@@ -213,18 +234,18 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i
                 mp_raise_ValueError(MP_ERROR_TEXT("negative power with no float support"));
                 #endif
             }
-            long long ans = 1;
-            while (rhs_val > 0) {
+            result = 1;
+            while (rhs_val > 0 && !overflow) {
                 if (rhs_val & 1) {
-                    ans *= lhs_val;
+                    overflow = mp_mul_ll_overflow(result, lhs_val, &result);
                 }
-                if (rhs_val == 1) {
+                if (rhs_val == 1 || overflow) {
                     break;
                 }
                 rhs_val /= 2;
-                lhs_val *= lhs_val;
+                overflow = mp_mul_ll_overflow(lhs_val, lhs_val, &lhs_val);
             }
-            return mp_obj_new_int_from_ll(ans);
+            break;
         }
 
         case MP_BINARY_OP_LESS:
@@ -242,6 +263,12 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i
             return MP_OBJ_NULL; // op not supported
     }
 
+    if (overflow) {
+        raise_long_long_overflow();
+    }
+
+    return mp_obj_new_int_from_ll(result);
+
 zero_division:
     mp_raise_msg(&mp_type_ZeroDivisionError, MP_ERROR_TEXT("divide by zero"));
 }
@@ -265,22 +292,12 @@ mp_obj_t mp_obj_new_int_from_ll(long long val) {
 }
 
 mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) {
-    // TODO raise an exception if the unsigned long long won't fit
     if (val >> (sizeof(unsigned long long) * 8 - 1) != 0) {
-        mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("ulonglong too large"));
+        raise_long_long_overflow();
     }
     return mp_obj_new_int_from_ll(val);
 }
 
-mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) {
-    // TODO this does not honor the given length of the string, but it all cases it should anyway be null terminated
-    // TODO check overflow
-    char *endptr;
-    mp_obj_t result = mp_obj_new_int_from_ll(strtoll(*str, &endptr, base));
-    *str = endptr;
-    return result;
-}
-
 mp_int_t mp_obj_int_get_truncated(mp_const_obj_t self_in) {
     if (mp_obj_is_small_int(self_in)) {
         return MP_OBJ_SMALL_INT_VALUE(self_in);

py/parsenum.c

@@ -46,6 +46,27 @@ static MP_NORETURN void raise_exc(mp_obj_t exc, mp_lexer_t *lex) {
     nlr_raise(exc);
 }
 
+#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG
+// For the common small integer parsing case, we parse directly to mp_int_t and
+// check that the value doesn't overflow a smallint (in which case we fail over
+// to bigint parsing if supported)
+typedef mp_int_t parsed_int_t;
+
+#define PARSED_INT_MUL_OVERFLOW mp_small_int_mul_overflow
+#define PARSED_INT_FITS MP_SMALL_INT_FITS
+#else
+// In the special case where bigint support is long long, we save code size by
+// parsing directly to long long and then return either a bigint or smallint
+// from the same result.
+//
+// To avoid pulling in (slow) signed 64-bit math routines we do the initial
+// parsing to an unsigned long long and only convert to signed at the end.
+typedef unsigned long long parsed_int_t;
+
+#define PARSED_INT_MUL_OVERFLOW mp_mul_ull_overflow
+#define PARSED_INT_FITS(I) ((I) <= (unsigned long long)LLONG_MAX)
+#endif
+
 mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, mp_lexer_t *lex) {
     const byte *restrict str = (const byte *)str_;
     const byte *restrict top = str + len;
@@ -76,7 +97,7 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m
     str += mp_parse_num_base((const char *)str, top - str, &base);
 
     // string should be an integer number
-    mp_int_t int_val = 0;
+    parsed_int_t parsed_val = 0;
     const byte *restrict str_val_start = str;
     for (; str < top; str++) {
         // get next digit as a value
@@ -98,25 +119,29 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m
             break;
         }
 
-        // add next digi and check for overflow
-        if (mp_small_int_mul_overflow(int_val, base)) {
+        // add next digit and check for overflow
+        if (PARSED_INT_MUL_OVERFLOW(parsed_val, base, &parsed_val)) {
             goto overflow;
         }
-        int_val = int_val * base + dig;
-        if (!MP_SMALL_INT_FITS(int_val)) {
+        parsed_val += dig;
+        if (!PARSED_INT_FITS(parsed_val)) {
             goto overflow;
         }
     }
 
-    // negate value if needed
-    if (neg) {
-        int_val = -int_val;
-    }
-
-    // create the small int
-    ret_val = MP_OBJ_NEW_SMALL_INT(int_val);
-
+    #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG
+    // The PARSED_INT_FITS check above ensures parsed_val fits in small int representation
+    ret_val = MP_OBJ_NEW_SMALL_INT(neg ? (-parsed_val) : parsed_val);
 have_ret_val:
+    #else
+    // The PARSED_INT_FITS check above ensures parsed_val won't overflow signed long long
+    long long signed_val = parsed_val;
+    if (neg) {
+        signed_val = -signed_val;
+    }
+    ret_val = mp_obj_new_int_from_ll(signed_val); // Could be large or small int
+    #endif
+
     // check we parsed something
     if (str == str_val_start) {
         goto value_error;
@@ -135,6 +160,7 @@ have_ret_val:
     return ret_val;
 
 overflow:
+    #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_LONGLONG
     // reparse using long int
     {
         const char *s2 = (const char *)str_val_start;
@@ -142,6 +168,9 @@ overflow:
         str = (const byte *)s2;
         goto have_ret_val;
     }
+    #else
+    mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("result overflows long long storage"));
+    #endif
 
 value_error:
     {

py/runtime.c

@@ -505,13 +505,14 @@ mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs
                     }
                     #endif
 
-                    if (mp_small_int_mul_overflow(lhs_val, rhs_val)) {
+                    mp_int_t int_res;
+                    if (mp_small_int_mul_overflow(lhs_val, rhs_val, &int_res)) {
                         // use higher precision
                         lhs = mp_obj_new_int_from_ll(lhs_val);
                         goto generic_binary_op;
                     } else {
                         // use standard precision
-                        return MP_OBJ_NEW_SMALL_INT(lhs_val * rhs_val);
+                        return MP_OBJ_NEW_SMALL_INT(int_res);
                     }
                 }
                 case MP_BINARY_OP_FLOOR_DIVIDE:
@@ -552,19 +553,19 @@ mp_obj_t MICROPY_WRAP_MP_BINARY_OP(mp_binary_op)(mp_binary_op_t op, mp_obj_t lhs
                         mp_int_t ans = 1;
                         while (rhs_val > 0) {
                             if (rhs_val & 1) {
-                                if (mp_small_int_mul_overflow(ans, lhs_val)) {
+                                if (mp_small_int_mul_overflow(ans, lhs_val, &ans)) {
                                     goto power_overflow;
                                 }
-                                ans *= lhs_val;
                             }
                             if (rhs_val == 1) {
                                 break;
                             }
                             rhs_val /= 2;
-                            if (mp_small_int_mul_overflow(lhs_val, lhs_val)) {
+                            mp_int_t int_res;
+                            if (mp_small_int_mul_overflow(lhs_val, lhs_val, &int_res)) {
                                 goto power_overflow;
                             }
-                            lhs_val *= lhs_val;
+                            lhs_val = int_res;
                         }
                         lhs_val = ans;
                     }

py/smallint.c

@@ -26,7 +26,7 @@
 
 #include "py/smallint.h"
 
-bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y) {
+bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y, mp_int_t *res) {
     // Check for multiply overflow; see CERT INT32-C
     if (x > 0) { // x is positive
         if (y > 0) { // x and y are positive
@@ -49,6 +49,9 @@ bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y) {
             }
         } // End if x and y are nonpositive
     } // End if x is nonpositive
+
+    // Result doesn't overflow
+    *res = x * y;
     return false;
 }
 

py/smallint.h

@@ -68,7 +68,10 @@
 // The number of bits in a MP_SMALL_INT including the sign bit.
 #define MP_SMALL_INT_BITS (MP_IMAX_BITS(MP_SMALL_INT_MAX) + 1)
 
-bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y);
+// Multiply two small ints.
+// If returns false, the correct result is stored in 'res'
+// If returns true, the multiplication would have overflowed. 'res' is unchanged.
+bool mp_small_int_mul_overflow(mp_int_t x, mp_int_t y, mp_int_t *res);
 mp_int_t mp_small_int_modulo(mp_int_t dividend, mp_int_t divisor);
 mp_int_t mp_small_int_floor_divide(mp_int_t num, mp_int_t denom);
 

py/vm.c

@@ -195,6 +195,22 @@
 #define TRACE_TICK(current_ip, current_sp, is_exception)
 #endif // MICROPY_PY_SYS_SETTRACE
 
+#if MICROPY_PY_BUILTINS_SLICE
+// This function is marked "no inline" so it doesn't increase the C stack usage of the main VM function.
+MP_NOINLINE static mp_obj_t *build_slice_stack_allocated(byte op, mp_obj_t *sp, mp_obj_t step) {
+    mp_obj_t stop = sp[2];
+    mp_obj_t start = sp[1];
+    mp_obj_slice_t slice = { .base = { .type = &mp_type_slice }, .start = start, .stop = stop, .step = step };
+    if (op == MP_BC_LOAD_SUBSCR) {
+        SET_TOP(mp_obj_subscr(TOP(), MP_OBJ_FROM_PTR(&slice), MP_OBJ_SENTINEL));
+    } else { // MP_BC_STORE_SUBSCR
+        mp_obj_subscr(TOP(), MP_OBJ_FROM_PTR(&slice), sp[-1]);
+        sp -= 2;
+    }
+    return sp;
+}
+#endif
+
 // fastn has items in reverse order (fastn[0] is local[0], fastn[-1] is local[1], etc)
 // sp points to bottom of stack which grows up
 // returns:
@@ -849,9 +865,19 @@ unwind_jump:;
                         // 3-argument slice includes step
                         step = POP();
                     }
-                    mp_obj_t stop = POP();
-                    mp_obj_t start = TOP();
-                    SET_TOP(mp_obj_new_slice(start, stop, step));
+                    if ((*ip == MP_BC_LOAD_SUBSCR || *ip == MP_BC_STORE_SUBSCR) && mp_obj_is_native_type(mp_obj_get_type(sp[-2]))) {
+                        // Fast path optimisation for when the BUILD_SLICE is immediately followed
+                        // by a LOAD/STORE_SUBSCR for a native type to avoid needing to allocate
+                        // the slice on the heap.  In some cases (e.g. a[1:3] = x) this can result
+                        // in no allocations at all.  We can't do this for instance types because
+                        // the get/set/delattr implementation may keep a reference to the slice.
+                        byte op = *ip++;
+                        sp = build_slice_stack_allocated(op, sp - 2, step);
+                    } else {
+                        mp_obj_t stop = POP();
+                        mp_obj_t start = TOP();
+                        SET_TOP(mp_obj_new_slice(start, stop, step));
+                    }
                     DISPATCH();
                 }
                 #endif

tests/basics/builtin_slice.py

@@ -1,11 +1,44 @@
 # test builtin slice
 
+# ensures that slices passed to user types are heap-allocated and can be
+# safely stored as well as not overriden by subsequent slices.
+
 # print slice
 class A:
     def __getitem__(self, idx):
-        print(idx)
+        print("get", idx)
+        print("abc"[1:])
+        print("get", idx)
         return idx
-s = A()[1:2:3]
+
+    def __setitem__(self, idx, value):
+        print("set", idx)
+        print("abc"[1:])
+        print("set", idx)
+        self.saved_idx = idx
+        return idx
+
+    def __delitem__(self, idx):
+        print("del", idx)
+        print("abc"[1:])
+        print("del", idx)
+        return idx
+
+
+a = A()
+s = a[1:2:3]
+a[4:5:6] = s
+del a[7:8:9]
+
+print(a.saved_idx)
+
+# nested slicing
+print(A()[1 : A()[A()[2:3:4] : 5]])
+
+# tuple slicing
+a[1:2, 4:5, 7:8]
+a[1, 4:5, 7:8, 2]
+a[1:2, a[3:4], 5:6]
 
 # check type
 print(type(s) is slice)

tests/basics/fun_code_full.py

@@ -6,12 +6,6 @@ except AttributeError:
     print("SKIP")
     raise SystemExit
 
-try:
-    import warnings
-    warnings.simplefilter("ignore") # ignore deprecation warning about co_lnotab
-except ImportError:
-    pass
-
 def f(x, y):
     a = x + y
     b = x - y
@@ -25,7 +19,6 @@ print(code.co_filename.rsplit('/')[-1]) # same terminal filename but might be di
 print(type(code.co_firstlineno)) # both ints (but mpy points to first line inside, cpy points to declaration)
 print(code.co_name)
 print(iter(code.co_names) is not None) # both iterable (but mpy returns dict with names as keys, cpy only the names; and not necessarily the same set)
-print(type(code.co_lnotab)) # both bytes
 
 co_lines = code.co_lines()
 

tests/basics/fun_code_lnotab.py

@@ -0,0 +1,34 @@
+# Test deprecation of co_lnotab
+
+try:
+    (lambda: 0).__code__.co_code
+except AttributeError:
+    print("SKIP")
+    raise SystemExit
+
+
+import unittest
+import sys
+
+
+mpy_is_v2 = getattr(sys.implementation, '_v2', False)
+
+
+def f():
+    pass
+
+
+class Test(unittest.TestCase):
+
+    @unittest.skipIf(mpy_is_v2, "Removed in MicroPython v2 and later.")
+    def test_co_lnotab_exists(self):
+        self.assertIsInstance(f.__code__.co_lnotab, bytes)
+
+    @unittest.skipUnless(mpy_is_v2, "Not removed before MicroPython v2.")
+    def test_co_lnotab_removed(self):
+        with self.assertRaises(AttributeError):
+            f.__code__.co_lnotab
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/basics/int_64_basics.py

@@ -0,0 +1,137 @@
+# test support for 64-bit long integers
+# (some ports don't support arbitrary precision but do support these)
+
+# this test is adapted from int_big1.py with numbers kept within 64-bit signed range
+
+# to test arbitrary precision integers
+
+x = 1000000000000000000
+xn = -1000000000000000000
+y = 2000000000000000000
+
+# printing
+print(x)
+print(y)
+print('%#X' % (x - x)) # print prefix
+print('{:#,}'.format(x)) # print with commas
+
+# construction
+print(int(x))
+
+# addition
+print(x + 1)
+print(x + y)
+print(x + xn == 0)
+print(bool(x + xn))
+
+# subtraction
+print(x - 1)
+print(x - y)
+print(y - x)
+print(x - x == 0)
+print(bool(x - x))
+
+# multiplication
+print(x * 2)
+print(1090511627776 * 1048500)
+
+# integer division
+print(x // 2)
+print(y // x)
+
+# bit inversion
+print(~x)
+print(~(-x))
+
+# left shift
+print("left shift positive")
+x = 0x40000000
+for i in range(32):
+    x = x << 1
+    print(x)
+
+# right shift
+print("right shift positive")
+x = 0x2000000000000000  # TODO: why can't second-tip bit be set?
+for i in range(64):
+    x = x >> 1
+    print(x)
+
+# left shift of a negative number
+print("left shift negative")
+for i in range(8):
+    print(-10000000000000000 << i)
+    print(-10000000000000001 << i)
+    print(-10000000000000002 << i)
+    print(-10000000000000003 << i)
+    print(-10000000000000004 << i)
+
+
+# right shift of a negative number
+print("right shift negative")
+for i in range(8):
+    print(-1000000000000000000 >> i)
+    print(-1000000000000000001 >> i)
+    print(-1000000000000000002 >> i)
+    print(-1000000000000000003 >> i)
+    print(-1000000000000000004 >> i)
+
+# conversion from string
+print(int("1234567890123456789"))
+print(int("-1234567890123456789"))
+print(int("1234567890abcdef", 16))
+print(int("1234567890ABCDEF", 16))
+print(int("-1234567890ABCDEF", 16))
+print(int("ijklmnopqrsz", 36))
+
+# numbers close to 64-bit limits
+print(int("-9111222333444555666"))
+print(int("9111222333444555666"))
+
+# numbers with preceding 0s
+print(int("-00000000000000000000009111222333444555666"))
+print(int("0000000000000000000000009111222333444555666"))
+
+# invalid characters in string
+try:
+    print(int("1234567890abcdef"))
+except ValueError:
+    print('ValueError');
+try:
+    print(int("123456789\x01"))
+except ValueError:
+    print('ValueError');
+
+# test parsing ints just on threshold of small to big
+# for 32 bit archs
+x = 1073741823 # small
+x = -1073741823 # small
+x = 1073741824 # big
+x = -1073741824 # big
+# for 64 bit archs
+x = 4611686018427387903 # small
+x = -4611686018427387903 # small
+x = 4611686018427387904 # big
+x = -4611686018427387904 # big
+
+# sys.maxsize is a constant bigint, so test it's compatible with dynamic ones
+import sys
+if hasattr(sys, "maxsize"):
+    print(sys.maxsize - 1 + 1 == sys.maxsize)
+else:
+    print(True)  # No maxsize property in this config
+
+# test extraction of big int value via mp_obj_get_int_maybe
+x = 1 << 62
+print('a' * (x + 4 - x))
+
+# negative shifts are invalid
+try:
+    print((1 << 48) >> -4)
+except ValueError as e:
+    print(e)
+
+try:
+    print((1 << 48) << -6)
+except ValueError as e:
+    print(e)

tests/extmod/json_loads_int_64.py

@@ -0,0 +1,16 @@
+# Parse 64-bit integers from JSON payloads.
+#
+# This also exercises parsing integers from strings
+# where the value may not be null terminated (last line)
+try:
+    import json
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+
+print(json.loads("9111222333444555666"))
+print(json.loads("-9111222333444555666"))
+print(json.loads("9111222333444555666"))
+print(json.loads("-9111222333444555666"))
+print(json.loads("[\"9111222333444555666777\",9111222333444555666]"))

tests/extmod/select_poll_udp.py

@@ -29,3 +29,5 @@ print(poll.poll(0)[0][1] == select.POLLOUT)
 if hasattr(select, "select"):
     r, w, e = select.select([s], [], [], 0)
     assert not r and not w and not e
+
+s.close()

tests/extmod/socket_udp_nonblock.py

@@ -19,3 +19,5 @@ try:
     s.recv(1)
 except OSError as er:
     print("EAGAIN:", er.errno == errno.EAGAIN)
+
+s.close()

tests/extmod/uctypes_addressof.py

@@ -12,5 +12,8 @@ for i in range(8):
     print(uctypes.addressof(uctypes.bytearray_at(1 << i, 8)))
 
 # Test address that is bigger than the greatest small-int but still within the address range.
-large_addr = maxsize + 1
-print(uctypes.addressof(uctypes.bytearray_at(large_addr, 8)) == large_addr)
+try:
+    large_addr = maxsize + 1
+    print(uctypes.addressof(uctypes.bytearray_at(large_addr, 8)) == large_addr)
+except OverflowError:
+    print(True)  # systems with 64-bit bigints will overflow on the above operation

tests/extmod/uctypes_array_load_store.py

@@ -6,6 +6,13 @@ except ImportError:
     print("SKIP")
     raise SystemExit
 
+# 'int' needs to be able to represent UINT64 for this test
+try:
+    int("FF" * 8, 16)
+except OverflowError:
+    print("SKIP")
+    raise SystemExit
+
 N = 5
 
 for endian in ("NATIVE", "LITTLE_ENDIAN", "BIG_ENDIAN"):

tests/feature_check/int_64.py

@@ -0,0 +1,2 @@
+# Check whether 64-bit long integers are supported
+print(1 << 62)

tests/feature_check/int_64.py.exp

@@ -0,0 +1 @@
+4611686018427387904

tests/micropython/heapalloc_slice.py

@@ -0,0 +1,18 @@
+# slice operations that don't require allocation
+try:
+    from micropython import heap_lock, heap_unlock
+except (ImportError, AttributeError):
+    heap_lock = heap_unlock = lambda: 0
+
+b = bytearray(range(10))
+
+m = memoryview(b)
+
+heap_lock()
+
+b[3:5] = b"aa"
+m[5:7] = b"bb"
+
+heap_unlock()
+
+print(b)

tests/perf_bench/bm_pidigits.py

@@ -5,6 +5,12 @@
 # This benchmark stresses big integer arithmetic.
 # Adapted from code on: http://benchmarksgame.alioth.debian.org/
 
+try:
+    int("0x10000000000000000", 16)
+except:
+    print("SKIP")  # No support for >64-bit integers
+    raise SystemExit
+
 
 def compose(a, b):
     aq, ar, as_, at = a

tests/ports/unix/extra_coverage.py.exp

@@ -2,19 +2,34 @@
 -123 +123  123
 -0123
 123
-123
-1ABCDEF
+123f
+123F
+7fffffffffffffff
+7FFFFFFFFFFFFFFF
+18446744073709551615
+789f
+789F
 ab abc ' abc' ' True' 'Tru'
 
 false true
 (null)
 -2147483648
 2147483648
-80000000
-80000000
+8000000f
+8000000F
 abc
 %
 .a  .
+<%>
+<aaaa>
+<true>
+<43690>
+<43690>
+<43690>
+<test>
+<1000.000000>
+<True>
+<9223372036854775807>
 # GC
 0
 0

tests/ports/webassembly/py_proxy_get.mjs

@@ -0,0 +1,14 @@
+// Test `<py-obj> get <attr>` on the JavaScript side, which tests PyProxy.get.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+x = {"a": 1}
+`);
+
+const x = mp.globals.get("x");
+console.log(x.a === 1);
+console.log(x.b === undefined);
+console.log(typeof x[Symbol.iterator] === "function");
+console.log(x[Symbol.toStringTag] === undefined);
+console.log(x.then === undefined);

tests/ports/webassembly/py_proxy_get.mjs.exp

@@ -0,0 +1,5 @@
+true
+true
+true
+true
+true

tests/run-tests.py

@@ -628,6 +628,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
     skip_tests = set()
     skip_native = False
     skip_int_big = False
+    skip_int_64 = False
     skip_bytearray = False
     skip_set_type = False
     skip_slice = False
@@ -658,6 +659,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         if output != b"1000000000000000000000000000000000000000000000\n":
             skip_int_big = True
 
+        # Check if 'long long' precision integers are supported, even if arbitrary precision is not
+        output = run_feature_check(pyb, args, "int_64.py")
+        if output != b"4611686018427387904\n":
+            skip_int_64 = True
+
         # Check if bytearray is supported, and skip such tests if it's not
         output = run_feature_check(pyb, args, "bytearray.py")
         if output != b"bytearray\n":
@@ -854,6 +860,9 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         skip_tests.add(
             "micropython/emg_exc.py"
         )  # because native doesn't have proper traceback info
+        skip_tests.add(
+            "micropython/heapalloc_slice.py"
+        )  # because native doesn't do the stack-allocated slice optimisation
         skip_tests.add(
             "micropython/heapalloc_traceback.py"
         )  # because native doesn't have proper traceback info
@@ -882,7 +891,12 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         test_name = os.path.splitext(os.path.basename(test_file))[0]
         is_native = test_name.startswith("native_") or test_name.startswith("viper_")
         is_endian = test_name.endswith("_endian")
-        is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
+        is_int_big = (
+            test_name.startswith("int_big")
+            or test_name.endswith("_intbig")
+            or test_name.startswith("ffi_int")  # these tests contain large integer literals
+        )
+        is_int_64 = test_name.startswith("int_64") or test_name.endswith("_int64")
         is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray")
         is_set_type = test_name.startswith(("set_", "frozenset")) or test_name.endswith("_set")
         is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests
@@ -896,6 +910,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         skip_it |= skip_native and is_native
         skip_it |= skip_endian and is_endian
         skip_it |= skip_int_big and is_int_big
+        skip_it |= skip_int_64 and is_int_64
         skip_it |= skip_bytearray and is_bytearray
         skip_it |= skip_set_type and is_set_type
         skip_it |= skip_slice and is_slice

tools/ci.sh

@@ -563,7 +563,7 @@ function ci_native_mpy_modules_build {
     else
         arch=$1
     fi
-    for natmod in deflate features1 features3 features4 framebuf heapq random re
+    for natmod in btree deflate features1 features3 features4 framebuf heapq random re
     do
         make -C examples/natmod/$natmod ARCH=$arch clean
         make -C examples/natmod/$natmod ARCH=$arch
@@ -576,12 +576,6 @@ function ci_native_mpy_modules_build {
     else
         make -C examples/natmod/features2 ARCH=$arch
     fi
-
-    # btree requires thread local storage support on rv32imc.
-    if [ $arch != "rv32imc" ]; then
-        make -C examples/natmod/btree ARCH=$arch clean
-        make -C examples/natmod/btree ARCH=$arch
-    fi
 }
 
 function ci_native_mpy_modules_32bit_build {
@@ -695,6 +689,14 @@ function ci_unix_nanbox_run_tests {
     ci_unix_run_tests_full_no_native_helper nanbox PYTHON=python2.7
 }
 
+function ci_unix_longlong_build {
+    ci_unix_build_helper VARIANT=longlong
+}
+
+function ci_unix_longlong_run_tests {
+    ci_unix_run_tests_full_helper longlong
+}
+
 function ci_unix_float_build {
     ci_unix_build_helper VARIANT=standard CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT"
     ci_unix_build_ffi_lib_helper gcc

tools/pyboard.py

@@ -282,7 +282,7 @@ class Pyboard:
         if device.startswith("exec:"):
             self.serial = ProcessToSerial(device[len("exec:") :])
         elif device.startswith("execpty:"):
-            self.serial = ProcessPtyToTerminal(device[len("qemupty:") :])
+            self.serial = ProcessPtyToTerminal(device[len("execpty:") :])
         elif device and device[0].isdigit() and device[-1].isdigit() and device.count(".") == 3:
             # device looks like an IP address
             self.serial = TelnetToSerial(device, user, password, read_timeout=10)