Lerking/micropython
1
0
Fork 0
mirror of https://github.com/micropython/micropython.git synced 2025-07-21 04:51:12 +02:00
Code Issues Actions 25 Packages Projects Releases Wiki Activity

tests/run-tests.py: Change --target/--device options to --test-instance.

Browse Source 
Previously to this commit, running the test suite on a bare-metal board
required specifying the target (really platform) and device, eg:

    $ ./run-tests.py --target pyboard --device /dev/ttyACM1

That's quite a lot to type, and you also need to know what the target
platform is, when a lot of the time you either don't care or it doesn't
matter.

This commit makes it easier to run the tests by replacing both of these
options with a single `--test-instance` (`-t` for short) option.  That
option specifies the executable/port/device to test.  Then the target
platform is automatically detected.

The `--test-instance` can be passed:
- "unix" (the default) to use the unix version of MicroPython
- "webassembly" to test the webassembly port
- anything else is considered a port/device to pass to Pyboard

There are also some shortcuts to specify a port/device, following
`mpremote`:
- a<n> is short for /dev/ttyACM<n>
- u<n> is short for /dev/ttyUSB<n>
- c<n> is short for COM<n>

For example:

    $ ./run-tests.py -t a1

Note that the default test instance is "unix" and so this commit does not
change the standard way to run tests on the unix port, by just doing
`./run-tests.py`.

As part of this change, the platform (and it's native architecture if it
supports importing native .mpy files) is show at the start of the test run.

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George
2024-08-18 16:21:39 +10:00
parent f2ac471989
commit 8978102f35
10 changed files with 116 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/develop/gettingstarted.rst
View File

@@ -278,7 +278,7 @@ To run a selection of tests on a board/device connected over USB use:
.. code-block:: bash
$ cd tests
$ ./run-tests.py --target minimal --device /dev/ttyACM0
$ ./run-tests.py -t /dev/ttyACM0
See also :ref:`writingtests`.

2
docs/develop/writingtests.rst
View File

@@ -60,7 +60,7 @@ Then to run on a board:
.. code-block:: bash
$ ./run-tests.py --target minimal --device /dev/ttyACM0
$ ./run-tests.py -t /dev/ttyACM0
And to run only a certain set of tests (eg a directory):

2
ports/cc3200/tools/smoke.py
View File

@@ -6,7 +6,7 @@ import os
"""
Execute it like this:
python3 run-tests.py --target wipy --device 192.168.1.1 ../cc3200/tools/smoke.py
python3 run-tests.py -t 192.168.1.1 ../cc3200/tools/smoke.py
"""
pin_map = [23, 24, 11, 12, 13, 14, 15, 16, 17, 22, 28, 10, 9, 8, 7, 6, 30, 31, 3, 0, 4, 5]

2
ports/qemu/Makefile
View File

@@ -166,7 +166,7 @@ run: $(BUILD)/firmware.elf
.PHONY: test
test: $(BUILD)/firmware.elf
$(eval DIRNAME=ports/$(notdir $(CURDIR)))
cd $(TOP)/tests && ./run-tests.py --target qemu --device execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" $(RUN_TESTS_ARGS) $(RUN_TESTS_EXTRA)
cd $(TOP)/tests && ./run-tests.py -t execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" $(RUN_TESTS_ARGS) $(RUN_TESTS_EXTRA)
$(BUILD)/firmware.elf: $(LDSCRIPT) $(OBJ)
$(Q)$(CC) $(LDFLAGS) -o $@ $(OBJ) $(LIBS)

2
ports/qemu/README.md
View File

@@ -95,7 +95,7 @@ Or manually by first starting the emulation with `make run` and then running the
tests against the serial device, for example:
$ cd ../../tests
$ ./run-tests.py --target qemu --device /dev/pts/1
$ ./run-tests.py -t /dev/pts/1
Extra make options
------------------

4
ports/webassembly/Makefile
View File

@@ -148,10 +148,10 @@ repl: $(BUILD)/micropython.mjs
min: $(BUILD)/micropython.min.mjs
test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly
test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly
################################################################################
# Remaining make rules.

16
tests/README.md
View File

@@ -1,7 +1,19 @@
# MicroPython Test Suite
This directory contains tests for various functionality areas of MicroPython.
To run all stable tests, run "run-tests.py" script in this directory.
This directory contains tests for most parts of MicroPython.
To run all stable tests, run the "run-tests.py" script in this directory. By default
that will run the test suite against the unix port of MicroPython.
To run the test suite against a bare-metal target (a board running MicroPython firmware)
use the `-t` option to specify the serial port. This will automatically detect the
target platform and run the appropriate set of tests for that platform. For example:
$ ./run-tests.py -t /dev/ttyACM0
That will run tests on the `/dev/ttyACM0` serial port. You can also use shortcut
device names like `a<n>` for `/dev/ttyACM<n>` and `c<n>` for `COM<n>`. Use
`./run-tests.py --help` to see all of the device possibilites, and other options.
Tests of capabilities not supported on all platforms should be written
to check for the capability being present. If it is not, the test

3
tests/feature_check/target_info.py
View File

@@ -4,6 +4,7 @@
import sys
platform = getattr(sys, "platform", "minimal")
sys_mpy = getattr(sys.implementation, "_mpy", 0)
arch = [
None,
@@ -19,4 +20,4 @@ arch = [
"xtensawin",
"rv32imc",
][sys_mpy >> 10]
print(arch)
print(platform, arch)

148
tests/run-tests.py
View File

@@ -89,9 +89,12 @@ os.chdir('/__vfstest')
__import__('__injected_test')
"""
# Platforms associated with the unix port, values of `sys.platform`.
PC_PLATFORMS = ("darwin", "linux", "win32")
# Tests to skip on specific targets.
# These are tests that are difficult to detect that they should not be run on the given target.
target_tests_to_skip = {
platform_tests_to_skip = {
"esp8266": (
"micropython/viper_args.py", # too large
"micropython/viper_binop_arith.py", # too large
@@ -161,7 +164,7 @@ target_tests_to_skip = {
"micropython/extreme_exc.py",
"micropython/heapalloc_exc_compressed_emg_exc.py",
),
"wipy": (
"WiPy": (
"misc/print_exception.py", # requires error reporting full
),
"zephyr": (
@@ -197,6 +200,53 @@ def convert_regex_escapes(line):
return bytes("".join(cs), "utf8")
def get_test_instance(test_instance, baudrate, user, password):
if test_instance.startswith("port:"):
_, port = test_instance.split(":", 1)
elif test_instance == "unix":
return None
elif test_instance == "webassembly":
return PyboardNodeRunner()
elif test_instance.startswith("a") and test_instance[1:].isdigit():
port = "/dev/ttyACM" + test_instance[1:]
elif test_instance.startswith("u") and test_instance[1:].isdigit():
port = "/dev/ttyUSB" + test_instance[1:]
elif test_instance.startswith("c") and test_instance[1:].isdigit():
port = "COM" + test_instance[1:]
else:
# Assume it's a device path.
port = test_instance
global pyboard
sys.path.append(base_path("../tools"))
import pyboard
pyb = pyboard.Pyboard(port, baudrate, user, password)
pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target
pyb.enter_raw_repl()
return pyb
def detect_test_platform(pyb, args):
# Run a script to detect various bits of information about the target test instance.
output = run_feature_check(pyb, args, "target_info.py")
if output.endswith(b"CRASH"):
raise ValueError("cannot detect platform: {}".format(output))
platform, arch = str(output, "ascii").strip().split()
if arch == "None":
arch = None
args.platform = platform
args.arch = arch
if arch and not args.mpy_cross_flags:
args.mpy_cross_flags = "-march=" + arch
print("platform={}".format(platform), end="")
if arch:
print(" arch={}".format(arch), end="")
print()
def prepare_script_for_target(args, *, script_filename=None, script_text=None, force_plain=False):
if force_plain or (not args.via_mpy and args.emit == "bytecode"):
if script_filename is not None:
@@ -706,18 +756,18 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
skip_tests.add("extmod/ssl_poll.py")
# Skip thread mutation tests on targets that don't have the GIL.
if args.target in ("rp2", "unix"):
if args.platform in PC_PLATFORMS + ("rp2",):
for t in tests:
if t.startswith("thread/mutate_"):
skip_tests.add(t)
# Some tests shouldn't be run on pyboard
if args.target != "unix":
if args.platform not in PC_PLATFORMS:
skip_tests.add("basics/exception_chain.py") # warning is not printed
skip_tests.add("micropython/meminfo.py") # output is very different to PC output
# Skip target-specific tests.
skip_tests.update(target_tests_to_skip.get(args.target, ()))
# Skip platform-specific tests.
skip_tests.update(platform_tests_to_skip.get(args.platform, ()))
# Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == "64bit":
@@ -932,17 +982,38 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Run and manage tests for MicroPython.
By default the tests are run against the unix port of MicroPython. To run it
against something else, use the -t option. See below for details.
Tests are discovered by scanning test directories for .py files or using the
specified test files. If test files nor directories are specified, the script
expects to be ran in the tests directory (where this file is located) and the
builtin tests suitable for the target platform are ran.
When running tests, run-tests.py compares the MicroPython output of the test with the output
produced by running the test through CPython unless a <test>.exp file is found, in which
case it is used as comparison.
If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
directory with the MicroPython output and the expectations, respectively.
""",
epilog="""\
The -t option accepts the following for the test instance:
- unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON
environment variable (which defaults to the standard variant of either the unix
or windows ports, depending on the host platform)
- webassembly - use the webassembly port of MicroPython, specified by the
MICROPY_MICROPYTHON_MJS environment variable (which defaults to the standard
variant of the webassembly port)
- port:<device> - connect to and use the given serial port device
- a<n> - connect to and use /dev/ttyACM<n>
- u<n> - connect to and use /dev/ttyUSB<n>
- c<n> - connect to and use COM<n>
- exec:<command> - execute a command and attach to its stdin/stdout
- execpty:<command> - execute a command and attach to the printed /dev/pts/<n> device
- <a>.<b>.<c>.<d> - connect to the given IPv4 address
- anything else specifies a serial port
Options -i and -e can be multiple and processed in the order given. Regex
"search" (vs "match") operation is used. An action (include/exclude) of
the last matching regex is used:
@@ -951,11 +1022,8 @@ the last matching regex is used:
run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo
""",
)
cmd_parser.add_argument("--target", default="unix", help="the target platform")
cmd_parser.add_argument(
"--device",
default="/dev/ttyACM0",
help="the serial device or the IP address of the pyboard",
"-t", "--test-instance", default="unix", help="the MicroPython instance to test"
)
cmd_parser.add_argument(
"-b", "--baudrate", default=115200, help="the baud rate of the serial device"
@@ -1039,43 +1107,11 @@ the last matching regex is used:
sys.exit(0)
LOCAL_TARGETS = (
"unix",
"webassembly",
)
EXTERNAL_TARGETS = (
"pyboard",
"wipy",
"esp8266",
"esp32",
"minimal",
"nrf",
"qemu",
"renesas-ra",
"rp2",
"zephyr",
)
if args.target in LOCAL_TARGETS:
pyb = None
if args.target == "webassembly":
pyb = PyboardNodeRunner()
elif args.target in EXTERNAL_TARGETS:
global pyboard
sys.path.append(base_path("../tools"))
import pyboard
# Get the test instance to run on.
pyb = get_test_instance(args.test_instance, args.baudrate, args.user, args.password)
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target
pyb.enter_raw_repl()
else:
raise ValueError("target must be one of %s" % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
# Automatically detect the native architecture for mpy-cross if not given.
if not args.mpy_cross_flags:
output = run_feature_check(pyb, args, "target_info.py")
arch = str(output, "ascii").strip()
if arch != "None":
args.mpy_cross_flags = "-march=" + arch
# Automatically detect the platform.
detect_test_platform(pyb, args)
if args.run_failures and (any(args.files) or args.test_dirs is not None):
raise ValueError(
@@ -1091,7 +1127,7 @@ the last matching regex is used:
tests = []
elif len(args.files) == 0:
test_extensions = ("*.py",)
if args.target == "webassembly":
if args.platform == "webassembly":
test_extensions += ("*.js", "*.mjs")
if args.test_dirs is None:
@@ -1101,23 +1137,23 @@ the last matching regex is used:
"misc",
"extmod",
)
if args.target == "pyboard":
if args.platform == "pyboard":
# run pyboard tests
test_dirs += ("float", "stress", "inlineasm", "ports/stm32")
elif args.target in ("renesas-ra"):
elif args.platform == "renesas-ra":
test_dirs += ("float", "inlineasm", "ports/renesas-ra")
elif args.target == "rp2":
elif args.platform == "rp2":
test_dirs += ("float", "stress", "thread", "ports/rp2")
if "arm" in args.mpy_cross_flags:
test_dirs += ("inlineasm",)
elif args.target == "esp32":
elif args.platform == "esp32":
test_dirs += ("float", "stress", "thread")
elif args.target in ("esp8266", "minimal", "nrf"):
elif args.platform in ("esp8266", "minimal", "nrf"):
test_dirs += ("float",)
elif args.target == "wipy":
elif args.platform == "WiPy":
# run WiPy tests
test_dirs += ("ports/cc3200",)
elif args.target == "unix":
elif args.platform in PC_PLATFORMS:
# run PC tests
test_dirs += (
"float",
@@ -1128,13 +1164,13 @@ the last matching regex is used:
"cmdline",
"ports/unix",
)
elif args.target == "qemu":
elif args.platform == "qemu":
test_dirs += (
"float",
"inlineasm",
"ports/qemu",
)
elif args.target == "webassembly":
elif args.platform == "webassembly":
test_dirs += ("float", "ports/webassembly")
else:
# run tests from these directories

2
tools/ci.sh
View File

@@ -785,5 +785,5 @@ function ci_zephyr_run_tests {
docker exec zephyr-ci west build -p auto -b qemu_cortex_m3 -- -DCONF_FILE=prj_minimal.conf
# Issues with zephyr tests:
# - inf_nan_arith fails pow(-1, nan) test
(cd tests && ./run-tests.py --target minimal --device execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)
(cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)
}