#!/usr/bin/env python3 # Author: Volodymyr Shymanskyy # Usage: # ./run-wasi-test.py # ./run-wasi-test.py --exec ../custom_build/wasm3 --timeout 120 # ./run-wasi-test.py --exec "wasmer run --mapdir=/:." --separate-args # ./run-wasi-test.py --exec "wasmer run --mapdir=/:. wasm3.wasm --" --fast import argparse import sys import subprocess import hashlib import fnmatch sys.path.append('../extra') from testutils import * from pprint import pprint # # Args handling # parser = argparse.ArgumentParser() parser.add_argument("--exec", metavar="", default="../build/wasm3") parser.add_argument("--separate-args", action='store_true') # use "--" separator for wasmer, wasmtime parser.add_argument("--timeout", type=int, default=120) parser.add_argument("--fast", action='store_true') args = parser.parse_args() stats = dotdict(total_run=0, failed=0, crashed=0, timeout=0) commands_full = [ { "name": "Simple WASI test", "wasm": "./wasi/simple/test.wasm", "args": ["cat", "/wasi/simple/0.txt"], "expect_pattern": "Hello world*Constructor OK*Args: *; cat; /wasi/simple/0.txt;*fib(20) = 6765* ms*48 65 6c 6c 6f 20 77 6f 72 6c 64*=== done ===*" }, { "name": "Simple WASI test (wasm-opt -O3)", "wasm": "./wasi/simple/test-opt.wasm", "args": ["cat", "./wasi/simple/0.txt"], "expect_pattern": "Hello world*Constructor OK*Args: *; cat; ./wasi/simple/0.txt;*fib(20) = 6765* ms*48 65 6c 6c 6f 20 77 6f 72 6c 64*=== done ===*" }, { "name": "mandelbrot", "wasm": "./wasi/mandelbrot/mandel.wasm", "args": ["128", "4e5"], "expect_sha1": "37091e7ce96adeea88f079ad95d239a651308a56" }, { "name": "mandelbrot (doubledouble)", "wasm": "./wasi/mandelbrot/mandel_dd.wasm", "args": ["128", "4e5"], "expect_sha1": "b3f904daf1c972b4f7d3f8996743cb5b5146b877" }, { "name": "C-Ray", "stdin": "./wasi/c-ray/scene", "wasm": "./wasi/c-ray/c-ray.wasm", "args": ["-s", "128x128"], "expect_sha1": "90f86845ae227466a06ea8db06e753af4838f2fa" }, { "name": "smallpt (explicit light sampling)", "wasm": "./wasi/smallpt/smallpt-ex.wasm", "args": ["16", "64"], "expect_sha1": "d85df3561eb15f6f0e6f20d5640e8e1306222c6d" }, { "name": "smallpt (explicit light sampling, multi-value)", "wasm": "./wasi/smallpt/smallpt-ex-mv.wasm", "args": ["16", "64"], "expect_sha1": "d85df3561eb15f6f0e6f20d5640e8e1306222c6d" }, { "name": "mal", "wasm": "./wasi/mal/mal.wasm", "args": ["./wasi/mal/test-fib.mal", "16"], "expect_pattern": "987\n", }, { "name": "STREAM", "wasm": "./wasi/stream/stream.wasm", "expect_pattern": "----*Solution Validates:*on all three arrays*----*" }, { # TODO "if": { "file_exists": "./self-hosting/wasm3-fib.wasm" }, "name": "Self-hosting", "wasm": "./self-hosting/wasm3-fib.wasm", "expect_pattern": "wasm3 on WASM*Result: 832040*Elapsed: * ms*" }, { "name": "Brotli", "stdin": "./wasi/brotli/alice29.txt", "wasm": "./wasi/brotli/brotli.wasm", "args": ["-c", "-f"], "expect_sha1": "8eacda4b80fc816cad185330caa7556e19643dff" }, { "name": "CoreMark", "wasm": "./wasi/coremark/coremark.wasm", "expect_pattern": "*Correct operation validated.*CoreMark 1.0 : * / Clang* / STATIC*" } ] commands_fast = [ { "name": "Simple WASI test", "wasm": "./wasi/simple/test.wasm", "args": ["cat", "./wasi/simple/0.txt"], "expect_pattern": "Hello world*Constructor OK*Args: *; cat; ./wasi/simple/0.txt;*fib(20) = 6765* ms*48 65 6c 6c 6f 20 77 6f 72 6c 64*=== done ===*" }, { "skip": True, # Backtraces not enabled by default "name": "Simple WASI test", "wasm": "./wasi/test.wasm", "args": ["trap"], "can_crash": True, "expect_pattern": "Hello world*Constructor OK*Args: *; trap;* wasm backtrace:* 6: 0x*Error:* unreachable executed*" }, { "name": "mandelbrot", "wasm": "./wasi/mandelbrot/mandel.wasm", "args": ["32", "4e5"], "expect_sha1": "1fdb7dea7ec0f2465054cc623dc5a7225a876361" }, { "name": "mandelbrot (doubledouble)", "wasm": "./wasi/mandelbrot/mandel_dd.wasm", "args": ["32", "4e5"], "expect_sha1": "b6d3c158a5c0dff1f6e82a3556c071e4f8b9e3f0" }, { "name": "C-Ray", "stdin": "./wasi/c-ray/scene", "wasm": "./wasi/c-ray/c-ray.wasm", "args": ["-s", "32x32"], "expect_sha1": "05af9604bf352234276e4d64e84b8d666574316c" }, { "name": "smallpt (explicit light sampling)", "wasm": "./wasi/smallpt/smallpt-ex.wasm", "args": ["4", "32"], "expect_sha1": "ea05d85998b2f453b588ef76a1256215bf9b851c" }, { "name": "smallpt (explicit light sampling, multi-value)", "wasm": "./wasi/smallpt/smallpt-ex-mv.wasm", "args": ["4", "32"], "expect_sha1": "ea05d85998b2f453b588ef76a1256215bf9b851c" }, { "name": "mal", "wasm": "./wasi/mal/mal.wasm", "args": ["./wasi/mal/test-fib.mal", "16"], "expect_pattern": "987\n", }, { "name": "Brotli", "stdin": "./wasi/brotli/alice29_small.txt", "wasm": "./wasi/brotli/brotli.wasm", "args": ["-c", "-f"], "expect_sha1": "0e8af02a7207c0c617d7d38eed92853c4a619987" } ] def fail(msg): print(f"{ansi.FAIL}FAIL:{ansi.ENDC} {msg}") stats.failed += 1 commands = commands_fast if args.fast else commands_full for cmd in commands: if "skip" in cmd: continue command = args.exec.split(' ') command.append(cmd['wasm']) if "args" in cmd: if args.separate_args: command.append("--") command.extend(cmd['args']) command = list(map(str, command)) print(f"=== {cmd['name']} ===") stats.total_run += 1 try: if "stdin" in cmd: fn = cmd['stdin'] f = open(fn, "rb") print(f"cat {fn} | {' '.join(command)}") output = subprocess.check_output(command, timeout=args.timeout, stdin=f) elif "can_crash" in cmd: print(f"{' '.join(command)}") output = subprocess.run(command, timeout=args.timeout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout else: print(f"{' '.join(command)}") output = subprocess.check_output(command, timeout=args.timeout) except subprocess.TimeoutExpired: stats.timeout += 1 fail("Timeout") continue except subprocess.CalledProcessError as e: stats.crashed += 1 fail(f"Exited with error code {e.returncode}") continue if "expect_sha1" in cmd: actual = hashlib.sha1(output).hexdigest() if actual != cmd['expect_sha1']: fail(f"Actual sha1: {actual}") if "expect_pattern" in cmd: actual = output.decode("utf-8") if not fnmatch.fnmatch(actual, cmd['expect_pattern']): fail(f"Output does not match pattern:\n{actual}") print() pprint(stats) if stats.failed: print(f"{ansi.FAIL}=======================") print(f" FAILED: {stats.failed}/{stats.total_run}") print(f"======================={ansi.ENDC}") sys.exit(1) else: print(f"{ansi.OKGREEN}=======================") print(f" All {stats.total_run} tests OK") print(f"======================={ansi.ENDC}")