Meta: Skip wasm tests whose modules cannot be loaded

Also add support for a few more assertion types to go along with it.

Meta/generate-libwasm-spec-test.py

@@ -50,6 +50,11 @@ def parse(sexp):
     return stack.pop()
 
 
+class TestGenerationError(Exception):
+    def __init__(self, message):
+        self.msg = message
+
+
 def parse_typed_value(ast):
     types = {
         'i32.const': 'i32',
@@ -162,6 +167,21 @@ def generate(ast):
                     "module": result['module'],
                 }]
             })
+        elif entry[0] in (('assert_malformed',), ('assert_invalid',)):
+            # (assert_malformed/invalid module message)
+            if len(entry) < 2 or not isinstance(entry[1], list) or entry[1][0] != ('module',):
+                print(f"Invalid argument to assert_malformed: {entry[1]}", file=stderr)
+                continue
+            result = generate_module(entry[1])
+            kind = entry[0][0][len('assert_'):]
+            tests.append({
+                'module': None,
+                'kind': kind,
+                'tests': [{
+                    "kind": kind,
+                    "module": result['module'],
+                }]
+            })
         elif len(entry) in [2, 3] and entry[0][0].startswith('assert_'):
             if entry[1][0] == ('invoke',):
                 arg, name, module = 0, None, None
@@ -171,14 +191,15 @@ def generate(ast):
                     name = entry[1][2]
                     module = named_modules[entry[1][1][0]]
                     arg = 1
+                kind = entry[0][0][len('assert_'):]
                 tests[-1]["tests"].append({
-                    "kind": entry[0][0][len('assert_'):],
+                    "kind": kind,
                     "function": {
                         "module": module,
                         "name": name,
                         "args": list(parse_typed_value(x) for x in entry[1][arg + 2:])
                     },
-                    "result": parse_typed_value(entry[2]) if len(entry) == 3 + arg else None
+                    "result": parse_typed_value(entry[2]) if len(entry) == 3 + arg and kind != 'exhaustion' else None
                 })
             elif entry[1][0] == ('get',):
                 arg, name, module = 0, None, None
@@ -324,7 +345,7 @@ all_names_in_main = {}
 
 
 def genresult(ident, entry, index):
-    expectation = f'expect().fail("Unknown result structure " + {json.dumps(entry)})'
+    expectation = None
     if "function" in entry:
         tmodule = 'module'
         if entry['function']['module'] is not None:
@@ -341,11 +362,6 @@ def genresult(ident, entry, index):
                 (f'expect({ident}_result).toBe({genarg(entry["result"])})\n    ' if entry["result"] is not None else '')
         )
 
-    if entry['kind'] == 'trap':
-        return (
-            f'expect(() => {expectation}).toThrow(TypeError, "Execution trapped");\n    '
-        )
-
     if entry['kind'] == 'ignore':
         return expectation
 
@@ -361,10 +377,21 @@ def genresult(ident, entry, index):
             f'    }}).toThrow(TypeError, "Linking failed");'
         )
 
-    if entry['kind'] == 'testgen_fail':
+    if entry['kind'] in ('exhaustion', 'trap', 'invalid'):
-        return f'throw Exception("Test Generator Failure: " + {json.dumps(entry["reason"])});\n    '
+        return (
+            f'expect(() => {expectation}.toThrow(TypeError, "Execution trapped"));\n    '
+        )
 
-    return f'throw Exception("(Test Generator) Unknown test kind {entry["kind"]}");\n    '
+    if entry['kind'] == 'malformed':
+        return ''
+
+    if entry['kind'] == 'testgen_fail':
+        raise TestGenerationError(entry["reason"])
+
+    if not expectation:
+        raise TestGenerationError(f"Unknown test result structure in {json.dumps(entry)}")
+
+    return expectation
 
 
 raw_test_number = 0
@@ -393,17 +420,22 @@ def gentest(entry, main_name):
         key = "function" if "function" in entry else "get"
         if entry[key]['module'] is not None:
             tmodule = f'namedModules[{json.dumps(named_modules_inverse[entry[key]["module"]][0])}]'
-    source = (
+    test = "_test"
-            f'test({json.dumps(test_name)}, () => {{\n' +
+    try:
+        result = genresult(ident, entry, count)
+    except TestGenerationError as e:
+        test = f"/* {e.msg} */ _test.skip"
+        result = ""
+    return (
+            f'{test}({json.dumps(test_name)}, () => {{\n' +
             (
                 f'let {ident} = {tmodule}.getExport({json.dumps(name)});\n        '
                 f'expect({ident}).not.toBeUndefined();\n        '
                 if not isempty else ''
             ) +
-            f'{genresult(ident, entry, count)}'
+            f'{result}'
             '});\n\n    '
     )
-    return source
 
 
 def gen_parse_module(name, index):
@@ -415,8 +447,11 @@ def gen_parse_module(name, index):
             export_string += f'globalImportObject[{json.dumps(entry[1])}] = module;\n    '
 
     return (
-        f'let content = readBinaryWasmFile("Fixtures/SpecTests/{name}.wasm");\n    '
+        'let content, module;\n    '
-        f'const module = parseWebAssemblyModule(content, globalImportObject)\n    '
+        'try {\n        '
+        f'content = readBinaryWasmFile("Fixtures/SpecTests/{name}.wasm");\n        '
+        f'module = parseWebAssemblyModule(content, globalImportObject)\n        '
+        '} catch(e) { _test("parse", () => expect().fail(e)); _test = test.skip; _test.skip = test.skip; }\n    '
         f'{export_string}\n     '
     )
 
@@ -456,11 +491,12 @@ def main():
         testname = f'{name}_{index}'
         outpath = path.join(module_output_path, f'{testname}.wasm')
         mod = description["module"]
-        if not compile_wasm_source(mod, outpath):
+        if not compile_wasm_source(mod, outpath) and ('kind' not in description or description["kind"] != "malformed"):
             print("Failed to compile", name, "module index", index, "skipping that test", file=stderr)
             continue
         sep = ""
         print(f'''describe({json.dumps(testname)}, () => {{
+let _test = test;
 {gen_parse_module(testname, index) if mod else ''}
 {sep.join(gentest(x, testname) for x in description["tests"])}
 }});