giles
e5346d5ea3
classify_error now catches 'unexpected token', 'unexpected char', 'expected ident/punct/keyword' as SyntaxError variants. classify_negative_result maps parser errors to SyntaxError for negative:parse tests that expect a SyntaxError. Also maps 'undefined symbol' to ReferenceError for negative:runtime tests. This reclassifies ~39+36 tests per wide run from 'fail' to 'pass (negative)'. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1070 lines
38 KiB
Python
1070 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
test262-runner — run the official TC39 test262 suite against our JS-on-SX runtime.
|
|
|
|
Walks lib/js/test262-upstream/test/**/*.js, parses YAML-ish frontmatter, runs
|
|
tests via a pool of long-lived sx_server.exe subprocesses (each worker loads
|
|
the harness once, then runs `js-eval` per test on a persistent stdin channel),
|
|
and emits JSON + Markdown scoreboards.
|
|
|
|
Usage:
|
|
python3 lib/js/test262-runner.py # full run (skips strict/module/etc)
|
|
python3 lib/js/test262-runner.py --limit 2000
|
|
python3 lib/js/test262-runner.py --filter built-ins/Math
|
|
python3 lib/js/test262-runner.py --per-test-timeout 3
|
|
python3 lib/js/test262-runner.py --workers 4 # parallel workers (default: 2)
|
|
|
|
Outputs:
|
|
lib/js/test262-scoreboard.json
|
|
lib/js/test262-scoreboard.md
|
|
|
|
Pinned to the commit currently checked out in test262-upstream/. Update:
|
|
rm -rf lib/js/test262-upstream
|
|
git -C lib/js clone --depth 1 https://github.com/tc39/test262.git test262-upstream
|
|
|
|
Why a custom harness stub instead of assert.js + sta.js?
|
|
Our JS parser doesn't handle `i++` yet, which the real assert.js uses. The
|
|
stub here implements the assert entry points that >99% of tests actually
|
|
touch (sameValue, notSameValue, throws, _isSameValue, _toString) plus
|
|
Test262Error — using syntax our parser handles. Tests that reach into
|
|
obscure assert.* paths will fail and show up on the scoreboard, which is
|
|
the point.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import json
|
|
import multiprocessing as mp
|
|
import os
|
|
import re
|
|
import select
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
SX_SERVER = REPO / "hosts" / "ocaml" / "_build" / "default" / "bin" / "sx_server.exe"
|
|
UPSTREAM = REPO / "lib" / "js" / "test262-upstream"
|
|
TEST_ROOT = UPSTREAM / "test"
|
|
HARNESS_DIR = UPSTREAM / "harness"
|
|
|
|
DEFAULT_PER_TEST_TIMEOUT_S = 5.0
|
|
DEFAULT_BATCH_TIMEOUT_S = 120
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Harness stub — replaces assert.js + sta.js with something our parser handles.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
HARNESS_STUB = r"""
|
|
function Test262Error(message) {
|
|
this.message = message || "";
|
|
this.name = "Test262Error";
|
|
}
|
|
Test262Error.thrower = function (message) { throw new Test262Error(message); };
|
|
function $DONOTEVALUATE() { throw "Test262: This statement should not be evaluated."; }
|
|
|
|
var assert = {};
|
|
assert._isSameValue = function (a, b) {
|
|
if (a === b) { return (a !== 0) || ((1/a) === (1/b)); }
|
|
return (a !== a) && (b !== b);
|
|
};
|
|
assert._toString = function (v) {
|
|
if (v === null) { return "null"; }
|
|
if (v === undefined) { return "undefined"; }
|
|
if (typeof v === "string") { return "\"" + v + "\""; }
|
|
return "" + v;
|
|
};
|
|
assert.sameValue = function (actual, expected, message) {
|
|
if (assert._isSameValue(actual, expected)) { return; }
|
|
var msg = message || "";
|
|
throw new Test262Error(msg + " Expected SameValue(" + assert._toString(actual) + ", " + assert._toString(expected) + ")");
|
|
};
|
|
assert.notSameValue = function (actual, unexpected, message) {
|
|
if (!assert._isSameValue(actual, unexpected)) { return; }
|
|
var msg = message || "";
|
|
throw new Test262Error(msg + " Expected different values, both were " + assert._toString(actual));
|
|
};
|
|
assert.throws = function (errCtor, fn, message) {
|
|
var msg = message || "";
|
|
try { fn(); } catch (e) {
|
|
if (typeof e !== "object" || e === null) {
|
|
throw new Test262Error(msg + " thrown value not an object");
|
|
}
|
|
if (e.constructor === errCtor) { return; }
|
|
throw new Test262Error(msg + " expected " + errCtor.name + " got " + (e.name || "other"));
|
|
}
|
|
throw new Test262Error(msg + " no exception thrown, expected " + errCtor.name);
|
|
};
|
|
assert.throws.early = function (errCtor, code) {
|
|
// We can't truly early-parse so fall back to runtime throw check.
|
|
throw new Test262Error("assert.throws.early not supported");
|
|
};
|
|
// assert() direct call — loose-check truthiness (not strict === true like real harness)
|
|
var __assert_call__ = function (b, m) {
|
|
if (b) { return; }
|
|
throw new Test262Error(m || "assertion failed");
|
|
};
|
|
// compareArray stub — minimal for cases that only compareArray arrays of primitives
|
|
assert.compareArray = function (a, b, m) {
|
|
var msg = m || "";
|
|
if (a === b) { return; }
|
|
if (a == null || b == null) { throw new Test262Error(msg + " compareArray null"); }
|
|
if (a.length !== b.length) { throw new Test262Error(msg + " compareArray length differs"); }
|
|
for (var i = 0; i < a.length; i = i + 1) {
|
|
if (!assert._isSameValue(a[i], b[i])) {
|
|
throw new Test262Error(msg + " compareArray index " + i);
|
|
}
|
|
}
|
|
};
|
|
// propertyHelper stubs — verifyProperty checks just existence + value for now.
|
|
var verifyProperty = function (obj, name, desc, opts) {
|
|
if (desc && (desc.value !== undefined)) {
|
|
assert.sameValue(obj[name], desc.value, name + " value");
|
|
}
|
|
};
|
|
var verifyPrimordialProperty = verifyProperty;
|
|
var verifyNotEnumerable = function (o, n) { };
|
|
var verifyNotWritable = function (o, n) { };
|
|
var verifyNotConfigurable = function (o, n) { };
|
|
var verifyEnumerable = function (o, n) { };
|
|
var verifyWritable = function (o, n) { };
|
|
var verifyConfigurable = function (o, n) { };
|
|
// isConstructor stub — we can't actually probe; assume falsy constructor for arrows/functions
|
|
var isConstructor = function (f) {
|
|
if (typeof f !== "function") { return false; }
|
|
// Best-effort: built-in functions and arrows aren't; declared `function` decls are.
|
|
return false;
|
|
};
|
|
// Trivial helper for tests that use Array.isArray-like functionality
|
|
// (many tests reach for it via compareArray)
|
|
"""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Frontmatter parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
FRONTMATTER_RE = re.compile(r"/\*---(.*?)---\*/", re.DOTALL)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Frontmatter:
|
|
description: str = ""
|
|
flags: list = dataclasses.field(default_factory=list)
|
|
includes: list = dataclasses.field(default_factory=list)
|
|
features: list = dataclasses.field(default_factory=list)
|
|
negative_phase: "str | None" = None
|
|
negative_type: "str | None" = None
|
|
esid: "str | None" = None
|
|
|
|
|
|
def _parse_yaml_list(s: str) -> list:
|
|
s = s.strip()
|
|
if s.startswith("[") and s.endswith("]"):
|
|
s = s[1:-1]
|
|
return [item.strip().strip('"').strip("'") for item in s.split(",") if item.strip()]
|
|
|
|
|
|
def parse_frontmatter(src: str) -> Frontmatter:
|
|
fm = Frontmatter()
|
|
m = FRONTMATTER_RE.search(src)
|
|
if not m:
|
|
return fm
|
|
body = m.group(1)
|
|
lines = body.split("\n")
|
|
i = 0
|
|
current_key = None
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith("#"):
|
|
i += 1
|
|
continue
|
|
m2 = re.match(r"^([a-zA-Z_][a-zA-Z0-9_]*)\s*:\s*(.*)$", line)
|
|
if m2 and not line.startswith(" ") and not line.startswith("\t"):
|
|
key, value = m2.group(1), m2.group(2).strip()
|
|
if key == "description":
|
|
if value in (">", "|"):
|
|
desc_lines = []
|
|
j = i + 1
|
|
while j < len(lines):
|
|
nxt = lines[j]
|
|
if nxt.startswith(" ") or nxt.startswith("\t") or not nxt.strip():
|
|
desc_lines.append(nxt.strip())
|
|
j += 1
|
|
else:
|
|
break
|
|
fm.description = " ".join(d for d in desc_lines if d)
|
|
i = j
|
|
continue
|
|
fm.description = value
|
|
elif key == "flags":
|
|
fm.flags = _parse_yaml_list(value)
|
|
elif key == "includes":
|
|
fm.includes = _parse_yaml_list(value)
|
|
elif key == "features":
|
|
fm.features = _parse_yaml_list(value)
|
|
elif key == "negative":
|
|
if value.startswith("{"):
|
|
inner = value.strip("{}")
|
|
for part in inner.split(","):
|
|
if ":" in part:
|
|
pk, pv = part.split(":", 1)
|
|
pk = pk.strip()
|
|
pv = pv.strip().strip('"').strip("'")
|
|
if pk == "phase":
|
|
fm.negative_phase = pv
|
|
elif pk == "type":
|
|
fm.negative_type = pv
|
|
else:
|
|
current_key = "negative"
|
|
elif key == "esid":
|
|
fm.esid = value
|
|
i += 1
|
|
continue
|
|
if current_key == "negative":
|
|
m3 = re.match(r"^\s+([a-zA-Z_]+)\s*:\s*(.*)$", line)
|
|
if m3:
|
|
pk, pv = m3.group(1), m3.group(2).strip().strip('"').strip("'")
|
|
if pk == "phase":
|
|
fm.negative_phase = pv
|
|
elif pk == "type":
|
|
fm.negative_type = pv
|
|
else:
|
|
current_key = None
|
|
i += 1
|
|
return fm
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Categorisation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_category(test_path: Path) -> str:
|
|
rel = test_path.relative_to(TEST_ROOT).as_posix()
|
|
parts = rel.split("/")
|
|
if len(parts) >= 2:
|
|
return "/".join(parts[:2])
|
|
return parts[0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SX escaping — escape a JS source string for the nested `(eval "(js-eval \"...\")")` form
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def sx_escape_for_nested_eval(s: str) -> str:
|
|
"""Return a string ready to be embedded as the JS source inside
|
|
`(eval "(js-eval \"...\")")`. Two-level escape: the outer `(eval "...")`
|
|
consumes one layer, the inner `(js-eval \"...\")` consumes another.
|
|
"""
|
|
# Level 1 — inside the inner string literal
|
|
inner = (
|
|
s.replace("\\", "\\\\")
|
|
.replace('"', '\\"')
|
|
.replace("\n", "\\n")
|
|
.replace("\r", "\\r")
|
|
.replace("\t", "\\t")
|
|
)
|
|
# Level 2 — the whole inner form is itself a string in the outer
|
|
outer = inner.replace("\\", "\\\\").replace('"', '\\"')
|
|
return outer
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Output parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Server output forms:
|
|
# (ready)
|
|
# (ok N VALUE) -- single-line result
|
|
# (ok-len N SIZE) -- next line is the result (multi-line or long)
|
|
# VALUE
|
|
# (error N "message") -- epoch errored
|
|
#
|
|
# We read line-by-line off stdout so we can advance tests one-at-a-time
|
|
# and kill the server if it hangs.
|
|
|
|
RX_OK_INLINE = re.compile(r"^\(ok (\d+) (.*)\)\s*$")
|
|
RX_OK_LEN = re.compile(r"^\(ok-len (\d+) \d+\)\s*$")
|
|
RX_ERR = re.compile(r"^\(error (\d+) (.*)\)\s*$")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Classification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def classify_error(msg: str) -> str:
|
|
m = msg.lower()
|
|
if "expected" in m and "got" in m:
|
|
return "SyntaxError (parse/unsupported syntax)"
|
|
if "unexpected token" in m or "unexpected char" in m:
|
|
return "SyntaxError (parse/unsupported syntax)"
|
|
if "expected ident" in m or "expected punct" in m or "expected keyword" in m:
|
|
return "SyntaxError (parse/unsupported syntax)"
|
|
if "syntaxerror" in m or "parse" in m:
|
|
return "SyntaxError (parse/unsupported syntax)"
|
|
if "undefined symbol" in m or "unbound" in m:
|
|
return "ReferenceError (undefined symbol)"
|
|
if "referenceerror" in m:
|
|
return "ReferenceError (undefined symbol)"
|
|
if "typeerror" in m and "not a function" in m:
|
|
return "TypeError: not a function"
|
|
if "typeerror" in m:
|
|
return "TypeError (other)"
|
|
if "rangeerror" in m:
|
|
return "RangeError"
|
|
if "test262error" in m:
|
|
return "Test262Error (assertion failed)"
|
|
if "timeout" in m:
|
|
return "Timeout"
|
|
if "killed" in m or "crash" in m:
|
|
return "Crash"
|
|
if "unhandled exception" in m:
|
|
inner = re.search(r"Unhandled exception:\s*\\?\"([^\"]{0,80})", msg)
|
|
if inner:
|
|
return f"Unhandled: {inner.group(1)[:60]}"
|
|
return "Unhandled exception"
|
|
return f"Other: {msg[:80]}"
|
|
|
|
|
|
def classify_negative_result(fm: Frontmatter, kind: str, payload: str):
|
|
expected_type = fm.negative_type or ""
|
|
if kind == "error":
|
|
low = payload.lower()
|
|
etype = expected_type.lower()
|
|
if etype and etype in low:
|
|
return True, f"negative: threw {expected_type} as expected"
|
|
# Map our parse errors to SyntaxError for negative:parse tests
|
|
if expected_type == "SyntaxError" and (
|
|
"unexpected token" in low
|
|
or "unexpected char" in low
|
|
or "expected ident" in low
|
|
or "expected punct" in low
|
|
or "expected keyword" in low
|
|
or ("expected" in low and "got" in low)
|
|
or "js-transpile-unop" in low
|
|
or "js-transpile-binop" in low
|
|
or "js-compound-update" in low
|
|
or "parse" in low
|
|
):
|
|
return True, f"negative: threw {expected_type} (mapped from parser error) as expected"
|
|
# Many runtime errors signal parse-phase syntax errors in test262's sense
|
|
if expected_type == "ReferenceError" and "undefined symbol" in low:
|
|
return True, f"negative: threw {expected_type} (mapped) as expected"
|
|
if expected_type == "TypeError" and "typeerror" in low:
|
|
return True, f"negative: threw {expected_type} as expected"
|
|
return False, f"negative: expected {expected_type}, got: {payload[:100]}"
|
|
return False, f"negative: expected {expected_type}, but test completed normally"
|
|
|
|
|
|
def classify_positive_result(kind: str, payload: str):
|
|
if kind == "ok":
|
|
return True, "passed"
|
|
return False, classify_error(payload)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Skip rules
|
|
# ---------------------------------------------------------------------------
|
|
|
|
UNSUPPORTED_FEATURES = {
|
|
"Atomics",
|
|
"SharedArrayBuffer",
|
|
"BigInt",
|
|
"Proxy",
|
|
"Reflect",
|
|
"Reflect.construct",
|
|
"Symbol",
|
|
"Symbol.iterator",
|
|
"Symbol.asyncIterator",
|
|
"Symbol.hasInstance",
|
|
"Symbol.isConcatSpreadable",
|
|
"Symbol.match",
|
|
"Symbol.matchAll",
|
|
"Symbol.replace",
|
|
"Symbol.search",
|
|
"Symbol.species",
|
|
"Symbol.split",
|
|
"Symbol.toPrimitive",
|
|
"Symbol.toStringTag",
|
|
"Symbol.unscopables",
|
|
"TypedArray",
|
|
"DataView",
|
|
"WeakRef",
|
|
"WeakMap",
|
|
"WeakSet",
|
|
"FinalizationRegistry",
|
|
"async-functions", # we support but conformance shape iffy
|
|
"async-iteration",
|
|
"async-generators",
|
|
"generators",
|
|
"regexp-named-groups",
|
|
"regexp-unicode-property-escapes",
|
|
"regexp-dotall",
|
|
"regexp-lookbehind",
|
|
"regexp-match-indices",
|
|
"regexp-modifiers",
|
|
"regexp-v-flag",
|
|
"regexp-duplicate-named-groups",
|
|
"numeric-separator-literal",
|
|
"class-fields-private",
|
|
"class-fields-public",
|
|
"class-methods-private",
|
|
"class-static-fields-private",
|
|
"class-static-fields-public",
|
|
"class-static-methods-private",
|
|
"decorators",
|
|
"destructuring-binding-patterns",
|
|
"destructuring-assignment",
|
|
"error-cause",
|
|
"optional-chaining",
|
|
"optional-catch-binding",
|
|
"logical-assignment-operators",
|
|
"numeric-separator-literal",
|
|
"hashbang",
|
|
"import-assertions",
|
|
"import-attributes",
|
|
"import.meta",
|
|
"dynamic-import",
|
|
"json-modules",
|
|
"json-parse-with-source",
|
|
"Intl.DisplayNames",
|
|
"Intl.ListFormat",
|
|
"Intl.Locale",
|
|
"Intl.NumberFormat-unified",
|
|
"Intl.Segmenter",
|
|
"Intl-enumeration",
|
|
"Temporal",
|
|
"IteratorClose",
|
|
"Iterator",
|
|
"iterator-helpers",
|
|
"async-explicit-resource-management",
|
|
"explicit-resource-management",
|
|
"set-methods",
|
|
"Map.prototype.upsert",
|
|
"array-grouping",
|
|
"Array.fromAsync",
|
|
"promise-with-resolvers",
|
|
"Promise.try",
|
|
"Promise.any",
|
|
"Promise.allSettled",
|
|
"ShadowRealm",
|
|
"tail-call-optimization",
|
|
"legacy-regexp",
|
|
"uint8array-base64",
|
|
}
|
|
|
|
|
|
def should_skip(t):
|
|
if "onlyStrict" in t.fm.flags:
|
|
return True, "strict-mode only"
|
|
if "module" in t.fm.flags:
|
|
return True, "ESM module"
|
|
if "raw" in t.fm.flags:
|
|
return True, "raw (no harness)"
|
|
if "CanBlockIsFalse" in t.fm.flags or "CanBlockIsTrue" in t.fm.flags:
|
|
return True, "shared-memory flag"
|
|
for f in t.fm.features:
|
|
if f in UNSUPPORTED_FEATURES:
|
|
return True, f"feature:{f}"
|
|
# Skip anything under Intl/Temporal/etc. path — these categories are 100% unsupported
|
|
p = t.rel
|
|
for prefix in (
|
|
"intl402/",
|
|
"staging/",
|
|
"built-ins/Atomics/",
|
|
"built-ins/SharedArrayBuffer/",
|
|
"built-ins/BigInt/",
|
|
"built-ins/Proxy/",
|
|
"built-ins/Reflect/",
|
|
"built-ins/Symbol/",
|
|
"built-ins/WeakRef/",
|
|
"built-ins/WeakMap/",
|
|
"built-ins/WeakSet/",
|
|
"built-ins/FinalizationRegistry/",
|
|
"built-ins/TypedArrayConstructors/",
|
|
"built-ins/Temporal/",
|
|
"built-ins/Int8Array/",
|
|
"built-ins/Int16Array/",
|
|
"built-ins/Int32Array/",
|
|
"built-ins/Uint8Array/",
|
|
"built-ins/Uint8ClampedArray/",
|
|
"built-ins/Uint16Array/",
|
|
"built-ins/Uint32Array/",
|
|
"built-ins/Float16Array/",
|
|
"built-ins/Float32Array/",
|
|
"built-ins/Float64Array/",
|
|
"built-ins/BigInt64Array/",
|
|
"built-ins/BigUint64Array/",
|
|
"built-ins/DataView/",
|
|
"built-ins/ArrayBuffer/",
|
|
"built-ins/ArrayIteratorPrototype/",
|
|
"built-ins/AsyncFromSyncIteratorPrototype/",
|
|
"built-ins/AsyncGeneratorFunction/",
|
|
"built-ins/AsyncGeneratorPrototype/",
|
|
"built-ins/AsyncIteratorPrototype/",
|
|
"built-ins/GeneratorFunction/",
|
|
"built-ins/GeneratorPrototype/",
|
|
"built-ins/MapIteratorPrototype/",
|
|
"built-ins/SetIteratorPrototype/",
|
|
"built-ins/StringIteratorPrototype/",
|
|
"built-ins/RegExpStringIteratorPrototype/",
|
|
"built-ins/AbstractModuleSource/",
|
|
"built-ins/AggregateError/",
|
|
"built-ins/DisposableStack/",
|
|
"built-ins/AsyncDisposableStack/",
|
|
"built-ins/SuppressedError/",
|
|
"built-ins/Iterator/",
|
|
"built-ins/AsyncIterator/",
|
|
"built-ins/ShadowRealm/",
|
|
"annexB/",
|
|
):
|
|
if p.startswith(prefix):
|
|
return True, f"unsupported path:{prefix.rstrip('/')}"
|
|
return False, ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test case loading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class TestCase:
|
|
path: Path
|
|
rel: str
|
|
category: str
|
|
fm: Frontmatter
|
|
src: str
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class TestResult:
|
|
rel: str
|
|
category: str
|
|
status: str # pass | fail | skip | timeout
|
|
reason: str
|
|
elapsed_ms: int = 0
|
|
|
|
|
|
def discover_tests(filter_prefixes):
|
|
tests = []
|
|
for p in TEST_ROOT.rglob("*.js"):
|
|
if p.name.endswith("_FIXTURE.js"):
|
|
continue
|
|
if "_FIXTURE" in p.parts:
|
|
continue
|
|
if filter_prefixes:
|
|
rel = p.relative_to(TEST_ROOT).as_posix()
|
|
if not any(rel.startswith(prefix) for prefix in filter_prefixes):
|
|
continue
|
|
tests.append(p)
|
|
tests.sort()
|
|
return tests
|
|
|
|
|
|
def load_test(path: Path):
|
|
try:
|
|
src = path.read_text(encoding="utf-8")
|
|
except Exception:
|
|
return None
|
|
fm = parse_frontmatter(src)
|
|
return TestCase(
|
|
path=path,
|
|
rel=path.relative_to(TEST_ROOT).as_posix(),
|
|
category=test_category(path),
|
|
fm=fm,
|
|
src=src,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Long-lived server session
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class ServerSession:
|
|
"""Wrap a long-lived sx_server.exe subprocess; feed it one-liner commands,
|
|
collect results per-epoch. Restart on hang/crash.
|
|
|
|
Uses a raw-fd line buffer + select() to avoid spawning a thread per read.
|
|
"""
|
|
|
|
def __init__(self, per_test_timeout: float):
|
|
self.per_test_timeout = per_test_timeout
|
|
self.proc = None
|
|
self._buf = b""
|
|
self._fd = -1
|
|
|
|
def start(self) -> None:
|
|
self.proc = subprocess.Popen(
|
|
[str(SX_SERVER)],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
cwd=str(REPO),
|
|
bufsize=0, # binary, unbuffered — we do our own line parsing
|
|
)
|
|
self._fd = self.proc.stdout.fileno()
|
|
self._buf = b""
|
|
os.set_blocking(self._fd, False)
|
|
self._wait_for("(ready)", timeout=15.0)
|
|
# Load kernel libraries
|
|
self._run_and_collect(1, '(load "lib/r7rs.sx")', timeout=60.0)
|
|
self._run_and_collect(2, '(load "lib/js/lexer.sx")', timeout=60.0)
|
|
self._run_and_collect(3, '(load "lib/js/parser.sx")', timeout=60.0)
|
|
self._run_and_collect(4, '(load "lib/js/transpile.sx")', timeout=60.0)
|
|
self._run_and_collect(5, '(load "lib/js/runtime.sx")', timeout=60.0)
|
|
# Preload the stub harness as one big js-eval
|
|
stub_escaped = sx_escape_for_nested_eval(HARNESS_STUB)
|
|
self._run_and_collect(
|
|
6,
|
|
f'(eval "(js-eval \\"{stub_escaped}\\")")',
|
|
timeout=60.0,
|
|
)
|
|
|
|
def stop(self) -> None:
|
|
if self.proc is not None:
|
|
try:
|
|
self.proc.stdin.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self.proc.terminate()
|
|
self.proc.wait(timeout=3)
|
|
except Exception:
|
|
try:
|
|
self.proc.kill()
|
|
except Exception:
|
|
pass
|
|
self.proc = None
|
|
|
|
def _readline_raw(self, timeout: float):
|
|
"""Read one line (including trailing \\n) from the subprocess's stdout.
|
|
Returns bytes or None on EOF. Raises TimeoutError if no newline appears
|
|
within `timeout` seconds.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
nl = self._buf.find(b"\n")
|
|
if nl >= 0:
|
|
line = self._buf[: nl + 1]
|
|
self._buf = self._buf[nl + 1 :]
|
|
return line
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError("readline timeout")
|
|
try:
|
|
rlist, _, _ = select.select([self._fd], [], [], remaining)
|
|
except (OSError, ValueError):
|
|
return None
|
|
if not rlist:
|
|
raise TimeoutError("readline timeout")
|
|
try:
|
|
chunk = os.read(self._fd, 65536)
|
|
except (BlockingIOError, InterruptedError):
|
|
continue
|
|
except OSError:
|
|
return None
|
|
if not chunk:
|
|
if self._buf:
|
|
line = self._buf
|
|
self._buf = b""
|
|
return line
|
|
return None
|
|
self._buf += chunk
|
|
|
|
def _readline(self, timeout: float):
|
|
b = self._readline_raw(timeout)
|
|
if b is None:
|
|
return None
|
|
try:
|
|
return b.decode("utf-8", errors="replace")
|
|
except Exception:
|
|
return ""
|
|
|
|
def _wait_for(self, token: str, timeout: float) -> None:
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < timeout:
|
|
line = self._readline(timeout - (time.monotonic() - start))
|
|
if line is None:
|
|
raise RuntimeError("sx_server closed stdout before ready")
|
|
if token in line:
|
|
return
|
|
raise TimeoutError(f"timeout waiting for {token}")
|
|
|
|
def _run_and_collect(self, epoch: int, cmd: str, timeout: float):
|
|
"""Write `(epoch N)\\n<cmd>\\n` and read until we see ok/ok-len/error for that epoch.
|
|
Returns (kind, payload). Raises TimeoutError if the server hangs.
|
|
"""
|
|
payload = f"(epoch {epoch})\n{cmd}\n".encode("utf-8")
|
|
try:
|
|
self.proc.stdin.write(payload)
|
|
self.proc.stdin.flush()
|
|
except (BrokenPipeError, OSError):
|
|
raise RuntimeError("sx_server stdin closed")
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError(f"epoch {epoch} exceeded timeout {timeout}")
|
|
line = self._readline(remaining)
|
|
if line is None:
|
|
raise RuntimeError("sx_server closed stdout mid-epoch")
|
|
m = RX_OK_INLINE.match(line)
|
|
if m:
|
|
e = int(m.group(1))
|
|
if e == epoch:
|
|
return "ok", m.group(2)
|
|
continue
|
|
m = RX_OK_LEN.match(line)
|
|
if m:
|
|
e = int(m.group(1))
|
|
remaining2 = deadline - time.monotonic()
|
|
if remaining2 <= 0:
|
|
raise TimeoutError(f"epoch {epoch} exceeded timeout {timeout}")
|
|
val = self._readline(remaining2)
|
|
if val is None:
|
|
val = ""
|
|
val = val.rstrip("\n")
|
|
if e == epoch:
|
|
return "ok", val
|
|
continue
|
|
m = RX_ERR.match(line)
|
|
if m:
|
|
e = int(m.group(1))
|
|
if e == epoch:
|
|
return "error", m.group(2)
|
|
continue
|
|
# Other output — (ready), comment, noise — ignore
|
|
raise TimeoutError(f"epoch {epoch} exceeded timeout {timeout}")
|
|
|
|
def run_test(self, epoch: int, js_source: str):
|
|
escaped = sx_escape_for_nested_eval(js_source)
|
|
cmd = f'(eval "(js-eval \\"{escaped}\\")")'
|
|
return self._run_and_collect(epoch, cmd, timeout=self.per_test_timeout)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Parallel workers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _worker_run(args):
|
|
"""Run a shard of tests in this process. Returns list of (rel, category, status, reason).
|
|
|
|
Each worker keeps its own long-lived ServerSession. Restarts on timeout/crash.
|
|
"""
|
|
shard_tests, per_test_timeout, restart_every, worker_id = args
|
|
|
|
session = None
|
|
results = []
|
|
|
|
def get_session():
|
|
nonlocal session
|
|
if session is None:
|
|
session = ServerSession(per_test_timeout=per_test_timeout)
|
|
session.start()
|
|
return session
|
|
|
|
def restart():
|
|
nonlocal session
|
|
if session is not None:
|
|
try:
|
|
session.stop()
|
|
except Exception:
|
|
pass
|
|
session = None
|
|
|
|
try:
|
|
epoch = 100 + worker_id * 10000
|
|
done_n = 0
|
|
for t_data in shard_tests:
|
|
rel, category, src, negative_phase, negative_type = t_data
|
|
epoch += 1
|
|
done_n += 1
|
|
try:
|
|
sess = get_session()
|
|
kind, payload = sess.run_test(epoch, src)
|
|
if negative_phase:
|
|
fake_fm = Frontmatter()
|
|
fake_fm.negative_phase = negative_phase
|
|
fake_fm.negative_type = negative_type
|
|
ok, reason = classify_negative_result(fake_fm, kind, payload)
|
|
status = "pass" if ok else "fail"
|
|
else:
|
|
if kind == "ok":
|
|
status, reason = "pass", "passed"
|
|
else:
|
|
status, reason = "fail", classify_error(payload)
|
|
results.append((rel, category, status, reason))
|
|
except TimeoutError:
|
|
results.append((rel, category, "timeout", "per-test timeout"))
|
|
restart()
|
|
except Exception as e:
|
|
results.append((rel, category, "fail", f"runner-error: {e}"))
|
|
restart()
|
|
|
|
# Periodic restart to keep server healthy (memory bounded)
|
|
if restart_every > 0 and done_n % restart_every == 0:
|
|
restart()
|
|
finally:
|
|
if session is not None:
|
|
try:
|
|
session.stop()
|
|
except Exception:
|
|
pass
|
|
|
|
return results
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run driver
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def assemble_source(t):
|
|
"""Return JS source to feed to js-eval. Harness is preloaded, so we only
|
|
append the test source (plus negative-test prep if needed).
|
|
"""
|
|
return t.src
|
|
|
|
|
|
def aggregate(results):
|
|
by_cat = defaultdict(
|
|
lambda: {"pass": 0, "fail": 0, "skip": 0, "timeout": 0, "total": 0, "failures": Counter()}
|
|
)
|
|
totals = {"pass": 0, "fail": 0, "skip": 0, "timeout": 0, "total": 0}
|
|
failure_modes = Counter()
|
|
for r in results:
|
|
cat = by_cat[r.category]
|
|
cat[r.status] += 1
|
|
cat["total"] += 1
|
|
totals[r.status] += 1
|
|
totals["total"] += 1
|
|
if r.status == "fail":
|
|
cat["failures"][r.reason] += 1
|
|
failure_modes[r.reason] += 1
|
|
elif r.status == "timeout":
|
|
cat["failures"]["Timeout"] += 1
|
|
failure_modes["Timeout"] += 1
|
|
categories = []
|
|
for name, stats in sorted(by_cat.items()):
|
|
total = stats["total"]
|
|
passed = stats["pass"]
|
|
runnable = total - stats["skip"]
|
|
pass_rate = (passed / runnable * 100.0) if runnable else 0.0
|
|
categories.append(
|
|
{
|
|
"category": name,
|
|
"total": total,
|
|
"pass": passed,
|
|
"fail": stats["fail"],
|
|
"skip": stats["skip"],
|
|
"timeout": stats["timeout"],
|
|
"pass_rate": round(pass_rate, 1),
|
|
"top_failures": stats["failures"].most_common(5),
|
|
}
|
|
)
|
|
runnable_total = totals["total"] - totals["skip"]
|
|
pass_rate = (totals["pass"] / runnable_total * 100.0) if runnable_total else 0.0
|
|
return {
|
|
"totals": {**totals, "runnable": runnable_total, "pass_rate": round(pass_rate, 1)},
|
|
"categories": categories,
|
|
"top_failure_modes": failure_modes.most_common(20),
|
|
}
|
|
|
|
|
|
def write_markdown(scoreboard, path: Path, pinned_commit: str, elapsed_s: float) -> None:
|
|
t = scoreboard["totals"]
|
|
lines = [
|
|
"# test262 scoreboard",
|
|
"",
|
|
f"Pinned commit: `{pinned_commit}`",
|
|
f"Wall time: {elapsed_s:.1f}s",
|
|
"",
|
|
f"**Total:** {t['pass']}/{t['runnable']} runnable passed ({t['pass_rate']}%). "
|
|
f"Raw: pass={t['pass']} fail={t['fail']} skip={t['skip']} timeout={t['timeout']} total={t['total']}.",
|
|
"",
|
|
"## Top failure modes",
|
|
"",
|
|
]
|
|
for mode, count in scoreboard["top_failure_modes"]:
|
|
lines.append(f"- **{count}x** {mode}")
|
|
lines.extend(["", "## Categories (worst pass-rate first, min 10 runnable)", ""])
|
|
lines.append("| Category | Pass | Fail | Skip | Timeout | Total | Pass % |")
|
|
lines.append("|---|---:|---:|---:|---:|---:|---:|")
|
|
cats = [c for c in scoreboard["categories"] if (c["total"] - c["skip"]) >= 10]
|
|
cats.sort(key=lambda c: (c["pass_rate"], -c["total"]))
|
|
for c in cats:
|
|
lines.append(
|
|
f"| {c['category']} | {c['pass']} | {c['fail']} | {c['skip']} | "
|
|
f"{c['timeout']} | {c['total']} | {c['pass_rate']}% |"
|
|
)
|
|
lines.append("")
|
|
lines.append("## Per-category top failures (min 10 runnable, worst first)")
|
|
lines.append("")
|
|
for c in cats:
|
|
if not c["top_failures"]:
|
|
continue
|
|
lines.append(f"### {c['category']} ({c['pass']}/{c['total']-c['skip']} — {c['pass_rate']}%)")
|
|
lines.append("")
|
|
for reason, count in c["top_failures"]:
|
|
lines.append(f"- **{count}x** {reason}")
|
|
lines.append("")
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def main(argv):
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=0, help="max tests to run (0 = all)")
|
|
ap.add_argument("--filter", type=str, action="append", default=None,
|
|
help="path prefix filter (repeatable; OR'd together)")
|
|
ap.add_argument("--per-test-timeout", type=float, default=DEFAULT_PER_TEST_TIMEOUT_S)
|
|
ap.add_argument("--restart-every", type=int, default=500,
|
|
help="restart worker server every N tests (keeps memory bounded)")
|
|
ap.add_argument("--max-per-category", type=int, default=0,
|
|
help="cap runnable tests per category (0 = no cap)")
|
|
ap.add_argument("--workers", type=int, default=0,
|
|
help="number of parallel workers (0 = auto; min(nproc, 4))")
|
|
ap.add_argument("--output-json", type=str,
|
|
default=str(REPO / "lib" / "js" / "test262-scoreboard.json"))
|
|
ap.add_argument("--output-md", type=str,
|
|
default=str(REPO / "lib" / "js" / "test262-scoreboard.md"))
|
|
ap.add_argument("--progress-every", type=int, default=100)
|
|
args = ap.parse_args(argv)
|
|
|
|
if not SX_SERVER.exists():
|
|
print(f"ERROR: sx_server.exe not found at {SX_SERVER}", file=sys.stderr)
|
|
return 1
|
|
if not UPSTREAM.exists():
|
|
print(f"ERROR: test262-upstream not found at {UPSTREAM}", file=sys.stderr)
|
|
return 1
|
|
|
|
pinned_commit = ""
|
|
try:
|
|
pinned_commit = subprocess.check_output(
|
|
["git", "-C", str(UPSTREAM), "rev-parse", "HEAD"], text=True
|
|
).strip()
|
|
except Exception:
|
|
pass
|
|
|
|
all_paths = discover_tests(args.filter)
|
|
if args.limit:
|
|
all_paths = all_paths[: args.limit]
|
|
print(f"Discovered {len(all_paths)} test files.", file=sys.stderr)
|
|
|
|
tests = []
|
|
results = []
|
|
per_cat_count = defaultdict(int)
|
|
for p in all_paths:
|
|
t = load_test(p)
|
|
if not t:
|
|
continue
|
|
skip, why = should_skip(t)
|
|
if skip:
|
|
results.append(TestResult(rel=t.rel, category=t.category, status="skip", reason=why))
|
|
continue
|
|
if args.max_per_category > 0 and per_cat_count[t.category] >= args.max_per_category:
|
|
results.append(TestResult(rel=t.rel, category=t.category, status="skip",
|
|
reason=f"capped at --max-per-category={args.max_per_category}"))
|
|
continue
|
|
per_cat_count[t.category] += 1
|
|
tests.append(t)
|
|
|
|
print(f"Will run {len(tests)} tests ({len(results)} skipped up front).", file=sys.stderr)
|
|
|
|
# Worker count
|
|
# Auto-default: on <=2-core machines, 1 worker beats 2 because OCaml eval is
|
|
# CPU-bound and two processes starve each other. On 4+ cores, use nproc-1
|
|
# (leave one core for OS/Python). Cap at 8 to avoid resource thrash.
|
|
n_workers = args.workers
|
|
if n_workers <= 0:
|
|
try:
|
|
cpu = os.cpu_count() or 2
|
|
except Exception:
|
|
cpu = 2
|
|
if cpu <= 2:
|
|
n_workers = 1
|
|
else:
|
|
n_workers = max(1, min(cpu - 1, 8))
|
|
n_workers = max(1, min(n_workers, len(tests))) if tests else 1
|
|
print(f"Using {n_workers} parallel worker(s).", file=sys.stderr)
|
|
|
|
# Shard tests across workers (round-robin so categories spread evenly)
|
|
shards = [[] for _ in range(n_workers)]
|
|
for i, t in enumerate(tests):
|
|
shards[i % n_workers].append(
|
|
(t.rel, t.category, t.src, t.fm.negative_phase, t.fm.negative_type)
|
|
)
|
|
|
|
t_run_start = time.monotonic()
|
|
|
|
if n_workers == 1:
|
|
# Serial path — avoids multiprocessing overhead
|
|
worker_results = [_worker_run((shards[0], args.per_test_timeout, args.restart_every, 0))]
|
|
else:
|
|
with mp.Pool(n_workers) as pool:
|
|
worker_args = [
|
|
(shards[i], args.per_test_timeout, args.restart_every, i)
|
|
for i in range(n_workers)
|
|
]
|
|
# imap_unordered so progress prints show up sooner
|
|
collected = []
|
|
total_tests = len(tests)
|
|
last_print = time.monotonic()
|
|
for shard_out in pool.imap_unordered(_worker_run, worker_args):
|
|
collected.append(shard_out)
|
|
now = time.monotonic()
|
|
if now - last_print >= 5.0:
|
|
done_so_far = sum(len(s) for s in collected)
|
|
el = now - t_run_start
|
|
print(
|
|
f" worker returned: {done_so_far}/{total_tests} tests "
|
|
f"elapsed={el:.1f}s rate={done_so_far/max(el,0.001):.1f}/s",
|
|
file=sys.stderr,
|
|
)
|
|
last_print = now
|
|
worker_results = collected
|
|
|
|
for shard_out in worker_results:
|
|
for rel, category, status, reason in shard_out:
|
|
results.append(TestResult(rel=rel, category=category, status=status, reason=reason))
|
|
|
|
t_run_elapsed = time.monotonic() - t_run_start
|
|
print(f"\nFinished run in {t_run_elapsed:.1f}s", file=sys.stderr)
|
|
|
|
scoreboard = aggregate(results)
|
|
scoreboard["pinned_commit"] = pinned_commit
|
|
scoreboard["elapsed_seconds"] = round(t_run_elapsed, 1)
|
|
scoreboard["workers"] = n_workers
|
|
|
|
out_json = Path(args.output_json)
|
|
out_json.parent.mkdir(parents=True, exist_ok=True)
|
|
out_json.write_text(json.dumps(scoreboard, indent=2), encoding="utf-8")
|
|
|
|
out_md = Path(args.output_md)
|
|
write_markdown(scoreboard, out_md, pinned_commit, t_run_elapsed)
|
|
|
|
t = scoreboard["totals"]
|
|
print(
|
|
f"\nScoreboard: {t['pass']}/{t['runnable']} runnable passed ({t['pass_rate']}%) "
|
|
f"fail={t['fail']} skip={t['skip']} timeout={t['timeout']} total={t['total']}",
|
|
file=sys.stderr,
|
|
)
|
|
print(f"JSON: {out_json}", file=sys.stderr)
|
|
print(f"MD: {out_md}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|