From eaf5af4cd885c6c15e88e4611bd89d4f94f4e831 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 11 Apr 2026 16:12:28 +0000 Subject: [PATCH] =?UTF-8?q?Step=2017:=20streaming=20render=20=E2=80=94=20c?= =?UTF-8?q?hunked=20transfer,=20shell-first=20suspense,=20resolve=20script?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server (sx_server.ml): - Chunked HTTP transport (Transfer-Encoding: chunked) - Streaming page detection via scan_defpages (:stream true) - Shell-first render: outer layout + shell AST → aser → SSR → flush - Data resolution: evaluate :data, render :content per slot, flush __sxResolve scripts - AJAX streaming: synchronous eval + OOB swaps for SPA navigation - SX URL → flat path conversion for defpage matching - Error boundaries per resolve section - streaming-demo-data helper for the demo page Client (sx-platform.js): - Sx.resolveSuspense: finds [data-suspense] element, parses SX, renders to DOM - Fallback define for resolve-suspense when boot.sx imports fail in WASM - __sxPending drain on boot (queued resolves from before sx.js loads) - __sxResolve direct dispatch after boot Tests (streaming.spec.js): - 5 sandbox tests using real WASM kernel - Suspense placeholder rendering, __sxResolve replacement, independent slot resolution - Full layout with gutters, end-to-end resolve with streaming-demo/chunk components Co-Authored-By: Claude Opus 4.6 (1M context) --- hosts/ocaml/bin/sx_server.ml | 322 ++++++++++++++++++++++++++ shared/static/wasm/sx-platform.js | 28 +++ tests/playwright/streaming.spec.js | 351 +++++++++++++++++++++++++++++ 3 files changed, 701 insertions(+) create mode 100644 tests/playwright/streaming.spec.js diff --git a/hosts/ocaml/bin/sx_server.ml b/hosts/ocaml/bin/sx_server.ml index 3f2ec014..164f3d4c 100644 --- a/hosts/ocaml/bin/sx_server.ml +++ b/hosts/ocaml/bin/sx_server.ml @@ -149,6 +149,8 @@ let _scope_stacks = Sx_primitives._scope_stacks (* Populated from __app-config dict after SX files load. *) let _app_config : (string, value) Hashtbl.t option ref = ref None let _defpage_paths : string list ref = ref [] +(* Streaming pages: path → page name, for pages with :stream true *) +let _streaming_pages : (string, string) Hashtbl.t = Hashtbl.create 8 let get_app_config key default = match !_app_config with @@ -1740,6 +1742,31 @@ let http_response ?(status=200) ?(content_type="text/html; charset=utf-8") body let http_redirect url = Printf.sprintf "HTTP/1.1 301 Moved Permanently\r\nLocation: %s\r\nContent-Length: 0\r\nConnection: keep-alive\r\n\r\n" url +(* Chunked transfer encoding helpers for streaming responses *) +let http_chunked_header ?(status=200) ?(content_type="text/html; charset=utf-8") () = + let status_text = match status with + | 200 -> "OK" | 404 -> "Not Found" | 500 -> "Internal Server Error" | _ -> "Unknown" in + Printf.sprintf "HTTP/1.1 %d %s\r\nContent-Type: %s\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\n\r\n" + status status_text content_type + +let write_chunk fd data = + if String.length data > 0 then begin + let chunk = Printf.sprintf "%x\r\n%s\r\n" (String.length data) data in + let bytes = Bytes.of_string chunk in + let total = Bytes.length bytes in + let written = ref 0 in + (try + while !written < total do + let n = Unix.write fd bytes !written (total - !written) in + written := !written + n + done + with Unix.Unix_error _ -> ()) + end + +let end_chunked fd = + (try ignore (Unix.write_substring fd "0\r\n\r\n" 0 5) with Unix.Unix_error _ -> ()); + (try Unix.close fd with _ -> ()) + let parse_http_request data = match String.index_opt data '\r' with | None -> (match String.index_opt data '\n' with @@ -1900,6 +1927,194 @@ let http_render_page env path headers = end end +(* JSON-encode a string for use in __sxResolve script tags *) +let json_encode_string s = + let buf = Buffer.create (String.length s + 16) in + Buffer.add_char buf '"'; + String.iter (fun c -> match c with + | '"' -> Buffer.add_string buf "\\\"" + | '\\' -> Buffer.add_string buf "\\\\" + | '\n' -> Buffer.add_string buf "\\n" + | '\r' -> Buffer.add_string buf "\\r" + | '\t' -> Buffer.add_string buf "\\t" + | c when Char.code c < 0x20 -> + Buffer.add_string buf (Printf.sprintf "\\u%04x" (Char.code c)) + | c -> Buffer.add_char buf c + ) s; + Buffer.add_char buf '"'; + Buffer.contents buf + +(* Bootstrap script that queues resolves arriving before sx.js loads. + Must match _SX_STREAMING_BOOTSTRAP in shared/sx/helpers.py *) +let _sx_streaming_bootstrap = + "" + +(* Build a resolve script tag. Must match _SX_STREAMING_RESOLVE in helpers.py *) +let sx_streaming_resolve_script id sx_source = + Printf.sprintf "" + (json_encode_string id) (json_encode_string sx_source) + +(* ====================================================================== *) +(* Streaming page render — shell-first with chunked transfer encoding *) +(* ====================================================================== *) + +let http_render_page_streaming env path _headers fd page_name = + let t0 = Unix.gettimeofday () in + let page_def = try + match env_get env ("page:" ^ page_name) with Dict d -> d | _ -> raise Not_found + with _ -> + Printf.eprintf "[sx-stream] page def not found: page:%s\n%!" page_name; + let err = http_response ~status:500 "

Streaming page def not found

" in + let bytes = Bytes.of_string err in + (try ignore (Unix.write fd bytes 0 (Bytes.length bytes)) with _ -> ()); + (try Unix.close fd with _ -> ()); + raise Exit + in + + (* Extract streaming fields from page def *) + let shell_ast = match Hashtbl.find_opt page_def "shell" with Some v -> v | None -> Nil in + let data_ast = match Hashtbl.find_opt page_def "data" with Some v -> v | None -> Nil in + let content_ast = match Hashtbl.find_opt page_def "content" with Some v -> v | None -> Nil in + + (* Phase 1: Evaluate shell AST — contains ~suspense placeholders with fallbacks. + The :shell expression already includes the inner layout (e.g. ~layouts/doc), + so we only wrap in the outer layout (~shared:layout/app-body) for gutters. + NO inner layout wrapping — shell_ast already has it. *) + let shell_html = try + let outer_layout = get_app_str "outer-layout" "~shared:layout/app-body" in + let full_ast = List [Symbol outer_layout; Keyword "content"; shell_ast] in + let page_source = serialize_value full_ast in + (* aser → SSR *) + let body_result = + let call = List [Symbol "aser"; List [Symbol "quote"; full_ast]; Env env] in + Sx_ref.eval_expr call (Env env) in + let body_str = match body_result with + | String s | SxExpr s -> s | _ -> serialize_value body_result in + let body_html = try + let body_expr = match Sx_parser.parse_all body_str with + | [e] -> e | [] -> Nil | es -> List (Symbol "<>" :: es) in + if env_has env "render-to-html" then + let render_call = List [Symbol "render-to-html"; + List [Symbol "quote"; body_expr]; Env env] in + (match Sx_ref.eval_expr render_call (Env env) with + | String s | RawHTML s -> s | v -> Sx_runtime.value_to_str v) + else Sx_render.sx_render_to_html env body_expr env + with e -> Printf.eprintf "[sx-stream] SSR failed: %s\n%!" (Printexc.to_string e); "" in + + (* Build full page shell with body HTML *) + let get_shell name = try env_get env ("__shell-" ^ name) with _ -> Nil in + let shell_args = [ + Keyword "title"; String (get_app_str "title" "SX"); Keyword "csrf"; String ""; + Keyword "page-sx"; String page_source; + Keyword "body-html"; String body_html; + Keyword "component-defs"; get_shell "component-defs"; + Keyword "component-hash"; get_shell "component-hash"; + Keyword "pages-sx"; get_shell "pages-sx"; + Keyword "sx-css"; get_shell "sx-css"; + Keyword "asset-url"; get_shell "asset-url"; + Keyword "wasm-hash"; get_shell "wasm-hash"; + Keyword "platform-hash"; get_shell "platform-hash"; + Keyword "sxbc-hash"; get_shell "sxbc-hash"; + Keyword "inline-css"; get_shell "inline-css"; + Keyword "inline-head-js"; get_shell "inline-head-js"; + Keyword "init-sx"; get_shell "init-sx"; + Keyword "meta-html"; String ""; + ] in + let shell_sym = get_app_str "shell" "~shared:shell/sx-page-shell" in + let shell_call = List (Symbol shell_sym :: shell_args) in + if env_has env "render-to-html" then + let render_call = List [Symbol "render-to-html"; + List [Symbol "quote"; shell_call]; Env env] in + (match Sx_ref.eval_expr render_call (Env env) with + | String s | RawHTML s -> s | v -> Sx_runtime.value_to_str v) + else Sx_render.sx_render_to_html env shell_call env + with e -> + Printf.eprintf "[sx-stream] shell render failed: %s\n%!" (Printexc.to_string e); + "

Streaming shell render failed

" + in + let t1 = Unix.gettimeofday () in + + (* Phase 2: Send chunked header + shell HTML *) + let header = http_chunked_header () in + let header_bytes = Bytes.of_string header in + (try ignore (Unix.write fd header_bytes 0 (Bytes.length header_bytes)) with _ -> ()); + write_chunk fd shell_html; + (* Bootstrap resolve script — must come after shell so suspense elements exist *) + write_chunk fd _sx_streaming_bootstrap; + let t2 = Unix.gettimeofday () in + + (* Phase 3: Evaluate :data, render :content, flush resolve scripts *) + let resolve_count = ref 0 in + if data_ast <> Nil && content_ast <> Nil then begin + (try + let data_result = Sx_ref.eval_expr data_ast (Env env) in + let t3_data = Unix.gettimeofday () in + + (* Determine single-stream vs multi-stream *) + let data_items = match data_result with + | Dict _ -> [(data_result, "stream-content")] + | List items -> + List.map (fun item -> + let stream_id = match item with + | Dict d -> (match Hashtbl.find_opt d "stream-id" with + | Some (String s) -> s | _ -> "stream-content") + | _ -> "stream-content" in + (item, stream_id) + ) items + | _ -> + Printf.eprintf "[sx-stream] :data returned %s, expected dict or list\n%!" + (Sx_runtime.type_of data_result |> Sx_runtime.value_to_str); + [] + in + + (* For each data item, bind values and render :content *) + List.iter (fun (item, stream_id) -> + (try + (* Create fresh env with data bindings *) + let content_env = { bindings = Hashtbl.create 16; parent = Some env } in + (match item with + | Dict d -> + Hashtbl.iter (fun k v -> + if k <> "stream-id" && k <> "__type" then begin + (* Normalize: underscores → hyphens *) + let norm_k = String.map (fun c -> if c = '_' then '-' else c) k in + ignore (env_bind content_env norm_k v); + if norm_k <> k then ignore (env_bind content_env k v) + end + ) d + | _ -> ()); + + (* aser :content in the data-bound env *) + let content_result = + let call = List [Symbol "aser"; List [Symbol "quote"; content_ast]; Env content_env] in + Sx_ref.eval_expr call (Env content_env) in + let sx_source = match content_result with + | String s | SxExpr s -> s | _ -> serialize_value content_result in + let resolve_script = sx_streaming_resolve_script stream_id sx_source in + write_chunk fd resolve_script; + incr resolve_count + with e -> + (* Error boundary: emit error fallback for this slot *) + let msg = Printexc.to_string e in + Printf.eprintf "[sx-stream] resolve error for %s: %s\n%!" stream_id msg; + let error_sx = Printf.sprintf "(div :class \"text-rose-600 p-4 text-sm\" \"Error: %s\")" + (String.map (fun c -> if c = '"' then '\'' else c) msg) in + write_chunk fd (sx_streaming_resolve_script stream_id error_sx); + incr resolve_count) + ) data_items; + let t3 = Unix.gettimeofday () in + Printf.eprintf "[sx-stream] %s shell=%.3fs flush=%.3fs data=%.3fs resolve=%.3fs total=%.3fs chunks=%d\n%!" + path (t1 -. t0) (t2 -. t1) (t3_data -. t2) (t3 -. t3_data) (t3 -. t0) !resolve_count + with e -> + Printf.eprintf "[sx-stream] data eval failed: %s\n%!" (Printexc.to_string e)) + end else + Printf.eprintf "[sx-stream] %s shell=%.3fs (no :data/:content)\n%!" path (t1 -. t0); + + (* Phase 4: End chunked response *) + end_chunked fd + (* ====================================================================== *) (* Static file serving + file hashing *) (* ====================================================================== *) @@ -2056,8 +2271,12 @@ let http_inject_shell_statics env static_dir sx_sxc = | Some v -> serialize_value v | _ -> "" in let has_data = match extract_kw "data" rest with | Some _ -> true | None -> false in + let is_stream = match extract_kw "stream" rest with + | Some (Symbol "true") | Some (Bool true) -> true | _ -> false in if path_val <> "" then begin _defpage_paths := path_val :: !_defpage_paths; + if is_stream then + Hashtbl.replace _streaming_pages path_val name; Buffer.add_string pages_buf (Printf.sprintf "{:name \"%s\" :path \"%s\" :auth \"public\" :has-data %s :content \"%s\"}\n" name path_val (if has_data then "true" else "false") @@ -2074,6 +2293,9 @@ let http_inject_shell_statics env static_dir sx_sxc = Printf.eprintf "[sx-http] pages-sx: %d bytes (%d lines)\n%!" (String.length pages_sx) (List.length (String.split_on_char '\n' pages_sx)); + if Hashtbl.length _streaming_pages > 0 then + Printf.eprintf "[sx-http] streaming pages: %s\n%!" + (String.concat ", " (Hashtbl.fold (fun p n acc -> (p ^ "→" ^ n) :: acc) _streaming_pages [])); ignore (env_bind env "__shell-pages-sx" (String pages_sx)); ignore (env_bind env "__shell-sx-css" (String sx_css)); ignore (env_bind env "__shell-asset-url" (String "/static")); @@ -2514,6 +2736,23 @@ let http_setup_page_helpers env = with _ -> String (";; component " ^ name ^ " not found")) | _ -> raise (Eval_error "component-source: expected (name)")); + (* Streaming demo data helper — returns list of dicts for multi-stream test. + Matches the Python async generator streaming-demo-data in sx_docs. *) + bind "streaming-demo-data" (fun _args -> + let mk_item id label color message time = + let d = Hashtbl.create 8 in + Hashtbl.replace d "stream-id" (String id); + Hashtbl.replace d "stream_label" (String label); + Hashtbl.replace d "stream_color" (String color); + Hashtbl.replace d "stream_message" (String message); + Hashtbl.replace d "stream_time" (String time); + Dict d in + List [ + mk_item "stream-fast" "Fast source" "emerald" "Resolved in ~0ms (sync)" "0ms"; + mk_item "stream-medium" "Medium source" "amber" "Resolved in ~0ms (sync)" "0ms"; + mk_item "stream-slow" "Slow source" "violet" "Resolved in ~0ms (sync)" "0ms"; + ]); + ignore bind (* suppress unused warning *) let http_mode port = @@ -3040,6 +3279,89 @@ let http_mode port = in write_response fd response; true end else if is_sx then begin + (* Streaming pages: chunked transfer, bypass cache. + Convert SX URL to flat defpage path: + /sx/(geography.(isomorphism.streaming)) → /geography/isomorphism/streaming + Strip prefix, remove parens, replace dots with slashes. *) + let page_path = + let raw = if String.length path > app_prefix_len + && String.sub path 0 app_prefix_len = app_prefix + then String.sub path app_prefix_len (String.length path - app_prefix_len) + else path in + let buf = Buffer.create (String.length raw + 1) in + Buffer.add_char buf '/'; + String.iter (fun c -> match c with + | '(' | ')' -> () + | '.' -> Buffer.add_char buf '/' + | c -> Buffer.add_char buf c + ) raw; + Buffer.contents buf in + let stream_page_name = Hashtbl.find_opt _streaming_pages page_path in + if stream_page_name <> None then begin + let sname = match stream_page_name with Some s -> s | None -> "" in + if is_ajax then begin + (* AJAX streaming: evaluate shell + data + content synchronously, + return fully-resolved SX wire format (no chunked transfer). *) + let response = try + let page_def = match env_get env ("page:" ^ sname) with Dict d -> d | _ -> raise Not_found in + let shell_ast = match Hashtbl.find_opt page_def "shell" with Some v -> v | None -> Nil in + let data_ast = match Hashtbl.find_opt page_def "data" with Some v -> v | None -> Nil in + let content_ast = match Hashtbl.find_opt page_def "content" with Some v -> v | None -> Nil in + (* Evaluate shell — provides nav + suspense skeletons *) + let shell_sx = + let call = List [Symbol "aser"; List [Symbol "quote"; shell_ast]; Env env] in + match Sx_ref.eval_expr call (Env env) with + | String s | SxExpr s -> s | v -> serialize_value v in + (* If we have data+content, resolve all slots and embed as OOB swaps *) + let resolve_oob = if data_ast <> Nil && content_ast <> Nil then begin + let data_result = try Sx_ref.eval_expr data_ast (Env env) with _ -> Nil in + let data_items = match data_result with + | Dict _ -> [(data_result, "stream-content")] + | List items -> List.map (fun item -> + let sid = match item with Dict d -> + (match Hashtbl.find_opt d "stream-id" with Some (String s) -> s | _ -> "stream-content") + | _ -> "stream-content" in (item, sid)) items + | _ -> [] in + let buf = Buffer.create 1024 in + List.iter (fun (item, stream_id) -> + try + let cenv = { bindings = Hashtbl.create 16; parent = Some env } in + (match item with Dict d -> + Hashtbl.iter (fun k v -> + if k <> "stream-id" && k <> "__type" then begin + let nk = String.map (fun c -> if c = '_' then '-' else c) k in + ignore (env_bind cenv nk v); + if nk <> k then ignore (env_bind cenv k v) + end) d | _ -> ()); + let cr = let call = List [Symbol "aser"; List [Symbol "quote"; content_ast]; Env cenv] in + Sx_ref.eval_expr call (Env cenv) in + let sx_src = match cr with String s | SxExpr s -> s | v -> serialize_value v in + (* OOB swap: replace suspense placeholder contents *) + Buffer.add_string buf + (Printf.sprintf "(div :id \"sx-suspense-%s\" :data-suspense \"%s\" :sx-swap-oob \"innerHTML\" :style \"display:contents\" %s)" + stream_id stream_id sx_src) + with e -> + Printf.eprintf "[sx-stream-ajax] resolve error %s: %s\n%!" stream_id (Printexc.to_string e) + ) data_items; + Buffer.contents buf + end else "" in + http_response ~content_type:"text/sx; charset=utf-8" (shell_sx ^ resolve_oob) + with e -> + Printf.eprintf "[sx-stream-ajax] error for %s: %s\n%!" path (Printexc.to_string e); + http_response ~status:500 ~content_type:"text/sx; charset=utf-8" + (Printf.sprintf "(div :class \"p-4 text-rose-600\" \"Streaming page error: %s\")" + (escape_sx_string (Printexc.to_string e))) + in + write_response fd response; true + end else begin + (* Full page streaming: chunked transfer encoding *) + (try http_render_page_streaming env path [] fd sname + with Exit -> () (* page def not found — already handled *) + | e -> Printf.eprintf "[sx-stream] unexpected error for %s: %s\n%!" path (Printexc.to_string e); + (try Unix.close fd with _ -> ())); + true + end + end else let has_state_cookie = Hashtbl.mem _request_cookies "sx-home-stepper" in let cache_key = if is_ajax then "ajax:" ^ path else path in match (if has_state_cookie then None diff --git a/shared/static/wasm/sx-platform.js b/shared/static/wasm/sx-platform.js index 50bfa3a7..e5080678 100644 --- a/shared/static/wasm/sx-platform.js +++ b/shared/static/wasm/sx-platform.js @@ -682,10 +682,38 @@ var scrollY = (state && state.scrollY) ? state.scrollY : 0; K.eval("(handle-popstate " + scrollY + ")"); }); + // Process any streaming suspense resolutions that arrived before boot + if (globalThis.__sxPending) { + for (var pi = 0; pi < globalThis.__sxPending.length; pi++) { + try { + Sx.resolveSuspense(globalThis.__sxPending[pi].id, globalThis.__sxPending[pi].sx); + } catch(e) { console.error("[sx] pending resolve error:", e); } + } + globalThis.__sxPending = null; + } + // Set up direct resolution for future streaming chunks + globalThis.__sxResolve = function(id, sx) { Sx.resolveSuspense(id, sx); }; // Signal boot complete document.documentElement.setAttribute("data-sx-ready", "true"); console.log("[sx] boot done"); } + }, + // Resolve a streaming suspense placeholder via the SX kernel. + // boot.sx defines resolve-suspense but its imports may fail in WASM, + // so we define it here as a fallback using primitives that DO load. + resolveSuspense: function(id, sx) { + try { + // Ensure resolve-suspense exists (boot.sx imports may not have loaded) + if (!Sx._resolveReady) { + try { K.eval('(type-of resolve-suspense)'); } catch(_) { + K.eval('(define resolve-suspense (fn (id sx) (let ((el (dom-query (str "[data-suspense=\\"" id "\\"]")))) (when el (let ((exprs (sx-parse sx)) (env (get-render-env nil))) (dom-set-text-content el "") (for-each (fn (expr) (dom-append el (render-to-dom expr env nil))) exprs))))))'); + } + Sx._resolveReady = true; + } + K.eval('(resolve-suspense "' + id.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '" "' + sx.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '")'); + } catch(e) { + console.error("[sx] resolveSuspense error for id=" + id, e); + } } }; diff --git a/tests/playwright/streaming.spec.js b/tests/playwright/streaming.spec.js new file mode 100644 index 00000000..c013b071 --- /dev/null +++ b/tests/playwright/streaming.spec.js @@ -0,0 +1,351 @@ +// @ts-check +/** + * Streaming page sandbox tests — verify suspense + resolve DOM behavior. + * + * Boots the WASM kernel in sandbox mode (no server), loads the streaming + * demo components, renders the shell with suspense placeholders, and tests + * that __sxResolve correctly replaces placeholders with resolved content. + */ +const { test, expect } = require('playwright/test'); +const fs = require('fs'); +const path = require('path'); + +const PROJECT_ROOT = path.resolve(__dirname, '../..'); +const WASM_DIR = path.join(PROJECT_ROOT, 'shared/static/wasm'); +const SX_DIR = path.join(WASM_DIR, 'sx'); + +const WEB_MODULES = [ + 'render', 'core-signals', 'signals', 'deps', 'router', + 'page-helpers', 'freeze', 'dom', 'browser', + 'adapter-html', 'adapter-sx', 'adapter-dom', + 'boot-helpers', 'hypersx', 'engine', 'orchestration', 'boot', +]; + +function readModule(mod) { + const sxPath = path.join(SX_DIR, mod + '.sx'); + try { return fs.readFileSync(sxPath, 'utf8'); } catch { return null; } +} + +function readFile(relPath) { + return fs.readFileSync(path.join(PROJECT_ROOT, relPath), 'utf8'); +} + +// Component files needed for streaming demo (order matters — deps first) +const COMPONENT_FILES = [ + 'shared/sx/templates/tw.sx', + 'shared/sx/templates/tw-layout.sx', + 'shared/sx/templates/tw-type.sx', + 'shared/sx/templates/layout.sx', + 'shared/sx/templates/pages.sx', + 'sx/sx/streaming-demo.sx', +]; + +async function bootSandbox(page) { + await page.goto('about:blank'); + + // Set up a minimal HTML page with a body container + await page.setContent(` + +
+ + `); + + const kernelSrc = fs.readFileSync(path.join(WASM_DIR, 'sx_browser.bc.js'), 'utf8'); + await page.addScriptTag({ content: kernelSrc }); + await page.waitForFunction('!!window.SxKernel', { timeout: 10000 }); + + // Register host FFI natives + await page.evaluate(() => { + const K = window.SxKernel; + K.registerNative('host-global', a => { const n=a[0]; return (n in globalThis)?globalThis[n]:null; }); + K.registerNative('host-get', a => { if(a[0]==null)return null; const v=a[0][a[1]]; return v===undefined?null:v; }); + K.registerNative('host-set!', a => { if(a[0]!=null)a[0][a[1]]=a[2]; return a[2]; }); + K.registerNative('host-call', a => { + const[o,m,...r]=a; + if(o==null){const f=globalThis[m];return typeof f==='function'?f.apply(null,r):null;} + if(typeof o[m]!=='function')return null; + try{const v=o[m].apply(o,r);return v===undefined?null:v;}catch(e){return null;} + }); + K.registerNative('host-new', a => { + const C=typeof a[0]==='string'?globalThis[a[0]]:a[0]; + return typeof C==='function'?new C(...a.slice(1)):null; + }); + K.registerNative('host-callback', a => { + const fn=a[0]; + if(typeof fn==='function'&&fn.__sx_handle===undefined)return fn; + if(fn&&fn.__sx_handle!==undefined){ + return function(){const r=K.callFn(fn,Array.from(arguments));if(window._driveAsync)window._driveAsync(r);return r;}; + } + return function(){}; + }); + K.registerNative('host-typeof', a => { + const o=a[0]; if(o==null)return'nil'; + if(o instanceof Element)return'element'; if(o instanceof Text)return'text'; + if(o instanceof DocumentFragment)return'fragment'; if(o instanceof Document)return'document'; + if(o instanceof Event)return'event'; if(o instanceof Promise)return'promise'; + return typeof o; + }); + K.registerNative('host-await', a => { + const[p,cb]=a;if(p&&typeof p.then==='function'){const f=(cb&&cb.__sx_handle!==undefined)?v=>K.callFn(cb,[v]):()=>{};p.then(f);} + }); + K.registerNative('load-library!', () => false); + window._driveAsync = function driveAsync(result) { + if(!result||!result.suspended)return; + const req=result.request;const items=req&&(req.items||req); + const op=items&&items[0];const opName=typeof op==='string'?op:(op&&op.name)||String(op); + const arg=items&&items[1]; + function doResume(val,delay){setTimeout(()=>{try{const r=result.resume(val);driveAsync(r);}catch(e){}},delay);} + if(opName==='io-sleep'||opName==='wait')doResume(null,Math.min(typeof arg==='number'?arg:0,10)); + else if(opName==='io-fetch')doResume({ok:true,text:''},1); + }; + K.eval('(define SX_VERSION "streaming-test-1.0")'); + K.eval('(define SX_ENGINE "ocaml-vm-sandbox")'); + K.eval('(define parse sx-parse)'); + K.eval('(define serialize sx-serialize)'); + }); + + // Load web modules (bytecode) + const loadErrors = []; + await page.evaluate(() => { if (window.SxKernel.beginModuleLoad) window.SxKernel.beginModuleLoad(); }); + for (const mod of WEB_MODULES) { + const src = readModule(mod); + if (!src) { loadErrors.push(mod); continue; } + const err = await page.evaluate(s => { + try { window.SxKernel.load(s); return null; } catch(e) { return e.message; } + }, src); + if (err) loadErrors.push(mod + ': ' + err); + } + await page.evaluate(() => { if (window.SxKernel.endModuleLoad) window.SxKernel.endModuleLoad(); }); + + // Load component files (.sx source) + for (const f of COMPONENT_FILES) { + const src = readFile(f); + const err = await page.evaluate(s => { + try { window.SxKernel.load(s); return null; } catch(e) { return e.message; } + }, src); + if (err) loadErrors.push(f + ': ' + err); + } + + // The boot module's imports prevent its defines from running in sandbox. + // Define resolve-suspense inline using the same deps that ARE available. + await page.evaluate(() => { + const K = window.SxKernel; + K.eval(`(define resolve-suspense (fn (id sx) + (let ((el (dom-query (str "[data-suspense=\\"" id "\\"]")))) + (when el + (let ((exprs (sx-parse sx)) + (env (get-render-env nil))) + (dom-set-text-content el "") + (for-each (fn (expr) (dom-append el (render-to-dom expr env nil))) exprs))))))`); + }); + + // Set up Sx.resolveSuspense using the kernel (mirrors sx-platform.js) + await page.evaluate(() => { + const K = window.SxKernel; + window.Sx = window.Sx || {}; + Sx.resolveSuspense = function(id, sx) { + try { + K.eval('(resolve-suspense "' + id.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '" "' + sx.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '")'); + } catch(e) { + console.error("[sx] resolveSuspense error:", e); + } + }; + // Bootstrap (same as server sends in streaming response) + window.__sxPending = []; + window.__sxResolve = function(id, sx) { + if (window.Sx && Sx.resolveSuspense) { + Sx.resolveSuspense(id, sx); + } else { + window.__sxPending.push({ id, sx }); + } + }; + }); + + return loadErrors; +} + +test.describe('Streaming sandbox', () => { + test.describe.configure({ timeout: 60000 }); + + test('suspense component renders placeholder with data-suspense attr', async ({ page }) => { + const errors = await bootSandbox(page); + expect(errors).toEqual([]); + + // Render a suspense component into the DOM + const html = await page.evaluate(() => { + const K = window.SxKernel; + return K.eval('(render-to-dom (~shared:pages/suspense :id "test-slot" :fallback (div "Loading...")))'); + }); + + // Mount it into the page + await page.evaluate((h) => { + const root = document.getElementById('sx-root'); + if (typeof h === 'object' && h instanceof Node) { + root.appendChild(h); + } else { + // render-to-dom returns a DOM node — check if Sx.render works + const node = window.SxKernel.eval('(render-to-dom (~shared:pages/suspense :id "test-slot" :fallback (div "Loading...")))'); + root.appendChild(node); + } + }, html); + + // Verify the placeholder exists with correct attributes + const suspense = page.locator('[data-suspense="test-slot"]'); + await expect(suspense).toHaveCount(1); + await expect(suspense).toContainText('Loading...'); + }); + + test('__sxResolve replaces suspense placeholder content', async ({ page }) => { + const errors = await bootSandbox(page); + expect(errors).toEqual([]); + + // Render suspense placeholder + await page.evaluate(() => { + const K = window.SxKernel; + const node = K.eval('(render-to-dom (~shared:pages/suspense :id "resolve-test" :fallback (div "skeleton")))'); + document.getElementById('sx-root').appendChild(node); + }); + + // Verify placeholder shows fallback + await expect(page.locator('[data-suspense="resolve-test"]')).toContainText('skeleton'); + + // Now resolve it — simulate server sending __sxResolve + // Uses the real Sx.resolveSuspense (calls kernel's resolve-suspense fn) + await page.evaluate(() => { + window.__sxResolve('resolve-test', '(div :class "resolved" "Content loaded!")'); + }); + + // Verify the placeholder content was replaced + const slot = page.locator('[data-suspense="resolve-test"]'); + await expect(slot).toContainText('Content loaded!'); + await expect(slot).not.toContainText('skeleton'); + }); + + test('multiple suspense slots resolve independently', async ({ page }) => { + const errors = await bootSandbox(page); + expect(errors).toEqual([]); + + // Render 3 suspense slots (like the streaming demo shell) + await page.evaluate(() => { + const K = window.SxKernel; + const ids = ['stream-fast', 'stream-medium', 'stream-slow']; + const root = document.getElementById('sx-root'); + for (const id of ids) { + const node = K.eval( + '(render-to-dom (~shared:pages/suspense :id "' + id + '" :fallback (div "Loading ' + id + '...")))' + ); + root.appendChild(node); + } + }); + + // All 3 placeholders should exist + await expect(page.locator('[data-suspense]')).toHaveCount(3); + await expect(page.locator('[data-suspense="stream-fast"]')).toContainText('Loading stream-fast'); + await expect(page.locator('[data-suspense="stream-slow"]')).toContainText('Loading stream-slow'); + + // Resolve middle slot only — uses real kernel resolve-suspense + await page.evaluate(() => { + window.__sxResolve('stream-medium', '(div "Medium resolved!")'); + }); + + // Medium resolved, others still loading + await expect(page.locator('[data-suspense="stream-medium"]')).toContainText('Medium resolved!'); + await expect(page.locator('[data-suspense="stream-fast"]')).toContainText('Loading stream-fast'); + await expect(page.locator('[data-suspense="stream-slow"]')).toContainText('Loading stream-slow'); + + // Resolve remaining slots + await page.evaluate(() => { + window.__sxResolve('stream-fast', '(div "Fast resolved!")'); + window.__sxResolve('stream-slow', '(div "Slow resolved!")'); + }); + + await expect(page.locator('[data-suspense="stream-fast"]')).toContainText('Fast resolved!'); + await expect(page.locator('[data-suspense="stream-slow"]')).toContainText('Slow resolved!'); + }); + + test('streaming shell renders with outer layout gutters', async ({ page }) => { + const errors = await bootSandbox(page); + expect(errors).toEqual([]); + + // Render shell wrapped in outer layout (like the server does) + const result = await page.evaluate(() => { + const K = window.SxKernel; + try { + const html = K.eval(`(render-to-html + (~shared:layout/app-body :content + (~streaming-demo/layout + (~shared:pages/suspense :id "stream-fast" + :fallback (~streaming-demo/stream-skeleton)) + (~shared:pages/suspense :id "stream-medium" + :fallback (~streaming-demo/stream-skeleton)) + (~shared:pages/suspense :id "stream-slow" + :fallback (~streaming-demo/stream-skeleton)))))`); + document.getElementById('sx-root').innerHTML = html; + return { ok: true }; + } catch (e) { + return { ok: false, error: e.message }; + } + }); + + expect(result.error || '').toBe(''); + + // Should have outer layout structure (gutters) + await expect(page.locator('#root-panel')).toHaveCount(1); + await expect(page.locator('#main-panel')).toHaveCount(1); + + // Should have 3 suspense slots inside the layout + await expect(page.locator('[data-suspense]')).toHaveCount(3); + + // Should have the demo heading + await expect(page.locator('h1')).toContainText('Streaming'); + }); + + test('resolve fills suspense slots with chunk content end-to-end', async ({ page }) => { + const errors = await bootSandbox(page); + expect(errors).toEqual([]); + + // Phase 1: Render shell with suspense skeletons + await page.evaluate(() => { + const K = window.SxKernel; + const html = K.eval(`(render-to-html + (~shared:layout/app-body :content + (~streaming-demo/layout + (~shared:pages/suspense :id "stream-fast" + :fallback (~streaming-demo/stream-skeleton)) + (~shared:pages/suspense :id "stream-medium" + :fallback (~streaming-demo/stream-skeleton)) + (~shared:pages/suspense :id "stream-slow" + :fallback (~streaming-demo/stream-skeleton)))))`); + document.getElementById('sx-root').innerHTML = html; + }); + + // Verify skeletons are showing + await expect(page.locator('[data-suspense="stream-fast"]')).toHaveCount(1); + + // Phase 2: Simulate server sending resolve scripts (like chunked transfer) + // Use the real Sx.resolveSuspense (calls kernel's resolve-suspense SX fn) + await page.evaluate(() => { + + // Simulate 3 resolve scripts arriving from chunked transfer + window.__sxResolve('stream-fast', + '(~streaming-demo/chunk :stream-label "Fast source" :stream-color "emerald" :stream-message "Resolved in 1ms" :stream-time "1ms")'); + window.__sxResolve('stream-medium', + '(~streaming-demo/chunk :stream-label "Medium source" :stream-color "amber" :stream-message "Resolved in 50ms" :stream-time "50ms")'); + window.__sxResolve('stream-slow', + '(~streaming-demo/chunk :stream-label "Slow source" :stream-color "violet" :stream-message "Resolved in 200ms" :stream-time "200ms")'); + }); + + // Phase 3: Verify all slots filled with actual content + await expect(page.locator('[data-suspense="stream-fast"]')).toContainText('Fast source'); + await expect(page.locator('[data-suspense="stream-fast"]')).toContainText('1ms'); + + await expect(page.locator('[data-suspense="stream-medium"]')).toContainText('Medium source'); + await expect(page.locator('[data-suspense="stream-medium"]')).toContainText('50ms'); + + await expect(page.locator('[data-suspense="stream-slow"]')).toContainText('Slow source'); + await expect(page.locator('[data-suspense="stream-slow"]')).toContainText('200ms'); + + // Skeletons should be gone — replaced by resolved content + await expect(page.locator('[data-suspense="stream-fast"]')).not.toContainText('Loading'); + }); +});