Add simultaneous streaming image and video comparisons

.gitattributes

@@ -37,3 +37,4 @@ docs/assets/demo-poster.jpg filter=lfs diff=lfs merge=lfs -text
 docs/assets/demo.mp4 filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
 gemma_rlcd/static/sample-street.jpg filter=lfs diff=lfs merge=lfs -text
+docs/assets/live-demo.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -26,9 +26,11 @@ Give the model one input and a set of questions. It encodes the input once, scor
 
 This download includes the complete 4-bit multimodal checkpoint and the `gemma_rlcd` runtime. The checkpoint preserves the pinned MLX quantization; the runtime implements parallel decision scoring. [Source code on GitHub](https://github.com/Larkooo/gemma-e2b-rlcd).
 
-<video controls playsinline preload="metadata" width="100%" poster="https://huggingface.co/larkooo/gemma-e2b-rlcd/resolve/main/docs/assets/demo-poster.jpg" src="https://huggingface.co/larkooo/gemma-e2b-rlcd/resolve/main/docs/assets/demo.mp4"></video>
+<video controls playsinline preload="metadata" width="100%" src="https://huggingface.co/larkooo/gemma-e2b-rlcd/resolve/main/docs/assets/live-demo.mp4"></video>
 
-**[Watch the demo](https://huggingface.co/larkooo/gemma-e2b-rlcd/resolve/main/docs/assets/demo.mp4)** · 28 matching outputs · 4.41 s vs 15.66 s · 3.55× faster on the support-triage workload.
+**128 visual checks, streaming live.** In this recorded simultaneous run: **11.57 s vs 54.10 s · 4.67× faster · 120/128 matching answers**. Both paths share one GPU. The full startup pause and original elapsed time are preserved.
+
+[Download the video](docs/assets/live-demo.mp4) · [Recorded results](reports/live-visual-demo.json)
 
 ## Quick start
 
@@ -56,6 +58,8 @@ The download is approximately 3.6 GB and includes the image and audio encoders,
 
 Open **http://127.0.0.1:8787/demo** for 32, 64, or 128 checks over an image or video. Use the included street photo or upload your own media. The parallel scorer streams completed field batches; normal Gemma streams its generated JSON. Live clocks, per-check probabilities, answer differences, and downloadable events make the comparison inspectable. Both paths start together and stream side by side, sharing the resident weights with separate processors and KV caches. Timings measure concurrent completion on one GPU, including resource contention. The playground’s ordinary comparison remains sequential for isolated timings.
 
+The first answer follows input preparation and a full multimodal prefill. The UI shows these stages, first-token time, and first-decision time separately. Repeated runs reuse compiled field definitions; media, input KV state, and answers are recomputed. Use **Focus view** to see all 128 outputs together.
+
 ## How it works
 
 ```mermaid

checkpoint-provenance.json

@@ -1,7 +1,7 @@
 {
   "repository": "larkooo/gemma-e2b-rlcd",
   "runtime_repository": "https://github.com/Larkooo/gemma-e2b-rlcd",
-  "runtime_commit": "e78b79605ac4e37b776727935e2de7dc8f259a81",
+  "runtime_commit": "4ae799d204d8b2f4a802b756b4236541af0de468",
   "checkpoint_repository": "mlx-community/gemma-4-e2b-it-4bit",
   "checkpoint_revision": "238767527555cb75a05732a84dff5d6ba0dd6809",
   "checkpoint_modified": false,

docs/architecture.md

@@ -68,3 +68,9 @@ Video sampling can miss brief events. The limits describe the current serving co
 | `--backend head` | `DecisionHeadBackend` | Candidate-conditioned head; requires a checkpoint |
 
 The cached path assigns tokenizer-verified codes to options and scores per-question suffixes against shared state. Historical answer-code and catalog measurements are retained for reproducing those experiments. The [training guide](training.md) describes the head separately.
+
+## Streaming startup
+
+The visual demo starts scoring and normal generation together on separate worker streams with separate processors and KV caches. Both process the complete media and question schema before producing answers; the UI reports input preparation, prefill, first token, and first completed decision separately. End-to-end concurrent timings include GPU contention.
+
+The scorer encodes the common prompt once for field compilation and keeps up to eight compiled schemas in an LRU cache. Cache keys include the complete prompt and candidate definitions. This cache contains only tokenized field definitions; every request recomputes media features, input KV state, and answer probabilities. Boundary-merge validation and complete-candidate scoring remain unchanged.

docs/assets/live-demo.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:803559adc8667d1cbf92070943d005ce8659a2e4e2498a83f84b74ac65d80f8b
+size 1843915

gemma_rlcd/comparison.py

@@ -144,13 +144,19 @@ def prepare_generation(backend, state: State, questions: dict) -> tuple[str, dic
     return prompt, inputs
 
 
-def generate_answers(backend, state: State, questions: dict, on_token=None) -> dict:
+def generate_answers(
+    backend, state: State, questions: dict, on_token=None, on_progress=None
+) -> dict:
     from mlx_vlm import generate, stream_generate
 
     started = time.perf_counter()
+    if on_progress:
+        on_progress("preparing", None)
     prompt, inputs = prepare_generation(backend, state, questions)
     budget = output_budget(backend.tokenizer, questions)
     prepared = time.perf_counter()
+    if on_progress:
+        on_progress("prefill", int(inputs["input_ids"].shape[-1]))
     options = dict(
         **inputs,
         max_tokens=budget,
@@ -166,6 +172,8 @@ def generate_answers(backend, state: State, questions: dict, on_token=None) -> d
         parts = []
         generated = None
         for chunk in stream_generate(backend.model, backend.processor, prompt, **options):
+            if generated is None and on_progress:
+                on_progress("generating", chunk.prompt_tokens)
             generated = chunk
             parts.append(chunk.text)
             on_token(chunk.text, chunk.generation_tokens)
@@ -266,10 +274,21 @@ def compare(
                 }
             )
 
+        def on_progress(stage, input_tokens):
+            emit(
+                {
+                    "type": "progress",
+                    "method": method,
+                    "stage": stage,
+                    "input_tokens": input_tokens,
+                    "seconds": media_seconds + time.perf_counter() - started,
+                }
+            )
+
         if method == "parallel":
             engine = DecisionEngine(worker_backend)
             output = (
-                engine.system_one(state, questions, on_answer=on_answer)
+                engine.system_one(state, questions, on_answer=on_answer, on_progress=on_progress)
                 if emit
                 else engine.system_one(state, questions)
             )
@@ -282,7 +301,9 @@ def compare(
             )
         else:
             output = (
-                generate_answers(worker_backend, state, questions, on_token=on_token)
+                generate_answers(
+                    worker_backend, state, questions, on_token=on_token, on_progress=on_progress
+                )
                 if emit
                 else generate_answers(worker_backend, state, questions)
             )

gemma_rlcd/core.py

@@ -216,7 +216,9 @@ class DecisionEngine:
     def decide(self, state: State, question: Question) -> dict:
         return self.system_one(state, {"answer": question})["answers"]["answer"]
 
-    def system_one(self, state: State, questions: Mapping[str, Question], on_answer=None) -> dict:
+    def system_one(
+        self, state: State, questions: Mapping[str, Question], on_answer=None, on_progress=None
+    ) -> dict:
         if not questions:
             raise ValueError("At least one question is required")
         jobs = []
@@ -268,11 +270,12 @@ class DecisionEngine:
                 )
 
         if question_score is not None:
-            scores = (
-                question_score(state, questions, on_scores=completed_scores)
-                if on_answer
-                else question_score(state, questions)
-            )
+            callbacks = {}
+            if on_answer:
+                callbacks["on_scores"] = completed_scores
+            if on_progress:
+                callbacks["on_progress"] = on_progress
+            scores = question_score(state, questions, **callbacks)
         elif batch_score is not None:
             scores = batch_score(state, requests)
         else:

gemma_rlcd/json_backend.py

@@ -2,6 +2,7 @@
 
 import math
 import time
+from collections import OrderedDict
 
 from .cached_backend import CachedMLXBackend, PreparedState
 from .comparison import prepare_generation
@@ -67,12 +68,28 @@ class JSONMLXBackend(CachedMLXBackend):
                 on_scores(completed)
         return output, batches
 
-    def score_questions(self, state, questions, on_scores=None):
+    def score_questions(self, state, questions, on_scores=None, on_progress=None):
         started = time.perf_counter()
+        if on_progress:
+            on_progress("preparing", None)
         prompt, inputs = prepare_generation(self, state, questions)
-        fields = [
-            compile_field(self.tokenizer, field, prompt) for field in candidate_fields(questions)
-        ]
+        candidates = tuple(candidate_fields(questions))
+        key = (prompt, candidates)
+        if not hasattr(self, "_field_cache"):
+            self._field_cache = OrderedDict()
+        fields = self._field_cache.get(key)
+        schema_cache_hit = fields is not None
+        if fields is None:
+            base_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+            fields = [
+                compile_field(self.tokenizer, field, prompt, base_ids=base_ids)
+                for field in candidates
+            ]
+            self._field_cache[key] = fields
+            if len(self._field_cache) > 8:
+                self._field_cache.popitem(last=False)
+        else:
+            self._field_cache.move_to_end(key)
         prefix_tokens = int(inputs["input_ids"].shape[1])
         if any(
             prefix_tokens + len(field.prefix) + max(map(len, field.candidates))
@@ -84,8 +101,12 @@ class JSONMLXBackend(CachedMLXBackend):
             )
         prepared = PreparedState(inputs, [], prefix_tokens)
         processed = time.perf_counter()
+        if on_progress:
+            on_progress("prefill", prefix_tokens)
         prefix_cache = self.prefill(prepared)
         prefilled = time.perf_counter()
+        if on_progress:
+            on_progress("scoring", prefix_tokens)
         self.last_stats = {}
         scores = [None] * len(fields)
         single = [
@@ -136,6 +157,7 @@ class JSONMLXBackend(CachedMLXBackend):
             "execution": "shared_json_prefix_gpu_batched_fields",
             "prefix_prefills": 1,
             "prefix_tokens": prefix_tokens,
+            "schema_cache_hit": schema_cache_hit,
             "primitive_fields": len(fields),
             "question_suffix_tokens": [len(field.prefix) for field in fields],
             "candidate_token_lengths": [

gemma_rlcd/json_scoring.py

@@ -36,11 +36,12 @@ def candidate_fields(questions: dict) -> list[JSONField]:
     return fields
 
 
-def compile_field(tokenizer, field: JSONField, prompt: str) -> FieldTokens:
+def compile_field(tokenizer, field: JSONField, prompt: str, *, base_ids=None) -> FieldTokens:
     # Whitespace is part of the model's answer context. In particular, scoring
     # digits immediately after ':' instead of ': ' can score a whitespace slot.
     opening = "".join("{" + json.dumps(key, ensure_ascii=False) + ": " for key in field.path)
-    base_ids = tokenizer.encode(prompt, add_special_tokens=False)
+    if base_ids is None:
+        base_ids = tokenizer.encode(prompt, add_special_tokens=False)
     sequences = []
     for value in field.values:
         suffix = opening + json.dumps(value, ensure_ascii=False)

gemma_rlcd/static/demo.css

@@ -1 +1,3 @@
 :root{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#1d2924;background:#f6f7f4;--green:#23674e;--muted:#69766d;--line:#dfe5dc}*{box-sizing:border-box}body{margin:0;font-size:14px;line-height:1.5}header{display:flex;align-items:center;justify-content:space-between;gap:20px;padding:20px 36px;border-bottom:1px solid var(--line);background:#fcfdfb}.brand{font-weight:650;font-size:17px;color:inherit;text-decoration:none}nav{display:flex;gap:24px;align-items:center;font-size:12px}a{color:var(--green)}#model-status{color:var(--muted)}main{max-width:1560px;margin:auto;padding:35px 36px 70px}h1,h2,h3,p{margin:0}h1{font-size:36px;letter-spacing:-1.4px;font-weight:580;line-height:1.25;margin:8px 0}h1>span{color:var(--green)}h2{font-size:15px;font-weight:650}h3{font-size:14px;font-weight:620}.eyebrow{font-size:10px;font-weight:650;letter-spacing:1.7px;color:var(--muted)}.heading{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-bottom:28px}.heading p{color:var(--muted);font-size:13px}.workspace{display:grid;grid-template-columns:minmax(280px,.75fr) minmax(0,1.6fr);gap:24px;align-items:start}.input-panel,.live-panel,.wall{border:1px solid var(--line);background:white;border-radius:10px;padding:22px}.section-title{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:16px}.small{font-size:11px;color:var(--muted);line-height:1.65}.preview{height:260px;background:#eff2ed;border-radius:6px;overflow:hidden;display:grid;place-items:center;cursor:pointer}.preview img,.preview video{width:100%;height:100%;max-height:260px;object-fit:contain}.preview.dragging{outline:3px solid #7aab8f}.media-controls{display:flex;gap:12px;align-items:center;margin-top:12px}.file-button{font-size:12px;font-weight:550;color:var(--green);cursor:pointer;position:relative;flex-shrink:0}.file-button input{position:absolute;inset:0;opacity:0;width:100%;cursor:pointer}#file-name{font-size:10px;color:var(--muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}#credit{margin-top:5px;min-height:18px}label:not(.file-button){display:block;font-size:11px;font-weight:550;margin:15px 0 6px}button,input,textarea,select{font:inherit;color:inherit}button,select{cursor:pointer}button{border:1px solid transparent;border-radius:6px;padding:10px 14px;font-size:12px;font-weight:550}button:disabled{opacity:.5;cursor:default}.primary{background:var(--green);color:white}.primary:hover:not(:disabled){background:#18513b}.secondary{background:white;border-color:var(--line)}.text-button{padding:2px;background:none;color:var(--green);font-size:11px}.input-panel select,.input-panel textarea{width:100%;border:1px solid var(--line);border-radius:6px;padding:10px;background:#fcfdfb;font-size:12px}.input-panel textarea{resize:vertical;line-height:1.5;margin-bottom:9px}.actions{display:flex;gap:10px;margin:18px 0 9px}.actions .primary{flex:1}.engines{display:grid;grid-template-columns:1fr 1fr;gap:24px}.engine{min-width:0}.engine-title{display:flex;align-items:center;justify-content:space-between;gap:8px}.engine-title>span{font-size:10px;color:var(--muted)}.parallel h3,.parallel .clock{color:var(--green)}.clock{display:block;font-size:50px;font-weight:450;letter-spacing:-2px;font-variant-numeric:tabular-nums;margin-top:8px}.clock>span{font-size:15px;letter-spacing:0;color:var(--muted);margin-left:7px}.progress{height:4px;background:#edf0e9;border-radius:3px;overflow:hidden;margin-top:12px}.progress i{display:block;height:100%;width:0;background:var(--green)}.normal .progress i{background:#4b5850}.engine-stats{display:flex;justify-content:space-between;gap:5px;color:var(--muted);font-size:10px;margin-top:7px}.engine>.small{margin-top:10px}.verdict{border-top:1px solid var(--line);border-bottom:1px solid var(--line);margin:22px 0;padding:15px 0;min-height:54px;font-size:13px;color:var(--green)}.verdict strong{font-size:22px;font-weight:600;margin-right:8px}.streams{display:grid;grid-template-columns:1fr 1fr;gap:20px;min-width:0}.streams>div{min-width:0}.stream-label{display:flex;justify-content:space-between;gap:10px;font-size:11px;color:var(--muted)}.live-dot{font-size:8px;letter-spacing:1px;color:var(--green)}pre{height:235px;margin:10px 0 0;background:#f6f8f3;border:1px solid #e9eee4;border-radius:6px;padding:12px;white-space:pre-wrap;overflow-wrap:anywhere;overflow:auto;font:11px/1.7 ui-monospace,SFMono-Regular,Menlo,monospace;color:#3e6550}.normal-stream{color:#46504a}.method-note{font-size:10px;color:var(--muted);line-height:1.8;margin-top:17px}.wall{margin-top:24px}.wall-heading{display:flex;justify-content:space-between;align-items:center;gap:20px;margin-bottom:20px}.wall-heading p{margin-top:5px}.filters{display:flex;align-items:center;gap:12px;flex-shrink:0}.filters select{font-size:11px;padding:8px;border:1px solid var(--line);border-radius:6px;background:white}.group{margin-top:19px}.group:first-child{margin-top:0}.group-title{font-size:11px;color:var(--muted);font-weight:550;margin-bottom:9px}.check-grid{display:grid;grid-template-columns:repeat(8,minmax(0,1fr));gap:7px}.check{border:1px solid #e3e8de;border-radius:5px;padding:8px 9px;min-width:0;min-height:59px;transition:background .2s,border-color .2s}.check-name{display:block;font-size:10px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:#53624f}.values{display:flex;gap:12px;justify-content:space-between;margin-top:6px;font:10px ui-monospace,monospace;color:#8a9585}.values b{font-weight:500}.check.detected{background:#f0f7ec;border-color:#c7dcc1}.check.different{background:#fff6e9;border-color:#e9c998}.values .yes{color:#23674e}.values .no{color:#727d70}.check.flash{animation:arrive .55s ease-out}.details{font-size:11px;color:var(--muted);margin-top:20px;max-width:1000px}.details p{margin-top:10px}.details summary{cursor:pointer}#error{margin-top:12px;padding:11px;border:1px solid #eccfc4;background:#fff6f0;color:#963e2e;font-size:12px;border-radius:6px}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}[hidden]{display:none!important}:focus-visible{outline:3px solid #75a58c;outline-offset:3px}@keyframes arrive{0%{background:#dceccd}100%{}}@media(prefers-reduced-motion:reduce){*{animation:none!important;transition:none!important}}@media(max-width:1200px){.check-grid{grid-template-columns:repeat(6,minmax(0,1fr))}.workspace{grid-template-columns:minmax(275px,.8fr) minmax(0,1.4fr)}.clock{font-size:43px}.engine-title{align-items:flex-start;flex-direction:column;gap:2px}}@media(max-width:850px){header{padding:17px 20px}main{padding:25px 20px 50px}.workspace{grid-template-columns:1fr}.preview{height:280px}.preview img,.preview video{max-height:280px}.input-panel{display:grid;grid-template-columns:1fr 1fr;column-gap:20px}.input-panel>*{grid-column:1/-1}.check-grid{grid-template-columns:repeat(4,minmax(0,1fr))}.engine-title{flex-direction:row}.wall-heading{align-items:flex-start;flex-direction:column}h1{font-size:32px}}@media(max-width:520px){header{align-items:flex-start}.brand{font-size:14px}nav{flex-direction:column;gap:3px;align-items:flex-end;font-size:10px}main{padding:22px 13px 40px}.heading{align-items:flex-start}.heading>button{padding:8px;font-size:10px;white-space:nowrap}h1{font-size:27px}.heading p{font-size:12px}.input-panel,.live-panel,.wall{padding:16px}.engines{gap:17px}.engine-title{align-items:flex-start;flex-direction:column}.clock{font-size:40px}.streams{gap:12px}pre{height:215px;font-size:10px;padding:9px}.check-grid{grid-template-columns:repeat(3,minmax(0,1fr))}.check{padding:7px}.values{gap:6px}.preview{height:230px}.preview img,.preview video{max-height:230px}.filters{flex-wrap:wrap}.wall-heading p{font-size:10px}}
+
+.view-actions{display:flex;gap:8px;flex-shrink:0}.engine>#normal-first{margin-top:2px}.focused header{padding:12px 26px}.focused main{max-width:1920px;padding:16px 26px}.focused .heading{margin-bottom:16px}.focused h1{font-size:28px}.focused .heading .eyebrow,.focused .heading p,.focused .input-panel textarea,.focused .input-panel label[for="instructions"],.focused .input-panel>p.small:not(#credit):not(#run-note),.focused .details{display:none}.focused .input-panel,.focused .live-panel,.focused .wall{padding:16px}.focused .workspace{grid-template-columns:320px minmax(0,1fr);gap:18px}.focused .preview,.focused .preview img,.focused .preview video{height:185px;max-height:185px}.focused .clock{font-size:42px;margin-top:0}.focused .verdict{margin:13px 0;padding:10px 0;min-height:48px}.focused pre{height:155px}.focused .wall{margin-top:16px}.focused .wall-heading{margin-bottom:10px}.focused #output-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:20px}.focused .group{margin:0}.focused .check-grid{grid-template-columns:repeat(4,minmax(0,1fr));gap:5px}.focused .check{min-height:43px;padding:5px 7px}.focused .values{margin-top:4px}.focused .actions{margin-top:12px}.focused .method-note{margin-top:10px}@media(max-width:1200px){.focused #output-grid{grid-template-columns:1fr 1fr}}@media(max-width:850px){.focused .workspace{grid-template-columns:1fr}.focused #output-grid{grid-template-columns:1fr}.view-actions{flex-direction:column}.focused main{padding:16px}}

gemma_rlcd/static/demo.html

@@ -8,7 +8,7 @@
 <body>
   <header><a class="brand" href="/demo">Gemma E2B RLCD</a><nav><span id="model-status" role="status">Connecting…</span><a href="/">Open playground ↗</a></nav></header>
   <main>
-    <div class="heading"><div><span class="eyebrow">LIVE MULTIMODAL COMPARISON</span><h1>One scene. <span id="headline-count">128</span> decisions.</h1><p>Both start together. Watch batched decisions race against normal Gemma’s streamed JSON.</p></div><button id="export" class="secondary" disabled>Export run ↓</button></div>
+    <div class="heading"><div><span class="eyebrow">LIVE MULTIMODAL COMPARISON</span><h1>One scene. <span id="headline-count">128</span> decisions.</h1><p>Both start together. Watch batched decisions race against normal Gemma’s streamed JSON.</p></div><div class="view-actions"><button id="focus" class="secondary" aria-pressed="false">Focus view</button><button id="export" class="secondary" disabled>Export run ↓</button></div></div>
     <div class="workspace">
       <section class="input-panel" aria-labelledby="input-title">
         <div class="section-title"><h2 id="input-title">The evidence</h2><button id="sample" class="text-button">Use sample photo</button></div>
@@ -25,8 +25,8 @@
       <section class="live-panel" aria-labelledby="live-title">
         <div class="section-title"><h2 id="live-title">Live output</h2><span id="run-phase" class="small">Ready</span></div>
         <div class="engines">
-          <article class="engine parallel"><div class="engine-title"><h3>Parallel scorer</h3><span id="parallel-state">Waiting</span></div><strong class="clock" id="parallel-clock">0.00<span>s</span></strong><div class="progress"><i id="parallel-progress"></i></div><div class="engine-stats"><span id="parallel-count">0 / 128 decisions</span><span>GPU batches</span></div><p id="parallel-first" class="small">First answer —</p></article>
-          <article class="engine normal"><div class="engine-title"><h3>Normal Gemma</h3><span id="normal-state">Waiting</span></div><strong class="clock" id="normal-clock">0.00<span>s</span></strong><div class="progress"><i id="normal-progress"></i></div><div class="engine-stats"><span id="normal-count">0 / 128 decisions</span><span id="token-count">0 tokens</span></div><p id="normal-first" class="small">First answer —</p></article>
+          <article class="engine parallel"><div class="engine-title"><h3>Parallel scorer</h3><span id="parallel-state">Waiting</span></div><strong class="clock" id="parallel-clock">0.00<span>s</span></strong><div class="progress"><i id="parallel-progress"></i></div><div class="engine-stats"><span id="parallel-count">0 / 128 decisions</span><span>GPU batches</span></div><p id="parallel-first" class="small">First decision —</p></article>
+          <article class="engine normal"><div class="engine-title"><h3>Normal Gemma</h3><span id="normal-state">Waiting</span></div><strong class="clock" id="normal-clock">0.00<span>s</span></strong><div class="progress"><i id="normal-progress"></i></div><div class="engine-stats"><span id="normal-count">0 / 128 decisions</span><span id="token-count">0 tokens</span></div><p id="normal-token-first" class="small">First token —</p><p id="normal-first" class="small">First decision —</p></article>
         </div>
         <div id="verdict" class="verdict" aria-live="polite">Upload a scene or try the sample. Results are measured live.</div>
         <div class="streams">
@@ -37,7 +37,7 @@
       </section>
     </div>
     <section class="wall" aria-labelledby="wall-title"><div class="wall-heading"><div><h2 id="wall-title">Every decision, as it lands</h2><p class="small">A check is “yes” when visually established. P = parallel probability of yes · G = Gemma’s boolean. Streaming JSON is provisional until validated.</p></div><div class="filters"><label for="filter" class="sr-only">Filter decisions</label><select id="filter"><option value="all">All checks</option><option value="yes">Detected by either</option><option value="different">Different answers</option></select><span id="agreement" class="small"></span></div></div><div id="output-grid"></div><p id="no-matches" class="small" hidden>No completed checks match this filter.</p></section>
-    <details class="details"><summary>What is being measured?</summary><p>Both paths process the same complete image or sampled video, instructions, and label descriptions. The parallel scorer returns probabilities as field batches finish. Normal Gemma is asked to generate one compact JSON object with boolean decisions, without explanations or probability prose.</p><p>The timers include input preparation and inference, with shared upload decoding added equally. They exclude model loading, upload transfer, and allocator reset. This is one simultaneous run per path; GPU contention and first-use effects can affect timing. The result measures completion time while both are running, not isolated throughput. Matching answers measures agreement, not correctness. Invalid normal output is shown without a speedup claim.</p><p>Video processing targets one frame per second with a 32-frame cap. Long media plus 128 questions can exceed the 8,192-token input limit; reduce the number of checks or use a shorter clip. Media is never silently trimmed.</p></details>
+    <details class="details"><summary>What is being measured?</summary><p>Both paths must read the complete image or sampled video and all question descriptions before producing an answer. The reading stage is shown live; no answers are buffered for playback. Only reusable tokenized field definitions are cached, never media features, input KV state, or answers. The parallel scorer returns probabilities as field batches finish. Normal Gemma is asked to generate one compact JSON object with boolean decisions, without explanations or probability prose.</p><p>The timers include input preparation and inference, with shared upload decoding added equally. They exclude model loading, upload transfer, and allocator reset. This is one simultaneous run per path; GPU contention and first-use effects can affect timing. The result measures completion time while both are running, not isolated throughput. Matching answers measures agreement, not correctness. Invalid normal output is shown without a speedup claim.</p><p>Video processing targets one frame per second with a 32-frame cap. Long media plus 128 questions can exceed the 8,192-token input limit; reduce the number of checks or use a shorter clip. Media is never silently trimmed.</p></details>
   </main>
 </body>
 </html>

gemma_rlcd/static/demo.js

@@ -3,6 +3,7 @@ const $ = (selector) => document.querySelector(selector);
 const escapeHTML = (value) => String(value).replace(/[&<>"']/g, (char) => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[char]));
 let config, media, mediaURL, active = false, ready = false, controller, finalResult = null;
 let checks = [], rows = new Map(), completed = {parallel: new Map(), normal: new Map()};
+let firstToken = null;
 let clocks = {}, rawText = "", parallelLines = [], events = [], firstAnswer = {}, frame = null, sampleVersion = 0;
 
 function error(message) { $("#error").textContent = message; $("#error").hidden = !message; }
@@ -39,20 +40,21 @@ async function sample() {
   finally { $("#sample").disabled = active; }
 }
 function reset() {
-  finalResult = null; rawText = ""; parallelLines = []; events = []; firstAnswer = {}; clocks = {};
+  finalResult = null; firstToken = null; $("#normal-token-first").textContent = "First token —"; rawText = ""; parallelLines = []; events = []; firstAnswer = {}; clocks = {};
   completed = {parallel: new Map(), normal: new Map()};
   $("#headline-count").textContent = count();
   $("#parallel-stream").textContent = "Answers appear as soon as each batch completes.";
   $("#normal-stream").textContent = "Real token output will stream here.";
   $("#token-count").textContent = "0 tokens";
   $("#run-phase").textContent = "Ready";
+  $("#run-note").textContent = "The model loads once. Every comparison uses fresh input state.";
   $("#agreement").textContent = "";
   $("#verdict").textContent = "Upload a scene or try the sample. Results are measured live.";
   $("#export").disabled = true;
   for (const method of ["parallel","normal"]) {
     $(`#${method}-clock`).innerHTML = seconds(0);
     $(`#${method}-state`).textContent = "Waiting";
-    $(`#${method}-first`).textContent = "First answer —";
+    $(`#${method}-first`).textContent = "First decision —";
     $(`#${method}-progress`).style.width = "0%";
     $(`#${method}-count`).textContent = `0 / ${count()} decisions`;
   }
@@ -83,7 +85,7 @@ function updateDecision(method, path, value, probability, elapsed) {
   row.classList.toggle("different", Boolean(parallel && normal && parallel.value !== normal.value));
   if (firstAnswer[method] === undefined) {
     firstAnswer[method] = elapsed;
-    $(`#${method}-first`).textContent = `First answer ${elapsed.toFixed(2)} s`;
+    $(`#${method}-first`).textContent = `First decision ${elapsed.toFixed(2)} s`;
   }
   $(`#${method}-count`).textContent = `${completed[method].size} / ${count()} decisions`;
   $(`#${method}-progress`).style.width = `${100 * completed[method].size / count()}%`;
@@ -122,6 +124,9 @@ function handle(event) {
   } else if (event.type === "phase_start") {
     if (!clocks[method]) clocks[method] = {running:true, start:performance.now() - event.media_seconds * 1000};
     $(`#${method}-state`).textContent = "Running";
+  } else if (event.type === "progress") {
+    const labels = {preparing:"Preparing input", prefill:`Reading ${(event.input_tokens || 0).toLocaleString()} tokens`, scoring:"Scoring fields", generating:"Generating"};
+    $(`#${method}-state`).textContent = labels[event.stage] || event.stage;
   } else if (event.type === "answer") {
     const path = event.path.join(".");
     const probability = event.answer.probabilities.yes;
@@ -130,6 +135,10 @@ function handle(event) {
     $("#parallel-stream").textContent = parallelLines.join("\n");
     $("#parallel-stream").scrollTop = $("#parallel-stream").scrollHeight;
   } else if (event.type === "token") {
+    if (firstToken === null && event.text) {
+      firstToken = event.seconds;
+      $("#normal-token-first").textContent = `First token ${firstToken.toFixed(2)} s`;
+    }
     rawText += event.text;
     $("#normal-stream").textContent = rawText;
     $("#normal-stream").scrollTop = $("#normal-stream").scrollHeight;
@@ -193,7 +202,7 @@ async function run() {
     }
     if(pending.trim()) handle(JSON.parse(pending));
     if(!finalResult) throw new Error("The stream ended before the comparison completed.");
-    finalResult = {request:spec,response:finalResult,stream_events:events,first_answer_seconds:firstAnswer};
+    finalResult = {request:spec,response:finalResult,stream_events:events,first_answer_seconds:firstAnswer,first_token_seconds:firstToken};
     $("#run-note").textContent = "Complete. Export preserves the answers, events, and measured timings.";
   } catch(failure) {
     controller.abort();
@@ -216,6 +225,11 @@ async function status() {
   } catch { ready=false; $("#model-status").textContent="Server disconnected"; }
   syncButtons();
 }
+$("#focus").addEventListener("click",()=>{
+  const focused = document.body.classList.toggle("focused");
+  $("#focus").setAttribute("aria-pressed",String(focused));
+  $("#focus").textContent = focused ? "Standard view" : "Focus view";
+});
 $("#run").addEventListener("click",run);
 $("#stop").addEventListener("click",()=>controller?.abort());
 $("#sample").addEventListener("click",sample);

reports/live-visual-demo.json

@@ -0,0 +1,23 @@
+{
+  "source": "Live browser capture of http://127.0.0.1:8787/demo",
+  "presentation": "Actual browser captures at original elapsed time; 4 fps capture; both inference paths share the GPU; model already loaded; only schema tokenization may be warm.",
+  "capture_duration_seconds": 58.189,
+  "metrics_displayed": {
+    "error": "",
+    "normal-clock": "54.10s",
+    "normal-count": "128 / 128 decisions",
+    "normal-first": "First decision 7.93 s",
+    "normal-token-first": "First token 7.45 s",
+    "parallel-clock": "11.57s",
+    "parallel-count": "128 / 128 decisions",
+    "parallel-first": "First decision 7.11 s",
+    "run-phase": "Comparison complete",
+    "verdict": "4.67\u00d7 faster this run \u00b7 120 / 128 matching answers"
+  },
+  "answers_displayed": {
+    "normal": "```json\n{\"people\": {\"person\": true, \"crowd\": true, \"walking\": false, \"sitting\": false, \"standing\": true, \"running\": false, \"cycling\": false, \"carrying_bag\": false, \"backpack\": false, \"handbag\": false, \"hat\": false, \"sunglasses\": false, \"umbrella\": false, \"phone_in_hand\": false, \"using_camera\": false, \"raised_hand\": false, \"waving\": false, \"pointing\": false, \"eating\": false, \"drinking\": false, \"stroller\": false, \"walking_dog\": false, \"uniform\": false, \"helmet\": false, \"high_vis_vest\": false, \"face_mask\": false, \"red_clothing\": false, \"blue_clothing\": false, \"white_clothing\": false, \"striped_clothing\": false, \"group_interaction\": false, \"person_lying_down\": false}, \"objects\": {\"car\": false, \"bus\": false, \"bicycle\": false, \"motorcycle\": false, \"truck\": false, \"taxi\": false, \"boat\": false, \"train\": false, \"traffic_light\": false, \"street_sign\": true, \"billboard\": true, \"bench\": false, \"chair\": false, \"table\": false, \"trash_bin\": false, \"traffic_cone\": false, \"bollard\": false, \"fence\": false, \"streetlamp\": false, \"shop_window\": false, \"door\": false, \"stairs\": false, \"ramp\": false, \"clock\": false, \"flag\": false, \"food_stall\": false, \"bottle\": false, \"cup\": false, \"suitcase\": false, \"screen\": true, \"fire_hydrant\": false, \"scaffolding\": false}, \"nature\": {\"dog\": false, \"cat\": false, \"bird\": false, \"horse\": false, \"cow\": false, \"sheep\": false, \"goat\": false, \"skunk\": false, \"rabbit\": false, \"squirrel\": false, \"duck\": false, \"fish\": false, \"tree\": false, \"grass\": false, \"flowers\": false, \"potted_plant\": false, \"bush\": false, \"mountain\": false, \"water\": false, \"beach\": false, \"snow\": false, \"rain\": false, \"clouds\": false, \"blue_sky\": false, \"sun\": false, \"moon\": false, \"smoke\": false, \"fire\": false, \"rocks\": false, \"fallen_leaves\": false, \"puddle\": false, \"animal_group\": false}, \"scene\": {\"outdoors\": false, \"indoors\": true, \"street\": true, \"sidewalk\": true, \"crosswalk\": false, \"buildings\": true, \"high_rise\": true, \"storefront\": true, \"park\": false, \"kitchen\": false, \"office\": false, \"living_room\": false, \"daylight\": false, \"night\": true, \"artificial_lighting\": true, \"shadows\": true, \"reflections\": true, \"wet_ground\": false, \"visible_text\": true, \"advertising\": true, \"road_markings\": false, \"red_dominant_area\": true, \"blue_dominant_area\": true, \"green_dominant_area\": false, \"yellow_dominant_area\": false, \"closeup\": false, \"wide_view\": false, \"blur\": false, \"occlusion\": true, \"dense_scene\": true, \"clear_foreground\": true, \"distant_background\": true}}\n```",
+    "parallel": "people.person: true  (99.8% yes)\npeople.crowd: false  (49.3% yes)\npeople.walking: false  (42.6% yes)\npeople.sitting: false  (0.1% yes)\npeople.standing: true  (99.9% yes)\npeople.running: false  (0.0% yes)\npeople.cycling: false  (0.0% yes)\npeople.carrying_bag: false  (6.9% yes)\npeople.backpack: false  (0.0% yes)\npeople.handbag: false  (0.0% yes)\npeople.hat: false  (0.0% yes)\npeople.sunglasses: false  (0.0% yes)\npeople.umbrella: false  (0.0% yes)\npeople.phone_in_hand: false  (0.0% yes)\npeople.using_camera: false  (0.0% yes)\npeople.raised_hand: false  (0.0% yes)\npeople.waving: false  (0.2% yes)\npeople.pointing: false  (0.1% yes)\npeople.eating: false  (0.0% yes)\npeople.drinking: false  (0.9% yes)\npeople.stroller: false  (0.0% yes)\npeople.walking_dog: false  (0.0% yes)\npeople.uniform: false  (0.0% yes)\npeople.helmet: false  (0.0% yes)\npeople.high_vis_vest: false  (0.0% yes)\npeople.face_mask: false  (0.0% yes)\npeople.red_clothing: false  (42.0% yes)\npeople.blue_clothing: true  (99.8% yes)\npeople.white_clothing: false  (11.4% yes)\npeople.striped_clothing: false  (0.0% yes)\npeople.group_interaction: false  (0.3% yes)\npeople.person_lying_down: false  (0.0% yes)\nobjects.car: false  (0.0% yes)\nobjects.bus: false  (0.0% yes)\nobjects.bicycle: false  (0.0% yes)\nobjects.motorcycle: false  (0.0% yes)\nobjects.truck: false  (0.0% yes)\nobjects.taxi: false  (0.0% yes)\nobjects.boat: false  (0.0% yes)\nobjects.train: false  (0.0% yes)\nobjects.traffic_light: false  (0.0% yes)\nobjects.street_sign: true  (99.8% yes)\nobjects.billboard: true  (99.9% yes)\nobjects.bench: false  (0.0% yes)\nobjects.chair: false  (0.0% yes)\nobjects.table: false  (0.7% yes)\nobjects.trash_bin: false  (0.0% yes)\nobjects.traffic_cone: false  (0.0% yes)\nobjects.bollard: false  (0.0% yes)\nobjects.fence: false  (0.0% yes)\nobjects.streetlamp: false  (3.1% yes)\nobjects.shop_window: true  (96.8% yes)\nobjects.door: false  (0.5% yes)\nobjects.stairs: false  (0.1% yes)\nobjects.ramp: false  (1.1% yes)\nobjects.clock: false  (0.0% yes)\nobjects.flag: false  (0.0% yes)\nobjects.food_stall: false  (0.0% yes)\nobjects.bottle: false  (0.1% yes)\nobjects.cup: true  (89.0% yes)\nobjects.suitcase: false  (0.0% yes)\nobjects.screen: true  (100.0% yes)\nobjects.fire_hydrant: false  (0.0% yes)\nobjects.scaffolding: false  (2.7% yes)\nnature.dog: false  (0.0% yes)\nnature.cat: false  (0.0% yes)\nnature.bird: false  (0.0% yes)\nnature.horse: false  (0.0% yes)\nnature.cow: false  (0.0% yes)\nnature.sheep: false  (0.0% yes)\nnature.goat: false  (0.0% yes)\nnature.skunk: false  (0.0% yes)\nnature.rabbit: false  (0.0% yes)\nnature.squirrel: false  (34.9% yes)\nnature.duck: false  (0.0% yes)\nnature.fish: false  (0.0% yes)\nnature.tree: false  (10.2% yes)\nnature.grass: false  (0.0% yes)\nnature.flowers: false  (0.0% yes)\nnature.potted_plant: false  (15.1% yes)\nnature.bush: false  (0.5% yes)\nnature.mountain: false  (0.0% yes)\nnature.water: false  (0.0% yes)\nnature.beach: false  (0.0% yes)\nnature.snow: false  (0.0% yes)\nnature.rain: false  (0.0% yes)\nnature.clouds: false  (0.0% yes)\nnature.blue_sky: false  (0.0% yes)\nnature.sun: false  (0.5% yes)\nnature.moon: false  (0.0% yes)\nnature.smoke: false  (0.0% yes)\nnature.fire: false  (0.0% yes)\nnature.rocks: true  (91.7% yes)\nnature.fallen_leaves: false  (0.0% yes)\nnature.puddle: false  (0.8% yes)\nnature.animal_group: false  (0.0% yes)\nscene.outdoors: false  (0.0% yes)\nscene.indoors: true  (100.0% yes)\nscene.street: true  (100.0% yes)\nscene.sidewalk: true  (92.8% yes)\nscene.crosswalk: false  (0.0% yes)\nscene.buildings: true  (100.0% yes)\nscene.high_rise: true  (100.0% yes)\nscene.storefront: true  (99.8% yes)\nscene.park: false  (0.0% yes)\nscene.kitchen: false  (0.0% yes)\nscene.office: true  (99.9% yes)\nscene.living_room: false  (0.0% yes)\nscene.daylight: false  (0.1% yes)\nscene.night: true  (99.9% yes)\nscene.artificial_lighting: true  (100.0% yes)\nscene.shadows: true  (99.6% yes)\nscene.reflections: true  (99.8% yes)\nscene.wet_ground: false  (0.3% yes)\nscene.visible_text: true  (100.0% yes)\nscene.advertising: true  (100.0% yes)\nscene.road_markings: false  (0.0% yes)\nscene.red_dominant_area: false  (11.2% yes)\nscene.blue_dominant_area: true  (98.6% yes)\nscene.green_dominant_area: false  (0.1% yes)\nscene.yellow_dominant_area: false  (1.5% yes)\nscene.closeup: false  (7.6% yes)\nscene.wide_view: true  (85.1% yes)\nscene.blur: false  (1.8% yes)\nscene.occlusion: true  (69.9% yes)\nscene.dense_scene: true  (100.0% yes)\nscene.clear_foreground: true  (78.3% yes)\nscene.distant_background: true  (100.0% yes)"
+  },
+  "accuracy_note": "Answer agreement is not accuracy; no ground truth was supplied for this scene.",
+  "github_video_asset": "https://github.com/user-attachments/assets/2dcac6b3-4dbe-4b1a-94fe-6c015f7defb0"
+}

tests/test_comparison.py

@@ -117,7 +117,7 @@ def test_concurrent_comparison_streams_both_paths_before_either_finishes(monkeyp
         def symbols(self, count):
             return ("A", "B")
 
-        def score_questions(self, state, questions, on_scores):
+        def score_questions(self, state, questions, on_scores, on_progress=None):
             assert generated.wait(2), "Generation never started alongside scoring"
             result = TokenScores((5, 0), 1, 20)
             on_scores([(0, result)])
@@ -126,7 +126,7 @@ def test_concurrent_comparison_streams_both_paths_before_either_finishes(monkeyp
 
     backend = Backend()
 
-    def generate(worker, state, questions, on_token):
+    def generate(worker, state, questions, on_token, on_progress=None):
         assert worker.processor is not backend.processor
         assert worker.processor.tokenizer is not backend.processor.tokenizer
         on_token('{"visible":', 1)

tests/test_json_backend.py

@@ -35,3 +35,43 @@ def test_complete_candidate_likelihood_uses_suffix_and_ignores_padding(batch_siz
     assert sum(batches) == 3
     assert max(batches) <= batch_size
     assert streamed == [[(0, results[0])]]
+
+
+def test_repeated_schema_reuses_only_tokenization_and_still_prefills_each_image(
+    monkeypatch, tmp_path
+):
+    from gemma_rlcd import json_backend
+    from gemma_rlcd.core import Noul, State, TokenScores
+
+    encoded, prefills = [], []
+    backend = JSONMLXBackend.__new__(JSONMLXBackend)
+    backend.tokenizer = SimpleNamespace(
+        encode=lambda text, **kwargs: encoded.append(text) or list(map(ord, text))
+    )
+    backend.max_input_tokens = 8192
+    backend.last_stats = {}
+    monkeypatch.setattr(
+        json_backend,
+        "prepare_generation",
+        lambda backend, state, questions: (
+            "prompt",
+            {"input_ids": SimpleNamespace(shape=(1, 6)), "image": state.images},
+        ),
+    )
+    backend.prefill = lambda prepared: prefills.append(prepared.inputs["image"]) or []
+    backend._sequence_scores = lambda cache, tokens, fields, on_scores: (
+        {index: TokenScores((float(len(prefills)), 0), 1, 10) for index, _ in fields},
+        [2],
+    )
+    questions = {"visible": Noul("Visible?")}
+    paths = [tmp_path / name for name in ("one.jpg", "two.jpg")]
+    for path in paths:
+        path.touch()
+    first = backend.score_questions(State(images=(str(paths[0]),)), questions)
+    assert not backend.last_stats["schema_cache_hit"]
+    count = len(encoded)
+    second = backend.score_questions(State(images=(str(paths[1]),)), questions)
+    assert backend.last_stats["schema_cache_hit"]
+    assert len(encoded) == count
+    assert prefills == [(str(paths[0]),), (str(paths[1]),)]
+    assert first[0].logits != second[0].logits

tests/test_web.py

@@ -474,7 +474,7 @@ def test_streaming_comparison_orders_events_and_cleans_uploads(playground, monke
 
     client, _, directory = playground
 
-    def generate(backend, state, questions, on_token=None):
+    def generate(backend, state, questions, on_token=None, on_progress=None):
         on_token('{"animal":', 1)
         on_token('"cat"}', 2)
         return {