Fix judge retry starvation on deterministic failures

- Widen max_tokens on each in-call retry so finish_reason=length
overflows don't fail identically every attempt
- Treat malformed JSON as a retryable failure instead of aborting
- Cap error re-queues at JUDGE_MAX_RETRIES per revision, then park the
logbook with a visible error status until it actually changes
- Show honest retry status instead of 're-judging (logbook updated)'

Reported in challenge discussion #22.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

README.md

@@ -26,10 +26,14 @@ awaiting a credit email to
 - `POST /run` — trigger a scan manually (for testing).
 
 Secrets: `HF_TOKEN` (write access to the org). Optional vars: `JUDGE_MODEL`
-(default `zai-org/GLM-5.2`), `JUDGE_MAX_TOKENS` (default 8000),
-`JUDGE_ATTEMPTS` (default 2), `SCAN_INTERVAL_S` (default 60),
-`JUDGE_INTERVAL_S` (default 300), and `TRIGGER_TOKEN` (if set, `POST /run`
-requires `?token=`).
+(default `zai-org/GLM-5.2`), `JUDGE_MAX_TOKENS` (default 8000; the output
+budget widens to `N × JUDGE_MAX_TOKENS` on the Nth in-call retry so
+`finish_reason=length` failures don't repeat deterministically),
+`JUDGE_ATTEMPTS` (default 2), `JUDGE_MAX_RETRIES` (default 3; errored
+logbooks are re-queued at most this many times per revision, then parked
+with a visible error until the logbook changes), `SCAN_INTERVAL_S`
+(default 60), `JUDGE_INTERVAL_S` (default 300), and `TRIGGER_TOKEN` (if
+set, `POST /run` requires `?token=`).
 
 The judge prompt explicitly treats the logbook's own verification statuses and
 verdicts as untrusted assertions. It must independently assess the concrete

app.py

@@ -32,6 +32,7 @@ CREDIT_EMAIL_ENABLED = os.environ.get("CREDIT_EMAIL_ENABLED", "false").lower() =
 JUDGE_MODEL = os.environ.get("JUDGE_MODEL", "zai-org/GLM-5.2")
 JUDGE_MAX_TOKENS = int(os.environ.get("JUDGE_MAX_TOKENS", "8000"))
 JUDGE_ATTEMPTS = int(os.environ.get("JUDGE_ATTEMPTS", "2"))
+JUDGE_MAX_RETRIES = int(os.environ.get("JUDGE_MAX_RETRIES", "3"))
 SCAN_INTERVAL_S = int(os.environ.get("SCAN_INTERVAL_S", "60"))
 JUDGE_INTERVAL_S = int(os.environ.get("JUDGE_INTERVAL_S", "300"))
 TRIGGER_TOKEN = os.environ.get("TRIGGER_TOKEN")
@@ -270,6 +271,9 @@ Judge each claim. Return ONLY a JSON object of this shape:
 def call_judge(client: httpx.Client, prompt: str) -> dict:
     last_error = None
     for attempt in range(1, JUDGE_ATTEMPTS + 1):
+        # A finish_reason=length overflow is deterministic at a fixed cap, so
+        # widen the output budget each retry instead of replaying the request.
+        max_tokens = JUDGE_MAX_TOKENS * attempt
         r = client.post(
             ROUTER_URL,
             headers={"Authorization": f"Bearer {HF_TOKEN}"},
@@ -280,7 +284,7 @@ def call_judge(client: httpx.Client, prompt: str) -> dict:
                     {"role": "user", "content": prompt},
                 ],
                 "temperature": 0.1,
-                "max_tokens": JUDGE_MAX_TOKENS,
+                "max_tokens": max_tokens,
             },
             timeout=600,
         )
@@ -289,13 +293,20 @@ def call_judge(client: httpx.Client, prompt: str) -> dict:
         content = choice.get("message", {}).get("content") or ""
         match = re.search(r"\{[\s\S]*\}", content)
         if match:
-            return json.loads(match.group(0))
-        finish_reason = choice.get("finish_reason", "unknown")
-        last_error = ValueError(
-            f"Judge returned no JSON (finish_reason={finish_reason}): {content[:200]}"
-        )
+            try:
+                return json.loads(match.group(0))
+            except json.JSONDecodeError as e:
+                last_error = e
+        else:
+            finish_reason = choice.get("finish_reason", "unknown")
+            last_error = ValueError(
+                f"Judge returned no JSON (finish_reason={finish_reason}): {content[:200]}"
+            )
         if attempt < JUDGE_ATTEMPTS:
-            log(f"judge attempt {attempt} returned no JSON; retrying")
+            log(
+                f"judge attempt {attempt} produced no valid JSON; "
+                f"retrying with max_tokens={JUDGE_MAX_TOKENS * (attempt + 1)}"
+            )
     raise last_error
 
 
@@ -406,11 +417,25 @@ def discover_and_queue() -> dict:
                 "skipped",
             }:
                 current.update(base)
+            elif current.get("sha") == lb["sha"] and current.get("status") == "error":
+                # Errors at the same revision are usually deterministic: retry a
+                # bounded number of times, then park the error until the logbook
+                # actually changes, so broken books don't starve the queue.
+                attempts = current.get("attempts", 0)
+                if attempts < JUDGE_MAX_RETRIES:
+                    mark(
+                        key,
+                        "queued",
+                        f"retrying after error (attempt {attempts + 1}/{JUDGE_MAX_RETRIES})",
+                        **base,
+                    )
+                    queued += 1
             else:
                 mark(
                     key,
                     "queued",
                     "re-judging (logbook updated)" if prev else "new logbook",
+                    attempts=0,
                     **base,
                 )
                 queued += 1
@@ -476,8 +501,15 @@ def judge_queued() -> dict:
                 mark(key, "judged", summary)
                 log(f"verdict {key}: {summary}")
             except Exception as e:
-                mark(key, "error", repr(e)[:200])
-                log(f"error judging {key}: {e!r}")
+                attempts = (STATE["spaces"].get(key) or {}).get("attempts", 0) + 1
+                mark(
+                    key,
+                    "error",
+                    f"attempt {attempts}/{JUDGE_MAX_RETRIES} failed: {repr(e)[:160]}",
+                    attempts=attempts,
+                    **base,
+                )
+                log(f"error judging {key} (attempt {attempts}/{JUDGE_MAX_RETRIES}): {e!r}")
 
         if judged:
             save_verdicts(VERDICTS)

test_app.py

@@ -101,6 +101,35 @@ class CallJudgeTest(unittest.TestCase):
         self.assertEqual(call_judge(client, "prompt"), {"claims": []})
         self.assertEqual(len(client.requests), 2)
         self.assertEqual(client.requests[0][1]["json"]["max_tokens"], 8000)
+        # length overflows are deterministic: the retry must widen the budget
+        self.assertEqual(client.requests[1][1]["json"]["max_tokens"], 16000)
+
+    @patch("app.JUDGE_ATTEMPTS", 2)
+    @patch("app.JUDGE_MAX_TOKENS", 8000)
+    def test_retries_malformed_json_response(self):
+        client = FakeClient(
+            [
+                {
+                    "choices": [
+                        {
+                            "finish_reason": "stop",
+                            "message": {"content": '{"claims": [,]}'},
+                        }
+                    ]
+                },
+                {
+                    "choices": [
+                        {
+                            "finish_reason": "stop",
+                            "message": {"content": '{"claims": []}'},
+                        }
+                    ]
+                },
+            ]
+        )
+
+        self.assertEqual(call_judge(client, "prompt"), {"claims": []})
+        self.assertEqual(len(client.requests), 2)
 
 
 class SchedulerConfigurationTest(unittest.TestCase):
@@ -195,6 +224,40 @@ class DiscoveryQueueTest(unittest.TestCase):
         self.assertEqual(STATE["spaces"]["owner/logbook"]["status"], "queued")
         self.assertEqual(discover_and_queue(), {"found": 1, "queued": 0})
 
+    @patch("app.discover_logbooks")
+    def test_errored_space_retries_bounded_then_parks(self, discover):
+        discover.return_value = [
+            {"space_id": "owner/logbook", "orid": "paper-1", "sha": "abc"}
+        ]
+        STATE["spaces"]["owner/logbook"] = {
+            "status": "error",
+            "sha": "abc",
+            "attempts": 1,
+        }
+
+        self.assertEqual(discover_and_queue(), {"found": 1, "queued": 1})
+        entry = STATE["spaces"]["owner/logbook"]
+        self.assertEqual(entry["status"], "queued")
+        self.assertEqual(entry["detail"], "retrying after error (attempt 2/3)")
+
+        # at the retry cap the error is parked until the logbook changes
+        STATE["spaces"]["owner/logbook"] = {
+            "status": "error",
+            "sha": "abc",
+            "attempts": 3,
+        }
+        self.assertEqual(discover_and_queue(), {"found": 1, "queued": 0})
+        self.assertEqual(STATE["spaces"]["owner/logbook"]["status"], "error")
+
+        # a new revision resets the counter and re-queues
+        discover.return_value = [
+            {"space_id": "owner/logbook", "orid": "paper-1", "sha": "def"}
+        ]
+        self.assertEqual(discover_and_queue(), {"found": 1, "queued": 1})
+        entry = STATE["spaces"]["owner/logbook"]
+        self.assertEqual(entry["status"], "queued")
+        self.assertEqual(entry["attempts"], 0)
+
     @patch("app.discover_logbooks")
     def test_matching_persisted_verdict_is_not_requeued(self, discover):
         discover.return_value = [