claude
01e80fce4a
The 2026-08-07 week of usage was typed by hand and cannot be replayed, so
it measured a build and not a change. scripts/usage-run.py drives the same
reach from a turns file, which makes the next run a diff.
Baseline is master at beb093a: 140 turns, p50 1.6s, zero errors. Three
defects to move. A parked reminder clarify contaminates 19 later turns and
survives a day boundary. Query sources that guess claim six turns they
cannot answer, which is the class V-655 removes. And one question was read
as a capture.
Also records the slot head: gemma distils 2178 spans, three heads score
intent 92.8%, destination 82.8%, slot span F1 72.4% over three seeds.
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
"""Drive a fortnight of conversation through POST /api/chat and record it.
|
|
|
|
The 2026-08-07 week of usage was typed by hand. This is the same reach and the
|
|
same turn source, tap:text, so it exercises the path the mic and telegram take.
|
|
|
|
The endpoint is a form POST that redirects to /chat with the reply in the query
|
|
string. Reading the Location header is the whole protocol, so nothing here
|
|
parses HTML.
|
|
|
|
This exists to be re-run. The baseline is 2026-08-08 against master at beb093a,
|
|
in docs/evals/2026-08-08-two-weeks.md. Re-running the same turns after a routing
|
|
change is the comparison, so edit the turns file by adding, never by rewriting.
|
|
|
|
python3 scripts/usage-run.py scripts/testdata/usage-turns.txt out-prefix
|
|
|
|
Input is one utterance per line. A line starting with "# " opens a day. A blank
|
|
line is ignored. Output is a markdown transcript and a jsonl log beside it.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
URL = "http://127.0.0.1:9201/api/chat"
|
|
TIMEOUT = 90
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""A 303 carries the reply. Following it would throw the reply away."""
|
|
|
|
def redirect_request(self, *a, **kw):
|
|
return None
|
|
|
|
|
|
# ProxyHandler({}) is not optional. This box exports http_proxy, urllib honours
|
|
# it, and the proxy answers 503 for a loopback address.
|
|
OPENER = urllib.request.build_opener(NoRedirect, urllib.request.ProxyHandler({}))
|
|
|
|
|
|
def turn(text):
|
|
body = urllib.parse.urlencode({"text": text}).encode()
|
|
t0 = time.perf_counter()
|
|
try:
|
|
OPENER.open(urllib.request.Request(URL, data=body), timeout=TIMEOUT)
|
|
return {"reply": "", "error": "no redirect", "secs": time.perf_counter() - t0}
|
|
except urllib.error.HTTPError as e:
|
|
dt = time.perf_counter() - t0
|
|
if e.code != 303:
|
|
return {"reply": "", "error": f"HTTP {e.code}", "secs": dt}
|
|
loc = e.headers.get("Location", "")
|
|
q = urllib.parse.parse_qs(urllib.parse.urlparse(loc).query)
|
|
return {
|
|
"reply": q.get("r", [""])[0],
|
|
"source": q.get("src", q.get("source", [""]))[0],
|
|
"trace": q.get("t", [""])[0],
|
|
"secs": dt,
|
|
}
|
|
except Exception as e: # a dead box must not lose the turns already done
|
|
return {"reply": "", "error": str(e), "secs": time.perf_counter() - t0}
|
|
|
|
|
|
def main():
|
|
lines = [l.rstrip("\n") for l in open(sys.argv[1])]
|
|
prefix = sys.argv[2]
|
|
md = open(prefix + "-transcript.md", "w")
|
|
log = open(prefix + ".jsonl", "w")
|
|
|
|
day = 0
|
|
n = 0
|
|
print(f"# Raw transcript, two weeks of usage\n", file=md)
|
|
for line in lines:
|
|
if not line.strip():
|
|
continue
|
|
if line.startswith("# "):
|
|
if day:
|
|
print("```\n", file=md)
|
|
day += 1
|
|
print(f"## {line[2:]}\n\n```", file=md)
|
|
continue
|
|
n += 1
|
|
r = turn(line)
|
|
r["day"] = day
|
|
r["n"] = n
|
|
r["utterance"] = line
|
|
log.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
log.flush()
|
|
reply = r.get("error") or r["reply"]
|
|
print(f"YOU: {line}", file=md)
|
|
print(f"MAVEN: {reply}", file=md)
|
|
tag = f"[{r['secs']:.1f}s"
|
|
if r.get("source"):
|
|
tag += f" src={r['source']}"
|
|
print(f" {tag} t={r.get('trace', '')}]\n", file=md)
|
|
md.flush()
|
|
print(f"{n:3} d{day} {r['secs']:5.1f}s {line[:40]:40s} -> {reply[:60]}",
|
|
flush=True)
|
|
print("```", file=md)
|
|
md.close()
|
|
log.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|