Record a fortnight of usage as a re-runnable baseline (V-661)
The 2026-08-07 week of usage was typed by hand and cannot be replayed, so
it measured a build and not a change. scripts/usage-run.py drives the same
reach from a turns file, which makes the next run a diff.
Baseline is master at beb093a: 140 turns, p50 1.6s, zero errors. Three
defects to move. A parked reminder clarify contaminates 19 later turns and
survives a day boundary. Query sources that guess claim six turns they
cannot answer, which is the class V-655 removes. And one question was read
as a capture.
Also records the slot head: gemma distils 2178 spans, three heads score
intent 92.8%, destination 82.8%, slot span F1 72.4% over three seeds.
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
"""Drive a fortnight of conversation through POST /api/chat and record it.
|
||||
|
||||
The 2026-08-07 week of usage was typed by hand. This is the same reach and the
|
||||
same turn source, tap:text, so it exercises the path the mic and telegram take.
|
||||
|
||||
The endpoint is a form POST that redirects to /chat with the reply in the query
|
||||
string. Reading the Location header is the whole protocol, so nothing here
|
||||
parses HTML.
|
||||
|
||||
This exists to be re-run. The baseline is 2026-08-08 against master at beb093a,
|
||||
in docs/evals/2026-08-08-two-weeks.md. Re-running the same turns after a routing
|
||||
change is the comparison, so edit the turns file by adding, never by rewriting.
|
||||
|
||||
python3 scripts/usage-run.py scripts/testdata/usage-turns.txt out-prefix
|
||||
|
||||
Input is one utterance per line. A line starting with "# " opens a day. A blank
|
||||
line is ignored. Output is a markdown transcript and a jsonl log beside it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
URL = "http://127.0.0.1:9201/api/chat"
|
||||
TIMEOUT = 90
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""A 303 carries the reply. Following it would throw the reply away."""
|
||||
|
||||
def redirect_request(self, *a, **kw):
|
||||
return None
|
||||
|
||||
|
||||
# ProxyHandler({}) is not optional. This box exports http_proxy, urllib honours
|
||||
# it, and the proxy answers 503 for a loopback address.
|
||||
OPENER = urllib.request.build_opener(NoRedirect, urllib.request.ProxyHandler({}))
|
||||
|
||||
|
||||
def turn(text):
|
||||
body = urllib.parse.urlencode({"text": text}).encode()
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
OPENER.open(urllib.request.Request(URL, data=body), timeout=TIMEOUT)
|
||||
return {"reply": "", "error": "no redirect", "secs": time.perf_counter() - t0}
|
||||
except urllib.error.HTTPError as e:
|
||||
dt = time.perf_counter() - t0
|
||||
if e.code != 303:
|
||||
return {"reply": "", "error": f"HTTP {e.code}", "secs": dt}
|
||||
loc = e.headers.get("Location", "")
|
||||
q = urllib.parse.parse_qs(urllib.parse.urlparse(loc).query)
|
||||
return {
|
||||
"reply": q.get("r", [""])[0],
|
||||
"source": q.get("src", q.get("source", [""]))[0],
|
||||
"trace": q.get("t", [""])[0],
|
||||
"secs": dt,
|
||||
}
|
||||
except Exception as e: # a dead box must not lose the turns already done
|
||||
return {"reply": "", "error": str(e), "secs": time.perf_counter() - t0}
|
||||
|
||||
|
||||
def main():
|
||||
lines = [l.rstrip("\n") for l in open(sys.argv[1])]
|
||||
prefix = sys.argv[2]
|
||||
md = open(prefix + "-transcript.md", "w")
|
||||
log = open(prefix + ".jsonl", "w")
|
||||
|
||||
day = 0
|
||||
n = 0
|
||||
print(f"# Raw transcript, two weeks of usage\n", file=md)
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
if line.startswith("# "):
|
||||
if day:
|
||||
print("```\n", file=md)
|
||||
day += 1
|
||||
print(f"## {line[2:]}\n\n```", file=md)
|
||||
continue
|
||||
n += 1
|
||||
r = turn(line)
|
||||
r["day"] = day
|
||||
r["n"] = n
|
||||
r["utterance"] = line
|
||||
log.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
log.flush()
|
||||
reply = r.get("error") or r["reply"]
|
||||
print(f"YOU: {line}", file=md)
|
||||
print(f"MAVEN: {reply}", file=md)
|
||||
tag = f"[{r['secs']:.1f}s"
|
||||
if r.get("source"):
|
||||
tag += f" src={r['source']}"
|
||||
print(f" {tag} t={r.get('trace', '')}]\n", file=md)
|
||||
md.flush()
|
||||
print(f"{n:3} d{day} {r['secs']:5.1f}s {line[:40]:40s} -> {reply[:60]}",
|
||||
flush=True)
|
||||
print("```", file=md)
|
||||
md.close()
|
||||
log.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user