claude
a37c4138a1
scripts/usage-run.py read the redirect parameter "src". cmd/mavweb/chat.go writes it as "s". So Source came back empty on all 140 turns of both fortnight runs, and every finding in those two docs is read off the reply wording instead of off the badge. Re-run confirms the column now arrives: 68 of 140 turns name a source. The two homelab misses are now direct evidence rather than inference. "какая скорость у меня сейчас?" is claimed by weather and "хватает ли места под новые бэкапы?" by feeds. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
111 lines
3.8 KiB
Python
111 lines
3.8 KiB
Python
"""Drive a fortnight of conversation through POST /api/chat and record it.
|
|
|
|
The 2026-08-07 week of usage was typed by hand. This is the same reach and the
|
|
same turn source, tap:text, so it exercises the path the mic and telegram take.
|
|
|
|
The endpoint is a form POST that redirects to /chat with the reply in the query
|
|
string. Reading the Location header is the whole protocol, so nothing here
|
|
parses HTML.
|
|
|
|
This exists to be re-run. The baseline is 2026-08-08 against master at beb093a,
|
|
in docs/evals/2026-08-08-two-weeks.md. Re-running the same turns after a routing
|
|
change is the comparison, so edit the turns file by adding, never by rewriting.
|
|
|
|
python3 scripts/usage-run.py scripts/testdata/usage-turns.txt out-prefix
|
|
|
|
Input is one utterance per line. A line starting with "# " opens a day. A blank
|
|
line is ignored. Output is a markdown transcript and a jsonl log beside it.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
URL = "http://127.0.0.1:9201/api/chat"
|
|
TIMEOUT = 90
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""A 303 carries the reply. Following it would throw the reply away."""
|
|
|
|
def redirect_request(self, *a, **kw):
|
|
return None
|
|
|
|
|
|
# ProxyHandler({}) is not optional. This box exports http_proxy, urllib honours
|
|
# it, and the proxy answers 503 for a loopback address.
|
|
OPENER = urllib.request.build_opener(NoRedirect, urllib.request.ProxyHandler({}))
|
|
|
|
|
|
def turn(text):
|
|
body = urllib.parse.urlencode({"text": text}).encode()
|
|
t0 = time.perf_counter()
|
|
try:
|
|
OPENER.open(urllib.request.Request(URL, data=body), timeout=TIMEOUT)
|
|
return {"reply": "", "error": "no redirect", "secs": time.perf_counter() - t0}
|
|
except urllib.error.HTTPError as e:
|
|
dt = time.perf_counter() - t0
|
|
if e.code != 303:
|
|
return {"reply": "", "error": f"HTTP {e.code}", "secs": dt}
|
|
loc = e.headers.get("Location", "")
|
|
q = urllib.parse.parse_qs(urllib.parse.urlparse(loc).query)
|
|
return {
|
|
"reply": q.get("r", [""])[0],
|
|
# "s", not "src". cmd/mavweb/chat.go writes the badge under that
|
|
# name, and reading the wrong one cost both fortnight runs their
|
|
# source column: every finding in those docs is inferred from the
|
|
# reply wording instead.
|
|
"source": q.get("s", [""])[0],
|
|
"trace": q.get("t", [""])[0],
|
|
"secs": dt,
|
|
}
|
|
except Exception as e: # a dead box must not lose the turns already done
|
|
return {"reply": "", "error": str(e), "secs": time.perf_counter() - t0}
|
|
|
|
|
|
def main():
|
|
lines = [l.rstrip("\n") for l in open(sys.argv[1])]
|
|
prefix = sys.argv[2]
|
|
md = open(prefix + "-transcript.md", "w")
|
|
log = open(prefix + ".jsonl", "w")
|
|
|
|
day = 0
|
|
n = 0
|
|
print(f"# Raw transcript, two weeks of usage\n", file=md)
|
|
for line in lines:
|
|
if not line.strip():
|
|
continue
|
|
if line.startswith("# "):
|
|
if day:
|
|
print("```\n", file=md)
|
|
day += 1
|
|
print(f"## {line[2:]}\n\n```", file=md)
|
|
continue
|
|
n += 1
|
|
r = turn(line)
|
|
r["day"] = day
|
|
r["n"] = n
|
|
r["utterance"] = line
|
|
log.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
log.flush()
|
|
reply = r.get("error") or r["reply"]
|
|
print(f"YOU: {line}", file=md)
|
|
print(f"MAVEN: {reply}", file=md)
|
|
tag = f"[{r['secs']:.1f}s"
|
|
if r.get("source"):
|
|
tag += f" src={r['source']}"
|
|
print(f" {tag} t={r.get('trace', '')}]\n", file=md)
|
|
md.flush()
|
|
print(f"{n:3} d{day} {r['secs']:5.1f}s {line[:40]:40s} -> {reply[:60]}",
|
|
flush=True)
|
|
print("```", file=md)
|
|
md.close()
|
|
log.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|