Reconstruct repo from Claude Code + codex transcripts
Working tree (including .git) was lost to an rm. Rebuilt by replaying Write/Edit/ Read/attachment events from 25 Claude sessions and 22 successful codex apply_patch blocks into one timestamp-ordered timeline. Verified against ground truth recorded in the transcripts: wc -l on 10 files and ls -l on 5 files at 2026-07-18T13:13:44Z both match exactly; 18 files are byte-identical to their newest ~/.claude/file-history blob. See HANDOFF.md for sources, gaps, and how to rebuild .venv. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+216
@@ -0,0 +1,216 @@
|
||||
# transport.py — minio artifact client, shared by all workers.
|
||||
# workers are stateless: pull inputs by uri to local disk, push outputs back, return uris.
|
||||
# no sqlite here (see plan phase 2b option A: homesrv orchestrator owns state).
|
||||
# uri form: "s3://<bucket>/<key...>" or bare "<bucket>/<key...>". first path segment = bucket.
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
from starlette.responses import Response
|
||||
|
||||
_client = None
|
||||
|
||||
|
||||
def _summarize(body: bytes, limit=6) -> str:
|
||||
"""compact one-line view of a json body for observability: uri inputs/outputs (basename, or
|
||||
key×N for lists) + short scalars; big lists/dicts (embeddings, panel arrays) shown as key[N].
|
||||
this is what makes the log answer 'what did the stage get / return / where did it go'."""
|
||||
try:
|
||||
obj = json.loads(body)
|
||||
except Exception:
|
||||
return "-"
|
||||
if not isinstance(obj, dict):
|
||||
return f"[{len(obj)}]" if isinstance(obj, list) else "-"
|
||||
parts = []
|
||||
for k, v in obj.items():
|
||||
if k == "panel_id":
|
||||
continue
|
||||
if isinstance(v, str) and (k.endswith("uri") or k.endswith("url")):
|
||||
parts.append(f"{k}={v.rsplit('/', 1)[-1]}")
|
||||
elif isinstance(v, list) and v and isinstance(v[0], str) and (k.endswith("uris") or k.endswith("urls")):
|
||||
parts.append(f"{k}×{len(v)}")
|
||||
elif isinstance(v, (int, float, bool)):
|
||||
parts.append(f"{k}={v}")
|
||||
elif isinstance(v, str):
|
||||
parts.append(f"{k}={v[:24]!r}" if len(v) > 24 else f"{k}={v!r}")
|
||||
elif isinstance(v, (list, dict)):
|
||||
parts.append(f"{k}[{len(v)}]")
|
||||
if len(parts) >= limit:
|
||||
parts.append("…")
|
||||
break
|
||||
return " ".join(parts) or "{}"
|
||||
|
||||
|
||||
def install_logging(app, name: str):
|
||||
"""one log line per request for any worker:
|
||||
`name METHOD /path panel=<id> in:{...} -> out:{...} 123ms 200` (or `... ERR: <exc>` on failure).
|
||||
call once right after app = FastAPI(). in/out summarize the json bodies (uris + short scalars,
|
||||
big arrays as key[N]); unparseable/empty bodies show `-`."""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
force=False,
|
||||
)
|
||||
log = logging.getLogger(name)
|
||||
|
||||
@app.middleware("http")
|
||||
async def _log(request, call_next):
|
||||
body = await request.body() # buffered; downstream re-reads from the cached bytes
|
||||
panel = "-"
|
||||
if body:
|
||||
try:
|
||||
panel = str(json.loads(body).get("panel_id") or "-")
|
||||
except Exception:
|
||||
pass
|
||||
req_in = _summarize(body) if body else "-"
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
resp = await call_next(request)
|
||||
except Exception as e:
|
||||
ms = int((time.perf_counter() - t0) * 1000)
|
||||
log.error("%s %s panel=%s in:{%s} %dms ERR: %r",
|
||||
request.method, request.url.path, panel, req_in, ms, e)
|
||||
raise
|
||||
# drain the response stream so we can summarize outputs, then hand back an identical response.
|
||||
out = "-"
|
||||
chunks = [c async for c in resp.body_iterator]
|
||||
raw = b"".join(chunks)
|
||||
if resp.headers.get("content-type", "").startswith("application/json"):
|
||||
out = _summarize(raw)
|
||||
resp = Response(content=raw, status_code=resp.status_code,
|
||||
headers=dict(resp.headers), media_type=resp.media_type)
|
||||
ms = int((time.perf_counter() - t0) * 1000)
|
||||
lvl = log.error if resp.status_code >= 500 else log.info
|
||||
lvl("%s %s panel=%s in:{%s} -> out:{%s} %dms %d",
|
||||
request.method, request.url.path, panel, req_in, out, ms, resp.status_code)
|
||||
return resp
|
||||
|
||||
|
||||
def _mc():
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = Minio(
|
||||
os.environ.get("MINIO_ENDPOINT", "192.168.1.104:9000"), # homesrv
|
||||
access_key=os.environ.get("MINIO_ACCESS_KEY", "admin"),
|
||||
secret_key=os.environ.get("MINIO_SECRET_KEY", "godforgiveus"),
|
||||
secure=False,
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def _split(uri: str):
|
||||
"""(bucket, key) from an s3-style or bare uri."""
|
||||
u = uri[5:] if uri.startswith("s3://") else uri
|
||||
bucket, _, key = u.partition("/")
|
||||
if not bucket or not key:
|
||||
raise ValueError(f"bad uri: {uri!r}")
|
||||
return bucket, key
|
||||
|
||||
|
||||
_known_buckets = set()
|
||||
|
||||
|
||||
def _ensure_bucket(c, bucket):
|
||||
# ponytail: process-local cache, buckets are never deleted at runtime
|
||||
if bucket in _known_buckets:
|
||||
return
|
||||
if not c.bucket_exists(bucket):
|
||||
c.make_bucket(bucket)
|
||||
_known_buckets.add(bucket)
|
||||
|
||||
|
||||
def put(local_path: str, uri: str, client=None) -> str:
|
||||
"""upload local file to minio at uri, return the s3:// uri. raises on failure."""
|
||||
c = client or _mc()
|
||||
bucket, key = _split(uri)
|
||||
_ensure_bucket(c, bucket)
|
||||
c.fput_object(bucket, key, local_path)
|
||||
return f"s3://{bucket}/{key}"
|
||||
|
||||
|
||||
def get(uri: str, local_path: str, client=None) -> str:
|
||||
"""download uri to local_path, return local_path. workers pull inputs to /dev/shm."""
|
||||
c = client or _mc()
|
||||
bucket, key = _split(uri)
|
||||
os.makedirs(os.path.dirname(local_path) or ".", exist_ok=True)
|
||||
c.fget_object(bucket, key, local_path)
|
||||
return local_path
|
||||
|
||||
|
||||
def exists(uri: str, client=None) -> bool:
|
||||
"""HEAD check for worker-level skip-if-exists (orchestrator is the real skip authority)."""
|
||||
c = client or _mc()
|
||||
bucket, key = _split(uri)
|
||||
try:
|
||||
c.stat_object(bucket, key)
|
||||
return True
|
||||
except S3Error:
|
||||
return False
|
||||
|
||||
|
||||
def put_bytes(data: bytes, uri: str, client=None) -> str:
|
||||
"""for generated artifacts that never touch disk (embeddings, json)."""
|
||||
import io
|
||||
c = client or _mc()
|
||||
bucket, key = _split(uri)
|
||||
_ensure_bucket(c, bucket)
|
||||
c.put_object(bucket, key, io.BytesIO(data), length=len(data))
|
||||
return f"s3://{bucket}/{key}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# self-check: fake in-memory minio, round-trip a file + bytes, exists true/false.
|
||||
import tempfile
|
||||
|
||||
class _Fake:
|
||||
def __init__(self):
|
||||
self.store = {}
|
||||
def bucket_exists(self, b):
|
||||
return any(k[0] == b for k in self.store)
|
||||
def make_bucket(self, b):
|
||||
pass
|
||||
def fput_object(self, b, k, path):
|
||||
with open(path, "rb") as f:
|
||||
self.store[(b, k)] = f.read()
|
||||
def fget_object(self, b, k, path):
|
||||
with open(path, "wb") as f:
|
||||
f.write(self.store[(b, k)])
|
||||
def put_object(self, b, k, stream, length):
|
||||
self.store[(b, k)] = stream.read()
|
||||
def stat_object(self, b, k):
|
||||
if (b, k) not in self.store:
|
||||
raise S3Error("NoSuchKey", "missing", "", "", "", None)
|
||||
return True
|
||||
|
||||
fake = _Fake()
|
||||
assert _split("s3://manga/a/b.png") == ("manga", "a/b.png")
|
||||
assert _split("manga/a/b.png") == ("manga", "a/b.png")
|
||||
|
||||
src = tempfile.NamedTemporaryFile(delete=False)
|
||||
src.write(b"hello panel"); src.close()
|
||||
uri = put(src.name, "s3://manga/x/p001.png", client=fake)
|
||||
assert uri == "s3://manga/x/p001.png", uri
|
||||
assert exists(uri, client=fake)
|
||||
assert not exists("s3://manga/x/nope.png", client=fake)
|
||||
dst = src.name + ".out"
|
||||
get(uri, dst, client=fake)
|
||||
assert open(dst, "rb").read() == b"hello panel"
|
||||
|
||||
put_bytes(b"\x01\x02", "s3://manga/x/e.npy", client=fake)
|
||||
assert exists("s3://manga/x/e.npy", client=fake)
|
||||
os.remove(src.name); os.remove(dst)
|
||||
|
||||
# _summarize: uris -> basename, uri lists -> key×N, scalars kept, big arrays -> key[N], panel_id dropped
|
||||
s = _summarize(json.dumps({
|
||||
"panel_id": "p1", "audio_uri": "s3://manga/x/cl.wav",
|
||||
"panel_uris": ["s3://a/1.png", "s3://a/2.png"], "active": 2, "rtl": True,
|
||||
"weights": [0.1, 0.2, 0.7], "narration_text": "a very long line of narration here indeed",
|
||||
}).encode())
|
||||
assert "audio_uri=cl.wav" in s and "panel_uris×2" in s and "active=2" in s, s
|
||||
assert "weights[3]" in s and "panel_id" not in s, s
|
||||
assert _summarize(b"clip_uri", ) == "-" # non-json
|
||||
assert _summarize(json.dumps({"clip_uri": "s3://m/c/clips/p1.mp4", "duration": 4.1}).encode()) \
|
||||
== "clip_uri=p1.mp4 duration=4.1"
|
||||
print("transport self-check ok")
|
||||
Reference in New Issue
Block a user