From 4b0024f7d98ca846855d2675a8f737186785fcab Mon Sep 17 00:00:00 2001 From: kami Date: Sun, 21 Jun 2026 07:07:23 +0000 Subject: [PATCH] feat(research): batch source-list fetch-approval (D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /sessions/{id}/approve-sources extracts distinct hosts from a source list (SourceListHosts.extract) and emits one EgressHostsGrantedEvent for the session, so the live per-session egress allowlist auto-clears subsequent web_fetches to those hosts — approve the list once instead of per-URL. Co-Authored-By: Claude Opus 4.8 --- .../server/research/SourceListApproval.kt | 37 ++++++++++ .../apps/server/research/SourceListHosts.kt | 34 +++++++++ .../apps/server/routes/SessionRoutes.kt | 27 ++++++++ .../server/research/SourceListApprovalTest.kt | 63 +++++++++++++++++ .../server/research/SourceListHostsTest.kt | 69 +++++++++++++++++++ 5 files changed, 230 insertions(+) create mode 100644 apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListApproval.kt create mode 100644 apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListHosts.kt create mode 100644 apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListApprovalTest.kt create mode 100644 apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListHostsTest.kt diff --git a/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListApproval.kt b/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListApproval.kt new file mode 100644 index 00000000..91dfe4a1 --- /dev/null +++ b/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListApproval.kt @@ -0,0 +1,37 @@ +package com.correx.apps.server.research + +import com.correx.core.events.EventDispatcher +import com.correx.core.events.events.EgressHostsGrantedEvent +import com.correx.core.events.types.SessionId + +/** Operator context recorded on the grant emitted by [approveSourceList]. */ +internal const val BATCH_SOURCE_LIST_REASON = "batch source-list approval" + +/** + * Batch fetch-approval at the source-list level (BACKLOG §D): the operator approves a whole research + * source list at once instead of clearing each `web_fetch` per-URL. Extracts the distinct hosts from + * [urls] (a source list's URLs, or its source_dossier source lines) and emits a single + * [EgressHostsGrantedEvent] for [sessionId]. That grant folds into the live per-session egress + * allowlist (commit ab7e4be) via [com.correx.core.sessions.projections.EgressAllowlistProjection], so + * subsequent fetches to those hosts auto-clear and no longer prompt. + * + * No-op when [urls] yields no parseable host: nothing to grant, so no event is emitted. Returns the + * granted host set (empty when nothing was emitted) for the caller to report back. + */ +suspend fun approveSourceList( + dispatcher: EventDispatcher, + sessionId: SessionId, + urls: List, +): Set { + val hosts = SourceListHosts.extract(urls) + if (hosts.isEmpty()) return emptySet() + dispatcher.emit( + payload = EgressHostsGrantedEvent( + sessionId = sessionId, + hosts = hosts, + reason = BATCH_SOURCE_LIST_REASON, + ), + sessionId = sessionId, + ) + return hosts +} diff --git a/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListHosts.kt b/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListHosts.kt new file mode 100644 index 00000000..f261deab --- /dev/null +++ b/apps/server/src/main/kotlin/com/correx/apps/server/research/SourceListHosts.kt @@ -0,0 +1,34 @@ +package com.correx.apps.server.research + +import java.net.URI + +/** + * Pure host extraction for batch source-list approval (BACKLOG §D). Given the URLs (or dossier + * source lines) of a research source list, returns the distinct, lower-cased hostnames the operator + * is approving egress to — the input set for a single [com.correx.core.events.events.EgressHostsGrantedEvent]. + * + * Mirrors [com.correx.core.toolintent.rules.NetworkHostRule]'s URL parsing so the granted hosts line + * up with what the network gate later matches: absolute URLs only (`scheme://host`), parsed via + * [java.net.URI], host lower-cased, port dropped (URI.host excludes it). Entries that don't parse to + * a host — relative paths, free text, malformed URLs — are ignored rather than failing the batch. + * + * source_dossier entries start with the URL but may carry a trailing note + * ("https://example.com/x — explains Y"); the leading whitespace-delimited token is taken so those + * lines still yield their host. + */ +object SourceListHosts { + + /** + * The distinct lower-cased hostnames drawn from [urls]. Deterministic; unparseable/relative + * entries are skipped. Duplicate hosts (and differing-case or differing-port variants of the + * same host) collapse to one. + */ + fun extract(urls: List): Set = + urls.mapNotNullTo(LinkedHashSet()) { hostOf(it) } + + private fun hostOf(raw: String): String? { + val token = raw.trim().substringBefore(' ').substringBefore('\t') + if (!token.contains("://")) return null + return runCatching { URI(token).host?.lowercase() }.getOrNull()?.takeIf { it.isNotEmpty() } + } +} diff --git a/apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt b/apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt index 0e0c693b..08dbdf4d 100644 --- a/apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt +++ b/apps/server/src/main/kotlin/com/correx/apps/server/routes/SessionRoutes.kt @@ -2,6 +2,8 @@ package com.correx.apps.server.routes import com.correx.apps.server.ServerModule import com.correx.apps.server.metrics.MetricsInspectionService +import com.correx.apps.server.research.approveSourceList +import com.correx.core.events.EventDispatcher import com.correx.apps.server.protocol.SessionConfigDto import com.correx.apps.server.replay.ReplayInspectionService import com.correx.apps.server.serialization.payloadDiscriminator @@ -56,6 +58,12 @@ data class SessionStateResponse(val sessionId: String, val status: String, val c @Serializable data class StartSessionResponse(val sessionId: String) +@Serializable +data class ApproveSourcesRequest(val urls: List) + +@Serializable +data class ApproveSourcesResponse(val grantedHosts: List) + private val log = LoggerFactory.getLogger("com.correx.apps.server.routes.SessionRoutes") fun Route.sessionRoutes(module: ServerModule) { @@ -79,6 +87,7 @@ fun Route.sessionRoutes(module: ServerModule) { route("/{id}") { getSessionRoute(module) cancelSessionRoute(module) + approveSourcesRoute(module) undoSessionRoute(module) resumeSessionRoute(module) getEventsRoute(module) @@ -129,6 +138,24 @@ private fun Route.cancelSessionRoute(module: ServerModule) { } } +// Batch fetch-approval at the source-list level (BACKLOG §D): the operator approves a research +// source list once and egress is granted for all its hosts in one EgressHostsGrantedEvent, so the +// session's subsequent web_fetches to those hosts auto-clear instead of prompting per URL. Body +// carries the source list's URLs (or its source_dossier source lines). +private fun Route.approveSourcesRoute(module: ServerModule) { + post("/approve-sources") { + val id = call.parameters["id"] + ?: return@post call.respond(HttpStatusCode.BadRequest, "Missing session id") + val body = call.receive() + val granted = approveSourceList( + dispatcher = EventDispatcher(module.eventStore), + sessionId = TypeId(id), + urls = body.urls, + ) + call.respond(HttpStatusCode.OK, ApproveSourcesResponse(grantedHosts = granted.sorted())) + } +} + private fun Route.undoSessionRoute(module: ServerModule) { post("/undo") { val id = call.parameters["id"] diff --git a/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListApprovalTest.kt b/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListApprovalTest.kt new file mode 100644 index 00000000..d0733a9a --- /dev/null +++ b/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListApprovalTest.kt @@ -0,0 +1,63 @@ +package com.correx.apps.server.research + +import com.correx.core.events.EventDispatcher +import com.correx.core.events.events.EgressHostsGrantedEvent +import com.correx.core.events.types.SessionId +import com.correx.infrastructure.persistence.InMemoryEventStore +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SourceListApprovalTest { + + private val sessionId = SessionId("s1") + + private fun grants(store: InMemoryEventStore, session: SessionId = sessionId): List = + store.read(session).map { it.payload }.filterIsInstance() + + @Test + fun `emits exactly one EgressHostsGrantedEvent with the extracted hosts for the session`() = runTest { + val store = InMemoryEventStore() + val granted = approveSourceList( + dispatcher = EventDispatcher(store), + sessionId = sessionId, + urls = listOf( + "https://example.com/a", + "https://example.com/b", + "https://docs.kotlinlang.org/spec", + ), + ) + + assertEquals(setOf("example.com", "docs.kotlinlang.org"), granted) + + val emitted = grants(store) + assertEquals(1, emitted.size) + val event = emitted.single() + assertEquals(sessionId, event.sessionId) + assertEquals(setOf("example.com", "docs.kotlinlang.org"), event.hosts) + assertEquals(BATCH_SOURCE_LIST_REASON, event.reason) + } + + @Test + fun `grant is recorded under the right session`() = runTest { + val store = InMemoryEventStore() + approveSourceList(EventDispatcher(store), sessionId, listOf("https://only-mine.com/x")) + + assertEquals(1, grants(store).size) + assertTrue(grants(store, SessionId("other")).isEmpty()) + } + + @Test + fun `no parseable host emits no event`() = runTest { + val store = InMemoryEventStore() + val granted = approveSourceList( + dispatcher = EventDispatcher(store), + sessionId = sessionId, + urls = listOf("/relative", "just text", ""), + ) + + assertTrue(granted.isEmpty()) + assertTrue(grants(store).isEmpty()) + } +} diff --git a/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListHostsTest.kt b/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListHostsTest.kt new file mode 100644 index 00000000..bd0e661d --- /dev/null +++ b/apps/server/src/test/kotlin/com/correx/apps/server/research/SourceListHostsTest.kt @@ -0,0 +1,69 @@ +package com.correx.apps.server.research + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SourceListHostsTest { + + @Test + fun `distinct hosts are extracted from multiple urls`() { + val hosts = SourceListHosts.extract( + listOf( + "https://example.com/a", + "https://docs.rust-lang.org/book", + "http://api.github.com/repos", + ), + ) + assertEquals(setOf("example.com", "docs.rust-lang.org", "api.github.com"), hosts) + } + + @Test + fun `duplicate hosts collapse`() { + val hosts = SourceListHosts.extract( + listOf( + "https://example.com/one", + "https://example.com/two", + "http://example.com/three", + ), + ) + assertEquals(setOf("example.com"), hosts) + } + + @Test + fun `ports are stripped`() { + val hosts = SourceListHosts.extract(listOf("https://example.com:8443/path")) + assertEquals(setOf("example.com"), hosts) + } + + @Test + fun `host case is normalised to lower case`() { + val hosts = SourceListHosts.extract( + listOf("https://Example.COM/a", "https://EXAMPLE.com/b"), + ) + assertEquals(setOf("example.com"), hosts) + } + + @Test + fun `unparseable and relative entries are ignored`() { + val hosts = SourceListHosts.extract( + listOf( + "/local/path", + "not a url at all", + "ftp ://broken", + "", + "https://kept.com/x", + ), + ) + assertEquals(setOf("kept.com"), hosts) + } + + @Test + fun `dossier source lines yield their leading url host`() { + val hosts = SourceListHosts.extract( + listOf("https://example.com/x — explains Y; bears on sub-question 2"), + ) + assertEquals(setOf("example.com"), hosts) + assertTrue("example.com" in hosts) + } +}