From 8d815d63dafc42078ca94966b2941ed9af8be23b Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 13 Jun 2026 22:53:21 +0400 Subject: [PATCH] =?UTF-8?q?feat(research):=20WebFetchTool=20=E2=80=94=20bo?= =?UTF-8?q?unded=20network=20fetch=20=E2=86=92=20extract=20(research-workf?= =?UTF-8?q?low=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first tool that punches through the sandbox to the network. T2, declares NETWORK_ACCESS with url marked NETWORK_TARGET, so egress is enforced by the harness (NetworkHostRule, plane 2) — not tool courtesy. Header-first guards (§3): content-type checked before the body is read (binary/media rejected at the header via the extractor's shared content-type policy); declared Content-Length capped; the read itself is bounded so a server lying about length can't blow the cap (default 10 MB). On success returns the extracted markdown plus metadata for citation lineage + replay: url, content_sha256 (over the raw bytes), fetched_bytes, extractor_version, quality. Consumes slice-1's HtmlMarkdownExtractor. Tested with Ktor MockEngine. NOT yet registered in the live tool registry: with an empty networkAllowedHosts the host rule allows everything, so exposing a fetch tool before the egress allowlist (SearXNG endpoint + approved-source hosts, per-session) is wired would be an open-egress hole. Registration + egress config land with the research workflow slice. --- infrastructure/research/build.gradle | 10 ++ .../research/extract/HtmlMarkdownExtractor.kt | 7 + .../research/web/WebFetchTool.kt | 147 ++++++++++++++++++ .../research/web/WebFetchToolTest.kt | Bin 0 -> 5258 bytes 4 files changed, 164 insertions(+) create mode 100644 infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt create mode 100644 infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt diff --git a/infrastructure/research/build.gradle b/infrastructure/research/build.gradle index cdef5d3d..fa460a0f 100644 --- a/infrastructure/research/build.gradle +++ b/infrastructure/research/build.gradle @@ -4,9 +4,19 @@ plugins { id 'org.jetbrains.kotlin.plugin.serialization' } +ext { + ktor_version = '3.0.3' +} + dependencies { implementation project(":core:events") + implementation project(":core:tools") + implementation project(":core:approvals") implementation 'org.jsoup:jsoup:1.20.1' + implementation "io.ktor:ktor-client-core:$ktor_version" + implementation "io.ktor:ktor-client-cio:$ktor_version" + + testImplementation "io.ktor:ktor-client-mock:$ktor_version" } // Pure deterministic extraction utility; coverage gate is enforced via its own thorough diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt index 3966c248..11b5f540 100644 --- a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt @@ -38,6 +38,13 @@ class HtmlMarkdownExtractor( ) } + /** + * Whether a body of this content-type can be extracted at all — the header check a fetcher runs + * before downloading the body (§3: "binary/media rejected at the header"). Single source of + * truth for the content-type policy, shared with the fetch tool. + */ + fun supports(contentType: String?): Boolean = classify(contentType) != ContentKind.UNSUPPORTED + private fun classify(contentType: String?): ContentKind { val type = contentType?.substringBefore(';')?.trim()?.lowercase() return when { diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt new file mode 100644 index 00000000..a68a5426 --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt @@ -0,0 +1,147 @@ +package com.correx.infrastructure.research.web + +import com.correx.core.approvals.Tier +import com.correx.core.events.events.ToolRequest +import com.correx.core.events.types.ToolInvocationId +import com.correx.core.tools.contract.ParamRole +import com.correx.core.tools.contract.Tool +import com.correx.core.tools.contract.ToolCapability +import com.correx.core.tools.contract.ToolExecutor +import com.correx.core.tools.contract.ToolResult +import com.correx.core.tools.contract.ValidationResult +import com.correx.infrastructure.research.extract.ExtractionQuality +import com.correx.infrastructure.research.extract.HtmlMarkdownExtractor +import io.ktor.client.HttpClient +import io.ktor.client.request.prepareGet +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.HttpHeaders +import io.ktor.http.isSuccess +import io.ktor.utils.io.readAvailable +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonObject +import java.security.MessageDigest + +/** + * Fetches an external URL and returns its main content as clean markdown + * (research-workflow-spec §3/§5). The first tool that punches through the sandbox to the network, + * so it is [Tier.T2] and declares [ToolCapability.NETWORK_ACCESS] with the URL marked + * [ParamRole.NETWORK_TARGET] — egress is enforced by the harness ([NetworkHostRule], plane 2), + * not by this tool's courtesy. + * + * Header-first guards (§3): content-type is checked before the body is read (binary/media rejected + * at the header) and the declared size is capped; the read itself is bounded so a server that lies + * about Content-Length cannot blow the cap. Extraction (§5) runs before anything leaves the tool; + * the result carries the content hash + pinned extractor version for citation lineage and replay. + */ +class WebFetchTool( + private val httpClient: HttpClient, + private val extractor: HtmlMarkdownExtractor = HtmlMarkdownExtractor(), + private val maxBytes: Long = DEFAULT_MAX_BYTES, +) : Tool, ToolExecutor { + + override val name: String = "web_fetch" + override val description: String = "Fetch a URL and return its main content as clean markdown." + override val tier: Tier = Tier.T2 + override val requiredCapabilities: Set = setOf(ToolCapability.NETWORK_ACCESS) + override val paramRoles: Map = mapOf("url" to ParamRole.NETWORK_TARGET) + override val parametersSchema: JsonObject = buildJsonObject { + put("type", "object") + putJsonObject("properties") { + putJsonObject("url") { + put("type", "string") + put("description", "Absolute http(s) URL to fetch.") + } + } + put("required", buildJsonArray { add(JsonPrimitive("url")) }) + } + + override fun validateRequest(request: ToolRequest): ValidationResult { + val url = request.parameters["url"] as? String + return when { + url.isNullOrBlank() -> ValidationResult.Invalid("Missing 'url' parameter.") + !isHttpUrl(url) -> ValidationResult.Invalid("Only http(s) URLs are supported: $url") + else -> ValidationResult.Valid + } + } + + override suspend fun execute(request: ToolRequest): ToolResult { + val invalid = validateRequest(request) as? ValidationResult.Invalid + if (invalid != null) return ToolResult.Failure(request.invocationId, invalid.reason, recoverable = false) + val url = request.parameters["url"] as String + return runCatching { fetch(request.invocationId, url) } + .getOrElse { ToolResult.Failure(request.invocationId, "fetch failed: ${it.message}", recoverable = true) } + } + + private suspend fun fetch(invocationId: ToolInvocationId, url: String): ToolResult = + httpClient.prepareGet(url).execute { response -> + val contentType = response.headers[HttpHeaders.ContentType] + val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + when { + !response.status.isSuccess() -> + fail(invocationId, "HTTP ${response.status.value} for $url", recoverable = true) + declaredLength != null && declaredLength > maxBytes -> + fail(invocationId, "content too large: $declaredLength bytes (cap $maxBytes)") + !extractor.supports(contentType) -> + fail(invocationId, "unsupported content-type: ${contentType ?: "unknown"}") + else -> readAndExtract(invocationId, url, contentType, response.bodyAsChannel()) + } + } + + private suspend fun readAndExtract( + invocationId: ToolInvocationId, + url: String, + contentType: String?, + channel: io.ktor.utils.io.ByteReadChannel, + ): ToolResult { + val bytes = readBounded(channel) + ?: return fail(invocationId, "content exceeded cap of $maxBytes bytes") + val result = extractor.extract(bytes.decodeToString(), contentType) + return if (result.quality == ExtractionQuality.REJECTED) { + fail(invocationId, result.reason ?: "rejected content") + } else { + ToolResult.Success( + invocationId = invocationId, + output = result.markdown, + metadata = mapOf( + "url" to url, + "content_type" to (contentType ?: ""), + "content_sha256" to sha256(bytes), + "fetched_bytes" to bytes.size.toString(), + "extractor_version" to result.extractorVersion, + "quality" to result.quality.name, + ), + ) + } + } + + private fun fail(invocationId: ToolInvocationId, reason: String, recoverable: Boolean = false): ToolResult.Failure = + ToolResult.Failure(invocationId, reason, recoverable) + + /** Reads the channel up to [maxBytes]; returns null if the body exceeds the cap. */ + private suspend fun readBounded(channel: io.ktor.utils.io.ByteReadChannel): ByteArray? { + val out = ArrayList() + val buffer = ByteArray(READ_CHUNK) + while (true) { + val read = channel.readAvailable(buffer, 0, buffer.size) + if (read == -1) break + if (out.size + read > maxBytes) return null + for (i in 0 until read) out.add(buffer[i]) + } + return out.toByteArray() + } + + private fun isHttpUrl(url: String): Boolean = + runCatching { java.net.URI(url).scheme?.lowercase() in setOf("http", "https") }.getOrDefault(false) + + private fun sha256(bytes: ByteArray): String = + MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + + companion object { + const val DEFAULT_MAX_BYTES: Long = 10_000_000 // 10 MB — a 200MB "page" is an attack or a mistake + private const val READ_CHUNK = 8_192 + } +} diff --git a/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt b/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..7c0e21c2388bcb5f51d6b4a1f96e5de22bd2028a GIT binary patch literal 5258 zcmc&&ZExE+628y*6@+#Vkg*P(_OACKaMMe($+c*^ImF)WE*6I@Eio2rnN&$Cv9X)~ ze$P;EmK~=la4!ZD%hGU2KJ&~oLlq)fh`D5mS@OhKD_30U8!Mc*W#UUKxs^@|n`Hb> z&JGULvM|dc`LPdy#AURlV8ctrF4GZ01p#~b!l%!;`u*? z70WA=%O~2?3}>PcGnFg9{)$0Y(v|sBSM`&~RVutP`tf0#^*|}h*$=@*V1uY>eo3IuJpWdWbB-ODz)l?qpJ0aFc(K&LI!K zS93BsNOEkNeU!5c>646jOhei42L}wF!m3;0CA%p#qZGL?HVzNy;moA#V>a;?5D(kC z*b;(s%!pxKpxDc;8~pNJH-_>KtR%)Z4ly9|a^**vU*^${>3i_gXUFWl(eqcZ)n%S{ zMoY1Jz4j8TpyznJxj+8>`Nipv@2A%vPJh0B{mb-xQVkp&_sjqRZIV8a8&>55Ty@E? zVbNG+6~STFr+Pdf;mX%rLueT8OP$69S3QQGb#M$JfGswIc11LhKJ~*2jaNLEh}26$5iE*=cneb4VbpHY6s{n_fYj_Vn*LkjSD+`22f3@WJLua)hYI}Wy*%Uf^y`Z%M~=n%XlLYfm~tC--99%XGu$bOC^K{oMKw~ zJ7X8jqnfj%Eul>4lzCz2V06IdH|)Kiku)dPOJfYkwh&C9Mxq!XsuIaDC?z#6G6gs6A~koK-6@~3Ot_4J zSOR^6*u@aN6zGKDUrA_O_%&o!780VLLi0>H&xm13j?Rqv4AfIvEz4!2JcoG@Vmv}H zGn1#BMF58!#DDDoAq6G9psB{R_>(dHJQ)jAUN}D)XU|Ti%I9)C!d1k+Yl%jbq1b*K zR!0FIkJ?dM+@9c%p{g&&BlzsjQBVLlNavX9+!;q;pp}2-K>*a*gd3&MNI^c z*|)#LPGm*+n&FDN?%wW2JUwK}QoLNK*_w$_EoZBuG{Dc2jK+`Jz9~looc4{ng&67Ezw4 zC924%(DP`EoHs({I4^yHoEMeVlgU|oMe+(~^ah6>*zh53m^jrRv`zYooSjW#5NI*W zWo1~`@GXgJaK6i=W_6k9kgnxMt`aFz7XU}mcskmY>mAfpKLj9O9QAsQXGi~E$WhZ6 z%Hw=5#*afMDy{VyDEURCW{Mi*Xw}O0v!kOfkdMixn(qf~G-BrvHq`W)#StfCaTCg> zViZ;;nM@YUz{Zq1MLvOh>3}K61d;)-2ojH5ly(pY-jnj0RR60SsDr`&_+yLm8+igA z?n{Y>!$so6q_qH`?xMravF#>Ln#rEdZ*7p68(ldaD<Rf+r&bpi4aQn4TF z|2D4e*1Bmo3S=i0TN?D@=r3P~EqF_>O3*4n*wNVF)e;Wcp=ENQ-BRvC`s99cqZ#;V z>x~S5tHaEF)g{loSy6KoX}V-OxehkuW~br!56ltW_yV2%YP3cz*+#yu`8(5u8+DVq zZ}IJ+!7S*!(0YI1cvklmIFnZgPaG_&i;Y7?@7)AP@83WjN0T?Fhv-S70Z+r5i`d}^ qONabZYsDI;SpLZE>=ON`@^sX_KlnF98Of#q literal 0 HcmV?d00001