feat(research): WebFetchTool — bounded network fetch → extract (research-workflow §3)

The first tool that punches through the sandbox to the network. T2, declares NETWORK_ACCESS with url marked NETWORK_TARGET, so egress is enforced by the harness (NetworkHostRule, plane 2) — not tool courtesy. Header-first guards (§3): content-type checked before the body is read (binary/media rejected at the header via the extractor's shared content-type policy); declared Content-Length capped; the read itself is bounded so a server lying about length can't blow the cap (default 10 MB). On success returns the extracted markdown plus metadata for citation lineage + replay: url, content_sha256 (over the raw bytes), fetched_bytes, extractor_version, quality. Consumes slice-1's HtmlMarkdownExtractor. Tested with Ktor MockEngine. NOT yet registered in the live tool registry: with an empty networkAllowedHosts the host rule allows everything, so exposing a fetch tool before the egress allowlist (SearXNG endpoint + approved-source hosts, per-session) is wired would be an open-egress hole. Registration + egress config land with the research workflow slice.
2026-06-13 22:53:21 +04:00
parent fb1d97058a
commit 8d815d63da
4 changed files with 164 additions and 0 deletions
@@ -4,9 +4,19 @@ plugins {
    id 'org.jetbrains.kotlin.plugin.serialization'
 }

+ext {
+    ktor_version = '3.0.3'
+}
+
 dependencies {
    implementation project(":core:events")
+    implementation project(":core:tools")
+    implementation project(":core:approvals")
    implementation 'org.jsoup:jsoup:1.20.1'
+    implementation "io.ktor:ktor-client-core:$ktor_version"
+    implementation "io.ktor:ktor-client-cio:$ktor_version"
+
+    testImplementation "io.ktor:ktor-client-mock:$ktor_version"
 }

 // Pure deterministic extraction utility; coverage gate is enforced via its own thorough
@@ -38,6 +38,13 @@ class HtmlMarkdownExtractor(
        )
    }

+    /**
+     * Whether a body of this content-type can be extracted at all — the header check a fetcher runs
+     * before downloading the body (§3: "binary/media rejected at the header"). Single source of
+     * truth for the content-type policy, shared with the fetch tool.
+     */
+    fun supports(contentType: String?): Boolean = classify(contentType) != ContentKind.UNSUPPORTED
+
    private fun classify(contentType: String?): ContentKind {
        val type = contentType?.substringBefore(';')?.trim()?.lowercase()
        return when {
@@ -0,0 +1,147 @@
+package com.correx.infrastructure.research.web
+
+import com.correx.core.approvals.Tier
+import com.correx.core.events.events.ToolRequest
+import com.correx.core.events.types.ToolInvocationId
+import com.correx.core.tools.contract.ParamRole
+import com.correx.core.tools.contract.Tool
+import com.correx.core.tools.contract.ToolCapability
+import com.correx.core.tools.contract.ToolExecutor
+import com.correx.core.tools.contract.ToolResult
+import com.correx.core.tools.contract.ValidationResult
+import com.correx.infrastructure.research.extract.ExtractionQuality
+import com.correx.infrastructure.research.extract.HtmlMarkdownExtractor
+import io.ktor.client.HttpClient
+import io.ktor.client.request.prepareGet
+import io.ktor.client.statement.bodyAsChannel
+import io.ktor.http.HttpHeaders
+import io.ktor.http.isSuccess
+import io.ktor.utils.io.readAvailable
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonArray
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import kotlinx.serialization.json.putJsonObject
+import java.security.MessageDigest
+
+/**
+ * Fetches an external URL and returns its main content as clean markdown
+ * (research-workflow-spec §3/§5). The first tool that punches through the sandbox to the network,
+ * so it is [Tier.T2] and declares [ToolCapability.NETWORK_ACCESS] with the URL marked
+ * [ParamRole.NETWORK_TARGET] — egress is enforced by the harness ([NetworkHostRule], plane 2),
+ * not by this tool's courtesy.
+ *
+ * Header-first guards (§3): content-type is checked before the body is read (binary/media rejected
+ * at the header) and the declared size is capped; the read itself is bounded so a server that lies
+ * about Content-Length cannot blow the cap. Extraction (§5) runs before anything leaves the tool;
+ * the result carries the content hash + pinned extractor version for citation lineage and replay.
+ */
+class WebFetchTool(
+    private val httpClient: HttpClient,
+    private val extractor: HtmlMarkdownExtractor = HtmlMarkdownExtractor(),
+    private val maxBytes: Long = DEFAULT_MAX_BYTES,
+) : Tool, ToolExecutor {
+
+    override val name: String = "web_fetch"
+    override val description: String = "Fetch a URL and return its main content as clean markdown."
+    override val tier: Tier = Tier.T2
+    override val requiredCapabilities: Set<ToolCapability> = setOf(ToolCapability.NETWORK_ACCESS)
+    override val paramRoles: Map<String, ParamRole> = mapOf("url" to ParamRole.NETWORK_TARGET)
+    override val parametersSchema: JsonObject = buildJsonObject {
+        put("type", "object")
+        putJsonObject("properties") {
+            putJsonObject("url") {
+                put("type", "string")
+                put("description", "Absolute http(s) URL to fetch.")
+            }
+        }
+        put("required", buildJsonArray { add(JsonPrimitive("url")) })
+    }
+
+    override fun validateRequest(request: ToolRequest): ValidationResult {
+        val url = request.parameters["url"] as? String
+        return when {
+            url.isNullOrBlank() -> ValidationResult.Invalid("Missing 'url' parameter.")
+            !isHttpUrl(url) -> ValidationResult.Invalid("Only http(s) URLs are supported: $url")
+            else -> ValidationResult.Valid
+        }
+    }
+
+    override suspend fun execute(request: ToolRequest): ToolResult {
+        val invalid = validateRequest(request) as? ValidationResult.Invalid
+        if (invalid != null) return ToolResult.Failure(request.invocationId, invalid.reason, recoverable = false)
+        val url = request.parameters["url"] as String
+        return runCatching { fetch(request.invocationId, url) }
+            .getOrElse { ToolResult.Failure(request.invocationId, "fetch failed: ${it.message}", recoverable = true) }
+    }
+
+    private suspend fun fetch(invocationId: ToolInvocationId, url: String): ToolResult =
+        httpClient.prepareGet(url).execute { response ->
+            val contentType = response.headers[HttpHeaders.ContentType]
+            val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
+            when {
+                !response.status.isSuccess() ->
+                    fail(invocationId, "HTTP ${response.status.value} for $url", recoverable = true)
+                declaredLength != null && declaredLength > maxBytes ->
+                    fail(invocationId, "content too large: $declaredLength bytes (cap $maxBytes)")
+                !extractor.supports(contentType) ->
+                    fail(invocationId, "unsupported content-type: ${contentType ?: "unknown"}")
+                else -> readAndExtract(invocationId, url, contentType, response.bodyAsChannel())
+            }
+        }
+
+    private suspend fun readAndExtract(
+        invocationId: ToolInvocationId,
+        url: String,
+        contentType: String?,
+        channel: io.ktor.utils.io.ByteReadChannel,
+    ): ToolResult {
+        val bytes = readBounded(channel)
+            ?: return fail(invocationId, "content exceeded cap of $maxBytes bytes")
+        val result = extractor.extract(bytes.decodeToString(), contentType)
+        return if (result.quality == ExtractionQuality.REJECTED) {
+            fail(invocationId, result.reason ?: "rejected content")
+        } else {
+            ToolResult.Success(
+                invocationId = invocationId,
+                output = result.markdown,
+                metadata = mapOf(
+                    "url" to url,
+                    "content_type" to (contentType ?: ""),
+                    "content_sha256" to sha256(bytes),
+                    "fetched_bytes" to bytes.size.toString(),
+                    "extractor_version" to result.extractorVersion,
+                    "quality" to result.quality.name,
+                ),
+            )
+        }
+    }
+
+    private fun fail(invocationId: ToolInvocationId, reason: String, recoverable: Boolean = false): ToolResult.Failure =
+        ToolResult.Failure(invocationId, reason, recoverable)
+
+    /** Reads the channel up to [maxBytes]; returns null if the body exceeds the cap. */
+    private suspend fun readBounded(channel: io.ktor.utils.io.ByteReadChannel): ByteArray? {
+        val out = ArrayList<Byte>()
+        val buffer = ByteArray(READ_CHUNK)
+        while (true) {
+            val read = channel.readAvailable(buffer, 0, buffer.size)
+            if (read == -1) break
+            if (out.size + read > maxBytes) return null
+            for (i in 0 until read) out.add(buffer[i])
+        }
+        return out.toByteArray()
+    }
+
+    private fun isHttpUrl(url: String): Boolean =
+        runCatching { java.net.URI(url).scheme?.lowercase() in setOf("http", "https") }.getOrDefault(false)
+
+    private fun sha256(bytes: ByteArray): String =
+        MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) }
+
+    companion object {
+        const val DEFAULT_MAX_BYTES: Long = 10_000_000 // 10 MB — a 200MB "page" is an attack or a mistake
+        private const val READ_CHUNK = 8_192
+    }
+}