diff --git a/infrastructure/research/build.gradle b/infrastructure/research/build.gradle index cdef5d3d..fa460a0f 100644 --- a/infrastructure/research/build.gradle +++ b/infrastructure/research/build.gradle @@ -4,9 +4,19 @@ plugins { id 'org.jetbrains.kotlin.plugin.serialization' } +ext { + ktor_version = '3.0.3' +} + dependencies { implementation project(":core:events") + implementation project(":core:tools") + implementation project(":core:approvals") implementation 'org.jsoup:jsoup:1.20.1' + implementation "io.ktor:ktor-client-core:$ktor_version" + implementation "io.ktor:ktor-client-cio:$ktor_version" + + testImplementation "io.ktor:ktor-client-mock:$ktor_version" } // Pure deterministic extraction utility; coverage gate is enforced via its own thorough diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt index 3966c248..11b5f540 100644 --- a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt @@ -38,6 +38,13 @@ class HtmlMarkdownExtractor( ) } + /** + * Whether a body of this content-type can be extracted at all — the header check a fetcher runs + * before downloading the body (§3: "binary/media rejected at the header"). Single source of + * truth for the content-type policy, shared with the fetch tool. + */ + fun supports(contentType: String?): Boolean = classify(contentType) != ContentKind.UNSUPPORTED + private fun classify(contentType: String?): ContentKind { val type = contentType?.substringBefore(';')?.trim()?.lowercase() return when { diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt new file mode 100644 index 00000000..a68a5426 --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/web/WebFetchTool.kt @@ -0,0 +1,147 @@ +package com.correx.infrastructure.research.web + +import com.correx.core.approvals.Tier +import com.correx.core.events.events.ToolRequest +import com.correx.core.events.types.ToolInvocationId +import com.correx.core.tools.contract.ParamRole +import com.correx.core.tools.contract.Tool +import com.correx.core.tools.contract.ToolCapability +import com.correx.core.tools.contract.ToolExecutor +import com.correx.core.tools.contract.ToolResult +import com.correx.core.tools.contract.ValidationResult +import com.correx.infrastructure.research.extract.ExtractionQuality +import com.correx.infrastructure.research.extract.HtmlMarkdownExtractor +import io.ktor.client.HttpClient +import io.ktor.client.request.prepareGet +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.HttpHeaders +import io.ktor.http.isSuccess +import io.ktor.utils.io.readAvailable +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonObject +import java.security.MessageDigest + +/** + * Fetches an external URL and returns its main content as clean markdown + * (research-workflow-spec §3/§5). The first tool that punches through the sandbox to the network, + * so it is [Tier.T2] and declares [ToolCapability.NETWORK_ACCESS] with the URL marked + * [ParamRole.NETWORK_TARGET] — egress is enforced by the harness ([NetworkHostRule], plane 2), + * not by this tool's courtesy. + * + * Header-first guards (§3): content-type is checked before the body is read (binary/media rejected + * at the header) and the declared size is capped; the read itself is bounded so a server that lies + * about Content-Length cannot blow the cap. Extraction (§5) runs before anything leaves the tool; + * the result carries the content hash + pinned extractor version for citation lineage and replay. + */ +class WebFetchTool( + private val httpClient: HttpClient, + private val extractor: HtmlMarkdownExtractor = HtmlMarkdownExtractor(), + private val maxBytes: Long = DEFAULT_MAX_BYTES, +) : Tool, ToolExecutor { + + override val name: String = "web_fetch" + override val description: String = "Fetch a URL and return its main content as clean markdown." + override val tier: Tier = Tier.T2 + override val requiredCapabilities: Set = setOf(ToolCapability.NETWORK_ACCESS) + override val paramRoles: Map = mapOf("url" to ParamRole.NETWORK_TARGET) + override val parametersSchema: JsonObject = buildJsonObject { + put("type", "object") + putJsonObject("properties") { + putJsonObject("url") { + put("type", "string") + put("description", "Absolute http(s) URL to fetch.") + } + } + put("required", buildJsonArray { add(JsonPrimitive("url")) }) + } + + override fun validateRequest(request: ToolRequest): ValidationResult { + val url = request.parameters["url"] as? String + return when { + url.isNullOrBlank() -> ValidationResult.Invalid("Missing 'url' parameter.") + !isHttpUrl(url) -> ValidationResult.Invalid("Only http(s) URLs are supported: $url") + else -> ValidationResult.Valid + } + } + + override suspend fun execute(request: ToolRequest): ToolResult { + val invalid = validateRequest(request) as? ValidationResult.Invalid + if (invalid != null) return ToolResult.Failure(request.invocationId, invalid.reason, recoverable = false) + val url = request.parameters["url"] as String + return runCatching { fetch(request.invocationId, url) } + .getOrElse { ToolResult.Failure(request.invocationId, "fetch failed: ${it.message}", recoverable = true) } + } + + private suspend fun fetch(invocationId: ToolInvocationId, url: String): ToolResult = + httpClient.prepareGet(url).execute { response -> + val contentType = response.headers[HttpHeaders.ContentType] + val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + when { + !response.status.isSuccess() -> + fail(invocationId, "HTTP ${response.status.value} for $url", recoverable = true) + declaredLength != null && declaredLength > maxBytes -> + fail(invocationId, "content too large: $declaredLength bytes (cap $maxBytes)") + !extractor.supports(contentType) -> + fail(invocationId, "unsupported content-type: ${contentType ?: "unknown"}") + else -> readAndExtract(invocationId, url, contentType, response.bodyAsChannel()) + } + } + + private suspend fun readAndExtract( + invocationId: ToolInvocationId, + url: String, + contentType: String?, + channel: io.ktor.utils.io.ByteReadChannel, + ): ToolResult { + val bytes = readBounded(channel) + ?: return fail(invocationId, "content exceeded cap of $maxBytes bytes") + val result = extractor.extract(bytes.decodeToString(), contentType) + return if (result.quality == ExtractionQuality.REJECTED) { + fail(invocationId, result.reason ?: "rejected content") + } else { + ToolResult.Success( + invocationId = invocationId, + output = result.markdown, + metadata = mapOf( + "url" to url, + "content_type" to (contentType ?: ""), + "content_sha256" to sha256(bytes), + "fetched_bytes" to bytes.size.toString(), + "extractor_version" to result.extractorVersion, + "quality" to result.quality.name, + ), + ) + } + } + + private fun fail(invocationId: ToolInvocationId, reason: String, recoverable: Boolean = false): ToolResult.Failure = + ToolResult.Failure(invocationId, reason, recoverable) + + /** Reads the channel up to [maxBytes]; returns null if the body exceeds the cap. */ + private suspend fun readBounded(channel: io.ktor.utils.io.ByteReadChannel): ByteArray? { + val out = ArrayList() + val buffer = ByteArray(READ_CHUNK) + while (true) { + val read = channel.readAvailable(buffer, 0, buffer.size) + if (read == -1) break + if (out.size + read > maxBytes) return null + for (i in 0 until read) out.add(buffer[i]) + } + return out.toByteArray() + } + + private fun isHttpUrl(url: String): Boolean = + runCatching { java.net.URI(url).scheme?.lowercase() in setOf("http", "https") }.getOrDefault(false) + + private fun sha256(bytes: ByteArray): String = + MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + + companion object { + const val DEFAULT_MAX_BYTES: Long = 10_000_000 // 10 MB — a 200MB "page" is an attack or a mistake + private const val READ_CHUNK = 8_192 + } +} diff --git a/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt b/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt new file mode 100644 index 00000000..7c0e21c2 Binary files /dev/null and b/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/web/WebFetchToolTest.kt differ