refactor(tools): fold research tools into :infrastructure:tools (drop new module)
The research feature is "no new architecture" (spec §1) — composition of existing
mechanisms. The only genuinely new code is the two web tools + the HTML→markdown
extractor (spec marks them "(new)"), and those belong with ShellTool/FileReadTool
in :infrastructure:tools, not in a separate module.
Moves extract/ + web/ from the short-lived :infrastructure:research module into
:infrastructure:tools (packages com.correx.infrastructure.tools.{extract,web}); adds
jsoup + ktor-client there; removes the module from settings.gradle. No behaviour change
— 44 tests green.
This commit is contained in:
@@ -4,6 +4,10 @@ plugins {
|
||||
id 'org.jetbrains.kotlin.plugin.serialization'
|
||||
}
|
||||
|
||||
ext {
|
||||
ktor_version = '3.0.3'
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation project(":core:tools")
|
||||
implementation project(":core:events")
|
||||
@@ -12,6 +16,13 @@ dependencies {
|
||||
implementation project(":infrastructure:tools:filesystem")
|
||||
implementation project(":core:artifacts")
|
||||
implementation project(":core:artifacts-store")
|
||||
|
||||
// Web research tools (web_search, web_fetch) + deterministic HTML→markdown extraction.
|
||||
implementation 'org.jsoup:jsoup:1.20.1'
|
||||
implementation "io.ktor:ktor-client-core:$ktor_version"
|
||||
implementation "io.ktor:ktor-client-cio:$ktor_version"
|
||||
|
||||
testImplementation "io.ktor:ktor-client-mock:$ktor_version"
|
||||
}
|
||||
|
||||
tasks.named("koverVerify").configure { enabled = false }
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.correx.infrastructure.tools.extract
|
||||
|
||||
/**
|
||||
* Outcome of extracting a fetched page (research-workflow-spec §5). The cleaned [markdown] is the
|
||||
* canonical fetch result that goes to CAS; raw HTML is discarded. [quality] tells the workflow
|
||||
* whether to route around a dead source ([LOW_QUALITY]) or reject an unsupported body ([REJECTED]).
|
||||
*
|
||||
* [extractorVersion] is pinned into every fetch event: heuristic improvements must never change
|
||||
* what an old event means, so replay reads the recorded artifact and never re-extracts — the same
|
||||
* discipline as the embedding-model hash.
|
||||
*/
|
||||
data class ExtractionResult(
|
||||
val markdown: String,
|
||||
val quality: ExtractionQuality,
|
||||
val extractorVersion: String,
|
||||
val reason: String? = null,
|
||||
)
|
||||
|
||||
enum class ExtractionQuality {
|
||||
/** Usable content of at least the minimum length. */
|
||||
OK,
|
||||
|
||||
/** Extracted, but below the minimum-length threshold — JS-rendered SPA or paywall; route around it. */
|
||||
LOW_QUALITY,
|
||||
|
||||
/** Content-type the deterministic path does not handle (binary/media/pdf) — dropped at the header. */
|
||||
REJECTED,
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package com.correx.infrastructure.tools.extract
|
||||
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
/**
|
||||
* Deterministic HTML → clean markdown extraction (research-workflow-spec §5). Zero inference,
|
||||
* zero network: structural strip + density extraction + content-type dispatch. The cleaned
|
||||
* markdown is the canonical fetch result destined for CAS; [ExtractionResult.extractorVersion]
|
||||
* is pinned so replay reads recorded artifacts and never re-extracts.
|
||||
*
|
||||
* Content-type dispatch (§5.3): HTML is extracted; JSON / plain text pass through unchanged;
|
||||
* everything else (binary, media, pdf) is [ExtractionQuality.REJECTED] at the header. Output below
|
||||
* [minContentLength] is [ExtractionQuality.LOW_QUALITY] so the workflow routes around dead sources
|
||||
* (JS-rendered SPAs, paywalls) instead of feeding empty pages to synthesis.
|
||||
*/
|
||||
class HtmlMarkdownExtractor(
|
||||
private val minContentLength: Int = DEFAULT_MIN_CONTENT_LENGTH,
|
||||
) {
|
||||
fun extract(body: String, contentType: String?): ExtractionResult =
|
||||
when (classify(contentType)) {
|
||||
ContentKind.HTML -> classifyQuality(MarkdownRenderer.render(MainContentSelector.select(Jsoup.parse(body))))
|
||||
ContentKind.PASSTHROUGH -> classifyQuality(body.trim())
|
||||
ContentKind.UNSUPPORTED -> ExtractionResult(
|
||||
markdown = "",
|
||||
quality = ExtractionQuality.REJECTED,
|
||||
extractorVersion = VERSION,
|
||||
reason = "unsupported content-type: ${contentType ?: "unknown"}",
|
||||
)
|
||||
}
|
||||
|
||||
private fun classifyQuality(markdown: String): ExtractionResult {
|
||||
val low = markdown.length < minContentLength
|
||||
return ExtractionResult(
|
||||
markdown = markdown,
|
||||
quality = if (low) ExtractionQuality.LOW_QUALITY else ExtractionQuality.OK,
|
||||
extractorVersion = VERSION,
|
||||
reason = if (low) "extracted ${markdown.length} chars, below minimum $minContentLength" else null,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a body of this content-type can be extracted at all — the header check a fetcher runs
|
||||
* before downloading the body (§3: "binary/media rejected at the header"). Single source of
|
||||
* truth for the content-type policy, shared with the fetch tool.
|
||||
*/
|
||||
fun supports(contentType: String?): Boolean = classify(contentType) != ContentKind.UNSUPPORTED
|
||||
|
||||
private fun classify(contentType: String?): ContentKind {
|
||||
val type = contentType?.substringBefore(';')?.trim()?.lowercase()
|
||||
return when {
|
||||
type.isNullOrBlank() -> ContentKind.HTML // best effort: most fetched bodies are HTML
|
||||
type.contains("json") -> ContentKind.PASSTHROUGH
|
||||
type in HTML_TYPES -> ContentKind.HTML
|
||||
type.startsWith("text/") -> ContentKind.PASSTHROUGH
|
||||
else -> ContentKind.UNSUPPORTED
|
||||
}
|
||||
}
|
||||
|
||||
private enum class ContentKind { HTML, PASSTHROUGH, UNSUPPORTED }
|
||||
|
||||
companion object {
|
||||
/** Bump on any heuristic change. Pinned into fetch events; old artifacts keep their meaning. */
|
||||
const val VERSION = "html-md-1"
|
||||
const val DEFAULT_MIN_CONTENT_LENGTH = 200
|
||||
|
||||
private val HTML_TYPES = setOf(
|
||||
"text/html", "application/xhtml+xml", "application/xml", "text/xml",
|
||||
)
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.correx.infrastructure.tools.extract
|
||||
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.nodes.Element
|
||||
|
||||
/**
|
||||
* Structural strip (§5.1) + density extraction (§5.2). Removes boilerplate by tag and by
|
||||
* class/id blocklist, then picks the main-content subtree by a text-density / link-density score
|
||||
* (readability/trafilatura family). Pure DOM transformation — no inference, no network.
|
||||
*/
|
||||
internal object MainContentSelector {
|
||||
|
||||
/** Whole-tag boilerplate removed outright. */
|
||||
private val STRIP_TAGS = listOf(
|
||||
"script", "style", "noscript", "template", "nav", "footer", "header",
|
||||
"aside", "iframe", "form", "svg", "button", "input", "select", "textarea",
|
||||
)
|
||||
|
||||
/** Substrings matched (case-insensitive) against class/id; a node carrying one is chrome, not content. */
|
||||
private val BLOCKLIST = listOf(
|
||||
"cookie", "banner", "sidebar", "related", "share", "comment", "newsletter",
|
||||
"promo", "ad-", "-ad", "advert", "social", "popup", "modal", "subscribe",
|
||||
"breadcrumb", "pagination", "menu", "nav-", "footer", "header",
|
||||
)
|
||||
|
||||
/** Containers that, when present and substantial, are the main content without scoring. */
|
||||
private val SEMANTIC_MAIN = listOf("article", "main", "[role=main]")
|
||||
|
||||
/** Strips boilerplate in place, then returns the best main-content element (falls back to body). */
|
||||
fun select(document: Document): Element {
|
||||
strip(document)
|
||||
val semantic = SEMANTIC_MAIN.firstNotNullOfOrNull { sel ->
|
||||
document.select(sel).maxByOrNull { textLength(it) }?.takeIf { textLength(it) > 0 }
|
||||
}
|
||||
if (semantic != null) return semantic
|
||||
return bestByDensity(document) ?: document.body() ?: document
|
||||
}
|
||||
|
||||
private fun strip(document: Document) {
|
||||
document.select(STRIP_TAGS.joinToString(", ")).remove()
|
||||
document.allElements
|
||||
.filter { el -> isBlocked(el) }
|
||||
.forEach { it.remove() }
|
||||
}
|
||||
|
||||
private fun isBlocked(element: Element): Boolean {
|
||||
val marker = (element.id() + " " + element.className()).lowercase()
|
||||
if (marker.isBlank()) return false
|
||||
return BLOCKLIST.any { marker.contains(it) }
|
||||
}
|
||||
|
||||
/** Highest score among block containers; score rewards text and penalises link-heavy nodes. */
|
||||
private fun bestByDensity(document: Document): Element? =
|
||||
document.select("div, section, td, article, main")
|
||||
.filter { textLength(it) > 0 }
|
||||
.maxByOrNull { score(it) }
|
||||
|
||||
/**
|
||||
* Density-adjusted text length: `text · (1 − linkDensity)` reduces to `text − linkText`, so a
|
||||
* nav block (text that is almost all link anchors) scores near zero while prose scores high.
|
||||
*/
|
||||
private fun score(element: Element): Int = textLength(element) - linkTextLength(element)
|
||||
|
||||
private fun textLength(element: Element): Int = element.text().length
|
||||
|
||||
private fun linkTextLength(element: Element): Int =
|
||||
element.select("a").sumOf { it.text().length }
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package com.correx.infrastructure.tools.extract
|
||||
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.nodes.Node
|
||||
import org.jsoup.nodes.TextNode
|
||||
|
||||
private val HEADINGS = setOf("h1", "h2", "h3", "h4", "h5", "h6")
|
||||
private val INLINE_PASSTHROUGH = setOf("span", "small", "mark", "sub", "sup", "u", "abbr", "label")
|
||||
private val LIST_TAGS = setOf("ul", "ol")
|
||||
private val WHITESPACE = Regex("[ \\t]+")
|
||||
private val BLANK_RUN = Regex("\n{3,}")
|
||||
|
||||
/**
|
||||
* Converts a cleaned main-content DOM subtree to markdown (§5.2: "Output is markdown, preserving
|
||||
* headings and tables — structure the synthesis model needs"). Block elements become markdown
|
||||
* blocks; inline elements (links, emphasis, code) render in place via [InlineRenderer]. Pure,
|
||||
* deterministic.
|
||||
*/
|
||||
internal object MarkdownRenderer {
|
||||
|
||||
fun render(root: Element): String {
|
||||
val blocks = mutableListOf<String>()
|
||||
appendBlocks(root, blocks)
|
||||
return normalize(blocks)
|
||||
}
|
||||
|
||||
private fun appendBlocks(container: Element, out: MutableList<String>) {
|
||||
for (node in container.childNodes()) {
|
||||
when (node) {
|
||||
is TextNode -> node.text().trim().takeIf { it.isNotEmpty() }?.let(out::add)
|
||||
is Element -> appendElement(node, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("CyclomaticComplexMethod")
|
||||
private fun appendElement(el: Element, out: MutableList<String>) {
|
||||
when (val tag = el.normalName()) {
|
||||
in HEADINGS -> heading(tag, el)?.let(out::add)
|
||||
"p" -> InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add)
|
||||
"pre" -> codeBlock(el)?.let(out::add)
|
||||
"blockquote" -> blockquote(el)?.let(out::add)
|
||||
"ul" -> renderList(el, ordered = false, indent = "")?.let(out::add)
|
||||
"ol" -> renderList(el, ordered = true, indent = "")?.let(out::add)
|
||||
"table" -> TableRenderer.render(el)?.let(out::add)
|
||||
"hr" -> out.add("---")
|
||||
"a", "strong", "b", "em", "i", "code", in INLINE_PASSTHROUGH ->
|
||||
InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add)
|
||||
else -> appendBlocks(el, out) // containers (div/section/article/main/…) recurse
|
||||
}
|
||||
}
|
||||
|
||||
private fun heading(tag: String, el: Element): String? {
|
||||
val text = InlineRenderer.inline(el)
|
||||
if (text.isBlank()) return null
|
||||
return "#".repeat(tag.last().digitToInt()) + " " + text
|
||||
}
|
||||
|
||||
private fun codeBlock(el: Element): String? {
|
||||
val code = el.wholeText().trimEnd('\n', ' ')
|
||||
return if (code.isBlank()) null else "```\n$code\n```"
|
||||
}
|
||||
|
||||
private fun blockquote(el: Element): String? {
|
||||
val inner = InlineRenderer.inline(el).trim()
|
||||
return if (inner.isBlank()) null else inner.lines().joinToString("\n") { "> $it" }
|
||||
}
|
||||
|
||||
private fun renderList(list: Element, ordered: Boolean, indent: String): String? {
|
||||
val items = list.children().filter { it.normalName() == "li" }
|
||||
if (items.isEmpty()) return null
|
||||
val out = mutableListOf<String>()
|
||||
items.forEachIndexed { index, li ->
|
||||
val marker = if (ordered) "${index + 1}. " else "- "
|
||||
InlineRenderer.inlineSkippingLists(li).takeIf { it.isNotBlank() }?.let { out.add(indent + marker + it) }
|
||||
li.children()
|
||||
.filter { it.normalName() in LIST_TAGS }
|
||||
.forEach { nested -> renderList(nested, nested.normalName() == "ol", "$indent ")?.let(out::add) }
|
||||
}
|
||||
return out.takeIf { it.isNotEmpty() }?.joinToString("\n")
|
||||
}
|
||||
|
||||
private fun normalize(blocks: List<String>): String {
|
||||
val deduped = mutableListOf<String>()
|
||||
blocks.map { it.trim() }.filter { it.isNotEmpty() }.forEach { block ->
|
||||
if (deduped.lastOrNull() != block) deduped.add(block) // drop consecutive repeats the strip missed
|
||||
}
|
||||
return deduped.joinToString("\n\n").replace(BLANK_RUN, "\n\n").trim()
|
||||
}
|
||||
}
|
||||
|
||||
/** Markdown table rendering (header row + separator + body), split out to keep each unit focused. */
|
||||
internal object TableRenderer {
|
||||
|
||||
fun render(table: Element): String? {
|
||||
val grid = table.select("tr")
|
||||
.map { tr -> tr.select("th, td").map { cell(it) } }
|
||||
.filter { it.isNotEmpty() }
|
||||
if (grid.isEmpty()) return null
|
||||
val width = grid.maxOf { it.size }
|
||||
return buildString {
|
||||
append(row(grid.first().pad(width))).append('\n')
|
||||
append(row(grid.first().map { "---" }.pad(width, "---"))).append('\n')
|
||||
grid.drop(1).forEach { append(row(it.pad(width))).append('\n') }
|
||||
}.trimEnd()
|
||||
}
|
||||
|
||||
private fun cell(el: Element): String = InlineRenderer.inline(el).replace("|", "\\|").ifBlank { " " }
|
||||
private fun row(cells: List<String>): String = "| " + cells.joinToString(" | ") + " |"
|
||||
private fun List<String>.pad(width: Int, fill: String = " "): List<String> =
|
||||
this + List((width - size).coerceAtLeast(0)) { fill }
|
||||
}
|
||||
|
||||
/** Inline span rendering: links, emphasis, code, line breaks, images. */
|
||||
internal object InlineRenderer {
|
||||
|
||||
fun inline(el: Element): String = collapseSpaces(
|
||||
buildString { for (node in el.childNodes()) append(inlineNode(node)) },
|
||||
).trim()
|
||||
|
||||
/** Inline content of a list item, skipping nested lists (those render as indented sub-lists). */
|
||||
fun inlineSkippingLists(el: Element): String = collapseSpaces(
|
||||
buildString {
|
||||
for (node in el.childNodes()) {
|
||||
if (node is Element && node.normalName() in LIST_TAGS) continue
|
||||
append(inlineNode(node))
|
||||
}
|
||||
},
|
||||
).trim()
|
||||
|
||||
private fun inlineNode(node: Node): String = when (node) {
|
||||
is TextNode -> node.text()
|
||||
is Element -> inlineElement(node)
|
||||
else -> ""
|
||||
}
|
||||
|
||||
private fun inlineElement(el: Element): String = when (el.normalName()) {
|
||||
"a" -> link(el)
|
||||
"strong", "b" -> inline(el).let { if (it.isBlank()) "" else "**$it**" }
|
||||
"em", "i" -> inline(el).let { if (it.isBlank()) "" else "*$it*" }
|
||||
"code" -> el.text().let { if (it.isBlank()) "" else "`$it`" }
|
||||
"br" -> "\n"
|
||||
"img" -> image(el)
|
||||
else -> inline(el)
|
||||
}
|
||||
|
||||
private fun link(el: Element): String {
|
||||
val href = el.attr("href").trim()
|
||||
val text = inline(el)
|
||||
return if (href.isNotBlank() && text.isNotBlank()) "[$text]($href)" else text
|
||||
}
|
||||
|
||||
private fun image(el: Element): String {
|
||||
val src = el.attr("src").trim()
|
||||
return if (src.isBlank()) "" else ""
|
||||
}
|
||||
|
||||
private fun collapseSpaces(text: String): String =
|
||||
text.replace(WHITESPACE, " ").lineSequence().joinToString("\n") { it.trim() }.trim()
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package com.correx.infrastructure.tools.web
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.events.types.ToolInvocationId
|
||||
import com.correx.core.tools.contract.ParamRole
|
||||
import com.correx.core.tools.contract.Tool
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.core.tools.contract.ToolExecutor
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import com.correx.core.tools.contract.ValidationResult
|
||||
import com.correx.infrastructure.tools.extract.ExtractionQuality
|
||||
import com.correx.infrastructure.tools.extract.HtmlMarkdownExtractor
|
||||
import io.ktor.client.HttpClient
|
||||
import io.ktor.client.request.prepareGet
|
||||
import io.ktor.client.statement.bodyAsChannel
|
||||
import io.ktor.http.HttpHeaders
|
||||
import io.ktor.http.isSuccess
|
||||
import io.ktor.utils.io.readAvailable
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
import kotlinx.serialization.json.putJsonObject
|
||||
import java.security.MessageDigest
|
||||
|
||||
/**
|
||||
* Fetches an external URL and returns its main content as clean markdown
|
||||
* (research-workflow-spec §3/§5). The first tool that punches through the sandbox to the network,
|
||||
* so it is [Tier.T2] and declares [ToolCapability.NETWORK_ACCESS] with the URL marked
|
||||
* [ParamRole.NETWORK_TARGET] — egress is enforced by the harness ([NetworkHostRule], plane 2),
|
||||
* not by this tool's courtesy.
|
||||
*
|
||||
* Header-first guards (§3): content-type is checked before the body is read (binary/media rejected
|
||||
* at the header) and the declared size is capped; the read itself is bounded so a server that lies
|
||||
* about Content-Length cannot blow the cap. Extraction (§5) runs before anything leaves the tool;
|
||||
* the result carries the content hash + pinned extractor version for citation lineage and replay.
|
||||
*/
|
||||
class WebFetchTool(
|
||||
private val httpClient: HttpClient,
|
||||
private val extractor: HtmlMarkdownExtractor = HtmlMarkdownExtractor(),
|
||||
private val maxBytes: Long = DEFAULT_MAX_BYTES,
|
||||
) : Tool, ToolExecutor {
|
||||
|
||||
override val name: String = "web_fetch"
|
||||
override val description: String = "Fetch a URL and return its main content as clean markdown."
|
||||
override val tier: Tier = Tier.T2
|
||||
override val requiredCapabilities: Set<ToolCapability> = setOf(ToolCapability.NETWORK_ACCESS)
|
||||
override val paramRoles: Map<String, ParamRole> = mapOf("url" to ParamRole.NETWORK_TARGET)
|
||||
override val parametersSchema: JsonObject = buildJsonObject {
|
||||
put("type", "object")
|
||||
putJsonObject("properties") {
|
||||
putJsonObject("url") {
|
||||
put("type", "string")
|
||||
put("description", "Absolute http(s) URL to fetch.")
|
||||
}
|
||||
}
|
||||
put("required", buildJsonArray { add(JsonPrimitive("url")) })
|
||||
}
|
||||
|
||||
override fun validateRequest(request: ToolRequest): ValidationResult {
|
||||
val url = request.parameters["url"] as? String
|
||||
return when {
|
||||
url.isNullOrBlank() -> ValidationResult.Invalid("Missing 'url' parameter.")
|
||||
!isHttpUrl(url) -> ValidationResult.Invalid("Only http(s) URLs are supported: $url")
|
||||
else -> ValidationResult.Valid
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun execute(request: ToolRequest): ToolResult {
|
||||
val invalid = validateRequest(request) as? ValidationResult.Invalid
|
||||
if (invalid != null) return ToolResult.Failure(request.invocationId, invalid.reason, recoverable = false)
|
||||
val url = request.parameters["url"] as String
|
||||
return runCatching { fetch(request.invocationId, url) }
|
||||
.getOrElse { ToolResult.Failure(request.invocationId, "fetch failed: ${it.message}", recoverable = true) }
|
||||
}
|
||||
|
||||
private suspend fun fetch(invocationId: ToolInvocationId, url: String): ToolResult =
|
||||
httpClient.prepareGet(url).execute { response ->
|
||||
val contentType = response.headers[HttpHeaders.ContentType]
|
||||
val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
|
||||
when {
|
||||
!response.status.isSuccess() ->
|
||||
fail(invocationId, "HTTP ${response.status.value} for $url", recoverable = true)
|
||||
declaredLength != null && declaredLength > maxBytes ->
|
||||
fail(invocationId, "content too large: $declaredLength bytes (cap $maxBytes)")
|
||||
!extractor.supports(contentType) ->
|
||||
fail(invocationId, "unsupported content-type: ${contentType ?: "unknown"}")
|
||||
else -> readAndExtract(invocationId, url, contentType, response.bodyAsChannel())
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun readAndExtract(
|
||||
invocationId: ToolInvocationId,
|
||||
url: String,
|
||||
contentType: String?,
|
||||
channel: io.ktor.utils.io.ByteReadChannel,
|
||||
): ToolResult {
|
||||
val bytes = readBounded(channel)
|
||||
?: return fail(invocationId, "content exceeded cap of $maxBytes bytes")
|
||||
val result = extractor.extract(bytes.decodeToString(), contentType)
|
||||
return if (result.quality == ExtractionQuality.REJECTED) {
|
||||
fail(invocationId, result.reason ?: "rejected content")
|
||||
} else {
|
||||
ToolResult.Success(
|
||||
invocationId = invocationId,
|
||||
output = result.markdown,
|
||||
metadata = mapOf(
|
||||
"url" to url,
|
||||
"content_type" to (contentType ?: ""),
|
||||
"content_sha256" to sha256(bytes),
|
||||
"fetched_bytes" to bytes.size.toString(),
|
||||
"extractor_version" to result.extractorVersion,
|
||||
"quality" to result.quality.name,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun fail(invocationId: ToolInvocationId, reason: String, recoverable: Boolean = false): ToolResult.Failure =
|
||||
ToolResult.Failure(invocationId, reason, recoverable)
|
||||
|
||||
/** Reads the channel up to [maxBytes]; returns null if the body exceeds the cap. */
|
||||
private suspend fun readBounded(channel: io.ktor.utils.io.ByteReadChannel): ByteArray? {
|
||||
val out = ArrayList<Byte>()
|
||||
val buffer = ByteArray(READ_CHUNK)
|
||||
while (true) {
|
||||
val read = channel.readAvailable(buffer, 0, buffer.size)
|
||||
if (read == -1) break
|
||||
if (out.size + read > maxBytes) return null
|
||||
for (i in 0 until read) out.add(buffer[i])
|
||||
}
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
private fun isHttpUrl(url: String): Boolean =
|
||||
runCatching { java.net.URI(url).scheme?.lowercase() in setOf("http", "https") }.getOrDefault(false)
|
||||
|
||||
private fun sha256(bytes: ByteArray): String =
|
||||
MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) }
|
||||
|
||||
companion object {
|
||||
const val DEFAULT_MAX_BYTES: Long = 10_000_000 // 10 MB — a 200MB "page" is an attack or a mistake
|
||||
private const val READ_CHUNK = 8_192
|
||||
}
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
package com.correx.infrastructure.tools.web
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.events.types.ToolInvocationId
|
||||
import com.correx.core.tools.contract.Tool
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.core.tools.contract.ToolExecutor
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import com.correx.core.tools.contract.ValidationResult
|
||||
import io.ktor.client.HttpClient
|
||||
import io.ktor.client.request.get
|
||||
import io.ktor.client.statement.bodyAsText
|
||||
import io.ktor.http.isSuccess
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
import kotlinx.serialization.json.putJsonObject
|
||||
import java.net.URLEncoder
|
||||
|
||||
/**
|
||||
* Searches the self-hosted SearXNG instance (research-workflow-spec §3). Local-first: no API keys,
|
||||
* no third-party telemetry. Search hits only the one configured [searxngBaseUrl], so unlike
|
||||
* [WebFetchTool] there is no arbitrary-host egress to police — it is [Tier.T1].
|
||||
*
|
||||
* Returns the top [maxResults] results as a readable list, with the result URLs in metadata so the
|
||||
* fetch stage can act on them (and the operator can approve the source list, not each URL).
|
||||
*/
|
||||
class WebSearchTool(
|
||||
private val httpClient: HttpClient,
|
||||
private val searxngBaseUrl: String,
|
||||
private val maxResults: Int = DEFAULT_MAX_RESULTS,
|
||||
) : Tool, ToolExecutor {
|
||||
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
override val name: String = "web_search"
|
||||
override val description: String = "Search the web (local SearXNG); returns titles, URLs, and snippets."
|
||||
override val tier: Tier = Tier.T1
|
||||
override val requiredCapabilities: Set<ToolCapability> = setOf(ToolCapability.NETWORK_ACCESS)
|
||||
override val parametersSchema: JsonObject = buildJsonObject {
|
||||
put("type", "object")
|
||||
putJsonObject("properties") {
|
||||
putJsonObject("query") {
|
||||
put("type", "string")
|
||||
put("description", "Search query.")
|
||||
}
|
||||
}
|
||||
put("required", buildJsonArray { add(JsonPrimitive("query")) })
|
||||
}
|
||||
|
||||
override fun validateRequest(request: ToolRequest): ValidationResult {
|
||||
val query = request.parameters["query"] as? String
|
||||
return if (query.isNullOrBlank()) {
|
||||
ValidationResult.Invalid("Missing 'query' parameter.")
|
||||
} else {
|
||||
ValidationResult.Valid
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun execute(request: ToolRequest): ToolResult {
|
||||
val invalid = validateRequest(request) as? ValidationResult.Invalid
|
||||
if (invalid != null) return ToolResult.Failure(request.invocationId, invalid.reason, recoverable = false)
|
||||
val query = request.parameters["query"] as String
|
||||
return runCatching { search(request.invocationId, query) }
|
||||
.getOrElse { ToolResult.Failure(request.invocationId, "search failed: ${it.message}", recoverable = true) }
|
||||
}
|
||||
|
||||
private suspend fun search(invocationId: ToolInvocationId, query: String): ToolResult {
|
||||
val url = "${searxngBaseUrl.trimEnd('/')}/search?q=${URLEncoder.encode(query, "UTF-8")}&format=json"
|
||||
val response = httpClient.get(url)
|
||||
if (!response.status.isSuccess()) {
|
||||
return ToolResult.Failure(invocationId, "SearXNG HTTP ${response.status.value}", recoverable = true)
|
||||
}
|
||||
val results = json.decodeFromString<SearxResponse>(response.bodyAsText()).results.take(maxResults)
|
||||
return ToolResult.Success(
|
||||
invocationId = invocationId,
|
||||
output = render(query, results),
|
||||
metadata = mapOf(
|
||||
"query" to query,
|
||||
"result_count" to results.size.toString(),
|
||||
"urls" to results.joinToString("\n") { it.url },
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
private fun render(query: String, results: List<SearxResult>): String {
|
||||
if (results.isEmpty()) return "No results for: $query"
|
||||
return results.mapIndexed { index, r ->
|
||||
buildString {
|
||||
append("${index + 1}. ${r.title.ifBlank { r.url }}\n")
|
||||
append(" ${r.url}")
|
||||
if (r.content.isNotBlank()) append("\n ${r.content.trim()}")
|
||||
}
|
||||
}.joinToString("\n\n")
|
||||
}
|
||||
|
||||
@Serializable
|
||||
private data class SearxResponse(val results: List<SearxResult> = emptyList())
|
||||
|
||||
@Serializable
|
||||
private data class SearxResult(
|
||||
val url: String = "",
|
||||
val title: String = "",
|
||||
val content: String = "",
|
||||
)
|
||||
|
||||
companion object {
|
||||
const val DEFAULT_MAX_RESULTS = 8
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
package com.correx.infrastructure.tools.extract
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class HtmlMarkdownExtractorTest {
|
||||
|
||||
// minContentLength=1 isolates conversion correctness from the quality threshold.
|
||||
private val structure = HtmlMarkdownExtractor(minContentLength = 1)
|
||||
|
||||
private fun md(html: String): String = structure.extract(html, "text/html").markdown
|
||||
|
||||
// ---- structural strip (§5.1) ----
|
||||
|
||||
@Test
|
||||
fun `strips script style nav footer and form`() {
|
||||
val out = md(
|
||||
"""
|
||||
<html><body>
|
||||
<nav>Home About Contact</nav>
|
||||
<script>var x = 1;</script>
|
||||
<style>.a{color:red}</style>
|
||||
<article><p>Real content here.</p></article>
|
||||
<footer>Copyright 2026</footer>
|
||||
<form><input name="q"></form>
|
||||
</body></html>
|
||||
""".trimIndent(),
|
||||
)
|
||||
assertEquals("Real content here.", out)
|
||||
assertFalse(out.contains("Home About"), "nav leaked")
|
||||
assertFalse(out.contains("Copyright"), "footer leaked")
|
||||
assertFalse(out.contains("var x"), "script leaked")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `removes nodes matched by class-id blocklist`() {
|
||||
val out = md(
|
||||
"""
|
||||
<body>
|
||||
<div class="cookie-banner">Accept cookies</div>
|
||||
<div class="content"><p>The actual article body that matters.</p></div>
|
||||
<aside class="sidebar related">Related links junk</aside>
|
||||
</body>
|
||||
""".trimIndent(),
|
||||
)
|
||||
assertTrue(out.contains("actual article body"), "main content lost: $out")
|
||||
assertFalse(out.contains("Accept cookies"), "cookie banner leaked")
|
||||
assertFalse(out.contains("Related links"), "sidebar leaked")
|
||||
}
|
||||
|
||||
// ---- density / main-content selection (§5.2) ----
|
||||
|
||||
@Test
|
||||
fun `density picks prose over link-heavy chrome`() {
|
||||
val out = md(
|
||||
"""
|
||||
<body>
|
||||
<div class="links-row"><a href="/1">L1</a><a href="/2">L2</a><a href="/3">L3</a></div>
|
||||
<div class="main-text"><p>A paragraph of genuine prose with enough words to win on text density.</p></div>
|
||||
</body>
|
||||
""".trimIndent(),
|
||||
)
|
||||
assertTrue(out.contains("genuine prose"), "prose not selected: $out")
|
||||
assertFalse(out.contains("[L1]"), "nav links selected over prose")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `prefers semantic article element`() {
|
||||
val out = md("<body><div>noise</div><article><h2>Heading</h2><p>Body.</p></article></body>")
|
||||
assertTrue(out.startsWith("## Heading"), "article not preferred: $out")
|
||||
}
|
||||
|
||||
// ---- markdown conversion ----
|
||||
|
||||
@Test
|
||||
fun `headings render with hash level`() {
|
||||
assertEquals("### Deep", md("<article><h3>Deep</h3></article>"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `unordered and ordered lists`() {
|
||||
assertEquals("- a\n- b", md("<article><ul><li>a</li><li>b</li></ul></article>"))
|
||||
assertEquals("1. x\n2. y", md("<article><ol><li>x</li><li>y</li></ol></article>"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `nested list is indented`() {
|
||||
assertEquals(
|
||||
"- a\n - b",
|
||||
md("<article><ul><li>a<ul><li>b</li></ul></li></ul></article>"),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `links emphasis and inline code`() {
|
||||
val link = """<a href="http://e.com">here</a>"""
|
||||
assertEquals("see [here](http://e.com)", md("<article><p>see $link</p></article>"))
|
||||
assertEquals("a **b** *c*", md("<article><p>a <strong>b</strong> <em>c</em></p></article>"))
|
||||
assertEquals("use `f()`", md("<article><p>use <code>f()</code></p></article>"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `pre becomes a fenced code block`() {
|
||||
assertEquals("```\nline1\nline2\n```", md("<article><pre><code>line1\nline2</code></pre></article>"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `blockquote and hr`() {
|
||||
assertEquals("> quoted", md("<article><blockquote>quoted</blockquote></article>"))
|
||||
assertEquals("a\n\n---\n\nb", md("<article><p>a</p><hr><p>b</p></article>"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `table renders header separator and body`() {
|
||||
val table = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>"
|
||||
val out = md("<article>$table</article>")
|
||||
assertEquals("| A | B |\n| --- | --- |\n| 1 | 2 |", out)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `consecutive duplicate blocks are deduplicated`() {
|
||||
assertEquals("repeat", md("<article><p>repeat</p><p>repeat</p></article>"))
|
||||
}
|
||||
|
||||
// ---- content-type dispatch (§5.3) ----
|
||||
|
||||
@Test
|
||||
fun `json passes through unchanged`() {
|
||||
val r = structure.extract("""{"a":1}""", "application/json; charset=utf-8")
|
||||
assertEquals("""{"a":1}""", r.markdown)
|
||||
assertEquals(ExtractionQuality.OK, r.quality)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `binary and media content types are rejected`() {
|
||||
for (ct in listOf("image/png", "application/pdf", "application/octet-stream", "video/mp4")) {
|
||||
val r = structure.extract("...bytes...", ct)
|
||||
assertEquals(ExtractionQuality.REJECTED, r.quality, "should reject $ct")
|
||||
assertTrue(r.markdown.isEmpty())
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `null content type is treated as html best-effort`() {
|
||||
assertEquals("Hello.", structure.extract("<article><p>Hello.</p></article>", null).markdown)
|
||||
}
|
||||
|
||||
// ---- quality threshold + version pinning (§5) ----
|
||||
|
||||
@Test
|
||||
fun `short extraction is flagged low quality`() {
|
||||
val r = HtmlMarkdownExtractor().extract("<article><p>tiny</p></article>", "text/html")
|
||||
assertEquals(ExtractionQuality.LOW_QUALITY, r.quality)
|
||||
assertTrue(r.reason!!.contains("below minimum"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `sufficiently long extraction is ok`() {
|
||||
val body = "word ".repeat(60).trim()
|
||||
val r = HtmlMarkdownExtractor().extract("<article><p>$body</p></article>", "text/html")
|
||||
assertEquals(ExtractionQuality.OK, r.quality)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `extractor version is pinned on every result`() {
|
||||
assertEquals("html-md-1", structure.extract("<p>x</p>", "text/html").extractorVersion)
|
||||
assertEquals("html-md-1", structure.extract("x", "image/png").extractorVersion)
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
+104
@@ -0,0 +1,104 @@
|
||||
package com.correx.infrastructure.tools.web
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import com.correx.core.tools.contract.ValidationResult
|
||||
import com.correx.core.utils.TypeId
|
||||
import io.ktor.client.HttpClient
|
||||
import io.ktor.client.engine.mock.MockEngine
|
||||
import io.ktor.client.engine.mock.respond
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.http.headersOf
|
||||
import io.ktor.http.HttpHeaders
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertInstanceOf
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class WebSearchToolTest {
|
||||
|
||||
private var lastRequestedUrl: String = ""
|
||||
|
||||
private fun toolFor(
|
||||
json: String,
|
||||
status: HttpStatusCode = HttpStatusCode.OK,
|
||||
maxResults: Int = 8,
|
||||
): WebSearchTool {
|
||||
val engine = MockEngine { request ->
|
||||
lastRequestedUrl = request.url.toString()
|
||||
respond(content = json, status = status, headers = headersOf(HttpHeaders.ContentType, "application/json"))
|
||||
}
|
||||
return WebSearchTool(HttpClient(engine), searxngBaseUrl = "http://searx.local:8888", maxResults = maxResults)
|
||||
}
|
||||
|
||||
private fun request(query: String?) = ToolRequest(
|
||||
invocationId = TypeId("inv-1"),
|
||||
sessionId = TypeId("s-1"),
|
||||
stageId = TypeId("stage-1"),
|
||||
toolName = "web_search",
|
||||
parameters = if (query == null) emptyMap() else mapOf("query" to query),
|
||||
)
|
||||
|
||||
private fun search(tool: WebSearchTool, query: String = "kotlin coroutines") =
|
||||
runBlocking { tool.execute(request(query)) }
|
||||
|
||||
private val sample = """
|
||||
{"results":[
|
||||
{"url":"https://a.com/1","title":"First","content":"about one"},
|
||||
{"url":"https://b.com/2","title":"Second","content":"about two"}
|
||||
]}
|
||||
""".trimIndent()
|
||||
|
||||
@Test
|
||||
fun `is tier one and network capable`() {
|
||||
val tool = toolFor(sample)
|
||||
assertEquals(Tier.T1, tool.tier)
|
||||
assertTrue(ToolCapability.NETWORK_ACCESS in tool.requiredCapabilities)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `queries searxng json endpoint with encoded query`() {
|
||||
search(toolFor(sample), "a b&c")
|
||||
assertTrue(lastRequestedUrl.contains("/search?q="), lastRequestedUrl)
|
||||
assertTrue(lastRequestedUrl.contains("format=json"), lastRequestedUrl)
|
||||
assertTrue(lastRequestedUrl.contains("a+b%26c"), "query not url-encoded: $lastRequestedUrl")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `renders results and exposes urls in metadata`() {
|
||||
val success = assertInstanceOf(ToolResult.Success::class.java, search(toolFor(sample)))
|
||||
assertTrue(success.output.contains("First"))
|
||||
assertTrue(success.output.contains("https://a.com/1"))
|
||||
assertEquals("2", success.metadata["result_count"])
|
||||
assertEquals("https://a.com/1\nhttps://b.com/2", success.metadata["urls"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `respects max results cap`() {
|
||||
val success = assertInstanceOf(ToolResult.Success::class.java, search(toolFor(sample, maxResults = 1)))
|
||||
assertEquals("1", success.metadata["result_count"])
|
||||
assertEquals("https://a.com/1", success.metadata["urls"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty results render a no-results message`() {
|
||||
val success = assertInstanceOf(ToolResult.Success::class.java, search(toolFor("""{"results":[]}""")))
|
||||
assertTrue(success.output.contains("No results"))
|
||||
assertEquals("0", success.metadata["result_count"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `searxng error is a recoverable failure`() {
|
||||
val result = search(toolFor("err", status = HttpStatusCode.BadGateway))
|
||||
val failure = assertInstanceOf(ToolResult.Failure::class.java, result)
|
||||
assertTrue(failure.recoverable)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `blank query is rejected`() {
|
||||
assertInstanceOf(ValidationResult.Invalid::class.java, toolFor(sample).validateRequest(request(" ")))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user