feat(research): deterministic HTML→markdown extraction (research-workflow §5)

First slice of the research workflow: the pure, zero-inference extraction pipeline
that the WebFetchTool will consume. New :infrastructure:research module (jsoup).

HtmlMarkdownExtractor: content-type dispatch (HTML extracted; JSON/text pass
through; binary/media/pdf rejected at the header), then structural strip
(MainContentSelector: drop script/style/nav/footer/aside/form + class/id blocklist)
+ density extraction (text − link-text score, semantic <article>/<main> preferred)
+ DOM→markdown (MarkdownRenderer/Table/Inline: headings, lists incl. nested, tables,
fenced code, blockquotes, links, emphasis). Output below the min-length threshold is
flagged LOW_QUALITY so the workflow can route around dead sources (SPAs, paywalls).

extractorVersion ("html-md-1") is pinned on every result so replay reads recorded
artifacts and never re-extracts — the embedding-hash discipline (§5, ADR-0000 §10).

Next slices: WebFetchTool + egress policy + CAS wiring (emits the fetch +
LowQualityExtraction events), then WebSearchTool/SearXNG, workflow graph, synthesis.

infrastructure/research/build.gradle

@@ -0,0 +1,14 @@
+plugins {
+    id 'java-library'
+    id 'org.jetbrains.kotlin.jvm'
+    id 'org.jetbrains.kotlin.plugin.serialization'
+}
+
+dependencies {
+    implementation project(":core:events")
+    implementation 'org.jsoup:jsoup:1.20.1'
+}
+
+// Pure deterministic extraction utility; coverage gate is enforced via its own thorough
+// unit tests, not the global line-count bound (same stance as :infrastructure:tools).
+tasks.named("koverVerify").configure { enabled = false }

infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/ExtractionResult.kt

@@ -0,0 +1,28 @@
+package com.correx.infrastructure.research.extract
+
+/**
+ * Outcome of extracting a fetched page (research-workflow-spec §5). The cleaned [markdown] is the
+ * canonical fetch result that goes to CAS; raw HTML is discarded. [quality] tells the workflow
+ * whether to route around a dead source ([LOW_QUALITY]) or reject an unsupported body ([REJECTED]).
+ *
+ * [extractorVersion] is pinned into every fetch event: heuristic improvements must never change
+ * what an old event means, so replay reads the recorded artifact and never re-extracts — the same
+ * discipline as the embedding-model hash.
+ */
+data class ExtractionResult(
+    val markdown: String,
+    val quality: ExtractionQuality,
+    val extractorVersion: String,
+    val reason: String? = null,
+)
+
+enum class ExtractionQuality {
+    /** Usable content of at least the minimum length. */
+    OK,
+
+    /** Extracted, but below the minimum-length threshold — JS-rendered SPA or paywall; route around it. */
+    LOW_QUALITY,
+
+    /** Content-type the deterministic path does not handle (binary/media/pdf) — dropped at the header. */
+    REJECTED,
+}

infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt

@@ -0,0 +1,63 @@
+package com.correx.infrastructure.research.extract
+
+import org.jsoup.Jsoup
+
+/**
+ * Deterministic HTML → clean markdown extraction (research-workflow-spec §5). Zero inference,
+ * zero network: structural strip + density extraction + content-type dispatch. The cleaned
+ * markdown is the canonical fetch result destined for CAS; [ExtractionResult.extractorVersion]
+ * is pinned so replay reads recorded artifacts and never re-extracts.
+ *
+ * Content-type dispatch (§5.3): HTML is extracted; JSON / plain text pass through unchanged;
+ * everything else (binary, media, pdf) is [ExtractionQuality.REJECTED] at the header. Output below
+ * [minContentLength] is [ExtractionQuality.LOW_QUALITY] so the workflow routes around dead sources
+ * (JS-rendered SPAs, paywalls) instead of feeding empty pages to synthesis.
+ */
+class HtmlMarkdownExtractor(
+    private val minContentLength: Int = DEFAULT_MIN_CONTENT_LENGTH,
+) {
+    fun extract(body: String, contentType: String?): ExtractionResult =
+        when (classify(contentType)) {
+            ContentKind.HTML -> classifyQuality(MarkdownRenderer.render(MainContentSelector.select(Jsoup.parse(body))))
+            ContentKind.PASSTHROUGH -> classifyQuality(body.trim())
+            ContentKind.UNSUPPORTED -> ExtractionResult(
+                markdown = "",
+                quality = ExtractionQuality.REJECTED,
+                extractorVersion = VERSION,
+                reason = "unsupported content-type: ${contentType ?: "unknown"}",
+            )
+        }
+
+    private fun classifyQuality(markdown: String): ExtractionResult {
+        val low = markdown.length < minContentLength
+        return ExtractionResult(
+            markdown = markdown,
+            quality = if (low) ExtractionQuality.LOW_QUALITY else ExtractionQuality.OK,
+            extractorVersion = VERSION,
+            reason = if (low) "extracted ${markdown.length} chars, below minimum $minContentLength" else null,
+        )
+    }
+
+    private fun classify(contentType: String?): ContentKind {
+        val type = contentType?.substringBefore(';')?.trim()?.lowercase()
+        return when {
+            type.isNullOrBlank() -> ContentKind.HTML // best effort: most fetched bodies are HTML
+            type.contains("json") -> ContentKind.PASSTHROUGH
+            type in HTML_TYPES -> ContentKind.HTML
+            type.startsWith("text/") -> ContentKind.PASSTHROUGH
+            else -> ContentKind.UNSUPPORTED
+        }
+    }
+
+    private enum class ContentKind { HTML, PASSTHROUGH, UNSUPPORTED }
+
+    companion object {
+        /** Bump on any heuristic change. Pinned into fetch events; old artifacts keep their meaning. */
+        const val VERSION = "html-md-1"
+        const val DEFAULT_MIN_CONTENT_LENGTH = 200
+
+        private val HTML_TYPES = setOf(
+            "text/html", "application/xhtml+xml", "application/xml", "text/xml",
+        )
+    }
+}

infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MainContentSelector.kt

@@ -0,0 +1,68 @@
+package com.correx.infrastructure.research.extract
+
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+/**
+ * Structural strip (§5.1) + density extraction (§5.2). Removes boilerplate by tag and by
+ * class/id blocklist, then picks the main-content subtree by a text-density / link-density score
+ * (readability/trafilatura family). Pure DOM transformation — no inference, no network.
+ */
+internal object MainContentSelector {
+
+    /** Whole-tag boilerplate removed outright. */
+    private val STRIP_TAGS = listOf(
+        "script", "style", "noscript", "template", "nav", "footer", "header",
+        "aside", "iframe", "form", "svg", "button", "input", "select", "textarea",
+    )
+
+    /** Substrings matched (case-insensitive) against class/id; a node carrying one is chrome, not content. */
+    private val BLOCKLIST = listOf(
+        "cookie", "banner", "sidebar", "related", "share", "comment", "newsletter",
+        "promo", "ad-", "-ad", "advert", "social", "popup", "modal", "subscribe",
+        "breadcrumb", "pagination", "menu", "nav-", "footer", "header",
+    )
+
+    /** Containers that, when present and substantial, are the main content without scoring. */
+    private val SEMANTIC_MAIN = listOf("article", "main", "[role=main]")
+
+    /** Strips boilerplate in place, then returns the best main-content element (falls back to body). */
+    fun select(document: Document): Element {
+        strip(document)
+        val semantic = SEMANTIC_MAIN.firstNotNullOfOrNull { sel ->
+            document.select(sel).maxByOrNull { textLength(it) }?.takeIf { textLength(it) > 0 }
+        }
+        if (semantic != null) return semantic
+        return bestByDensity(document) ?: document.body() ?: document
+    }
+
+    private fun strip(document: Document) {
+        document.select(STRIP_TAGS.joinToString(", ")).remove()
+        document.allElements
+            .filter { el -> isBlocked(el) }
+            .forEach { it.remove() }
+    }
+
+    private fun isBlocked(element: Element): Boolean {
+        val marker = (element.id() + " " + element.className()).lowercase()
+        if (marker.isBlank()) return false
+        return BLOCKLIST.any { marker.contains(it) }
+    }
+
+    /** Highest score among block containers; score rewards text and penalises link-heavy nodes. */
+    private fun bestByDensity(document: Document): Element? =
+        document.select("div, section, td, article, main")
+            .filter { textLength(it) > 0 }
+            .maxByOrNull { score(it) }
+
+    /**
+     * Density-adjusted text length: `text · (1 − linkDensity)` reduces to `text − linkText`, so a
+     * nav block (text that is almost all link anchors) scores near zero while prose scores high.
+     */
+    private fun score(element: Element): Int = textLength(element) - linkTextLength(element)
+
+    private fun textLength(element: Element): Int = element.text().length
+
+    private fun linkTextLength(element: Element): Int =
+        element.select("a").sumOf { it.text().length }
+}

infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MarkdownRenderer.kt

@@ -0,0 +1,160 @@
+package com.correx.infrastructure.research.extract
+
+import org.jsoup.nodes.Element
+import org.jsoup.nodes.Node
+import org.jsoup.nodes.TextNode
+
+private val HEADINGS = setOf("h1", "h2", "h3", "h4", "h5", "h6")
+private val INLINE_PASSTHROUGH = setOf("span", "small", "mark", "sub", "sup", "u", "abbr", "label")
+private val LIST_TAGS = setOf("ul", "ol")
+private val WHITESPACE = Regex("[ \\t]+")
+private val BLANK_RUN = Regex("\n{3,}")
+
+/**
+ * Converts a cleaned main-content DOM subtree to markdown (§5.2: "Output is markdown, preserving
+ * headings and tables — structure the synthesis model needs"). Block elements become markdown
+ * blocks; inline elements (links, emphasis, code) render in place via [InlineRenderer]. Pure,
+ * deterministic.
+ */
+internal object MarkdownRenderer {
+
+    fun render(root: Element): String {
+        val blocks = mutableListOf<String>()
+        appendBlocks(root, blocks)
+        return normalize(blocks)
+    }
+
+    private fun appendBlocks(container: Element, out: MutableList<String>) {
+        for (node in container.childNodes()) {
+            when (node) {
+                is TextNode -> node.text().trim().takeIf { it.isNotEmpty() }?.let(out::add)
+                is Element -> appendElement(node, out)
+            }
+        }
+    }
+
+    @Suppress("CyclomaticComplexMethod")
+    private fun appendElement(el: Element, out: MutableList<String>) {
+        when (val tag = el.normalName()) {
+            in HEADINGS -> heading(tag, el)?.let(out::add)
+            "p" -> InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add)
+            "pre" -> codeBlock(el)?.let(out::add)
+            "blockquote" -> blockquote(el)?.let(out::add)
+            "ul" -> renderList(el, ordered = false, indent = "")?.let(out::add)
+            "ol" -> renderList(el, ordered = true, indent = "")?.let(out::add)
+            "table" -> TableRenderer.render(el)?.let(out::add)
+            "hr" -> out.add("---")
+            "a", "strong", "b", "em", "i", "code", in INLINE_PASSTHROUGH ->
+                InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add)
+            else -> appendBlocks(el, out) // containers (div/section/article/main/…) recurse
+        }
+    }
+
+    private fun heading(tag: String, el: Element): String? {
+        val text = InlineRenderer.inline(el)
+        if (text.isBlank()) return null
+        return "#".repeat(tag.last().digitToInt()) + " " + text
+    }
+
+    private fun codeBlock(el: Element): String? {
+        val code = el.wholeText().trimEnd('\n', ' ')
+        return if (code.isBlank()) null else "```\n$code\n```"
+    }
+
+    private fun blockquote(el: Element): String? {
+        val inner = InlineRenderer.inline(el).trim()
+        return if (inner.isBlank()) null else inner.lines().joinToString("\n") { "> $it" }
+    }
+
+    private fun renderList(list: Element, ordered: Boolean, indent: String): String? {
+        val items = list.children().filter { it.normalName() == "li" }
+        if (items.isEmpty()) return null
+        val out = mutableListOf<String>()
+        items.forEachIndexed { index, li ->
+            val marker = if (ordered) "${index + 1}. " else "- "
+            InlineRenderer.inlineSkippingLists(li).takeIf { it.isNotBlank() }?.let { out.add(indent + marker + it) }
+            li.children()
+                .filter { it.normalName() in LIST_TAGS }
+                .forEach { nested -> renderList(nested, nested.normalName() == "ol", "$indent  ")?.let(out::add) }
+        }
+        return out.takeIf { it.isNotEmpty() }?.joinToString("\n")
+    }
+
+    private fun normalize(blocks: List<String>): String {
+        val deduped = mutableListOf<String>()
+        blocks.map { it.trim() }.filter { it.isNotEmpty() }.forEach { block ->
+            if (deduped.lastOrNull() != block) deduped.add(block) // drop consecutive repeats the strip missed
+        }
+        return deduped.joinToString("\n\n").replace(BLANK_RUN, "\n\n").trim()
+    }
+}
+
+/** Markdown table rendering (header row + separator + body), split out to keep each unit focused. */
+internal object TableRenderer {
+
+    fun render(table: Element): String? {
+        val grid = table.select("tr")
+            .map { tr -> tr.select("th, td").map { cell(it) } }
+            .filter { it.isNotEmpty() }
+        if (grid.isEmpty()) return null
+        val width = grid.maxOf { it.size }
+        return buildString {
+            append(row(grid.first().pad(width))).append('\n')
+            append(row(grid.first().map { "---" }.pad(width, "---"))).append('\n')
+            grid.drop(1).forEach { append(row(it.pad(width))).append('\n') }
+        }.trimEnd()
+    }
+
+    private fun cell(el: Element): String = InlineRenderer.inline(el).replace("|", "\\|").ifBlank { " " }
+    private fun row(cells: List<String>): String = "| " + cells.joinToString(" | ") + " |"
+    private fun List<String>.pad(width: Int, fill: String = " "): List<String> =
+        this + List((width - size).coerceAtLeast(0)) { fill }
+}
+
+/** Inline span rendering: links, emphasis, code, line breaks, images. */
+internal object InlineRenderer {
+
+    fun inline(el: Element): String = collapseSpaces(
+        buildString { for (node in el.childNodes()) append(inlineNode(node)) },
+    ).trim()
+
+    /** Inline content of a list item, skipping nested lists (those render as indented sub-lists). */
+    fun inlineSkippingLists(el: Element): String = collapseSpaces(
+        buildString {
+            for (node in el.childNodes()) {
+                if (node is Element && node.normalName() in LIST_TAGS) continue
+                append(inlineNode(node))
+            }
+        },
+    ).trim()
+
+    private fun inlineNode(node: Node): String = when (node) {
+        is TextNode -> node.text()
+        is Element -> inlineElement(node)
+        else -> ""
+    }
+
+    private fun inlineElement(el: Element): String = when (el.normalName()) {
+        "a" -> link(el)
+        "strong", "b" -> inline(el).let { if (it.isBlank()) "" else "**$it**" }
+        "em", "i" -> inline(el).let { if (it.isBlank()) "" else "*$it*" }
+        "code" -> el.text().let { if (it.isBlank()) "" else "`$it`" }
+        "br" -> "\n"
+        "img" -> image(el)
+        else -> inline(el)
+    }
+
+    private fun link(el: Element): String {
+        val href = el.attr("href").trim()
+        val text = inline(el)
+        return if (href.isNotBlank() && text.isNotBlank()) "[$text]($href)" else text
+    }
+
+    private fun image(el: Element): String {
+        val src = el.attr("src").trim()
+        return if (src.isBlank()) "" else "![${el.attr("alt")}]($src)"
+    }
+
+    private fun collapseSpaces(text: String): String =
+        text.replace(WHITESPACE, " ").lineSequence().joinToString("\n") { it.trim() }.trim()
+}

infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractorTest.kt

@@ -0,0 +1,171 @@
+package com.correx.infrastructure.research.extract
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertFalse
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Test
+
+class HtmlMarkdownExtractorTest {
+
+    // minContentLength=1 isolates conversion correctness from the quality threshold.
+    private val structure = HtmlMarkdownExtractor(minContentLength = 1)
+
+    private fun md(html: String): String = structure.extract(html, "text/html").markdown
+
+    // ---- structural strip (§5.1) ----
+
+    @Test
+    fun `strips script style nav footer and form`() {
+        val out = md(
+            """
+            <html><body>
+              <nav>Home About Contact</nav>
+              <script>var x = 1;</script>
+              <style>.a{color:red}</style>
+              <article><p>Real content here.</p></article>
+              <footer>Copyright 2026</footer>
+              <form><input name="q"></form>
+            </body></html>
+            """.trimIndent(),
+        )
+        assertEquals("Real content here.", out)
+        assertFalse(out.contains("Home About"), "nav leaked")
+        assertFalse(out.contains("Copyright"), "footer leaked")
+        assertFalse(out.contains("var x"), "script leaked")
+    }
+
+    @Test
+    fun `removes nodes matched by class-id blocklist`() {
+        val out = md(
+            """
+            <body>
+              <div class="cookie-banner">Accept cookies</div>
+              <div class="content"><p>The actual article body that matters.</p></div>
+              <aside class="sidebar related">Related links junk</aside>
+            </body>
+            """.trimIndent(),
+        )
+        assertTrue(out.contains("actual article body"), "main content lost: $out")
+        assertFalse(out.contains("Accept cookies"), "cookie banner leaked")
+        assertFalse(out.contains("Related links"), "sidebar leaked")
+    }
+
+    // ---- density / main-content selection (§5.2) ----
+
+    @Test
+    fun `density picks prose over link-heavy chrome`() {
+        val out = md(
+            """
+            <body>
+              <div class="links-row"><a href="/1">L1</a><a href="/2">L2</a><a href="/3">L3</a></div>
+              <div class="main-text"><p>A paragraph of genuine prose with enough words to win on text density.</p></div>
+            </body>
+            """.trimIndent(),
+        )
+        assertTrue(out.contains("genuine prose"), "prose not selected: $out")
+        assertFalse(out.contains("[L1]"), "nav links selected over prose")
+    }
+
+    @Test
+    fun `prefers semantic article element`() {
+        val out = md("<body><div>noise</div><article><h2>Heading</h2><p>Body.</p></article></body>")
+        assertTrue(out.startsWith("## Heading"), "article not preferred: $out")
+    }
+
+    // ---- markdown conversion ----
+
+    @Test
+    fun `headings render with hash level`() {
+        assertEquals("### Deep", md("<article><h3>Deep</h3></article>"))
+    }
+
+    @Test
+    fun `unordered and ordered lists`() {
+        assertEquals("- a\n- b", md("<article><ul><li>a</li><li>b</li></ul></article>"))
+        assertEquals("1. x\n2. y", md("<article><ol><li>x</li><li>y</li></ol></article>"))
+    }
+
+    @Test
+    fun `nested list is indented`() {
+        assertEquals(
+            "- a\n  - b",
+            md("<article><ul><li>a<ul><li>b</li></ul></li></ul></article>"),
+        )
+    }
+
+    @Test
+    fun `links emphasis and inline code`() {
+        val link = """<a href="http://e.com">here</a>"""
+        assertEquals("see [here](http://e.com)", md("<article><p>see $link</p></article>"))
+        assertEquals("a **b** *c*", md("<article><p>a <strong>b</strong> <em>c</em></p></article>"))
+        assertEquals("use `f()`", md("<article><p>use <code>f()</code></p></article>"))
+    }
+
+    @Test
+    fun `pre becomes a fenced code block`() {
+        assertEquals("```\nline1\nline2\n```", md("<article><pre><code>line1\nline2</code></pre></article>"))
+    }
+
+    @Test
+    fun `blockquote and hr`() {
+        assertEquals("> quoted", md("<article><blockquote>quoted</blockquote></article>"))
+        assertEquals("a\n\n---\n\nb", md("<article><p>a</p><hr><p>b</p></article>"))
+    }
+
+    @Test
+    fun `table renders header separator and body`() {
+        val table = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>"
+        val out = md("<article>$table</article>")
+        assertEquals("| A | B |\n| --- | --- |\n| 1 | 2 |", out)
+    }
+
+    @Test
+    fun `consecutive duplicate blocks are deduplicated`() {
+        assertEquals("repeat", md("<article><p>repeat</p><p>repeat</p></article>"))
+    }
+
+    // ---- content-type dispatch (§5.3) ----
+
+    @Test
+    fun `json passes through unchanged`() {
+        val r = structure.extract("""{"a":1}""", "application/json; charset=utf-8")
+        assertEquals("""{"a":1}""", r.markdown)
+        assertEquals(ExtractionQuality.OK, r.quality)
+    }
+
+    @Test
+    fun `binary and media content types are rejected`() {
+        for (ct in listOf("image/png", "application/pdf", "application/octet-stream", "video/mp4")) {
+            val r = structure.extract("...bytes...", ct)
+            assertEquals(ExtractionQuality.REJECTED, r.quality, "should reject $ct")
+            assertTrue(r.markdown.isEmpty())
+        }
+    }
+
+    @Test
+    fun `null content type is treated as html best-effort`() {
+        assertEquals("Hello.", structure.extract("<article><p>Hello.</p></article>", null).markdown)
+    }
+
+    // ---- quality threshold + version pinning (§5) ----
+
+    @Test
+    fun `short extraction is flagged low quality`() {
+        val r = HtmlMarkdownExtractor().extract("<article><p>tiny</p></article>", "text/html")
+        assertEquals(ExtractionQuality.LOW_QUALITY, r.quality)
+        assertTrue(r.reason!!.contains("below minimum"))
+    }
+
+    @Test
+    fun `sufficiently long extraction is ok`() {
+        val body = "word ".repeat(60).trim()
+        val r = HtmlMarkdownExtractor().extract("<article><p>$body</p></article>", "text/html")
+        assertEquals(ExtractionQuality.OK, r.quality)
+    }
+
+    @Test
+    fun `extractor version is pinned on every result`() {
+        assertEquals("html-md-1", structure.extract("<p>x</p>", "text/html").extractorVersion)
+        assertEquals("html-md-1", structure.extract("x", "image/png").extractorVersion)
+    }
+}

settings.gradle

@@ -39,6 +39,7 @@ include ':infrastructure:tools'
 include ':infrastructure:tools:filesystem'
 include ':infrastructure:workflow'
 include ':infrastructure:artifacts-cas'
+include ':infrastructure:research'
 
 include ':testing:replay'
 include ':testing:contracts'