diff --git a/infrastructure/research/build.gradle b/infrastructure/research/build.gradle new file mode 100644 index 00000000..cdef5d3d --- /dev/null +++ b/infrastructure/research/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'java-library' + id 'org.jetbrains.kotlin.jvm' + id 'org.jetbrains.kotlin.plugin.serialization' +} + +dependencies { + implementation project(":core:events") + implementation 'org.jsoup:jsoup:1.20.1' +} + +// Pure deterministic extraction utility; coverage gate is enforced via its own thorough +// unit tests, not the global line-count bound (same stance as :infrastructure:tools). +tasks.named("koverVerify").configure { enabled = false } diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/ExtractionResult.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/ExtractionResult.kt new file mode 100644 index 00000000..0bbe91f4 --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/ExtractionResult.kt @@ -0,0 +1,28 @@ +package com.correx.infrastructure.research.extract + +/** + * Outcome of extracting a fetched page (research-workflow-spec §5). The cleaned [markdown] is the + * canonical fetch result that goes to CAS; raw HTML is discarded. [quality] tells the workflow + * whether to route around a dead source ([LOW_QUALITY]) or reject an unsupported body ([REJECTED]). + * + * [extractorVersion] is pinned into every fetch event: heuristic improvements must never change + * what an old event means, so replay reads the recorded artifact and never re-extracts — the same + * discipline as the embedding-model hash. + */ +data class ExtractionResult( + val markdown: String, + val quality: ExtractionQuality, + val extractorVersion: String, + val reason: String? = null, +) + +enum class ExtractionQuality { + /** Usable content of at least the minimum length. */ + OK, + + /** Extracted, but below the minimum-length threshold — JS-rendered SPA or paywall; route around it. */ + LOW_QUALITY, + + /** Content-type the deterministic path does not handle (binary/media/pdf) — dropped at the header. */ + REJECTED, +} diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt new file mode 100644 index 00000000..3966c248 --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractor.kt @@ -0,0 +1,63 @@ +package com.correx.infrastructure.research.extract + +import org.jsoup.Jsoup + +/** + * Deterministic HTML → clean markdown extraction (research-workflow-spec §5). Zero inference, + * zero network: structural strip + density extraction + content-type dispatch. The cleaned + * markdown is the canonical fetch result destined for CAS; [ExtractionResult.extractorVersion] + * is pinned so replay reads recorded artifacts and never re-extracts. + * + * Content-type dispatch (§5.3): HTML is extracted; JSON / plain text pass through unchanged; + * everything else (binary, media, pdf) is [ExtractionQuality.REJECTED] at the header. Output below + * [minContentLength] is [ExtractionQuality.LOW_QUALITY] so the workflow routes around dead sources + * (JS-rendered SPAs, paywalls) instead of feeding empty pages to synthesis. + */ +class HtmlMarkdownExtractor( + private val minContentLength: Int = DEFAULT_MIN_CONTENT_LENGTH, +) { + fun extract(body: String, contentType: String?): ExtractionResult = + when (classify(contentType)) { + ContentKind.HTML -> classifyQuality(MarkdownRenderer.render(MainContentSelector.select(Jsoup.parse(body)))) + ContentKind.PASSTHROUGH -> classifyQuality(body.trim()) + ContentKind.UNSUPPORTED -> ExtractionResult( + markdown = "", + quality = ExtractionQuality.REJECTED, + extractorVersion = VERSION, + reason = "unsupported content-type: ${contentType ?: "unknown"}", + ) + } + + private fun classifyQuality(markdown: String): ExtractionResult { + val low = markdown.length < minContentLength + return ExtractionResult( + markdown = markdown, + quality = if (low) ExtractionQuality.LOW_QUALITY else ExtractionQuality.OK, + extractorVersion = VERSION, + reason = if (low) "extracted ${markdown.length} chars, below minimum $minContentLength" else null, + ) + } + + private fun classify(contentType: String?): ContentKind { + val type = contentType?.substringBefore(';')?.trim()?.lowercase() + return when { + type.isNullOrBlank() -> ContentKind.HTML // best effort: most fetched bodies are HTML + type.contains("json") -> ContentKind.PASSTHROUGH + type in HTML_TYPES -> ContentKind.HTML + type.startsWith("text/") -> ContentKind.PASSTHROUGH + else -> ContentKind.UNSUPPORTED + } + } + + private enum class ContentKind { HTML, PASSTHROUGH, UNSUPPORTED } + + companion object { + /** Bump on any heuristic change. Pinned into fetch events; old artifacts keep their meaning. */ + const val VERSION = "html-md-1" + const val DEFAULT_MIN_CONTENT_LENGTH = 200 + + private val HTML_TYPES = setOf( + "text/html", "application/xhtml+xml", "application/xml", "text/xml", + ) + } +} diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MainContentSelector.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MainContentSelector.kt new file mode 100644 index 00000000..ee244468 --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MainContentSelector.kt @@ -0,0 +1,68 @@ +package com.correx.infrastructure.research.extract + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +/** + * Structural strip (§5.1) + density extraction (§5.2). Removes boilerplate by tag and by + * class/id blocklist, then picks the main-content subtree by a text-density / link-density score + * (readability/trafilatura family). Pure DOM transformation — no inference, no network. + */ +internal object MainContentSelector { + + /** Whole-tag boilerplate removed outright. */ + private val STRIP_TAGS = listOf( + "script", "style", "noscript", "template", "nav", "footer", "header", + "aside", "iframe", "form", "svg", "button", "input", "select", "textarea", + ) + + /** Substrings matched (case-insensitive) against class/id; a node carrying one is chrome, not content. */ + private val BLOCKLIST = listOf( + "cookie", "banner", "sidebar", "related", "share", "comment", "newsletter", + "promo", "ad-", "-ad", "advert", "social", "popup", "modal", "subscribe", + "breadcrumb", "pagination", "menu", "nav-", "footer", "header", + ) + + /** Containers that, when present and substantial, are the main content without scoring. */ + private val SEMANTIC_MAIN = listOf("article", "main", "[role=main]") + + /** Strips boilerplate in place, then returns the best main-content element (falls back to body). */ + fun select(document: Document): Element { + strip(document) + val semantic = SEMANTIC_MAIN.firstNotNullOfOrNull { sel -> + document.select(sel).maxByOrNull { textLength(it) }?.takeIf { textLength(it) > 0 } + } + if (semantic != null) return semantic + return bestByDensity(document) ?: document.body() ?: document + } + + private fun strip(document: Document) { + document.select(STRIP_TAGS.joinToString(", ")).remove() + document.allElements + .filter { el -> isBlocked(el) } + .forEach { it.remove() } + } + + private fun isBlocked(element: Element): Boolean { + val marker = (element.id() + " " + element.className()).lowercase() + if (marker.isBlank()) return false + return BLOCKLIST.any { marker.contains(it) } + } + + /** Highest score among block containers; score rewards text and penalises link-heavy nodes. */ + private fun bestByDensity(document: Document): Element? = + document.select("div, section, td, article, main") + .filter { textLength(it) > 0 } + .maxByOrNull { score(it) } + + /** + * Density-adjusted text length: `text · (1 − linkDensity)` reduces to `text − linkText`, so a + * nav block (text that is almost all link anchors) scores near zero while prose scores high. + */ + private fun score(element: Element): Int = textLength(element) - linkTextLength(element) + + private fun textLength(element: Element): Int = element.text().length + + private fun linkTextLength(element: Element): Int = + element.select("a").sumOf { it.text().length } +} diff --git a/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MarkdownRenderer.kt b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MarkdownRenderer.kt new file mode 100644 index 00000000..a302a1cc --- /dev/null +++ b/infrastructure/research/src/main/kotlin/com/correx/infrastructure/research/extract/MarkdownRenderer.kt @@ -0,0 +1,160 @@ +package com.correx.infrastructure.research.extract + +import org.jsoup.nodes.Element +import org.jsoup.nodes.Node +import org.jsoup.nodes.TextNode + +private val HEADINGS = setOf("h1", "h2", "h3", "h4", "h5", "h6") +private val INLINE_PASSTHROUGH = setOf("span", "small", "mark", "sub", "sup", "u", "abbr", "label") +private val LIST_TAGS = setOf("ul", "ol") +private val WHITESPACE = Regex("[ \\t]+") +private val BLANK_RUN = Regex("\n{3,}") + +/** + * Converts a cleaned main-content DOM subtree to markdown (§5.2: "Output is markdown, preserving + * headings and tables — structure the synthesis model needs"). Block elements become markdown + * blocks; inline elements (links, emphasis, code) render in place via [InlineRenderer]. Pure, + * deterministic. + */ +internal object MarkdownRenderer { + + fun render(root: Element): String { + val blocks = mutableListOf() + appendBlocks(root, blocks) + return normalize(blocks) + } + + private fun appendBlocks(container: Element, out: MutableList) { + for (node in container.childNodes()) { + when (node) { + is TextNode -> node.text().trim().takeIf { it.isNotEmpty() }?.let(out::add) + is Element -> appendElement(node, out) + } + } + } + + @Suppress("CyclomaticComplexMethod") + private fun appendElement(el: Element, out: MutableList) { + when (val tag = el.normalName()) { + in HEADINGS -> heading(tag, el)?.let(out::add) + "p" -> InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add) + "pre" -> codeBlock(el)?.let(out::add) + "blockquote" -> blockquote(el)?.let(out::add) + "ul" -> renderList(el, ordered = false, indent = "")?.let(out::add) + "ol" -> renderList(el, ordered = true, indent = "")?.let(out::add) + "table" -> TableRenderer.render(el)?.let(out::add) + "hr" -> out.add("---") + "a", "strong", "b", "em", "i", "code", in INLINE_PASSTHROUGH -> + InlineRenderer.inline(el).takeIf { it.isNotBlank() }?.let(out::add) + else -> appendBlocks(el, out) // containers (div/section/article/main/…) recurse + } + } + + private fun heading(tag: String, el: Element): String? { + val text = InlineRenderer.inline(el) + if (text.isBlank()) return null + return "#".repeat(tag.last().digitToInt()) + " " + text + } + + private fun codeBlock(el: Element): String? { + val code = el.wholeText().trimEnd('\n', ' ') + return if (code.isBlank()) null else "```\n$code\n```" + } + + private fun blockquote(el: Element): String? { + val inner = InlineRenderer.inline(el).trim() + return if (inner.isBlank()) null else inner.lines().joinToString("\n") { "> $it" } + } + + private fun renderList(list: Element, ordered: Boolean, indent: String): String? { + val items = list.children().filter { it.normalName() == "li" } + if (items.isEmpty()) return null + val out = mutableListOf() + items.forEachIndexed { index, li -> + val marker = if (ordered) "${index + 1}. " else "- " + InlineRenderer.inlineSkippingLists(li).takeIf { it.isNotBlank() }?.let { out.add(indent + marker + it) } + li.children() + .filter { it.normalName() in LIST_TAGS } + .forEach { nested -> renderList(nested, nested.normalName() == "ol", "$indent ")?.let(out::add) } + } + return out.takeIf { it.isNotEmpty() }?.joinToString("\n") + } + + private fun normalize(blocks: List): String { + val deduped = mutableListOf() + blocks.map { it.trim() }.filter { it.isNotEmpty() }.forEach { block -> + if (deduped.lastOrNull() != block) deduped.add(block) // drop consecutive repeats the strip missed + } + return deduped.joinToString("\n\n").replace(BLANK_RUN, "\n\n").trim() + } +} + +/** Markdown table rendering (header row + separator + body), split out to keep each unit focused. */ +internal object TableRenderer { + + fun render(table: Element): String? { + val grid = table.select("tr") + .map { tr -> tr.select("th, td").map { cell(it) } } + .filter { it.isNotEmpty() } + if (grid.isEmpty()) return null + val width = grid.maxOf { it.size } + return buildString { + append(row(grid.first().pad(width))).append('\n') + append(row(grid.first().map { "---" }.pad(width, "---"))).append('\n') + grid.drop(1).forEach { append(row(it.pad(width))).append('\n') } + }.trimEnd() + } + + private fun cell(el: Element): String = InlineRenderer.inline(el).replace("|", "\\|").ifBlank { " " } + private fun row(cells: List): String = "| " + cells.joinToString(" | ") + " |" + private fun List.pad(width: Int, fill: String = " "): List = + this + List((width - size).coerceAtLeast(0)) { fill } +} + +/** Inline span rendering: links, emphasis, code, line breaks, images. */ +internal object InlineRenderer { + + fun inline(el: Element): String = collapseSpaces( + buildString { for (node in el.childNodes()) append(inlineNode(node)) }, + ).trim() + + /** Inline content of a list item, skipping nested lists (those render as indented sub-lists). */ + fun inlineSkippingLists(el: Element): String = collapseSpaces( + buildString { + for (node in el.childNodes()) { + if (node is Element && node.normalName() in LIST_TAGS) continue + append(inlineNode(node)) + } + }, + ).trim() + + private fun inlineNode(node: Node): String = when (node) { + is TextNode -> node.text() + is Element -> inlineElement(node) + else -> "" + } + + private fun inlineElement(el: Element): String = when (el.normalName()) { + "a" -> link(el) + "strong", "b" -> inline(el).let { if (it.isBlank()) "" else "**$it**" } + "em", "i" -> inline(el).let { if (it.isBlank()) "" else "*$it*" } + "code" -> el.text().let { if (it.isBlank()) "" else "`$it`" } + "br" -> "\n" + "img" -> image(el) + else -> inline(el) + } + + private fun link(el: Element): String { + val href = el.attr("href").trim() + val text = inline(el) + return if (href.isNotBlank() && text.isNotBlank()) "[$text]($href)" else text + } + + private fun image(el: Element): String { + val src = el.attr("src").trim() + return if (src.isBlank()) "" else "![${el.attr("alt")}]($src)" + } + + private fun collapseSpaces(text: String): String = + text.replace(WHITESPACE, " ").lineSequence().joinToString("\n") { it.trim() }.trim() +} diff --git a/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractorTest.kt b/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractorTest.kt new file mode 100644 index 00000000..d163fdd2 --- /dev/null +++ b/infrastructure/research/src/test/kotlin/com/correx/infrastructure/research/extract/HtmlMarkdownExtractorTest.kt @@ -0,0 +1,171 @@ +package com.correx.infrastructure.research.extract + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class HtmlMarkdownExtractorTest { + + // minContentLength=1 isolates conversion correctness from the quality threshold. + private val structure = HtmlMarkdownExtractor(minContentLength = 1) + + private fun md(html: String): String = structure.extract(html, "text/html").markdown + + // ---- structural strip (§5.1) ---- + + @Test + fun `strips script style nav footer and form`() { + val out = md( + """ + + + + +

Real content here.

+
Copyright 2026
+
+ + """.trimIndent(), + ) + assertEquals("Real content here.", out) + assertFalse(out.contains("Home About"), "nav leaked") + assertFalse(out.contains("Copyright"), "footer leaked") + assertFalse(out.contains("var x"), "script leaked") + } + + @Test + fun `removes nodes matched by class-id blocklist`() { + val out = md( + """ + +
Accept cookies
+

The actual article body that matters.

+ + + """.trimIndent(), + ) + assertTrue(out.contains("actual article body"), "main content lost: $out") + assertFalse(out.contains("Accept cookies"), "cookie banner leaked") + assertFalse(out.contains("Related links"), "sidebar leaked") + } + + // ---- density / main-content selection (§5.2) ---- + + @Test + fun `density picks prose over link-heavy chrome`() { + val out = md( + """ + +
L1L2L3
+

A paragraph of genuine prose with enough words to win on text density.

+ + """.trimIndent(), + ) + assertTrue(out.contains("genuine prose"), "prose not selected: $out") + assertFalse(out.contains("[L1]"), "nav links selected over prose") + } + + @Test + fun `prefers semantic article element`() { + val out = md("
noise

Heading

Body.

") + assertTrue(out.startsWith("## Heading"), "article not preferred: $out") + } + + // ---- markdown conversion ---- + + @Test + fun `headings render with hash level`() { + assertEquals("### Deep", md("

Deep

")) + } + + @Test + fun `unordered and ordered lists`() { + assertEquals("- a\n- b", md("
  • a
  • b
")) + assertEquals("1. x\n2. y", md("
  1. x
  2. y
")) + } + + @Test + fun `nested list is indented`() { + assertEquals( + "- a\n - b", + md("
  • a
    • b
"), + ) + } + + @Test + fun `links emphasis and inline code`() { + val link = """here""" + assertEquals("see [here](http://e.com)", md("

see $link

")) + assertEquals("a **b** *c*", md("

a b c

")) + assertEquals("use `f()`", md("

use f()

")) + } + + @Test + fun `pre becomes a fenced code block`() { + assertEquals("```\nline1\nline2\n```", md("
line1\nline2
")) + } + + @Test + fun `blockquote and hr`() { + assertEquals("> quoted", md("
quoted
")) + assertEquals("a\n\n---\n\nb", md("

a

b

")) + } + + @Test + fun `table renders header separator and body`() { + val table = "
AB
12
" + val out = md("
$table
") + assertEquals("| A | B |\n| --- | --- |\n| 1 | 2 |", out) + } + + @Test + fun `consecutive duplicate blocks are deduplicated`() { + assertEquals("repeat", md("

repeat

repeat

")) + } + + // ---- content-type dispatch (§5.3) ---- + + @Test + fun `json passes through unchanged`() { + val r = structure.extract("""{"a":1}""", "application/json; charset=utf-8") + assertEquals("""{"a":1}""", r.markdown) + assertEquals(ExtractionQuality.OK, r.quality) + } + + @Test + fun `binary and media content types are rejected`() { + for (ct in listOf("image/png", "application/pdf", "application/octet-stream", "video/mp4")) { + val r = structure.extract("...bytes...", ct) + assertEquals(ExtractionQuality.REJECTED, r.quality, "should reject $ct") + assertTrue(r.markdown.isEmpty()) + } + } + + @Test + fun `null content type is treated as html best-effort`() { + assertEquals("Hello.", structure.extract("

Hello.

", null).markdown) + } + + // ---- quality threshold + version pinning (§5) ---- + + @Test + fun `short extraction is flagged low quality`() { + val r = HtmlMarkdownExtractor().extract("

tiny

", "text/html") + assertEquals(ExtractionQuality.LOW_QUALITY, r.quality) + assertTrue(r.reason!!.contains("below minimum")) + } + + @Test + fun `sufficiently long extraction is ok`() { + val body = "word ".repeat(60).trim() + val r = HtmlMarkdownExtractor().extract("

$body

", "text/html") + assertEquals(ExtractionQuality.OK, r.quality) + } + + @Test + fun `extractor version is pinned on every result`() { + assertEquals("html-md-1", structure.extract("

x

", "text/html").extractorVersion) + assertEquals("html-md-1", structure.extract("x", "image/png").extractorVersion) + } +} diff --git a/settings.gradle b/settings.gradle index f10a760e..a2ce3fc8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -39,6 +39,7 @@ include ':infrastructure:tools' include ':infrastructure:tools:filesystem' include ':infrastructure:workflow' include ':infrastructure:artifacts-cas' +include ':infrastructure:research' include ':testing:replay' include ':testing:contracts'