feat(cas): content-addressed artifact store (steps 1–8)
New core:artifacts-store interface + infrastructure/artifacts-cas implementation: segment files + SQLite index, Blake3 hashing, group-commit fsync via flushBefore, recovery tail-scan, manual compactor, and oldest-first disk-cap evictor. Inference events now carry promptArtifactId / responseArtifactId; orchestrators put bytes before emitting. SqliteEventStore wraps its txn in artifactStore.flushBefore so segment data is fsynced before the event commit, making the crash window non-corrupting (TailScanner re-indexes orphan tail records on reopen). Compactor and evictor are mutually exclusive via maintenanceMutex. Step 9 (cloud sync) deferred to a later epic. See docs/reviews/2026-05-18-cas-steps-1-8-review.md for the final review.
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
plugins {
|
||||
id 'java-library'
|
||||
id 'org.jetbrains.kotlin.jvm'
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinx_coroutines_version"
|
||||
implementation(project(":core:events"))
|
||||
implementation(project(":core:artifacts-store"))
|
||||
implementation "org.xerial:sqlite-jdbc"
|
||||
implementation "org.bouncycastle:bcprov-jdk18on:1.78.1"
|
||||
testImplementation "org.junit.jupiter:junit-jupiter"
|
||||
testImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinx_coroutines_version"
|
||||
testImplementation(project(":infrastructure:persistence"))
|
||||
testImplementation(testFixtures(project(":testing:contracts")))
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package com.correx.infrastructure.artifactscas
|
||||
|
||||
import com.correx.core.artifactstore.ArtifactStore
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.events.types.ArtifactId
|
||||
import com.correx.core.utils.TypeId
|
||||
import com.correx.infrastructure.artifactscas.compact.CompactionReport
|
||||
import com.correx.infrastructure.artifactscas.compact.Compactor
|
||||
import com.correx.infrastructure.artifactscas.compact.LivenessScanner
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.evict.Evictor
|
||||
import com.correx.infrastructure.artifactscas.evict.EvictionReport
|
||||
import com.correx.infrastructure.artifactscas.index.ArtifactIndex
|
||||
import com.correx.infrastructure.artifactscas.recovery.TailScanReport
|
||||
import com.correx.infrastructure.artifactscas.recovery.TailScanner
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentLayout
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentReader
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentWriter
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import java.nio.file.Files
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
class CasArtifactStore(
|
||||
private val config: CasConfig,
|
||||
private val index: ArtifactIndex,
|
||||
) : ArtifactStore, AutoCloseable {
|
||||
private val segmentsDir = config.rootDir.resolve("segments").also { Files.createDirectories(it) }
|
||||
private val writer = SegmentWriter(segmentsDir, config.maxSegmentBytes)
|
||||
private val reader = SegmentReader()
|
||||
private val mutex = Mutex()
|
||||
private val maintenanceMutex = Mutex()
|
||||
|
||||
override suspend fun put(bytes: ByteArray): ArtifactId {
|
||||
val hash = SegmentLayout.blake3(bytes)
|
||||
return mutex.withLock {
|
||||
val existing = index.lookup(hash)
|
||||
if (existing == null) {
|
||||
val loc = writer.append(bytes, hash)
|
||||
index.insert(hash, loc)
|
||||
}
|
||||
TypeId(hash.toHex())
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun get(id: ArtifactId): ByteArray? {
|
||||
val hash = id.value.hexToBytes()
|
||||
val loc = mutex.withLock { index.lookup(hash) } ?: return null
|
||||
return reader.read(loc, segmentsDir)
|
||||
}
|
||||
|
||||
override suspend fun flushBefore(commit: suspend () -> Unit) {
|
||||
mutex.withLock {
|
||||
writer.fsync()
|
||||
index.flush()
|
||||
commit()
|
||||
}
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
writer.close()
|
||||
index.close()
|
||||
}
|
||||
|
||||
suspend fun recover(): TailScanReport? = TailScanner(segmentsDir, index).recover()
|
||||
|
||||
fun compactor(eventStore: EventStore): Compactor =
|
||||
Compactor(config, index, LivenessScanner(eventStore, index), storeMutex = mutex)
|
||||
|
||||
// Run synchronously; safe to call alongside puts (acquires storeMutex briefly when swapping segments).
|
||||
// maintenanceMutex makes compact and evict mutually exclusive so they cannot race on liveness/segment files.
|
||||
suspend fun compact(eventStore: EventStore): CompactionReport =
|
||||
maintenanceMutex.withLock { compactor(eventStore).compact() }
|
||||
|
||||
suspend fun evict(eventStore: EventStore): EvictionReport =
|
||||
maintenanceMutex.withLock {
|
||||
val evictor = Evictor(config, index, LivenessScanner(eventStore, index))
|
||||
mutex.withLock {
|
||||
val activeId = segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { runCatching { it.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull() }
|
||||
.maxOrNull() ?: return@withLock EvictionReport(emptyList(), 0L)
|
||||
evictor.evict(activeId)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
suspend fun open(config: CasConfig, index: ArtifactIndex): CasArtifactStore {
|
||||
val segmentsDir = config.rootDir.resolve("segments").also { Files.createDirectories(it) }
|
||||
TailScanner(segmentsDir, index).recover()
|
||||
return CasArtifactStore(config, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun ByteArray.toHex(): String = joinToString("") { "%02x".format(it) }
|
||||
|
||||
private const val HEX_RADIX = 16
|
||||
private const val HEX_CHARS_PER_BYTE = 2
|
||||
private const val HEX_NIBBLE_BITS = 4
|
||||
|
||||
private fun String.hexToBytes(): ByteArray {
|
||||
require(length % HEX_CHARS_PER_BYTE == 0) { "hex string must have even length" }
|
||||
return ByteArray(length / HEX_CHARS_PER_BYTE) { i ->
|
||||
val hi = Character.digit(this[HEX_CHARS_PER_BYTE * i], HEX_RADIX)
|
||||
val lo = Character.digit(this[HEX_CHARS_PER_BYTE * i + 1], HEX_RADIX)
|
||||
((hi shl HEX_NIBBLE_BITS) + lo).toByte()
|
||||
}
|
||||
}
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package com.correx.infrastructure.artifactscas.compact
|
||||
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.ArtifactIndex
|
||||
import com.correx.infrastructure.artifactscas.segment.Location
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentLayout
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentReader
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentWriter
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
data class CompactionReport(
|
||||
val compactedSegmentIds: List<Long>,
|
||||
val newSegmentId: Long?,
|
||||
val bytesReclaimed: Long,
|
||||
)
|
||||
|
||||
class Compactor(
|
||||
config: CasConfig,
|
||||
private val index: ArtifactIndex,
|
||||
private val liveness: LivenessScanner,
|
||||
private val storeMutex: Mutex,
|
||||
private val liveRatioThreshold: Double = DEFAULT_LIVE_RATIO_THRESHOLD,
|
||||
) {
|
||||
private val segmentsDir: Path = config.rootDir.resolve("segments")
|
||||
private val reader = SegmentReader()
|
||||
|
||||
suspend fun compact(): CompactionReport = withContext(Dispatchers.IO) {
|
||||
if (!Files.exists(segmentsDir)) return@withContext empty()
|
||||
val segmentIds = listSegmentIds()
|
||||
if (segmentIds.isEmpty()) return@withContext empty()
|
||||
val activeId = segmentIds.max()
|
||||
val scan = liveness.scan(segmentsDir)
|
||||
val eligible = selectEligible(scan, activeId)
|
||||
if (eligible.isEmpty()) return@withContext empty()
|
||||
|
||||
val liveEntries = readLiveEntries(eligible, scan.liveIds)
|
||||
val newSegmentId = activeId + 1
|
||||
val (bytesWritten, newLocations) = writeNewSegment(newSegmentId, liveEntries)
|
||||
|
||||
val reclaimedBefore = eligible.sumOf { id -> scan.perSegment[id]?.totalBytes ?: 0L }
|
||||
applyAtomicSwap(newLocations, eligible)
|
||||
|
||||
val effectiveNew = if (liveEntries.isEmpty()) {
|
||||
runCatching { Files.deleteIfExists(segmentsDir.resolve(SegmentWriter.segmentFileName(newSegmentId))) }
|
||||
null
|
||||
} else {
|
||||
newSegmentId
|
||||
}
|
||||
|
||||
CompactionReport(
|
||||
compactedSegmentIds = eligible,
|
||||
newSegmentId = effectiveNew,
|
||||
bytesReclaimed = reclaimedBefore - bytesWritten,
|
||||
)
|
||||
}
|
||||
|
||||
private fun selectEligible(scan: LivenessScanner.Liveness, activeId: Long): List<Long> =
|
||||
scan.perSegment
|
||||
.filter { (id, stats) ->
|
||||
id != activeId && stats.totalBytes > 0 && stats.liveRatio < liveRatioThreshold
|
||||
}
|
||||
.keys
|
||||
.sorted()
|
||||
|
||||
private suspend fun readLiveEntries(
|
||||
eligible: List<Long>,
|
||||
liveIds: Set<com.correx.core.events.types.ArtifactId>,
|
||||
): List<LiveEntry> {
|
||||
val liveHashHex = liveIds.map { it.value }.toHashSet()
|
||||
val out = mutableListOf<LiveEntry>()
|
||||
for (segId in eligible) {
|
||||
for ((hash, loc) in index.entriesIn(segId)) {
|
||||
if (hash.toHex() in liveHashHex) {
|
||||
val bytes = reader.read(loc, segmentsDir)
|
||||
out.add(LiveEntry(hash, bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private suspend fun writeNewSegment(
|
||||
newSegmentId: Long,
|
||||
entries: List<LiveEntry>,
|
||||
): Pair<Long, List<Pair<ByteArray, Location>>> {
|
||||
if (entries.isEmpty()) return 0L to emptyList()
|
||||
val newPath = segmentsDir.resolve(SegmentWriter.segmentFileName(newSegmentId))
|
||||
var bytesWritten = 0L
|
||||
val newLocations = mutableListOf<Pair<ByteArray, Location>>()
|
||||
withContext(Dispatchers.IO) {
|
||||
RandomAccessFile(newPath.toFile(), "rw").use { raf ->
|
||||
for (entry in entries) {
|
||||
val header = SegmentLayout.encodeHeader(entry.payload, entry.hash)
|
||||
val offset = raf.filePointer
|
||||
raf.write(header)
|
||||
raf.write(entry.payload)
|
||||
newLocations.add(entry.hash to Location(newSegmentId, offset, entry.payload.size))
|
||||
bytesWritten += header.size + entry.payload.size
|
||||
}
|
||||
raf.fd.sync()
|
||||
}
|
||||
}
|
||||
return bytesWritten to newLocations
|
||||
}
|
||||
|
||||
private suspend fun applyAtomicSwap(newLocations: List<Pair<ByteArray, Location>>, eligible: List<Long>) {
|
||||
storeMutex.withLock {
|
||||
for ((hash, loc) in newLocations) {
|
||||
index.update(hash, loc)
|
||||
}
|
||||
for (segId in eligible) {
|
||||
runCatching {
|
||||
Files.deleteIfExists(segmentsDir.resolve(SegmentWriter.segmentFileName(segId)))
|
||||
}
|
||||
}
|
||||
for (segId in eligible) {
|
||||
index.deleteEntriesIn(segId)
|
||||
}
|
||||
index.flush()
|
||||
}
|
||||
}
|
||||
|
||||
private fun listSegmentIds(): List<Long> =
|
||||
segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { p -> runCatching { p.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull() }
|
||||
|
||||
private fun empty(): CompactionReport = CompactionReport(emptyList(), null, 0L)
|
||||
|
||||
private class LiveEntry(
|
||||
val hash: ByteArray,
|
||||
val payload: ByteArray,
|
||||
)
|
||||
|
||||
companion object {
|
||||
const val DEFAULT_LIVE_RATIO_THRESHOLD = 0.5
|
||||
}
|
||||
}
|
||||
|
||||
private fun ByteArray.toHex(): String = joinToString("") { "%02x".format(it) }
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.correx.infrastructure.artifactscas.compact
|
||||
|
||||
import com.correx.core.events.events.InferenceCompletedEvent
|
||||
import com.correx.core.events.events.InferenceStartedEvent
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.events.types.ArtifactId
|
||||
import com.correx.infrastructure.artifactscas.index.ArtifactIndex
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
class LivenessScanner(
|
||||
private val eventStore: EventStore,
|
||||
private val index: ArtifactIndex,
|
||||
) {
|
||||
data class SegmentStats(val liveBytes: Long, val totalBytes: Long) {
|
||||
val liveRatio: Double get() = if (totalBytes == 0L) 1.0 else liveBytes.toDouble() / totalBytes
|
||||
}
|
||||
|
||||
data class Liveness(
|
||||
val liveIds: Set<ArtifactId>,
|
||||
val perSegment: Map<Long, SegmentStats>,
|
||||
)
|
||||
|
||||
suspend fun scan(): Liveness = scan(segmentsDir = null)
|
||||
|
||||
suspend fun scan(segmentsDir: Path?): Liveness {
|
||||
val liveIds = collectLiveIds()
|
||||
val perSegmentLive = mutableMapOf<Long, Long>()
|
||||
for (id in liveIds) {
|
||||
val hash = id.value.hexToBytes()
|
||||
val loc = index.lookup(hash) ?: continue
|
||||
perSegmentLive.merge(loc.segmentId, loc.length.toLong()) { a, b -> a + b }
|
||||
}
|
||||
val perSegment = if (segmentsDir != null && Files.exists(segmentsDir)) {
|
||||
segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { p ->
|
||||
val id = runCatching { p.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull()
|
||||
?: return@mapNotNull null
|
||||
val total = runCatching { Files.size(p) }.getOrDefault(0L)
|
||||
val live = perSegmentLive[id] ?: 0L
|
||||
id to SegmentStats(live, total)
|
||||
}.toMap()
|
||||
} else {
|
||||
perSegmentLive.mapValues { (_, live) -> SegmentStats(live, live) }
|
||||
}
|
||||
return Liveness(liveIds, perSegment)
|
||||
}
|
||||
|
||||
private suspend fun collectLiveIds(): Set<ArtifactId> = withContext(Dispatchers.IO) {
|
||||
val ids = mutableSetOf<ArtifactId>()
|
||||
for (event in eventStore.allEvents()) {
|
||||
when (val p = event.payload) {
|
||||
is InferenceStartedEvent -> ids.add(p.promptArtifactId)
|
||||
is InferenceCompletedEvent -> ids.add(p.responseArtifactId)
|
||||
else -> Unit
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private const val HEX_RADIX = 16
|
||||
private const val HEX_CHARS_PER_BYTE = 2
|
||||
private const val HEX_NIBBLE_BITS = 4
|
||||
|
||||
internal fun String.hexToBytes(): ByteArray {
|
||||
require(length % HEX_CHARS_PER_BYTE == 0) { "hex string must have even length" }
|
||||
return ByteArray(length / HEX_CHARS_PER_BYTE) { i ->
|
||||
val hi = Character.digit(this[HEX_CHARS_PER_BYTE * i], HEX_RADIX)
|
||||
val lo = Character.digit(this[HEX_CHARS_PER_BYTE * i + 1], HEX_RADIX)
|
||||
((hi shl HEX_NIBBLE_BITS) + lo).toByte()
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.correx.infrastructure.artifactscas.config
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
data class CasConfig(
|
||||
val rootDir: Path,
|
||||
val maxSegmentBytes: Long = DEFAULT_MAX_SEGMENT_BYTES,
|
||||
val maxTotalBytes: Long = DEFAULT_MAX_TOTAL_BYTES,
|
||||
) {
|
||||
companion object {
|
||||
const val DEFAULT_MAX_SEGMENT_BYTES: Long = 64L * 1024 * 1024
|
||||
const val DEFAULT_MAX_TOTAL_BYTES: Long = Long.MAX_VALUE
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.correx.infrastructure.artifactscas.evict
|
||||
|
||||
import com.correx.infrastructure.artifactscas.compact.LivenessScanner
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.ArtifactIndex
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentWriter
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
data class EvictionReport(
|
||||
val evictedSegmentIds: List<Long>,
|
||||
val bytesReclaimed: Long,
|
||||
)
|
||||
|
||||
class Evictor(
|
||||
config: CasConfig,
|
||||
private val index: ArtifactIndex,
|
||||
private val liveness: LivenessScanner,
|
||||
) {
|
||||
private val segmentsDir: Path = config.rootDir.resolve("segments")
|
||||
private val maxTotalBytes: Long = config.maxTotalBytes
|
||||
|
||||
suspend fun evict(activeSegmentId: Long): EvictionReport =
|
||||
withContext(Dispatchers.IO) {
|
||||
if (!Files.exists(segmentsDir)) return@withContext empty()
|
||||
val sizes = listSegmentSizes()
|
||||
if (sizes.isEmpty()) return@withContext empty()
|
||||
var totalBytes = sizes.values.sum()
|
||||
if (totalBytes <= maxTotalBytes) return@withContext empty()
|
||||
|
||||
val scan = liveness.scan(segmentsDir)
|
||||
val evicted = mutableListOf<Long>()
|
||||
var reclaimed = 0L
|
||||
val candidates = sizes.keys.sorted()
|
||||
.filter { it != activeSegmentId && (scan.perSegment[it]?.liveBytes ?: 0L) == 0L }
|
||||
for (id in candidates) {
|
||||
if (totalBytes <= maxTotalBytes) break
|
||||
val size = sizes.getValue(id)
|
||||
index.deleteEntriesIn(id)
|
||||
runCatching { Files.deleteIfExists(segmentsDir.resolve(SegmentWriter.segmentFileName(id))) }
|
||||
evicted.add(id)
|
||||
reclaimed += size
|
||||
totalBytes -= size
|
||||
}
|
||||
if (evicted.isEmpty()) empty() else EvictionReport(evicted, reclaimed)
|
||||
}
|
||||
|
||||
private fun listSegmentSizes(): Map<Long, Long> =
|
||||
segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { p ->
|
||||
val id = runCatching { p.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull()
|
||||
?: return@mapNotNull null
|
||||
val size = runCatching { Files.size(p) }.getOrDefault(0L)
|
||||
id to size
|
||||
}.toMap()
|
||||
|
||||
private fun empty(): EvictionReport = EvictionReport(emptyList(), 0L)
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package com.correx.infrastructure.artifactscas.index
|
||||
|
||||
import com.correx.infrastructure.artifactscas.segment.Location
|
||||
|
||||
interface ArtifactIndex : AutoCloseable {
|
||||
suspend fun lookup(hash: ByteArray): Location?
|
||||
suspend fun insert(hash: ByteArray, loc: Location)
|
||||
suspend fun update(hash: ByteArray, loc: Location)
|
||||
suspend fun entriesIn(segmentId: Long): List<Pair<ByteArray, Location>>
|
||||
suspend fun deleteEntriesIn(segmentId: Long)
|
||||
suspend fun flush()
|
||||
suspend fun maxKnownTailFor(segmentId: Long): Long
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package com.correx.infrastructure.artifactscas.index
|
||||
|
||||
import com.correx.infrastructure.artifactscas.segment.Location
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentLayout
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.nio.file.Path
|
||||
import java.sql.Connection
|
||||
import java.sql.DriverManager
|
||||
|
||||
class SqliteArtifactIndex(dbPath: Path) : ArtifactIndex {
|
||||
private val connection: Connection = DriverManager.getConnection("jdbc:sqlite:${dbPath.toAbsolutePath()}")
|
||||
|
||||
init {
|
||||
connection.createStatement().use { stmt ->
|
||||
stmt.execute("PRAGMA journal_mode=WAL;")
|
||||
stmt.execute("PRAGMA synchronous=NORMAL;")
|
||||
stmt.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
hash BLOB PRIMARY KEY,
|
||||
segment_id INTEGER NOT NULL,
|
||||
offset INTEGER NOT NULL,
|
||||
length INTEGER NOT NULL
|
||||
);
|
||||
""".trimIndent()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun lookup(hash: ByteArray): Location? = withContext(Dispatchers.IO) {
|
||||
connection.prepareStatement(
|
||||
"SELECT segment_id, offset, length FROM artifacts WHERE hash = ?"
|
||||
).use { ps ->
|
||||
ps.setBytes(PARAM_HASH, hash)
|
||||
ps.executeQuery().use { rs ->
|
||||
if (!rs.next()) {
|
||||
null
|
||||
} else {
|
||||
Location(rs.getLong(COL_SEGMENT_ID), rs.getLong(COL_OFFSET), rs.getInt(COL_LENGTH))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun insert(hash: ByteArray, loc: Location): Unit = withContext(Dispatchers.IO) {
|
||||
// Same hash always maps to same location; duplicate inserts are no-ops.
|
||||
connection.prepareStatement(
|
||||
"INSERT OR IGNORE INTO artifacts (hash, segment_id, offset, length) VALUES (?, ?, ?, ?)"
|
||||
).use { ps ->
|
||||
ps.setBytes(PARAM_HASH, hash)
|
||||
ps.setLong(PARAM_SEGMENT_ID, loc.segmentId)
|
||||
ps.setLong(PARAM_OFFSET, loc.offset)
|
||||
ps.setInt(PARAM_LENGTH, loc.length)
|
||||
ps.executeUpdate()
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun update(hash: ByteArray, loc: Location): Unit = withContext(Dispatchers.IO) {
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
INSERT INTO artifacts (hash, segment_id, offset, length) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(hash) DO UPDATE SET
|
||||
segment_id = excluded.segment_id,
|
||||
offset = excluded.offset,
|
||||
length = excluded.length
|
||||
""".trimIndent()
|
||||
).use { ps ->
|
||||
ps.setBytes(PARAM_HASH, hash)
|
||||
ps.setLong(PARAM_SEGMENT_ID, loc.segmentId)
|
||||
ps.setLong(PARAM_OFFSET, loc.offset)
|
||||
ps.setInt(PARAM_LENGTH, loc.length)
|
||||
ps.executeUpdate()
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun entriesIn(segmentId: Long): List<Pair<ByteArray, Location>> = withContext(Dispatchers.IO) {
|
||||
connection.prepareStatement(
|
||||
"SELECT hash, offset, length FROM artifacts WHERE segment_id = ? ORDER BY offset ASC"
|
||||
).use { ps ->
|
||||
ps.setLong(1, segmentId)
|
||||
ps.executeQuery().use { rs ->
|
||||
buildList {
|
||||
while (rs.next()) {
|
||||
val hash = rs.getBytes(COL_ENTRY_HASH)
|
||||
val loc = Location(segmentId, rs.getLong(COL_ENTRY_OFFSET), rs.getInt(COL_ENTRY_LENGTH))
|
||||
add(hash to loc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun deleteEntriesIn(segmentId: Long): Unit = withContext(Dispatchers.IO) {
|
||||
connection.prepareStatement("DELETE FROM artifacts WHERE segment_id = ?").use { ps ->
|
||||
ps.setLong(1, segmentId)
|
||||
ps.executeUpdate()
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun maxKnownTailFor(segmentId: Long): Long = withContext(Dispatchers.IO) {
|
||||
connection.prepareStatement(
|
||||
"SELECT offset, length FROM artifacts WHERE segment_id = ? ORDER BY offset DESC LIMIT 1"
|
||||
).use { ps ->
|
||||
ps.setLong(1, segmentId)
|
||||
ps.executeQuery().use { rs ->
|
||||
if (rs.next()) {
|
||||
rs.getLong(1) + SegmentLayout.HEADER_SIZE + rs.getInt(2)
|
||||
} else {
|
||||
0L
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun flush(): Unit = withContext(Dispatchers.IO) {
|
||||
connection.createStatement().use { it.execute("PRAGMA wal_checkpoint(TRUNCATE);") }
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val PARAM_HASH = 1
|
||||
const val PARAM_SEGMENT_ID = 2
|
||||
const val PARAM_OFFSET = 3
|
||||
const val PARAM_LENGTH = 4
|
||||
const val COL_SEGMENT_ID = 1
|
||||
const val COL_OFFSET = 2
|
||||
const val COL_LENGTH = 3
|
||||
const val COL_ENTRY_HASH = 1
|
||||
const val COL_ENTRY_OFFSET = 2
|
||||
const val COL_ENTRY_LENGTH = 3
|
||||
}
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.correx.infrastructure.artifactscas.recovery
|
||||
|
||||
import com.correx.infrastructure.artifactscas.index.ArtifactIndex
|
||||
import com.correx.infrastructure.artifactscas.segment.Location
|
||||
import com.correx.infrastructure.artifactscas.segment.SegmentLayout
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
data class TailScanReport(
|
||||
val activeSegmentId: Long,
|
||||
val knownTailOffset: Long,
|
||||
val recoveredRecords: Int,
|
||||
val truncatedAt: Long?,
|
||||
)
|
||||
|
||||
class TailScanner(
|
||||
private val segmentsDir: Path,
|
||||
private val index: ArtifactIndex,
|
||||
) {
|
||||
suspend fun recover(): TailScanReport? = withContext(Dispatchers.IO) {
|
||||
if (!Files.exists(segmentsDir)) return@withContext null
|
||||
val active = segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { p ->
|
||||
runCatching { p.fileName.toString().removeSuffix(".seg").toLong() to p }.getOrNull()
|
||||
}
|
||||
.maxByOrNull { it.first }
|
||||
?: return@withContext null
|
||||
val (activeId, activePath) = active
|
||||
val knownTail = index.maxKnownTailFor(activeId)
|
||||
scanAndRecover(activeId, activePath, knownTail)
|
||||
}
|
||||
|
||||
private suspend fun scanAndRecover(activeId: Long, path: Path, startOffset: Long): TailScanReport {
|
||||
var recovered = 0
|
||||
var truncatedAt: Long? = null
|
||||
RandomAccessFile(path.toFile(), "rw").use { raf ->
|
||||
val fileLen = raf.length()
|
||||
var pos = startOffset
|
||||
while (pos < fileLen) {
|
||||
val step = tryReadRecord(raf, pos, fileLen)
|
||||
if (step == null) {
|
||||
truncatedAt = pos
|
||||
break
|
||||
}
|
||||
index.insert(step.hash, Location(activeId, pos, step.length))
|
||||
recovered += 1
|
||||
pos += SegmentLayout.HEADER_SIZE + step.length
|
||||
}
|
||||
// Truncate at first bad/partial record so writer resumes at a clean boundary.
|
||||
truncatedAt?.let { raf.setLength(it) }
|
||||
}
|
||||
return TailScanReport(activeId, startOffset, recovered, truncatedAt)
|
||||
}
|
||||
|
||||
private fun tryReadRecord(raf: RandomAccessFile, pos: Long, fileLen: Long): ValidRecord? =
|
||||
runCatching {
|
||||
require(fileLen - pos >= SegmentLayout.HEADER_SIZE)
|
||||
raf.seek(pos)
|
||||
val header = ByteArray(SegmentLayout.HEADER_SIZE)
|
||||
raf.readFully(header)
|
||||
val (length, crc, hash) = SegmentLayout.decodeHeader(header)
|
||||
require(length >= 0 && pos + SegmentLayout.HEADER_SIZE + length <= fileLen)
|
||||
val payload = ByteArray(length)
|
||||
raf.readFully(payload)
|
||||
require(SegmentLayout.crc32c(payload) == crc)
|
||||
require(SegmentLayout.blake3(payload).contentEquals(hash))
|
||||
ValidRecord(length, hash)
|
||||
}.getOrNull()
|
||||
|
||||
private class ValidRecord(val length: Int, val hash: ByteArray)
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.correx.infrastructure.artifactscas.segment
|
||||
|
||||
import org.bouncycastle.crypto.digests.Blake3Digest
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.zip.CRC32C
|
||||
|
||||
object SegmentLayout {
|
||||
const val HASH_SIZE: Int = 32
|
||||
const val HEADER_SIZE: Int = 4 + 4 + HASH_SIZE
|
||||
private const val BLAKE3_DIGEST_BITS: Int = 256
|
||||
|
||||
fun encodeHeader(payload: ByteArray, hash: ByteArray): ByteArray {
|
||||
require(hash.size == HASH_SIZE) { "hash must be $HASH_SIZE bytes" }
|
||||
val crc = CRC32C().apply { update(payload, 0, payload.size) }.value.toInt()
|
||||
return ByteBuffer.allocate(HEADER_SIZE)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
.putInt(payload.size)
|
||||
.putInt(crc)
|
||||
.put(hash)
|
||||
.array()
|
||||
}
|
||||
|
||||
fun decodeHeader(header: ByteArray): Triple<Int, Int, ByteArray> {
|
||||
require(header.size == HEADER_SIZE) { "header must be $HEADER_SIZE bytes" }
|
||||
val buf = ByteBuffer.wrap(header).order(ByteOrder.BIG_ENDIAN)
|
||||
val length = buf.int
|
||||
val crc = buf.int
|
||||
val hash = ByteArray(HASH_SIZE)
|
||||
buf.get(hash)
|
||||
return Triple(length, crc, hash)
|
||||
}
|
||||
|
||||
fun blake3(bytes: ByteArray): ByteArray {
|
||||
val digest = Blake3Digest(BLAKE3_DIGEST_BITS)
|
||||
digest.update(bytes, 0, bytes.size)
|
||||
val out = ByteArray(HASH_SIZE)
|
||||
digest.doFinal(out, 0)
|
||||
return out
|
||||
}
|
||||
|
||||
fun crc32c(payload: ByteArray): Int =
|
||||
CRC32C().apply { update(payload, 0, payload.size) }.value.toInt()
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.correx.infrastructure.artifactscas.segment
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.file.Path
|
||||
|
||||
class CorruptRecordException(message: String) : RuntimeException(message)
|
||||
|
||||
class SegmentReader {
|
||||
suspend fun read(loc: Location, segmentsDir: Path): ByteArray = withContext(Dispatchers.IO) {
|
||||
val path = segmentsDir.resolve(SegmentWriter.segmentFileName(loc.segmentId))
|
||||
RandomAccessFile(path.toFile(), "r").use { raf ->
|
||||
raf.seek(loc.offset)
|
||||
val header = ByteArray(SegmentLayout.HEADER_SIZE)
|
||||
raf.readFully(header)
|
||||
val (length, crc, hash) = SegmentLayout.decodeHeader(header)
|
||||
if (length != loc.length) {
|
||||
throw CorruptRecordException("length mismatch at seg=${loc.segmentId} off=${loc.offset}")
|
||||
}
|
||||
val payload = ByteArray(length)
|
||||
raf.readFully(payload)
|
||||
if (SegmentLayout.crc32c(payload) != crc) {
|
||||
throw CorruptRecordException("crc mismatch at seg=${loc.segmentId} off=${loc.offset}")
|
||||
}
|
||||
if (!SegmentLayout.blake3(payload).contentEquals(hash)) {
|
||||
throw CorruptRecordException("hash mismatch at seg=${loc.segmentId} off=${loc.offset}")
|
||||
}
|
||||
payload
|
||||
}
|
||||
}
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package com.correx.infrastructure.artifactscas.segment
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
data class Location(val segmentId: Long, val offset: Long, val length: Int)
|
||||
|
||||
class SegmentWriter(
|
||||
private val segmentsDir: Path,
|
||||
private val maxSegmentBytes: Long,
|
||||
) : AutoCloseable {
|
||||
private var currentSegmentId: Long = 0
|
||||
private var currentRaf: RandomAccessFile? = null
|
||||
private var currentOffset: Long = 0
|
||||
|
||||
init {
|
||||
Files.createDirectories(segmentsDir)
|
||||
openOrCreateActive()
|
||||
}
|
||||
|
||||
suspend fun append(payload: ByteArray, hash: ByteArray): Location = withContext(Dispatchers.IO) {
|
||||
rotateIfNeeded(payload.size)
|
||||
val raf = currentRaf!!
|
||||
val offset = currentOffset
|
||||
val header = SegmentLayout.encodeHeader(payload, hash)
|
||||
raf.write(header)
|
||||
raf.write(payload)
|
||||
currentOffset += header.size + payload.size
|
||||
Location(currentSegmentId, offset, payload.size)
|
||||
}
|
||||
|
||||
suspend fun fsync() = withContext(Dispatchers.IO) {
|
||||
currentRaf?.fd?.sync()
|
||||
Unit
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
currentRaf?.close()
|
||||
currentRaf = null
|
||||
}
|
||||
|
||||
private fun openOrCreateActive() {
|
||||
val highest = segmentsDir.listDirectoryEntries("*.seg")
|
||||
.mapNotNull { runCatching { it.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull() }
|
||||
.maxOrNull()
|
||||
currentSegmentId = highest ?: 1L
|
||||
val path = segmentsDir.resolve(segmentFileName(currentSegmentId))
|
||||
val raf = RandomAccessFile(path.toFile(), "rw")
|
||||
raf.seek(raf.length())
|
||||
currentRaf = raf
|
||||
currentOffset = raf.length()
|
||||
}
|
||||
|
||||
private fun rotateIfNeeded(incomingPayloadBytes: Int) {
|
||||
val recordBytes = SegmentLayout.HEADER_SIZE.toLong() + incomingPayloadBytes
|
||||
if (currentOffset == 0L) return
|
||||
if (currentOffset + recordBytes <= maxSegmentBytes) return
|
||||
currentRaf?.fd?.sync()
|
||||
currentRaf?.close()
|
||||
currentSegmentId += 1
|
||||
val path = segmentsDir.resolve(segmentFileName(currentSegmentId))
|
||||
val raf = RandomAccessFile(path.toFile(), "rw")
|
||||
currentRaf = raf
|
||||
currentOffset = 0
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun segmentFileName(id: Long): String = "%08d.seg".format(id)
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.correx.infrastructure.artifactscas
|
||||
|
||||
import com.correx.core.utils.TypeId
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.SqliteArtifactIndex
|
||||
import com.correx.infrastructure.artifactscas.segment.CorruptRecordException
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertArrayEquals
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertThrows
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
import kotlin.random.Random
|
||||
|
||||
class CasArtifactStoreTest {
|
||||
|
||||
private fun newStore(dir: Path, maxSegmentBytes: Long = 64L * 1024 * 1024): CasArtifactStore {
|
||||
val index = SqliteArtifactIndex(dir.resolve("index.sqlite"))
|
||||
return CasArtifactStore(CasConfig(dir, maxSegmentBytes), index)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `roundtrip put then get returns same bytes`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir).use { store ->
|
||||
val small = "hello cas".toByteArray()
|
||||
val big = Random(1).nextBytes(1024 * 1024)
|
||||
val idSmall = store.put(small)
|
||||
val idBig = store.put(big)
|
||||
assertArrayEquals(small, store.get(idSmall))
|
||||
assertArrayEquals(big, store.get(idBig))
|
||||
}
|
||||
}
|
||||
|
||||
private fun segmentsTotalSize(dir: Path): Long =
|
||||
Files.walk(dir.resolve("segments"))
|
||||
.filter { Files.isRegularFile(it) }
|
||||
.mapToLong { Files.size(it) }
|
||||
.sum()
|
||||
|
||||
@Test
|
||||
fun `idempotent put returns same id and does not grow segments`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir).use { store ->
|
||||
val payload = Random(2).nextBytes(500)
|
||||
val id1 = store.put(payload)
|
||||
val sizeAfterFirst = segmentsTotalSize(dir)
|
||||
val id2 = store.put(payload)
|
||||
val sizeAfterSecond = segmentsTotalSize(dir)
|
||||
assertEquals(id1, id2)
|
||||
assertEquals(sizeAfterFirst, sizeAfterSecond)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `get for unknown id returns null`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir).use { store ->
|
||||
val bogus = TypeId("00".repeat(32))
|
||||
assertNull(store.get(bogus))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `restart after fsync reopened store reads previous bytes`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val payload = "persistent".toByteArray()
|
||||
val id = newStore(dir).use { store ->
|
||||
val id = store.put(payload)
|
||||
store.flushBefore { /* no-op event commit */ }
|
||||
id
|
||||
}
|
||||
newStore(dir).use { reopened ->
|
||||
assertArrayEquals(payload, reopened.get(id))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `rotation exceeding threshold opens a new segment`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 1024).use { store ->
|
||||
store.put(Random(3).nextBytes(600))
|
||||
store.put(Random(4).nextBytes(600))
|
||||
val files = dir.resolve("segments").listDirectoryEntries("*.seg")
|
||||
assertEquals(2, files.size, "expected rotation, found segments: ${files.map { it.fileName }}")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `get throws CorruptRecordException when segment payload is tampered`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val id = newStore(dir).use { store ->
|
||||
store.put("tamper me".toByteArray()).also { store.flushBefore {} }
|
||||
}
|
||||
val segFile = dir.resolve("segments").listDirectoryEntries("*.seg").single()
|
||||
val bytes = Files.readAllBytes(segFile)
|
||||
bytes[bytes.size - 1] = (bytes[bytes.size - 1].toInt() xor 0xFF).toByte()
|
||||
Files.write(segFile, bytes)
|
||||
newStore(dir).use { reopened ->
|
||||
assertThrows(CorruptRecordException::class.java) { runBlocking { reopened.get(id) } }
|
||||
}
|
||||
}
|
||||
}
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
package com.correx.infrastructure.artifactscas.compact
|
||||
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.InferenceCompletedEvent
|
||||
import com.correx.core.events.events.InferenceStartedEvent
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.types.ArtifactId
|
||||
import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.inference.TokenUsage
|
||||
import com.correx.core.utils.TypeId
|
||||
import com.correx.infrastructure.artifactscas.CasArtifactStore
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.SqliteArtifactIndex
|
||||
import com.correx.infrastructure.persistence.InMemoryEventStore
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.datetime.Clock
|
||||
import org.junit.jupiter.api.Assertions.assertArrayEquals
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.UUID
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
class CompactorTest {
|
||||
|
||||
private val sessionId: SessionId = TypeId("session-1")
|
||||
|
||||
private fun newStore(dir: Path, maxSegmentBytes: Long = 64L * 1024 * 1024): CasArtifactStore {
|
||||
val index = SqliteArtifactIndex(dir.resolve("index.sqlite"))
|
||||
return CasArtifactStore(CasConfig(dir, maxSegmentBytes), index)
|
||||
}
|
||||
|
||||
private suspend fun emitStarted(store: InMemoryEventStore, prompt: ArtifactId) {
|
||||
store.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = InferenceStartedEvent(
|
||||
requestId = TypeId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
stageId = TypeId("stage"),
|
||||
providerId = TypeId("prov"),
|
||||
promptArtifactId = prompt,
|
||||
),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun emitCompleted(store: InMemoryEventStore, response: ArtifactId) {
|
||||
store.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = InferenceCompletedEvent(
|
||||
requestId = TypeId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
stageId = TypeId("stage"),
|
||||
providerId = TypeId("prov"),
|
||||
tokensUsed = TokenUsage(0, 0),
|
||||
latencyMs = 0L,
|
||||
responseArtifactId = response,
|
||||
),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `no compaction when all artifacts live`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 256).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
val a = store.put("aaaa".toByteArray())
|
||||
val b = store.put("bbbb".toByteArray())
|
||||
val c = store.put("cccc".toByteArray())
|
||||
emitCompleted(eventStore, a)
|
||||
emitCompleted(eventStore, b)
|
||||
emitCompleted(eventStore, c)
|
||||
val report = store.compact(eventStore)
|
||||
assertTrue(report.compactedSegmentIds.isEmpty(), "report: $report")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `compacts a mostly-dead sealed segment`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
val live = store.put(ByteArray(40) { 1 })
|
||||
val dead = store.put(ByteArray(40) { 2 })
|
||||
store.put(ByteArray(40) { 3 })
|
||||
emitCompleted(eventStore, live)
|
||||
|
||||
val segsBefore = dir.resolve("segments").listDirectoryEntries("*.seg")
|
||||
.map { it.fileName.toString() }.toSet()
|
||||
assertTrue(segsBefore.size >= 3, "expected rotation, got $segsBefore")
|
||||
|
||||
val report = store.compact(eventStore)
|
||||
assertFalse(report.compactedSegmentIds.isEmpty(), "expected sealed compactions, got $report")
|
||||
|
||||
assertArrayEquals(ByteArray(40) { 1 }, store.get(live))
|
||||
assertNull(store.get(dead))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `fully dead sealed segment is deleted with no new segment written`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val sealedName: String
|
||||
val sealedBytes: Long
|
||||
val sealedSegId: Long
|
||||
newStore(dir, maxSegmentBytes = 64).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
val dead = store.put(ByteArray(40) { 1 })
|
||||
store.put(ByteArray(40) { 2 })
|
||||
|
||||
val segsDir = dir.resolve("segments")
|
||||
val segsBefore = segsDir.listDirectoryEntries("*.seg")
|
||||
.associate { it.fileName.toString() to Files.size(it) }
|
||||
assertEquals(2, segsBefore.size, "expected 2 segments before compaction, got $segsBefore")
|
||||
sealedName = segsBefore.keys.min()
|
||||
sealedBytes = segsBefore.getValue(sealedName)
|
||||
sealedSegId = sealedName.removeSuffix(".seg").toLong()
|
||||
|
||||
val report = store.compact(eventStore)
|
||||
|
||||
assertEquals(listOf(sealedSegId), report.compactedSegmentIds, "report: $report")
|
||||
assertNull(report.newSegmentId, "no new segment should be written, got $report")
|
||||
assertEquals(sealedBytes, report.bytesReclaimed, "report: $report")
|
||||
|
||||
val segsAfter = segsDir.listDirectoryEntries("*.seg")
|
||||
.map { it.fileName.toString() }.toSet()
|
||||
assertFalse(sealedName in segsAfter, "sealed segment file should be deleted, got $segsAfter")
|
||||
assertEquals(1, segsAfter.size, "only active segment should remain, got $segsAfter")
|
||||
|
||||
assertNull(store.get(dead), "dead artifact must not be retrievable")
|
||||
}
|
||||
SqliteArtifactIndex(dir.resolve("index.sqlite")).use { reopened ->
|
||||
assertTrue(
|
||||
reopened.entriesIn(sealedSegId).isEmpty(),
|
||||
"index entries for sealed segment must be removed",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `never touches the active segment`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64L * 1024 * 1024).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put("x".toByteArray())
|
||||
store.put("y".toByteArray())
|
||||
store.put("z".toByteArray())
|
||||
|
||||
val segsDir = dir.resolve("segments")
|
||||
val before = segsDir.listDirectoryEntries("*.seg")
|
||||
.associate { it.fileName.toString() to Files.size(it) }
|
||||
|
||||
val report = store.compact(eventStore)
|
||||
assertTrue(report.compactedSegmentIds.isEmpty(), "active must not be compacted, got $report")
|
||||
|
||||
val after = segsDir.listDirectoryEntries("*.seg")
|
||||
.associate { it.fileName.toString() to Files.size(it) }
|
||||
assertEquals(before, after)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `liveness scanner counts both prompt and response ids`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val eventStore = InMemoryEventStore()
|
||||
val id1: ArtifactId
|
||||
val id2: ArtifactId
|
||||
newStore(dir).use { store ->
|
||||
id1 = store.put("p1".toByteArray())
|
||||
id2 = store.put("p2".toByteArray())
|
||||
emitStarted(eventStore, id1)
|
||||
emitCompleted(eventStore, id2)
|
||||
}
|
||||
val index = SqliteArtifactIndex(dir.resolve("index.sqlite"))
|
||||
index.use {
|
||||
val scanner = LivenessScanner(eventStore, index)
|
||||
val live = scanner.scan().liveIds
|
||||
assertEquals(setOf(id1, id2), live)
|
||||
}
|
||||
}
|
||||
}
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
package com.correx.infrastructure.artifactscas.evict
|
||||
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.InferenceCompletedEvent
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.types.ArtifactId
|
||||
import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.inference.TokenUsage
|
||||
import com.correx.core.utils.TypeId
|
||||
import com.correx.infrastructure.artifactscas.CasArtifactStore
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.SqliteArtifactIndex
|
||||
import com.correx.infrastructure.persistence.InMemoryEventStore
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.datetime.Clock
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.UUID
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
class EvictorTest {
|
||||
|
||||
private val sessionId: SessionId = TypeId("session-evict")
|
||||
|
||||
private fun newStore(dir: Path, maxSegmentBytes: Long, maxTotalBytes: Long): CasArtifactStore {
|
||||
val index = SqliteArtifactIndex(dir.resolve("index.sqlite"))
|
||||
return CasArtifactStore(CasConfig(dir, maxSegmentBytes, maxTotalBytes), index)
|
||||
}
|
||||
|
||||
private suspend fun emitCompleted(store: InMemoryEventStore, response: ArtifactId) {
|
||||
store.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = InferenceCompletedEvent(
|
||||
requestId = TypeId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
stageId = TypeId("stage"),
|
||||
providerId = TypeId("prov"),
|
||||
tokensUsed = TokenUsage(0, 0),
|
||||
latencyMs = 0L,
|
||||
responseArtifactId = response,
|
||||
),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private fun segNames(dir: Path): Set<String> =
|
||||
dir.resolve("segments").listDirectoryEntries("*.seg").map { it.fileName.toString() }.toSet()
|
||||
|
||||
private fun segIds(dir: Path): List<Long> =
|
||||
dir.resolve("segments").listDirectoryEntries("*.seg")
|
||||
.mapNotNull { runCatching { it.fileName.toString().removeSuffix(".seg").toLong() }.getOrNull() }
|
||||
.sorted()
|
||||
|
||||
@Test
|
||||
fun `under cap - no eviction`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 10_000).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 })
|
||||
store.put(ByteArray(40) { 2 })
|
||||
store.put(ByteArray(40) { 3 })
|
||||
|
||||
val before = segNames(dir)
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertTrue(report.evictedSegmentIds.isEmpty(), "report: $report")
|
||||
assertEquals(0L, report.bytesReclaimed)
|
||||
assertEquals(before, segNames(dir))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `over cap, oldest dead segment evicted first`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 80).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 }) // dead, oldest
|
||||
store.put(ByteArray(40) { 2 }) // dead
|
||||
store.put(ByteArray(40) { 3 }) // active segment
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
assertTrue(idsBefore.size >= 3, "expected rotation, got $idsBefore")
|
||||
val oldest = idsBefore.first()
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertTrue(oldest in report.evictedSegmentIds, "expected oldest=$oldest evicted, got $report")
|
||||
assertTrue(report.bytesReclaimed > 0, "report: $report")
|
||||
assertFalse(
|
||||
"%08d.seg".format(oldest) in segNames(dir),
|
||||
"oldest segment file should be deleted",
|
||||
)
|
||||
}
|
||||
SqliteArtifactIndex(dir.resolve("index.sqlite")).use { reopened ->
|
||||
val oldestId = 1L
|
||||
assertTrue(
|
||||
reopened.entriesIn(oldestId).isEmpty(),
|
||||
"index entries for evicted segment must be removed",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `live segment never evicted even if oldest`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 50).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
val live = store.put(ByteArray(40) { 1 }) // oldest, but LIVE
|
||||
store.put(ByteArray(40) { 2 }) // dead
|
||||
store.put(ByteArray(40) { 3 }) // active
|
||||
emitCompleted(eventStore, live)
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
val oldest = idsBefore.first()
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertFalse(
|
||||
oldest in report.evictedSegmentIds,
|
||||
"live oldest must not be evicted, got $report",
|
||||
)
|
||||
assertTrue(
|
||||
"%08d.seg".format(oldest) in segNames(dir),
|
||||
"live oldest segment file must remain",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `active segment never evicted even if dead`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 1).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 })
|
||||
store.put(ByteArray(40) { 2 })
|
||||
store.put(ByteArray(40) { 3 }) // active
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
val active = idsBefore.last()
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertFalse(
|
||||
active in report.evictedSegmentIds,
|
||||
"active must not be evicted, got $report",
|
||||
)
|
||||
assertTrue(
|
||||
"%08d.seg".format(active) in segNames(dir),
|
||||
"active segment file must remain",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `evicts strictly oldest-first when multiple dead segments exist`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 200).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 }) // dead, oldest
|
||||
store.put(ByteArray(40) { 2 }) // dead
|
||||
store.put(ByteArray(40) { 3 }) // dead, youngest dead
|
||||
store.put(ByteArray(40) { 4 }) // active
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
assertTrue(idsBefore.size >= 4, "expected >=4 segments, got $idsBefore")
|
||||
val oldest = idsBefore[0]
|
||||
val secondOldest = idsBefore[1]
|
||||
val youngestDead = idsBefore[2]
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertEquals(
|
||||
listOf(oldest, secondOldest),
|
||||
report.evictedSegmentIds,
|
||||
"expected strictly oldest-first eviction, got $report",
|
||||
)
|
||||
assertTrue(
|
||||
"%08d.seg".format(youngestDead) in segNames(dir),
|
||||
"youngest dead segment must survive, ids=${segIds(dir)}",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `live segment in the middle is skipped and younger dead segment is evicted`(
|
||||
@TempDir dir: Path,
|
||||
): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 50).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 }) // dead, oldest
|
||||
val live = store.put(ByteArray(40) { 2 }) // live, middle
|
||||
store.put(ByteArray(40) { 3 }) // dead, newer
|
||||
store.put(ByteArray(40) { 4 }) // active
|
||||
emitCompleted(eventStore, live)
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
assertTrue(idsBefore.size >= 4, "expected >=4 segments, got $idsBefore")
|
||||
val oldest = idsBefore[0]
|
||||
val middle = idsBefore[1]
|
||||
val newerDead = idsBefore[2]
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertFalse(
|
||||
middle in report.evictedSegmentIds,
|
||||
"live middle segment must not be evicted, got $report",
|
||||
)
|
||||
assertTrue(
|
||||
"%08d.seg".format(middle) in segNames(dir),
|
||||
"live middle segment file must remain",
|
||||
)
|
||||
assertTrue(
|
||||
oldest in report.evictedSegmentIds,
|
||||
"expected oldest dead evicted, got $report",
|
||||
)
|
||||
// If more than one eviction was needed, the next must be the newer dead — in age order.
|
||||
if (report.evictedSegmentIds.size >= 2) {
|
||||
assertEquals(
|
||||
listOf(oldest, newerDead),
|
||||
report.evictedSegmentIds,
|
||||
"evictions must be in age order, skipping live middle",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `evicts multiple segments until under cap`(@TempDir dir: Path): Unit = runBlocking {
|
||||
newStore(dir, maxSegmentBytes = 64, maxTotalBytes = 100).use { store ->
|
||||
val eventStore = InMemoryEventStore()
|
||||
store.put(ByteArray(40) { 1 })
|
||||
store.put(ByteArray(40) { 2 })
|
||||
store.put(ByteArray(40) { 3 })
|
||||
store.put(ByteArray(40) { 4 })
|
||||
store.put(ByteArray(40) { 5 }) // active
|
||||
|
||||
val idsBefore = segIds(dir)
|
||||
assertTrue(idsBefore.size >= 4, "expected several segments, got $idsBefore")
|
||||
|
||||
val report = store.evict(eventStore)
|
||||
|
||||
assertTrue(report.evictedSegmentIds.size >= 2, "expected >=2 evicted, got $report")
|
||||
|
||||
val segsDir = dir.resolve("segments")
|
||||
val totalAfter = segsDir.listDirectoryEntries("*.seg").sumOf { Files.size(it) }
|
||||
assertTrue(
|
||||
totalAfter <= 100 || segIds(dir).size == 1,
|
||||
"expected under cap or only active remaining, totalAfter=$totalAfter ids=${segIds(dir)}",
|
||||
)
|
||||
for (evicted in report.evictedSegmentIds) {
|
||||
assertFalse(
|
||||
"%08d.seg".format(evicted) in segNames(dir),
|
||||
"evicted segment $evicted must be deleted",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.correx.infrastructure.artifactscas.recovery
|
||||
|
||||
import com.correx.infrastructure.artifactscas.CasArtifactStore
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.SqliteArtifactIndex
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertArrayEquals
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.sql.DriverManager
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
class TailScannerTest {
|
||||
|
||||
private fun newIndex(dir: Path) = SqliteArtifactIndex(dir.resolve("index.sqlite"))
|
||||
|
||||
@Test
|
||||
fun `tail beyond known index is re-added on open`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val payload = "recover me".toByteArray()
|
||||
val id = CasArtifactStore.open(CasConfig(dir), newIndex(dir)).use { store ->
|
||||
val id = store.put(payload)
|
||||
store.flushBefore {}
|
||||
id
|
||||
}
|
||||
// Simulate lost index: wipe the rows.
|
||||
DriverManager.getConnection("jdbc:sqlite:${dir.resolve("index.sqlite").toAbsolutePath()}").use { conn ->
|
||||
conn.createStatement().use { it.execute("DELETE FROM artifacts;") }
|
||||
}
|
||||
CasArtifactStore.open(CasConfig(dir), newIndex(dir)).use { reopened ->
|
||||
assertArrayEquals(payload, reopened.get(id))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `corrupt tail is truncated on open`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val p1 = "first".toByteArray()
|
||||
val p2 = "second".toByteArray()
|
||||
val (id1, id2) = CasArtifactStore.open(CasConfig(dir), newIndex(dir)).use { store ->
|
||||
val a = store.put(p1)
|
||||
val b = store.put(p2)
|
||||
store.flushBefore {}
|
||||
a to b
|
||||
}
|
||||
val segFile = dir.resolve("segments").listDirectoryEntries("*.seg").single()
|
||||
val cleanSize = Files.size(segFile)
|
||||
// Append garbage that will fail header/length validation.
|
||||
Files.write(
|
||||
segFile,
|
||||
ByteArray(GARBAGE_BYTES) { 0xAB.toByte() },
|
||||
java.nio.file.StandardOpenOption.APPEND,
|
||||
)
|
||||
CasArtifactStore.open(CasConfig(dir), newIndex(dir)).use { reopened ->
|
||||
assertArrayEquals(p1, reopened.get(id1))
|
||||
assertArrayEquals(p2, reopened.get(id2))
|
||||
}
|
||||
assertEquals(cleanSize, Files.size(segFile))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `recover returns null when no segments dir exists`(@TempDir dir: Path): Unit = runBlocking {
|
||||
val root = dir.resolve("fresh")
|
||||
Files.createDirectories(root)
|
||||
newIndex(root).use { idx ->
|
||||
val scanner = TailScanner(root.resolve("segments"), idx)
|
||||
assertNull(scanner.recover())
|
||||
}
|
||||
// And open() still succeeds despite no prior segments.
|
||||
CasArtifactStore.open(CasConfig(root), newIndex(root)).use { store ->
|
||||
assertNotNull(store)
|
||||
}
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val GARBAGE_BYTES = 20
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,8 @@ dependencies {
|
||||
implementation project(":infrastructure:tools:filesystem")
|
||||
implementation project(":infrastructure:workflow")
|
||||
implementation project(":core:artifacts")
|
||||
implementation project(":core:artifacts-store")
|
||||
implementation project(":infrastructure:artifacts-cas")
|
||||
implementation "io.ktor:ktor-client-core:$ktor_version"
|
||||
implementation "io.ktor:ktor-client-cio:$ktor_version"
|
||||
implementation "io.ktor:ktor-client-content-negotiation:$ktor_version"
|
||||
|
||||
+2
-1
@@ -30,12 +30,13 @@ import java.io.File
|
||||
*
|
||||
* Thread-safe: all public methods use Mutex for synchronization.
|
||||
*/
|
||||
@Suppress("LongParameterList", "TooGenericExceptionCaught", "UnusedPrivateProperty")
|
||||
@Suppress("LongParameterList", "TooGenericExceptionCaught")
|
||||
class DefaultModelManager(
|
||||
private val llamaServerBin: String = "llama-server",
|
||||
private val host: String = "127.0.0.1",
|
||||
private val port: Int = 10000,
|
||||
private val healthTimeoutMs: Long = 30_000L,
|
||||
@Suppress("UnusedPrivateProperty")
|
||||
private val eventStore: EventStore,
|
||||
private val httpClient: HttpClient,
|
||||
private val eventDispatcher: EventDispatcher,
|
||||
|
||||
+6
@@ -15,6 +15,7 @@ import com.correx.infrastructure.inference.commons.ModelDescriptor
|
||||
import io.ktor.client.HttpClient
|
||||
import io.ktor.client.call.body
|
||||
import io.ktor.client.engine.cio.CIO
|
||||
import io.ktor.client.plugins.HttpTimeout
|
||||
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
|
||||
import io.ktor.client.request.accept
|
||||
import io.ktor.client.request.get
|
||||
@@ -25,6 +26,8 @@ import io.ktor.http.contentType
|
||||
import io.ktor.serialization.kotlinx.json.json
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
private const val DEFAULT_REQUEST_TIMEOUT_MS = 60_000L
|
||||
|
||||
private fun defaultHttpClient(): HttpClient = HttpClient(CIO) {
|
||||
install(ContentNegotiation) {
|
||||
json(
|
||||
@@ -35,6 +38,9 @@ private fun defaultHttpClient(): HttpClient = HttpClient(CIO) {
|
||||
},
|
||||
)
|
||||
}
|
||||
install(HttpTimeout) {
|
||||
requestTimeoutMillis = DEFAULT_REQUEST_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("TooGenericExceptionCaught", "MagicNumber")
|
||||
|
||||
@@ -9,6 +9,7 @@ dependencies {
|
||||
implementation(project(":core:events"))
|
||||
implementation(project(":core:sessions"))
|
||||
implementation(project(":core:artifacts"))
|
||||
implementation(project(":core:artifacts-store"))
|
||||
implementation "org.xerial:sqlite-jdbc"
|
||||
testImplementation(testFixtures(project(":testing:contracts")))
|
||||
testImplementation "org.junit.jupiter:junit-jupiter"
|
||||
|
||||
+7
-2
@@ -16,7 +16,7 @@ class InMemoryEventStore : EventStore {
|
||||
private val seenEventIds = ConcurrentHashMap.newKeySet<EventId>()
|
||||
private val subscriptions = ConcurrentHashMap<SessionId, MutableSharedFlow<StoredEvent>>()
|
||||
|
||||
override fun append(event: NewEvent): StoredEvent {
|
||||
override suspend fun append(event: NewEvent): StoredEvent {
|
||||
val stream = streams.computeIfAbsent(event.metadata.sessionId) { mutableListOf() }
|
||||
|
||||
synchronized(stream) {
|
||||
@@ -27,7 +27,7 @@ class InMemoryEventStore : EventStore {
|
||||
}
|
||||
}
|
||||
|
||||
override fun appendAll(events: List<NewEvent>): List<StoredEvent> {
|
||||
override suspend fun appendAll(events: List<NewEvent>): List<StoredEvent> {
|
||||
if (events.isEmpty()) return emptyList()
|
||||
|
||||
val sessionId = events.first().metadata.sessionId
|
||||
@@ -58,6 +58,11 @@ class InMemoryEventStore : EventStore {
|
||||
override fun subscribe(sessionId: SessionId): Flow<StoredEvent> =
|
||||
subscriptions.computeIfAbsent(sessionId) { MutableSharedFlow(replay = 0, extraBufferCapacity = 64) }
|
||||
|
||||
override fun allEvents(): Sequence<StoredEvent> =
|
||||
streams.values
|
||||
.flatMap { stream -> synchronized(stream) { stream.toList() } }
|
||||
.asSequence()
|
||||
|
||||
private fun doAppend(event: NewEvent, stream: MutableList<StoredEvent>): StoredEvent {
|
||||
val seq = sequences.computeIfAbsent(event.metadata.sessionId) { AtomicLong(0) }
|
||||
.incrementAndGet()
|
||||
|
||||
+77
-45
@@ -1,5 +1,6 @@
|
||||
package com.correx.infrastructure.persistence
|
||||
|
||||
import com.correx.core.artifactstore.ArtifactStore
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
@@ -11,8 +12,10 @@ import com.correx.core.events.types.CorrelationId
|
||||
import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.infrastructure.persistence.util.JDBCHelper.transaction
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.datetime.Instant
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -20,7 +23,8 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
class SqliteEventStore(
|
||||
private val connection: Connection,
|
||||
private val jsonSerializer: JsonEventSerializer = JsonEventSerializer(eventJson)
|
||||
private val jsonSerializer: JsonEventSerializer = JsonEventSerializer(eventJson),
|
||||
private val artifactStore: ArtifactStore,
|
||||
) : EventStore {
|
||||
private val subscriptions = ConcurrentHashMap<SessionId, MutableSharedFlow<StoredEvent>>()
|
||||
|
||||
@@ -43,46 +47,57 @@ class SqliteEventStore(
|
||||
}
|
||||
}
|
||||
|
||||
override fun append(event: NewEvent): StoredEvent {
|
||||
val stored = connection.transaction {
|
||||
val existing = findByEventId(event.metadata.eventId)
|
||||
check(existing == null) { "duplicate event_id: ${event.metadata.eventId}" }
|
||||
override suspend fun append(event: NewEvent): StoredEvent {
|
||||
var stored: StoredEvent? = null
|
||||
artifactStore.flushBefore {
|
||||
withContext(Dispatchers.IO) {
|
||||
stored = connection.transaction {
|
||||
val existing = findByEventId(event.metadata.eventId)
|
||||
check(existing == null) { "duplicate event_id: ${event.metadata.eventId}" }
|
||||
|
||||
val seq = nextSequence(event.metadata.sessionId)
|
||||
val seq = nextSequence(event.metadata.sessionId)
|
||||
|
||||
val s = StoredEvent(
|
||||
metadata = event.metadata,
|
||||
sequence = seq,
|
||||
payload = event.payload
|
||||
)
|
||||
val s = StoredEvent(
|
||||
metadata = event.metadata,
|
||||
sequence = seq,
|
||||
payload = event.payload
|
||||
)
|
||||
|
||||
insert(s)
|
||||
s
|
||||
insert(s)
|
||||
s
|
||||
}
|
||||
}
|
||||
}
|
||||
subscriptions[event.metadata.sessionId]?.tryEmit(stored)
|
||||
return stored
|
||||
val result = checkNotNull(stored)
|
||||
subscriptions[event.metadata.sessionId]?.tryEmit(result)
|
||||
return result
|
||||
}
|
||||
|
||||
override fun appendAll(events: List<NewEvent>): List<StoredEvent> {
|
||||
override suspend fun appendAll(events: List<NewEvent>): List<StoredEvent> {
|
||||
if (events.isEmpty()) return emptyList()
|
||||
|
||||
val sessionId = events.first().metadata.sessionId
|
||||
|
||||
val stored = connection.transaction {
|
||||
events.map { event ->
|
||||
val existing = findByEventId(event.metadata.eventId)
|
||||
check(existing == null) { "duplicate event_id: ${event.metadata.eventId}" }
|
||||
var stored: List<StoredEvent> = emptyList()
|
||||
artifactStore.flushBefore {
|
||||
withContext(Dispatchers.IO) {
|
||||
stored = connection.transaction {
|
||||
events.map { event ->
|
||||
val existing = findByEventId(event.metadata.eventId)
|
||||
check(existing == null) { "duplicate event_id: ${event.metadata.eventId}" }
|
||||
|
||||
val seq = nextSequence(sessionId)
|
||||
val seq = nextSequence(sessionId)
|
||||
|
||||
val s = StoredEvent(
|
||||
metadata = event.metadata,
|
||||
sequence = seq,
|
||||
payload = event.payload
|
||||
)
|
||||
val s = StoredEvent(
|
||||
metadata = event.metadata,
|
||||
sequence = seq,
|
||||
payload = event.payload
|
||||
)
|
||||
|
||||
insert(s)
|
||||
s
|
||||
insert(s)
|
||||
s
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val flow = subscriptions[sessionId]
|
||||
@@ -103,7 +118,7 @@ class SqliteEventStore(
|
||||
ps.executeQuery().use { rs ->
|
||||
buildList {
|
||||
while (rs.next()) {
|
||||
add(map(rs))
|
||||
add(rs.toStoredEvent(jsonSerializer))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -124,7 +139,7 @@ class SqliteEventStore(
|
||||
ps.executeQuery().use { rs ->
|
||||
buildList {
|
||||
while (rs.next()) {
|
||||
add(map(rs))
|
||||
add(rs.toStoredEvent(jsonSerializer))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,6 +164,22 @@ class SqliteEventStore(
|
||||
override fun subscribe(sessionId: SessionId): Flow<StoredEvent> =
|
||||
subscriptions.computeIfAbsent(sessionId) { MutableSharedFlow(replay = 0, extraBufferCapacity = 64) }
|
||||
|
||||
override fun allEvents(): Sequence<StoredEvent> =
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
SELECT * FROM events
|
||||
ORDER BY session_id ASC, sequence ASC
|
||||
""".trimIndent()
|
||||
).use { ps ->
|
||||
ps.executeQuery().use { rs ->
|
||||
buildList {
|
||||
while (rs.next()) {
|
||||
add(rs.toStoredEvent(jsonSerializer))
|
||||
}
|
||||
}
|
||||
}
|
||||
}.asSequence()
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
private fun nextSequence(sessionId: SessionId): Long =
|
||||
@@ -205,21 +236,22 @@ class SqliteEventStore(
|
||||
ps.setString(1, eventId.value)
|
||||
|
||||
ps.executeQuery().use { rs ->
|
||||
if (rs.next()) map(rs) else null
|
||||
if (rs.next()) rs.toStoredEvent(jsonSerializer) else null
|
||||
}
|
||||
}
|
||||
|
||||
private fun map(rs: ResultSet): StoredEvent =
|
||||
StoredEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(rs.getString("event_id")),
|
||||
sessionId = SessionId(rs.getString("session_id")),
|
||||
timestamp = Instant.parse(rs.getString("timestamp")),
|
||||
schemaVersion = rs.getInt("schema_version"),
|
||||
causationId = rs.getString("causation_id")?.let { CausationId(it) },
|
||||
correlationId = rs.getString("correlation_id")?.let { CorrelationId(it) }
|
||||
),
|
||||
sequence = rs.getLong("sequence"),
|
||||
payload = jsonSerializer.deserialize(rs.getString("payload"))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun ResultSet.toStoredEvent(jsonSerializer: JsonEventSerializer): StoredEvent =
|
||||
StoredEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(getString("event_id")),
|
||||
sessionId = SessionId(getString("session_id")),
|
||||
timestamp = Instant.parse(getString("timestamp")),
|
||||
schemaVersion = getInt("schema_version"),
|
||||
causationId = getString("causation_id")?.let { CausationId(it) },
|
||||
correlationId = getString("correlation_id")?.let { CorrelationId(it) }
|
||||
),
|
||||
sequence = getLong("sequence"),
|
||||
payload = jsonSerializer.deserialize(getString("payload"))
|
||||
)
|
||||
+2
-1
@@ -1,12 +1,13 @@
|
||||
package com.correx.infrastructure.persistence
|
||||
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.testing.contracts.fixtures.artifactstore.NoopArtifactStore
|
||||
import com.correx.testing.contracts.fixtures.events.store.EventStoreContractTest
|
||||
import java.sql.DriverManager
|
||||
|
||||
class SqliteEventStoreTest : EventStoreContractTest() {
|
||||
override fun store(): EventStore {
|
||||
val conn = DriverManager.getConnection("jdbc:sqlite::memory:")
|
||||
return SqliteEventStore(conn)
|
||||
return SqliteEventStore(conn, artifactStore = NoopArtifactStore())
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -5,6 +5,7 @@ import com.correx.core.sessions.SessionCounterProjection
|
||||
import com.correx.core.sessions.SessionCounterState
|
||||
import com.correx.core.sessions.projections.replay.DefaultEventReplayer
|
||||
import com.correx.core.sessions.projections.replay.EventReplayer
|
||||
import com.correx.testing.contracts.fixtures.artifactstore.NoopArtifactStore
|
||||
import com.correx.testing.contracts.fixtures.projections.SessionReplayContractTest
|
||||
import java.sql.DriverManager
|
||||
|
||||
@@ -14,7 +15,7 @@ class SqliteReplayTest : SessionReplayContractTest() {
|
||||
val connection =
|
||||
DriverManager.getConnection("jdbc:sqlite::memory:")
|
||||
|
||||
return SqliteEventStore(connection)
|
||||
return SqliteEventStore(connection, artifactStore = NoopArtifactStore())
|
||||
}
|
||||
|
||||
override fun replayer(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.correx.infrastructure
|
||||
|
||||
import com.correx.core.approvals.domain.ApprovalEngine
|
||||
import com.correx.core.artifactstore.ArtifactStore
|
||||
import com.correx.core.events.EventDispatcher
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.inference.DefaultInferenceRouter
|
||||
@@ -17,6 +18,10 @@ import com.correx.infrastructure.inference.commons.ModelManager
|
||||
import com.correx.infrastructure.inference.commons.ResidencyMode
|
||||
import com.correx.infrastructure.inference.llama.cpp.DefaultModelManager
|
||||
import com.correx.infrastructure.inference.llama.cpp.LlamaCppInferenceProvider
|
||||
import com.correx.infrastructure.artifactscas.CasArtifactStore
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import com.correx.infrastructure.artifactscas.config.CasConfig
|
||||
import com.correx.infrastructure.artifactscas.index.SqliteArtifactIndex
|
||||
import com.correx.infrastructure.persistence.SqliteEventStore
|
||||
import com.correx.infrastructure.tools.DefaultToolRegistry
|
||||
import com.correx.infrastructure.tools.DispatchingToolExecutor
|
||||
@@ -31,17 +36,32 @@ import com.correx.infrastructure.persistence.artifact.LiveArtifactRepository
|
||||
import com.correx.core.artifacts.repository.ArtifactRepository
|
||||
import com.correx.core.artifacts.DefaultArtifactReducer
|
||||
import io.ktor.client.HttpClient
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.sql.DriverManager
|
||||
|
||||
@Suppress("TooManyFunctions")
|
||||
object InfrastructureModule {
|
||||
private val defaultDbPath: String =
|
||||
"${System.getProperty("user.home")}/.config/correx/correx.db"
|
||||
|
||||
fun createEventStore(dbPath: String = defaultDbPath): EventStore {
|
||||
private val defaultArtifactsRoot: String =
|
||||
"${System.getProperty("user.home")}/.config/correx/artifacts"
|
||||
|
||||
fun createEventStore(artifactStore: ArtifactStore, dbPath: String = defaultDbPath): EventStore {
|
||||
val dir = java.io.File(dbPath).parentFile
|
||||
if (!dir.exists()) dir.mkdirs()
|
||||
return SqliteEventStore(DriverManager.getConnection("jdbc:sqlite:$dbPath"))
|
||||
return SqliteEventStore(
|
||||
connection = DriverManager.getConnection("jdbc:sqlite:$dbPath"),
|
||||
artifactStore = artifactStore,
|
||||
)
|
||||
}
|
||||
|
||||
fun createArtifactStore(rootDir: Path = Paths.get(defaultArtifactsRoot)): ArtifactStore {
|
||||
Files.createDirectories(rootDir)
|
||||
val index = SqliteArtifactIndex(rootDir.resolve("index.sqlite"))
|
||||
return runBlocking { CasArtifactStore.open(CasConfig(rootDir), index) }
|
||||
}
|
||||
|
||||
fun createModelManager(
|
||||
|
||||
+5
-1
@@ -24,6 +24,7 @@ private data class WorkflowFile(
|
||||
private data class StageSection(
|
||||
val id: String = "",
|
||||
val prompt: String? = null,
|
||||
val systemPrompt: String? = null,
|
||||
val produces: List<String> = emptyList(),
|
||||
val needs: List<String> = emptyList(),
|
||||
@JsonProperty("allowed_tools") val allowedTools: List<String> = emptyList(),
|
||||
@@ -66,7 +67,10 @@ class TomlWorkflowLoader : WorkflowLoader {
|
||||
allowedTools = s.allowedTools.toSet(),
|
||||
tokenBudget = s.tokenBudget,
|
||||
maxRetries = s.maxRetries,
|
||||
metadata = buildMap { s.prompt?.let { put("prompt", it) } },
|
||||
metadata = buildMap {
|
||||
s.prompt?.let { put("prompt", it) }
|
||||
?: s.systemPrompt?.let { put("systemPrompt", it) }
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user