feat(server,cli): observability metrics as a replayable projection + correx stats

Implements observability-spec §2 + §3-tier-1: metrics as a pure fold over the
event log, no third pillar, no OTel. Every metric is replayable over the full
historical log because it derives from events other contexts already emit — no
new EventPayload subclasses, so no serialization registry change.

- MetricsProjection: Projection<MetricsState> folding inference/tool/approval/
  stage/workflow events into raw sums; approval request→decision latency is
  correlated in-fold via a transient pending-request map.
- MetricsInspectionService: rebuilds via DefaultEventReplayer, derives read-side
  ratios (token throughput, avg approval wait, session wall-time accounting split
  into inference / tool / approval-wait / other) into MetricsReport.
- GET /sessions/{id}/metrics route, mirroring the replay endpoint.
- `correx stats <sessionId>` CLI client (--json passthrough), mirroring replay.
- Tests: seeded-event fold correctness with controlled timestamps (7) + CLI
  render smoke tests (4).
This commit is contained in:
2026-06-13 10:23:40 +04:00
parent 75703ec0c8
commit fe941408a7
8 changed files with 876 additions and 0 deletions
@@ -0,0 +1,148 @@
package com.correx.apps.server.metrics
import com.correx.core.events.stores.EventStore
import com.correx.core.events.types.SessionId
import com.correx.core.sessions.projections.replay.DefaultEventReplayer
import kotlinx.serialization.Serializable
private const val MS_PER_SECOND = 1000.0
private const val PERCENT = 100.0
@Serializable
data class ProviderStatsDto(
val provider: String,
val completedCount: Long,
val totalLatencyMs: Long,
val promptTokens: Long,
val completionTokens: Long,
val tokensPerSecond: Double,
)
@Serializable
data class ToolStatsDto(
val toolName: String,
val completedCount: Long,
val totalDurationMs: Long,
)
@Serializable
data class TierApprovalStatsDto(
val tier: String,
val requestedCount: Long,
val resolvedCount: Long,
val totalWaitMs: Long,
val avgWaitMs: Long,
)
/**
* Render-friendly view of a session's metrics. Raw sums plus the derived ratios the operator
* actually reads; the time-accounting percentages split session wall time into inference /
* tool execution / awaiting-approval, with the remainder ("other": context build, orchestration,
* idle) recoverable as `100 (inference + tool + approvalWait)`.
*/
@Serializable
data class MetricsReport(
val sessionId: String,
val eventCount: Long,
val sessionDurationMs: Long,
val inferenceCount: Long,
val inferenceMs: Long,
val promptTokens: Long,
val completionTokens: Long,
val tokensPerSecond: Double,
val perProvider: List<ProviderStatsDto>,
val toolCount: Long,
val toolMs: Long,
val perTool: List<ToolStatsDto>,
val approvalsRequested: Long,
val approvalsResolved: Long,
val approvalsPending: Long,
val approvalWaitMs: Long,
val avgApprovalWaitMs: Long,
val perTier: List<TierApprovalStatsDto>,
val failures: FailureMetrics,
val inferencePct: Double,
val toolPct: Double,
val approvalWaitPct: Double,
)
/**
* Rebuilds [MetricsState] by replaying a session through [MetricsProjection], then derives the
* read-side ratios into a [MetricsReport]. Replay-only: no network, no live inference
* (Hard Invariant #8).
*/
class MetricsInspectionService(private val eventStore: EventStore) {
private val replayer = DefaultEventReplayer(eventStore, MetricsProjection())
fun inspect(sessionId: SessionId): MetricsReport {
val state = replayer.rebuild(sessionId)
return toReport(sessionId, state)
}
private fun toReport(sessionId: SessionId, s: MetricsState): MetricsReport {
val durationMs = sessionDurationMs(s)
return MetricsReport(
sessionId = sessionId.value,
eventCount = s.eventCount,
sessionDurationMs = durationMs,
inferenceCount = s.inference.completedCount,
inferenceMs = s.inference.totalLatencyMs,
promptTokens = s.inference.promptTokens,
completionTokens = s.inference.completionTokens,
tokensPerSecond = tokensPerSecond(s.inference.completionTokens, s.inference.totalLatencyMs),
perProvider = s.inference.perProvider.entries
.sortedByDescending { it.value.totalLatencyMs }
.map { (provider, m) ->
ProviderStatsDto(
provider = provider,
completedCount = m.completedCount,
totalLatencyMs = m.totalLatencyMs,
promptTokens = m.promptTokens,
completionTokens = m.completionTokens,
tokensPerSecond = tokensPerSecond(m.completionTokens, m.totalLatencyMs),
)
},
toolCount = s.tools.completedCount,
toolMs = s.tools.totalDurationMs,
perTool = s.tools.perTool.entries
.sortedByDescending { it.value.totalDurationMs }
.map { (name, m) -> ToolStatsDto(name, m.completedCount, m.totalDurationMs) },
approvalsRequested = s.approvals.requestedCount,
approvalsResolved = s.approvals.resolvedCount,
approvalsPending = s.pendingApprovals.size.toLong(),
approvalWaitMs = s.approvals.totalWaitMs,
avgApprovalWaitMs = avg(s.approvals.totalWaitMs, s.approvals.resolvedCount),
perTier = s.approvals.perTier.entries
.sortedBy { it.key }
.map { (tier, m) ->
TierApprovalStatsDto(
tier = tier,
requestedCount = m.requestedCount,
resolvedCount = m.resolvedCount,
totalWaitMs = m.totalWaitMs,
avgWaitMs = avg(m.totalWaitMs, m.resolvedCount),
)
},
failures = s.failures,
inferencePct = pct(s.inference.totalLatencyMs, durationMs),
toolPct = pct(s.tools.totalDurationMs, durationMs),
approvalWaitPct = pct(s.approvals.totalWaitMs, durationMs),
)
}
private fun sessionDurationMs(s: MetricsState): Long {
val first = s.firstEventAt ?: return 0L
val last = s.lastEventAt ?: return 0L
return (last - first).inWholeMilliseconds.coerceAtLeast(0L)
}
private fun tokensPerSecond(completionTokens: Long, latencyMs: Long): Double =
if (latencyMs <= 0L) 0.0 else completionTokens.toDouble() / (latencyMs / MS_PER_SECOND)
private fun pct(part: Long, whole: Long): Double =
if (whole <= 0L) 0.0 else (part.toDouble() / whole.toDouble()) * PERCENT
private fun avg(total: Long, count: Long): Long =
if (count <= 0L) 0L else total / count
}
@@ -0,0 +1,123 @@
package com.correx.apps.server.metrics
import com.correx.core.events.events.ApprovalDecisionResolvedEvent
import com.correx.core.events.events.ApprovalRequestedEvent
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.InferenceFailedEvent
import com.correx.core.events.events.InferenceTimeoutEvent
import com.correx.core.events.events.StageFailedEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.ToolExecutionCompletedEvent
import com.correx.core.events.events.ToolExecutionFailedEvent
import com.correx.core.events.events.ToolExecutionRejectedEvent
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.sessions.projections.Projection
import kotlinx.datetime.Instant
/**
* Pure fold deriving [MetricsState] from a session's event log
* (observability-spec §2: observability-as-projection — no separate trace store, no OTel).
*
* Every metric here is replayable: a metric added later can be recomputed against events
* recorded before it existed. This projection records no new events and reads only events
* other bounded contexts already emit; it is a disposable read-model (Hard Invariant #2).
*/
class MetricsProjection : Projection<MetricsState> {
override fun initial(): MetricsState = MetricsState()
override fun apply(state: MetricsState, event: StoredEvent): MetricsState {
val at = event.metadata.timestamp
val spanned = state.copy(
firstEventAt = state.firstEventAt ?: at,
lastEventAt = at,
eventCount = state.eventCount + 1,
)
return when (val payload = event.payload) {
is InferenceCompletedEvent -> spanned.applyInferenceCompleted(payload)
is InferenceFailedEvent -> spanned.bumpFailure { it.copy(inferenceFailures = it.inferenceFailures + 1) }
is InferenceTimeoutEvent -> spanned.bumpFailure { it.copy(inferenceTimeouts = it.inferenceTimeouts + 1) }
is ToolExecutionCompletedEvent -> spanned.applyToolCompleted(payload)
is ToolExecutionFailedEvent -> spanned.bumpFailure { it.copy(toolFailures = it.toolFailures + 1) }
is ToolExecutionRejectedEvent -> spanned.bumpFailure { it.copy(toolRejections = it.toolRejections + 1) }
is ApprovalRequestedEvent -> spanned.applyApprovalRequested(payload, at)
is ApprovalDecisionResolvedEvent -> spanned.applyApprovalResolved(payload)
is StageFailedEvent -> spanned.bumpFailure { it.copy(stageFailures = it.stageFailures + 1) }
is WorkflowFailedEvent -> spanned.bumpFailure { it.copy(workflowFailures = it.workflowFailures + 1) }
else -> spanned
}
}
private fun MetricsState.bumpFailure(transform: (FailureMetrics) -> FailureMetrics): MetricsState =
copy(failures = transform(failures))
private fun MetricsState.applyInferenceCompleted(e: InferenceCompletedEvent): MetricsState {
val provider = e.providerId.value
val prev = inference.perProvider[provider] ?: ProviderInferenceMetrics()
val updated = prev.copy(
completedCount = prev.completedCount + 1,
totalLatencyMs = prev.totalLatencyMs + e.latencyMs,
promptTokens = prev.promptTokens + e.tokensUsed.promptTokens,
completionTokens = prev.completionTokens + e.tokensUsed.completionTokens,
)
return copy(
inference = inference.copy(
completedCount = inference.completedCount + 1,
totalLatencyMs = inference.totalLatencyMs + e.latencyMs,
promptTokens = inference.promptTokens + e.tokensUsed.promptTokens,
completionTokens = inference.completionTokens + e.tokensUsed.completionTokens,
perProvider = inference.perProvider + (provider to updated),
),
)
}
private fun MetricsState.applyToolCompleted(e: ToolExecutionCompletedEvent): MetricsState {
val durationMs = e.receipt.durationMs
val prev = tools.perTool[e.toolName] ?: ToolUsageMetrics()
val updated = prev.copy(
completedCount = prev.completedCount + 1,
totalDurationMs = prev.totalDurationMs + durationMs,
)
return copy(
tools = tools.copy(
completedCount = tools.completedCount + 1,
totalDurationMs = tools.totalDurationMs + durationMs,
perTool = tools.perTool + (e.toolName to updated),
),
)
}
private fun MetricsState.applyApprovalRequested(e: ApprovalRequestedEvent, at: Instant): MetricsState {
val tier = e.tier.name
val prev = approvals.perTier[tier] ?: TierApprovalMetrics()
return copy(
approvals = approvals.copy(
requestedCount = approvals.requestedCount + 1,
perTier = approvals.perTier + (tier to prev.copy(requestedCount = prev.requestedCount + 1)),
),
pendingApprovals = pendingApprovals + (e.requestId.value to at),
)
}
private fun MetricsState.applyApprovalResolved(e: ApprovalDecisionResolvedEvent): MetricsState {
val requestedAt = pendingApprovals[e.requestId.value]
val waitMs = requestedAt
?.let { (e.resolutionTimestamp - it).inWholeMilliseconds.coerceAtLeast(0L) }
?: 0L
val tier = e.tier.name
val prev = approvals.perTier[tier] ?: TierApprovalMetrics()
return copy(
approvals = approvals.copy(
resolvedCount = approvals.resolvedCount + 1,
totalWaitMs = approvals.totalWaitMs + waitMs,
perTier = approvals.perTier + (
tier to prev.copy(
resolvedCount = prev.resolvedCount + 1,
totalWaitMs = prev.totalWaitMs + waitMs,
)
),
),
pendingApprovals = pendingApprovals - e.requestId.value,
)
}
}
@@ -0,0 +1,85 @@
package com.correx.apps.server.metrics
import kotlinx.datetime.Instant
import kotlinx.serialization.Serializable
/**
* Read-model accumulated by [MetricsProjection] — a pure fold over a single session's event log.
*
* Holds raw sums only. Derived ratios (token throughput, average wait, time-accounting
* percentages) are computed at render time by [MetricsInspectionService] so this stays a
* stable, replayable accumulator. Never persisted — rebuilt from events on demand
* (Hard Invariant #1: the event log is the only source of truth).
*/
@Serializable
data class MetricsState(
val firstEventAt: Instant? = null,
val lastEventAt: Instant? = null,
val eventCount: Long = 0,
val inference: InferenceMetrics = InferenceMetrics(),
val tools: ToolMetrics = ToolMetrics(),
val approvals: ApprovalMetrics = ApprovalMetrics(),
val failures: FailureMetrics = FailureMetrics(),
/**
* In-flight approval requests awaiting resolution: requestId → request-event timestamp.
* Transient correlation bookkeeping that lets request→decision latency be computed in a
* single forward fold. Resolved requests are removed; entries still present after the
* full replay are unresolved (their wait time is unknown and excluded from totals).
*/
val pendingApprovals: Map<String, Instant> = emptyMap(),
)
@Serializable
data class InferenceMetrics(
val completedCount: Long = 0,
val totalLatencyMs: Long = 0,
val promptTokens: Long = 0,
val completionTokens: Long = 0,
val perProvider: Map<String, ProviderInferenceMetrics> = emptyMap(),
)
@Serializable
data class ProviderInferenceMetrics(
val completedCount: Long = 0,
val totalLatencyMs: Long = 0,
val promptTokens: Long = 0,
val completionTokens: Long = 0,
)
@Serializable
data class ToolMetrics(
val completedCount: Long = 0,
val totalDurationMs: Long = 0,
val perTool: Map<String, ToolUsageMetrics> = emptyMap(),
)
@Serializable
data class ToolUsageMetrics(
val completedCount: Long = 0,
val totalDurationMs: Long = 0,
)
@Serializable
data class ApprovalMetrics(
val requestedCount: Long = 0,
val resolvedCount: Long = 0,
val totalWaitMs: Long = 0,
val perTier: Map<String, TierApprovalMetrics> = emptyMap(),
)
@Serializable
data class TierApprovalMetrics(
val requestedCount: Long = 0,
val resolvedCount: Long = 0,
val totalWaitMs: Long = 0,
)
@Serializable
data class FailureMetrics(
val inferenceFailures: Long = 0,
val inferenceTimeouts: Long = 0,
val toolFailures: Long = 0,
val toolRejections: Long = 0,
val stageFailures: Long = 0,
val workflowFailures: Long = 0,
)
@@ -1,6 +1,7 @@
package com.correx.apps.server.routes
import com.correx.apps.server.ServerModule
import com.correx.apps.server.metrics.MetricsInspectionService
import com.correx.apps.server.protocol.SessionConfigDto
import com.correx.apps.server.replay.ReplayInspectionService
import com.correx.apps.server.serialization.payloadDiscriminator
@@ -82,6 +83,7 @@ fun Route.sessionRoutes(module: ServerModule) {
resumeSessionRoute(module)
getEventsRoute(module)
getReplayRoute(module)
getMetricsRoute(module)
webSocket("/stream") {
val id = call.parameters["id"] ?: return@webSocket
val lastEventId = call.request.queryParameters["lastEventId"]?.toLongOrNull()
@@ -190,3 +192,13 @@ private fun Route.getReplayRoute(module: ServerModule) {
call.respond(report)
}
}
private fun Route.getMetricsRoute(module: ServerModule) {
val service = MetricsInspectionService(module.eventStore)
get("/metrics") {
val id = call.parameters["id"]
?: return@get call.respond(HttpStatusCode.BadRequest, "Missing session id")
val report = service.inspect(TypeId(id))
call.respond(report)
}
}
@@ -0,0 +1,257 @@
package com.correx.apps.server.metrics
import com.correx.core.approvals.ApprovalOutcome
import com.correx.core.approvals.ApprovalStatus
import com.correx.core.approvals.Tier
import com.correx.core.events.events.ApprovalDecisionResolvedEvent
import com.correx.core.events.events.ApprovalRequestedEvent
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.InferenceCompletedEvent
import com.correx.core.events.events.InferenceFailedEvent
import com.correx.core.events.events.NewEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.ToolExecutionCompletedEvent
import com.correx.core.events.events.ToolExecutionFailedEvent
import com.correx.core.events.events.ToolReceipt
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.events.stores.EventStore
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.inference.TokenUsage
import com.correx.core.utils.TypeId
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.datetime.Instant
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
class MetricsProjectionTest {
private val sessionId: SessionId = TypeId("metrics-session-1")
private val stageId = TypeId("stage-1")
private val t0 = Instant.parse("2026-01-01T00:00:00Z")
private fun at(seconds: Long): Instant = t0.plus(kotlin.time.Duration.parse("${seconds}s"))
private fun stored(payload: EventPayload, seq: Long, at: Instant): StoredEvent =
StoredEvent(
metadata = EventMetadata(
eventId = EventId("e$seq"),
sessionId = sessionId,
timestamp = at,
schemaVersion = 1,
causationId = null,
correlationId = null,
),
sequence = seq,
sessionSequence = seq,
payload = payload,
)
private fun inferenceDone(provider: String, latencyMs: Long, prompt: Int, completion: Int) =
InferenceCompletedEvent(
requestId = TypeId("inf-$provider-$latencyMs"),
sessionId = sessionId,
stageId = stageId,
providerId = TypeId(provider),
tokensUsed = TokenUsage(promptTokens = prompt, completionTokens = completion),
latencyMs = latencyMs,
responseArtifactId = TypeId("art-$provider-$latencyMs"),
)
private fun toolDone(name: String, durationMs: Long) =
ToolExecutionCompletedEvent(
invocationId = TypeId("inv-$name-$durationMs"),
sessionId = sessionId,
toolName = name,
receipt = ToolReceipt(
invocationId = TypeId("inv-$name-$durationMs"),
toolName = name,
exitCode = 0,
outputSummary = "ok",
durationMs = durationMs,
tier = Tier.T1,
timestamp = t0,
),
)
private val seed = listOf(
stored(inferenceDone("lfm:a", 2000, 100, 50), 1L, at(0)),
stored(inferenceDone("lfm:a", 3000, 200, 100), 2L, at(1)),
stored(inferenceDone("lfm:b", 1000, 10, 20), 3L, at(2)),
stored(toolDone("shell.read", 500), 4L, at(3)),
stored(toolDone("shell.read", 700), 5L, at(4)),
stored(toolDone("fs.write", 300), 6L, at(5)),
stored(
ApprovalRequestedEvent(
requestId = TypeId("req-1"),
tier = Tier.T2,
validationReportId = TypeId("vr-1"),
riskSummaryId = null,
sessionId = sessionId,
stageId = stageId,
projectId = null,
),
7L, at(10),
),
stored(
ApprovalDecisionResolvedEvent(
decisionId = TypeId("dec-1"),
requestId = TypeId("req-1"),
outcome = ApprovalOutcome.APPROVED,
status = ApprovalStatus.COMPLETED,
tier = Tier.T2,
resolutionTimestamp = at(40),
reason = null,
),
8L, at(40),
),
stored(
ApprovalRequestedEvent(
requestId = TypeId("req-2"),
tier = Tier.T2,
validationReportId = TypeId("vr-2"),
riskSummaryId = null,
sessionId = sessionId,
stageId = stageId,
projectId = null,
),
9L, at(50),
),
stored(
InferenceFailedEvent(
requestId = TypeId("inf-fail"),
sessionId = sessionId,
stageId = stageId,
providerId = TypeId("lfm:a"),
reason = "boom",
),
10L, at(51),
),
stored(
ToolExecutionFailedEvent(
invocationId = TypeId("inv-fail"),
sessionId = sessionId,
toolName = "shell.read",
reason = "nonzero exit",
),
11L, at(52),
),
stored(
WorkflowFailedEvent(
sessionId = sessionId,
stageId = stageId,
reason = "aborted",
retryExhausted = true,
),
12L, at(60),
),
)
private class SeededEventStore(private val events: List<StoredEvent>) : EventStore {
override suspend fun append(event: NewEvent): StoredEvent = error("unused")
override suspend fun appendAll(events: List<NewEvent>): List<StoredEvent> = error("unused")
override fun read(sessionId: SessionId): List<StoredEvent> =
events.filter { it.metadata.sessionId == sessionId }
override fun readFrom(sessionId: SessionId, fromSequence: Long): List<StoredEvent> =
events.filter { it.metadata.sessionId == sessionId && it.sequence >= fromSequence }
override fun lastSequence(sessionId: SessionId): Long? =
events.filter { it.metadata.sessionId == sessionId }.lastOrNull()?.sequence
override fun subscribe(sessionId: SessionId): Flow<StoredEvent> = emptyFlow()
override fun subscribeAll(): Flow<StoredEvent> = emptyFlow()
override suspend fun lastGlobalSequence(): Long = 0L
override fun allEvents(): Sequence<StoredEvent> = emptySequence()
override fun allSessionIds(): Set<SessionId> = events.map { it.metadata.sessionId }.toSet()
}
private fun report() = MetricsInspectionService(SeededEventStore(seed)).inspect(sessionId)
@Test
fun `session span and event count come from event metadata`() {
val r = report()
assertEquals(sessionId.value, r.sessionId)
assertEquals(12L, r.eventCount)
assertEquals(60_000L, r.sessionDurationMs)
}
@Test
fun `inference totals and per-provider breakdown accumulate`() {
val r = report()
assertEquals(3L, r.inferenceCount)
assertEquals(6000L, r.inferenceMs)
assertEquals(310L, r.promptTokens)
assertEquals(170L, r.completionTokens)
val a = r.perProvider.single { it.provider == "lfm:a" }
assertEquals(2L, a.completedCount)
assertEquals(5000L, a.totalLatencyMs)
assertEquals(300L, a.promptTokens)
assertEquals(150L, a.completionTokens)
val b = r.perProvider.single { it.provider == "lfm:b" }
assertEquals(1L, b.completedCount)
assertEquals(1000L, b.totalLatencyMs)
}
@Test
fun `tool totals and per-tool breakdown accumulate`() {
val r = report()
assertEquals(3L, r.toolCount)
assertEquals(1500L, r.toolMs)
val read = r.perTool.single { it.toolName == "shell.read" }
assertEquals(2L, read.completedCount)
assertEquals(1200L, read.totalDurationMs)
val write = r.perTool.single { it.toolName == "fs.write" }
assertEquals(1L, write.completedCount)
assertEquals(300L, write.totalDurationMs)
}
@Test
fun `approval wait is correlated request-to-resolution and pending excludes wait`() {
val r = report()
assertEquals(2L, r.approvalsRequested)
assertEquals(1L, r.approvalsResolved)
assertEquals(1L, r.approvalsPending)
// req-1: requested at t+10s, resolved at t+40s -> 30s. req-2 unresolved -> contributes 0.
assertEquals(30_000L, r.approvalWaitMs)
assertEquals(30_000L, r.avgApprovalWaitMs)
val t2 = r.perTier.single { it.tier == "T2" }
assertEquals(2L, t2.requestedCount)
assertEquals(1L, t2.resolvedCount)
assertEquals(30_000L, t2.totalWaitMs)
}
@Test
fun `failure counts are classified by event type`() {
val f = report().failures
assertEquals(1L, f.inferenceFailures)
assertEquals(1L, f.toolFailures)
assertEquals(1L, f.workflowFailures)
assertEquals(0L, f.inferenceTimeouts)
assertEquals(0L, f.toolRejections)
assertEquals(0L, f.stageFailures)
}
@Test
fun `time accounting splits session wall time`() {
val r = report()
// inference 6000/60000, tools 1500/60000, approval-wait 30000/60000
assertEquals(10.0, r.inferencePct, 0.001)
assertEquals(2.5, r.toolPct, 0.001)
assertEquals(50.0, r.approvalWaitPct, 0.001)
}
@Test
fun `empty session yields zeroed report`() {
val r = MetricsInspectionService(SeededEventStore(emptyList())).inspect(sessionId)
assertEquals(0L, r.eventCount)
assertEquals(0L, r.sessionDurationMs)
assertEquals(0L, r.inferenceCount)
assertEquals(0.0, r.tokensPerSecond, 0.001)
assertEquals(0.0, r.inferencePct, 0.001)
}
}