From 97e56b9275eb59ca4d0af33dec4efaeb65284818 Mon Sep 17 00:00:00 2001 From: kami Date: Sun, 12 Jul 2026 13:24:50 +0400 Subject: [PATCH] refactor(kernel): use real tokenizer for journal budgeting estimates (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real LLM Usage is already recorded (InferenceCompletedEvent.tokensUsed), aggregated per-session (MetricsProjection), exposed via correx stats + server MetricsInspectionService, and rendered in tui-go. The only remaining hardcoded length/4 estimates were the journal context-entry budget and the compaction threshold — both pre-injection (no LLM response exists for the journal text), so estimation is legitimate, but they now use the real tokenizer estimateTokens() for consistency with sibling context entries instead of length/4. --- .../core/kernel/orchestration/DefaultSessionOrchestrator.kt | 2 +- .../com/correx/core/kernel/orchestration/SessionOrchestrator.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt index ab6ee780..997600a7 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/DefaultSessionOrchestrator.kt @@ -699,7 +699,7 @@ class DefaultSessionOrchestrator( compactionService?.let { svc -> val journalState = decisionJournalRepository.getJournal(ctx.sessionId) val journalText = DecisionJournalRenderer().render(journalState) - val tokenEstimate = journalText.length / 4 + val tokenEstimate = estimateTokens(journalText) svc.compactIfNeeded( sessionId = ctx.sessionId, state = journalState, diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt index 7553c93b..e87ef556 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt @@ -511,7 +511,7 @@ abstract class SessionOrchestrator( content = journalText, sourceType = "decisionJournal", sourceId = "decision-journal", - tokenEstimate = journalText.length / 4, + tokenEstimate = estimateTokens(journalText), role = EntryRole.SYSTEM, ), )