refactor(kernel): use real tokenizer for journal budgeting estimates (#58)
Real LLM Usage is already recorded (InferenceCompletedEvent.tokensUsed), aggregated per-session (MetricsProjection), exposed via correx stats + server MetricsInspectionService, and rendered in tui-go. The only remaining hardcoded length/4 estimates were the journal context-entry budget and the compaction threshold — both pre-injection (no LLM response exists for the journal text), so estimation is legitimate, but they now use the real tokenizer estimateTokens() for consistency with sibling context entries instead of length/4.
This commit is contained in:
+1
-1
@@ -699,7 +699,7 @@ class DefaultSessionOrchestrator(
|
||||
compactionService?.let { svc ->
|
||||
val journalState = decisionJournalRepository.getJournal(ctx.sessionId)
|
||||
val journalText = DecisionJournalRenderer().render(journalState)
|
||||
val tokenEstimate = journalText.length / 4
|
||||
val tokenEstimate = estimateTokens(journalText)
|
||||
svc.compactIfNeeded(
|
||||
sessionId = ctx.sessionId,
|
||||
state = journalState,
|
||||
|
||||
+1
-1
@@ -511,7 +511,7 @@ abstract class SessionOrchestrator(
|
||||
content = journalText,
|
||||
sourceType = "decisionJournal",
|
||||
sourceId = "decision-journal",
|
||||
tokenEstimate = journalText.length / 4,
|
||||
tokenEstimate = estimateTokens(journalText),
|
||||
role = EntryRole.SYSTEM,
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user