package com.correx.infrastructure.inference.openai

import com.correx.core.inference.Token
import com.correx.core.inference.Tokenizer

/**
 * Remote OpenAI-compatible backends (NIM, OpenAI) expose no `/tokenize` endpoint, so we can't
 * get exact token ids without shipping a tokenizer. This is a length-based approximation
 * (~4 chars/token) used only for context-budget estimates; the synthesized [Token] ids are
 * placeholders and must not be treated as real model tokens.
 */
class HeuristicTokenizer(private val charsPerToken: Int = 4) : Tokenizer {

    override suspend fun tokenize(text: String): List<Token> {
        val count = countTokens(text)
        return List(count) { Token(it) }
    }

    override suspend fun countTokens(text: String): Int =
        if (text.isEmpty()) 0 else (text.length + charsPerToken - 1) / charsPerToken
}
