package com.correx.core.inference

@JvmInline
value class Token(val id: Int)

/**
 * Provider-owned. Tokenization is model-family-specific;
 * two providers for the same capability may not share a tokenizer.
 */
interface Tokenizer {
    suspend fun tokenize(text: String): List<Token>
    suspend fun countTokens(text: String): Int
}
