Estimate LLM token count and context window usage
# StatsEstimate token usage for a system prompt
$ echo "You are a helpful assistant. Answer the user's question conc..." | token-count (input) => {
if (!input.trim()) return "(empty input)";
const chars = input.length;
const words = input.trim().split(/\s+/).filter(Boolean).length;
const bytes = new Blob([input]).size;
// ~4 chars/token is the standard heuristic for cl100k_base / o200k_base
const tokens = Math.ceil(chars / 4);
const ratio = tokens > 0 ? (chars / tokens).toFixed(1) : "—";
const pct = (ctx) => {
const p = (tokens / ctx) * 100;
return p < 0.01
? "<0.01%"
: p < 1
? p.toFixed(2) + "%"
: p.toFixed(1) + "%";
};
return [
` Tokens: ~${tokens.toLocaleString()}`,
` Chars: ${chars.toLocaleString()}`,
` Words: ${words.toLocaleString()}`,
` Bytes: ${bytes.toLocaleString()}`,
` Ratio: ${ratio} chars/tok`,
``,
` ── Context window usage ──`,
` GPT-4o 128K ${pct(128_000).padStart(7)}`,
` Claude 3.5 200K ${pct(200_000).padStart(7)}`,
` Gemini 1.5 1M ${pct(1_000_000).padStart(7)}`,
` Llama 3 128K ${pct(128_000).padStart(7)}`,
].join("\n");
}