pipr.tools

token-count

Estimate LLM token count and context window usage

# Stats

Try it

stdin0 chars
stdout0 chars

Example

Estimate token usage for a system prompt

Usage
$ echo "You are a helpful assistant. Answer the user's question conc..." | token-count
View source
(input) => {
      if (!input.trim()) return "(empty input)";
      const chars = input.length;
      const words = input.trim().split(/\s+/).filter(Boolean).length;
      const bytes = new Blob([input]).size;
      // ~4 chars/token is the standard heuristic for cl100k_base / o200k_base
      const tokens = Math.ceil(chars / 4);
      const ratio = tokens > 0 ? (chars / tokens).toFixed(1) : "—";
      const pct = (ctx) => {
        const p = (tokens / ctx) * 100;
        return p < 0.01
          ? "<0.01%"
          : p < 1
            ? p.toFixed(2) + "%"
            : p.toFixed(1) + "%";
      };
      return [
        `  Tokens: ~${tokens.toLocaleString()}`,
        `  Chars:   ${chars.toLocaleString()}`,
        `  Words:   ${words.toLocaleString()}`,
        `  Bytes:   ${bytes.toLocaleString()}`,
        `  Ratio:   ${ratio} chars/tok`,
        ``,
        `  ── Context window usage ──`,
        `  GPT-4o        128K  ${pct(128_000).padStart(7)}`,
        `  Claude 3.5    200K  ${pct(200_000).padStart(7)}`,
        `  Gemini 1.5      1M  ${pct(1_000_000).padStart(7)}`,
        `  Llama 3       128K  ${pct(128_000).padStart(7)}`,
      ].join("\n");
    }

Suggested Pipelines

Related Tools