{ "methodology_version": 1, "checked_date": "2026-07-19", "basis": "Plain text only; no chat templates, roles, tools, images, or API wrappers.", "tokenizers": [ { "id": "openai-current", "name": "OpenAI Current", "basis": "o200k_base", "href": "/tools/openai-tokenizer", "source_url": "https://platform.openai.com/tokenizer" }, { "id": "openai-legacy", "name": "OpenAI Legacy", "basis": "cl100k_base", "href": "/tools/openai-tokenizer", "source_url": "https://platform.openai.com/tokenizer" }, { "id": "gemma-3", "name": "Gemma 3", "basis": "Pinned SentencePiece model", "href": "/tools/gemma-tokenizer", "source_url": "https://huggingface.co/google/gemma-3-1b-it/tree/main" }, { "id": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "basis": "Pinned tokenizer JSON", "href": "/tools/deepseek-tokenizer", "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/tokenizer.json" }, { "id": "qwen-3-5", "name": "Qwen 3.5", "basis": "Pinned tokenizer JSON", "href": "/tools/qwen-tokenizer", "source_url": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/main/tokenizer.json" }, { "id": "mistral-medium-3-5", "name": "Mistral Medium 3.5", "basis": "Pinned tokenizer JSON", "href": "/tools/mistral-tokenizer", "source_url": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/blob/main/tokenizer.json" }, { "id": "llama-3", "name": "Llama 3", "basis": "Pinned tokenizer JSON", "href": "/tools/llama-tokenizer", "source_url": "https://huggingface.co/Xenova/llama3-tokenizer/blob/main/tokenizer.json" } ], "samples": [ { "id": "general-text", "label": "General text", "description": "A bilingual support instruction that shows how one input can contain English, Chinese, or both.", "text": "A support agent should summarize the customer issue and ask for confirmation before issuing a refund.\n请总结客户反馈中的主要问题，并在执行退款前请求确认。", "characters": 128, "lines": 2, "results": [ { "tokenizer_id": "openai-current", "tokens": 32, "tokens_per_100_characters": 25, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "openai-legacy", "tokens": 39, "tokens_per_100_characters": 30.5, "difference_from_openai_current": 7, "percent_from_openai_current": 21.9 }, { "tokenizer_id": "gemma-3", "tokens": 34, "tokens_per_100_characters": 26.6, "difference_from_openai_current": 2, "percent_from_openai_current": 6.3 }, { "tokenizer_id": "deepseek-v4-pro", "tokens": 32, "tokens_per_100_characters": 25, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "qwen-3-5", "tokens": 33, "tokens_per_100_characters": 25.8, "difference_from_openai_current": 1, "percent_from_openai_current": 3.1 }, { "tokenizer_id": "mistral-medium-3-5", "tokens": 41, "tokens_per_100_characters": 32, "difference_from_openai_current": 9, "percent_from_openai_current": 28.1 }, { "tokenizer_id": "llama-3", "tokens": 36, "tokens_per_100_characters": 28.1, "difference_from_openai_current": 4, "percent_from_openai_current": 12.5 } ] }, { "id": "json-payload", "label": "JSON payload", "description": "A formatted JSON object with identifiers, arrays, booleans, and snake-case keys.", "text": "{\n \"customer_id\": \"cus_1042\",\n \"priority\": \"high\",\n \"tags\": [\"billing\", \"refund\"],\n \"requires_review\": true\n}", "characters": 113, "lines": 6, "results": [ { "tokenizer_id": "openai-current", "tokens": 37, "tokens_per_100_characters": 32.7, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "openai-legacy", "tokens": 37, "tokens_per_100_characters": 32.7, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "gemma-3", "tokens": 45, "tokens_per_100_characters": 39.8, "difference_from_openai_current": 8, "percent_from_openai_current": 21.6 }, { "tokenizer_id": "deepseek-v4-pro", "tokens": 40, "tokens_per_100_characters": 35.4, "difference_from_openai_current": 3, "percent_from_openai_current": 8.1 }, { "tokenizer_id": "qwen-3-5", "tokens": 43, "tokens_per_100_characters": 38.1, "difference_from_openai_current": 6, "percent_from_openai_current": 16.2 }, { "tokenizer_id": "mistral-medium-3-5", "tokens": 42, "tokens_per_100_characters": 37.2, "difference_from_openai_current": 5, "percent_from_openai_current": 13.5 }, { "tokenizer_id": "llama-3", "tokens": 37, "tokens_per_100_characters": 32.7, "difference_from_openai_current": 0, "percent_from_openai_current": 0 } ] }, { "id": "typescript-cost", "label": "TypeScript code", "description": "A small cost-calculation function with camel-case names and arithmetic.", "text": "export function monthlyCost(inputTokens, outputTokens, requests, rates) {\n return requests * (\n inputTokens * rates.input +\n outputTokens * rates.output\n ) / 1_000_000;\n}", "characters": 178, "lines": 6, "results": [ { "tokenizer_id": "openai-current", "tokens": 45, "tokens_per_100_characters": 25.3, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "openai-legacy", "tokens": 45, "tokens_per_100_characters": 25.3, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "gemma-3", "tokens": 56, "tokens_per_100_characters": 31.5, "difference_from_openai_current": 11, "percent_from_openai_current": 24.4 }, { "tokenizer_id": "deepseek-v4-pro", "tokens": 45, "tokens_per_100_characters": 25.3, "difference_from_openai_current": 0, "percent_from_openai_current": 0 }, { "tokenizer_id": "qwen-3-5", "tokens": 53, "tokens_per_100_characters": 29.8, "difference_from_openai_current": 8, "percent_from_openai_current": 17.8 }, { "tokenizer_id": "mistral-medium-3-5", "tokens": 49, "tokens_per_100_characters": 27.5, "difference_from_openai_current": 4, "percent_from_openai_current": 8.9 }, { "tokenizer_id": "llama-3", "tokens": 45, "tokens_per_100_characters": 25.3, "difference_from_openai_current": 0, "percent_from_openai_current": 0 } ] } ] }