大模型应用中统计token数量

Fuly1024

645人浏览 · 2025-11-03 16:52:48

Fuly1024 · 2025-11-03 16:52:48 发布

使用OpenAI接口
调用chat.completions.create接口时,返回的字段中
返回结果中的usage字段包含prompt_tokens（输入）、completion_tokens（输出）和total_tokens（总计）。流式输出需在stream_options中设置include_usage: true以获取完整统计。

    client = get_deepseek_openai_Client()
    response = client.chat.completions.create(
        model="deepseek-chat",
        messages=[
            {"role": "system", "content": "你是一个智能助手"},
            {"role": "user", "content": "你是谁?"},
        ],
        stream=False,
    )
    print(json.dumps(response.model_dump(), indent=2, ensure_ascii=False))
    
    usage = response.usage
    print(f"输入token: {usage.prompt_tokens}")
    print(f"输出token: {usage.completion_tokens}")
    print(f"总token: {usage.total_tokens}")

返回结果


{
  "id": "8be878f8-b39b-4683-9ffd-b07ddc354fd4",
  "choices": [
    {
      "finish_reason": "stop",
      "index": 0,
      "logprobs": null,
      "message": {
        "content": "你好！我是一个智能助手，可以帮助你解答问题、提供信息或协助完成任务。如果你有任何需要，随时告诉我，我会尽力帮助你！ 
        "content": "你好！我是一个智能助手，可以帮助你解答问题、提供信息或协助完成任务。如果你有任何需要，随时告诉我，我会尽力帮助你！ 
😊",
        "refusal": null,
        "role": "assistant",
        "annotations": null,
        "audio": null,
😊",
        "refusal": null,
        "role": "assistant",
        "annotations": null,
        "audio": null,
        "refusal": null,
        "role": "assistant",
        "annotations": null,
        "audio": null,
        "role": "assistant",
        "annotations": null,
        "audio": null,
        "annotations": null,
        "audio": null,
        "audio": null,
        "function_call": null,
        "tool_calls": null
      }
    }
  ],
  "created": 1762151814,
  "model": "deepseek-chat",
  "object": "chat.completion",
  "service_tier": null,
  "system_fingerprint": "fp_ffc7281d48_prod0820_fp8_kvcache",
  "usage": {
    "completion_tokens": 30,
    "prompt_tokens": 10,
    "total_tokens": 40,
  ],
  "created": 1762151814,
  "model": "deepseek-chat",
  "object": "chat.completion",
  "service_tier": null,
  "system_fingerprint": "fp_ffc7281d48_prod0820_fp8_kvcache",
  "usage": {
    "completion_tokens": 30,
    "prompt_tokens": 10,
    "total_tokens": 40,
  "object": "chat.completion",
  "service_tier": null,
  "system_fingerprint": "fp_ffc7281d48_prod0820_fp8_kvcache",
  "usage": {
    "completion_tokens": 30,
    "prompt_tokens": 10,
    "total_tokens": 40,
  "system_fingerprint": "fp_ffc7281d48_prod0820_fp8_kvcache",
  "usage": {
    "completion_tokens": 30,
    "prompt_tokens": 10,
    "total_tokens": 40,
    "completion_tokens": 30,
    "prompt_tokens": 10,
    "total_tokens": 40,
    "prompt_tokens": 10,
    "total_tokens": 40,
    "total_tokens": 40,
    "completion_tokens_details": null,
    "prompt_tokens_details": {
      "audio_tokens": null,
    "completion_tokens_details": null,
    "prompt_tokens_details": {
      "audio_tokens": null,
      "audio_tokens": null,
      "cached_tokens": 0
      "cached_tokens": 0
    },
    },
    "prompt_cache_hit_tokens": 0,
    "prompt_cache_miss_tokens": 10
  }
}
输入token: 10
输出token: 30
总token: 40

流式输出需在stream_options中设置include_usage: true以获取完整统计。
流式输出统计:

   client = get_deepseek_openai_Client()
    stream = client.chat.completions.create(
        model="deepseek-chat",
        messages=[
            {"role": "system", "content": "你是一个智能助手"},
            {"role": "user", "content": "你是谁?"},
        ],
        stream=True,  # 启用流式输出
        stream_options={"include_usage": True},  # 要求返回usage统计
    )

    full_response = ""  # 用于拼接完整响应内容
usage = None  # 用于存储 Token 统计

for chunk in stream:
    # 提取当前分块的内容（非空时拼接）
    if chunk.choices[0].delta.content:
        content = chunk.choices[0].delta.content
        full_response += content
        print(content, end="", flush=True)  # 实时打印流式内容

    # 当流式结束时（finish_reason 为 stop），获取 usage 统计
    if chunk.choices[0].finish_reason == "stop":
        usage = chunk.usage

# 打印完整结果和 Token 统计
print("\n\n完整响应：", full_response)
if usage:
    print("Token 统计：", usage)
else:
    print("当前模型不支持流式输出返回 usage 统计")

2.使用tiktoken库

import tiktoken
encoding = tiktoken.get_encoding("cl100k_base")  # GPT-4使用的编码器
tokens = encoding.encode("你的文本内容")
print(len(tokens))  # 输出Token数量

按模型统计

def count_tokens(text, model_name="gpt-3.5-turbo"):
    encoding = tiktoken.encoding_for_model(model_name)
    return len(encoding.encode(text))

# 使用示例
text = "需要统计的文本内容"
print(count_tokens(text, "gpt-4"))

但并不是所有模型都在这个库里面

MODEL_TO_ENCODING: dict[str, str] = {
    # reasoning
    "o1": "o200k_base",
    "o3": "o200k_base",
    "o4-mini": "o200k_base",
    # chat
    "gpt-5": "o200k_base",
    "gpt-4.1": "o200k_base",
    "gpt-4o": "o200k_base",
    "gpt-4": "cl100k_base",
    "gpt-3.5-turbo": "cl100k_base",
    "gpt-3.5": "cl100k_base",  # Common shorthand
    "gpt-35-turbo": "cl100k_base",  # Azure deployment name
    # base
    "davinci-002": "cl100k_base",
    "babbage-002": "cl100k_base",
    # embeddings
    "text-embedding-ada-002": "cl100k_base",
    "text-embedding-3-small": "cl100k_base",
    "text-embedding-3-large": "cl100k_base",
    # DEPRECATED MODELS
    # text (DEPRECATED)
    "text-davinci-003": "p50k_base",
    "text-davinci-002": "p50k_base",
    "text-davinci-001": "r50k_base",
    "text-curie-001": "r50k_base",
    "text-babbage-001": "r50k_base",
    "text-ada-001": "r50k_base",
    "davinci": "r50k_base",
    "curie": "r50k_base",
    "babbage": "r50k_base",
    "ada": "r50k_base",
    # code (DEPRECATED)
    "code-davinci-002": "p50k_base",
    "code-davinci-001": "p50k_base",
    "code-cushman-002": "p50k_base",
    "code-cushman-001": "p50k_base",
    "davinci-codex": "p50k_base",
    "cushman-codex": "p50k_base",
    # edit (DEPRECATED)
    "text-davinci-edit-001": "p50k_edit",
    "code-davinci-edit-001": "p50k_edit",
    # old embeddings (DEPRECATED)
    "text-similarity-davinci-001": "r50k_base",
    "text-similarity-curie-001": "r50k_base",
    "text-similarity-babbage-001": "r50k_base",
    "text-similarity-ada-001": "r50k_base",
    "text-search-davinci-doc-001": "r50k_base",
    "text-search-curie-doc-001": "r50k_base",
    "text-search-babbage-doc-001": "r50k_base",
    "text-search-ada-doc-001": "r50k_base",
    "code-search-babbage-code-001": "r50k_base",
    "code-search-ada-code-001": "r50k_base",
    # open source
    "gpt2": "gpt2",
    "gpt-2": "gpt2",  # Maintains consistency with gpt-4
}

MODEL_PREFIX_TO_ENCODING: dict[str, str] = {
    "o1-": "o200k_base",
    "o3-": "o200k_base",
    "o4-mini-": "o200k_base",
    # chat
    "gpt-5-": "o200k_base",
    "gpt-4.5-": "o200k_base",
    "gpt-4.1-": "o200k_base",
    "chatgpt-4o-": "o200k_base",
    "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13
    "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
    "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
    "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
    "gpt-oss-": "o200k_harmony",
    # fine-tuned
    "ft:gpt-4o": "o200k_base",
    "ft:gpt-4": "cl100k_base",
    "ft:gpt-3.5-turbo": "cl100k_base",
    "ft:davinci-002": "cl100k_base",
    "ft:babbage-002": "cl100k_base",
}

所以指定一下编码比较好

在线API统计(在线 Tokenizer 工具)

了解一下就行

亚马逊云科技技术品牌专区

更多推荐

open-code-review Codex 插件教程：对话式调用评审技能指南

本文是一份面向新手的 **open-code-review Codex 插件教程**，教你在 Codex 中通过对话式方式调用 open-code-review 的 AI 代码评审技能。open-code-review 是阿里巴巴开源的 AI 代码评审工具，采用"确定性流水线 + LLM Agent"混合架构，内置多语言评审规则（NPE、线程安全、XSS、SQL 注入等），可在 Codex 里用一

亚马逊云科技技术品牌专区

OBS Studio 完整指南：免费开源录屏与直播软件从零上手

想做一场像模像样的直播，或者把屏幕内容录成教程视频，但一想到商用软件的订阅费就有些犹豫？其实你不需要花一分钱。OBS Studio 是一款免费开源的直播与录屏软件，支持 Windows、macOS 和 Linux，采集、混音、编码、推流、录制，一条流水线全包了。## 先搞清楚 OBS Studio 是什么OBS Studio 是一个"场景化"的采集合成工具：你把摄像头、窗口、浏览器、图片、

亚马逊云科技技术品牌专区

如何用MediaCrawler爬取小红书抖音数据？7大平台爬虫从安装到实战完整指南

MediaCrawler 是一个支持小红书、抖音、快手、B站、微博、百度贴吧、知乎的开源自媒体数据采集工具。它能按关键词、按帖子、按创作者主页三种方式抓取笔记/视频内容、评论甚至二级评论，并把结果存成 JSON、CSV、Excel 或数据库。适合做市场调研、竞品监控、内容选题的运营人员，以及想要现成爬虫框架而不想从零写 JS 逆向的开发者。## 从一次"手动抄数据"说起想象这个场景：你负责