jim/openai-cookbook
1
0
Fork 0
mirror of https://github.com/james-m-jordan/openai-cookbook.git synced 2025-05-09 19:32:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update How_to_count_tokens_with_tiktoken.ipynb (#511)

Browse Source 
Add support for gpt-3.5-*-0613 models
This commit is contained in:
Avery Khoo 2023-06-24 07:10:30 +08:00 committed by GitHub
parent b587f9b3d5
commit 26978e9ed1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
examples/How_to_count_tokens_with_tiktoken.ipynb
View File

@ -441,13 +441,16 @@
" if model == \"gpt-3.5-turbo\":\n",
" print(\"Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.\")\n",
" return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0301\")\n",
" elif model == \"gpt-3.5-turbo-16k\":\n",
" print(\"Warning: gpt-3.5-turbo-16k may change over time. Returning num tokens assuming gpt-3.5-turbo-16k-0613.\")\n",
" return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-16k-0613\")\n",
" elif model == \"gpt-4\":\n",
" print(\"Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.\")\n",
" return num_tokens_from_messages(messages, model=\"gpt-4-0314\")\n",
" elif model == \"gpt-3.5-turbo-0301\":\n",
" tokens_per_message = 4 # every message follows <|start|>{role/name}\\n{content}<|end|>\\n\n",
" tokens_per_name = -1 # if there's a name, the role is omitted\n",
" elif model == \"gpt-4-0314\":\n",
" elif model in {\"gpt-4-0314\", \"gpt-3.5-turbo-0613\", \"gpt-3.5-turbo-16k-0613\"}:\n",
" tokens_per_message = 3\n",
" tokens_per_name = 1\n",
" else:\n",