1movmedia/pyworker
2
0
Fork 0
Code Pull Requests Activity

update tokenizers

Browse Source
This commit is contained in:
Nader Arbabian
2025-06-10 17:01:28 -07:00
committed by Nader Arbabian
parent d99adcfb36
commit 4bac805093
4 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
+9 -9
View File
@@ -1,9 +1,9 @@
aiohttp==3.11.16
anyio==4.4.0
lib==4.0.0
nltk==3.9.1
psutil==6.0.0
pycryptodome==3.20.0
Requests==2.32.4
transformers==4.50.0
utils==1.0.2
aiohttp~=3.11
anyio~=4.4
lib~=4.0
nltk~=3.9
psutil~=6.0
pycryptodome~=3.20
Requests~=2.32
transformers~=4.52
utils~=1.0
workers/hello_world/README.md
+2 -2
View File
@@ -56,7 +56,7 @@ import dataclasses
import random
from typing import Dict, Any
from transformers import AutoTokenizer # used to count tokens in a prompt
from transformers import OpenAIGPTTokenizer # used to count tokens in a prompt
import nltk # used to download a list of all words to generate a random prompt and benchmark the LLM model
from lib.data_types import ApiPayload
@@ -65,7 +65,7 @@ nltk.download("words")
WORD_LIST = nltk.corpus.words.words()
# you can use any tokenizer that fits your LLM. `openai-gpt` is free to use and is a good fit for most LLMs
tokenizer = AutoTokenizer.from_pretrained("openai-community/openai-gpt")
tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
@dataclasses.dataclass
class InputData(ApiPayload):
workers/hello_world/data_types.py
+2 -2
View File
@@ -3,7 +3,7 @@ import random
import inspect
from typing import Dict, Any
from transformers import AutoTokenizer
from transformers import OpenAIGPTTokenizer
import nltk
from lib.data_types import ApiPayload, JsonDataException
@@ -12,7 +12,7 @@ nltk.download("words")
WORD_LIST = nltk.corpus.words.words()
# used to count to count tokens and workload for LLM
tokenizer = AutoTokenizer.from_pretrained("openai-community/openai-gpt")
tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
@dataclasses.dataclass
workers/tgi/data_types.py
+2 -2
View File
@@ -3,7 +3,7 @@ import random
import inspect
from typing import Dict, Any
from transformers import AutoTokenizer
from transformers import OpenAIGPTTokenizer
import nltk
from lib.data_types import ApiPayload, JsonDataException
@@ -11,7 +11,7 @@ from lib.data_types import ApiPayload, JsonDataException
nltk.download("words")
WORD_LIST = nltk.corpus.words.words()
tokenizer = AutoTokenizer.from_pretrained("openai-community/openai-gpt")
tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
@dataclasses.dataclass