1movmedia/pyworker
2
0
Fork 0
Code Pull Requests Activity

update tokenizers

Browse Source
This commit is contained in:
Nader Arbabian
2025-06-10 17:01:28 -07:00
committed by Nader Arbabian
parent d99adcfb36
commit 4bac805093
4 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/tgi/data_types.py
+2 -2
View File
@@ -3,7 +3,7 @@ import random
import inspect
from typing import Dict, Any
from transformers import AutoTokenizer
from transformers import OpenAIGPTTokenizer
import nltk
from lib.data_types import ApiPayload, JsonDataException
@@ -11,7 +11,7 @@ from lib.data_types import ApiPayload, JsonDataException
nltk.download("words")
WORD_LIST = nltk.corpus.words.words()
tokenizer = AutoTokenizer.from_pretrained("openai-community/openai-gpt")
tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
@dataclasses.dataclass