1movmedia/pyworker
2
0
Fork 0
Code Pull Requests Activity

Remove nltk dep

Browse Source
This commit is contained in:
Mikhail Yevchenko
2026-05-21 19:11:53 +00:00
parent 0b47ef80fb
commit 976622a594
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
-1
View File
@@ -1,2 +1 @@
vastai-sdk vastai-sdk
nltk
workers/openai/worker.py
+8 -4
View File
@@ -1,4 +1,3 @@
import nltk
import random import random
import os import os
@@ -21,9 +20,6 @@ MODEL_ERROR_LOG_MSGS = [
MODEL_INFO_LOG_MSGS = [ MODEL_INFO_LOG_MSGS = [
] ]
nltk.download("words")
WORD_LIST = nltk.corpus.words.words()
def request_parser(request): def request_parser(request):
data = request data = request
if request.get("input") is not None: if request.get("input") is not None:
@@ -32,6 +28,14 @@ def request_parser(request):
def completions_benchmark_generator() -> dict: def completions_benchmark_generator() -> dict:
# extract words from the python source code of the worker to create a list of words for generating prompts
WORD_LIST = []
with open(__file__, 'r') as f:
for line in f:
WORD_LIST.extend(line.strip().split())
prompt = " ".join(random.choices(WORD_LIST, k=int(250))) prompt = " ".join(random.choices(WORD_LIST, k=int(250)))
model = os.environ.get("MODEL_NAME") model = os.environ.get("MODEL_NAME")
if not model: if not model: