1movmedia/pyworker
2
0
Fork 0
Code Pull Requests Activity

Merge pull request #68 from vast-ai/fix-vllm-concurrency

Browse Source 
Increase model wait time for vLLM
This commit is contained in:
LucasArmandVast
2025-12-03 16:13:51 -05:00
committed by GitHub
parent 0339b471c5 0bcd2219ea
commit 2f543c01ad
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/openai/server.py
+1
View File
@@ -35,6 +35,7 @@ backend = Backend(
model_server_url=os.environ["MODEL_SERVER_URL"], model_server_url=os.environ["MODEL_SERVER_URL"],
model_log_file=os.environ["MODEL_LOG"], model_log_file=os.environ["MODEL_LOG"],
allow_parallel_requests=True, allow_parallel_requests=True,
max_wait_time=600.0,
benchmark_handler=CompletionsHandler(benchmark_runs=3, benchmark_words=256), benchmark_handler=CompletionsHandler(benchmark_runs=3, benchmark_words=256),
log_actions=[ log_actions=[
*[(LogAction.ModelLoaded, info_msg) for info_msg in MODEL_SERVER_START_LOG_MSG], *[(LogAction.ModelLoaded, info_msg) for info_msg in MODEL_SERVER_START_LOG_MSG],