1movmedia/pyworker
2
0
Fork 0
Code Pull Requests Activity

Increase queue limits for vLLM and TGI

Browse Source
This commit is contained in:
Lucas Armand
2025-12-17 11:38:55 -08:00
parent 29f836eb1a
commit 9daf171487
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/openai/worker.py
+2 -2
View File
@@ -60,8 +60,8 @@ worker_config = WorkerConfig(
route="/v1/completions", route="/v1/completions",
workload_calculator= lambda data: data.get("max_tokens", 0), workload_calculator= lambda data: data.get("max_tokens", 0),
allow_parallel_requests=True, allow_parallel_requests=True,
max_queue_time=60.0,
request_parser=request_parser, request_parser=request_parser,
max_queue_time=600.0,
benchmark_config=BenchmarkConfig( benchmark_config=BenchmarkConfig(
generator=completions_benchmark_generator, generator=completions_benchmark_generator,
concurrency=100, concurrency=100,
@@ -72,8 +72,8 @@ worker_config = WorkerConfig(
route="/v1/chat/completions", route="/v1/chat/completions",
workload_calculator= lambda data: data.get("max_tokens", 0), workload_calculator= lambda data: data.get("max_tokens", 0),
allow_parallel_requests=True, allow_parallel_requests=True,
max_queue_time=60.0,
request_parser=request_parser request_parser=request_parser
max_queue_time=600.0,
) )
], ],
log_action_config=LogActionConfig( log_action_config=LogActionConfig(
workers/tgi/worker.py
+2 -2
View File
@@ -52,7 +52,7 @@ worker_config = WorkerConfig(
HandlerConfig( HandlerConfig(
route="/generate", route="/generate",
allow_parallel_requests=True, allow_parallel_requests=True,
max_queue_time=60.0, max_queue_time=600.0,
benchmark_config=BenchmarkConfig( benchmark_config=BenchmarkConfig(
generator=benchmark_generator, generator=benchmark_generator,
concurrency=50 concurrency=50
@@ -62,7 +62,7 @@ worker_config = WorkerConfig(
HandlerConfig( HandlerConfig(
route="/generate_stream", route="/generate_stream",
allow_parallel_requests=True, allow_parallel_requests=True,
max_queue_time=60.0, max_queue_time=600.0,
workload_calculator= lambda x: x["parameters"]["max_new_tokens"] workload_calculator= lambda x: x["parameters"]["max_new_tokens"]
) )
], ],