diff --git a/workers/openai/worker.py b/workers/openai/worker.py index 64992eb..46d7ef0 100644 --- a/workers/openai/worker.py +++ b/workers/openai/worker.py @@ -12,6 +12,7 @@ MODEL_SERVER_URL = 'http://127.0.0.1' MODEL_SERVER_PORT = 11434 MODEL_LOG_FILE = os.environ.get('MODEL_LOG', '/var/log/ollama.log') MODEL_HEALTHCHECK_ENDPOINT = "/" +BENCHMARK_CONCURRENCY = max(1, int(os.environ.get("OLLAMA_NUM_PARALLEL", "1"))) # Ollama-specific log messages def request_parser(request): @@ -66,7 +67,7 @@ worker_config = WorkerConfig( max_queue_time=600.0, benchmark_config=BenchmarkConfig( generator=completions_benchmark_generator, - concurrency=10, + concurrency=BENCHMARK_CONCURRENCY, runs=3 ) ), @@ -85,4 +86,4 @@ worker_config = WorkerConfig( ) ) -Worker(worker_config).run() \ No newline at end of file +Worker(worker_config).run()