Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0bcd2219ea | |||
| 0339b471c5 | |||
| 7a792fd176 | |||
| e0449cb3c7 |
@@ -11,6 +11,7 @@ MODEL_SERVER_START_LOG_MSG = [
|
|||||||
"llama runner started", # Ollama
|
"llama runner started", # Ollama
|
||||||
'"message":"Connected","target":"text_generation_router"', # TGI
|
'"message":"Connected","target":"text_generation_router"', # TGI
|
||||||
'"message":"Connected","target":"text_generation_router::server"', # TGI
|
'"message":"Connected","target":"text_generation_router::server"', # TGI
|
||||||
|
"main: model loaded" # llama.cpp
|
||||||
]
|
]
|
||||||
|
|
||||||
MODEL_SERVER_ERROR_LOG_MSGS = [
|
MODEL_SERVER_ERROR_LOG_MSGS = [
|
||||||
@@ -34,6 +35,7 @@ backend = Backend(
|
|||||||
model_server_url=os.environ["MODEL_SERVER_URL"],
|
model_server_url=os.environ["MODEL_SERVER_URL"],
|
||||||
model_log_file=os.environ["MODEL_LOG"],
|
model_log_file=os.environ["MODEL_LOG"],
|
||||||
allow_parallel_requests=True,
|
allow_parallel_requests=True,
|
||||||
|
max_wait_time=600.0,
|
||||||
benchmark_handler=CompletionsHandler(benchmark_runs=3, benchmark_words=256),
|
benchmark_handler=CompletionsHandler(benchmark_runs=3, benchmark_words=256),
|
||||||
log_actions=[
|
log_actions=[
|
||||||
*[(LogAction.ModelLoaded, info_msg) for info_msg in MODEL_SERVER_START_LOG_MSG],
|
*[(LogAction.ModelLoaded, info_msg) for info_msg in MODEL_SERVER_START_LOG_MSG],
|
||||||
|
|||||||
Reference in New Issue
Block a user