Compare commits

..

1 Commits

Author SHA1 Message Date
Edgar Lin fdd50a2aaa correct version pin for aiohttp 2025-12-10 19:14:34 -08:00
+57 -110
View File
@@ -22,49 +22,10 @@ function echo_var(){
echo "$1: ${!1}" echo "$1: ${!1}"
} }
function report_error_and_exit(){ [ -z "$BACKEND" ] && echo "BACKEND must be set!" && exit 1
local error_msg="$1" [ -z "$MODEL_LOG" ] && echo "MODEL_LOG must be set!" && exit 1
echo "ERROR: $error_msg" [ -z "$HF_TOKEN" ] && echo "HF_TOKEN must be set!" && exit 1
[ "$BACKEND" = "comfyui" ] && [ -z "$COMFY_MODEL" ] && echo "For comfyui backends, COMFY_MODEL must be set!" && exit 1
# Report error to autoscaler
MTOKEN="${MASTER_TOKEN:-}"
VERSION="${PYWORKER_VERSION:-0}"
IFS=',' read -r -a REPORT_ADDRS <<< "${REPORT_ADDR}"
for addr in "${REPORT_ADDRS[@]}"; do
curl -sS -X POST -H 'Content-Type: application/json' \
-d "$(cat <<JSON
{
"id": ${CONTAINER_ID:-0},
"mtoken": "${MTOKEN}",
"version": "${VERSION}",
"loadtime": 0,
"new_load": 0,
"cur_load": 0,
"rej_load": 0,
"max_perf": 0,
"cur_perf": 0,
"error_msg": "${error_msg}",
"num_requests_working": 0,
"num_requests_recieved": 0,
"additional_disk_usage": 0,
"working_request_idxs": [],
"cur_capacity": 0,
"max_capacity": 0,
"url": "${URL:-}"
}
JSON
)" "${addr%/}/worker_status/" || true
done
exit 1
}
[ -z "$BACKEND" ] && report_error_and_exit "BACKEND must be set!"
[ -z "$MODEL_LOG" ] && report_error_and_exit "MODEL_LOG must be set!"
[ -z "$HF_TOKEN" ] && report_error_and_exit "HF_TOKEN must be set!"
[ -z "$CONTAINER_ID" ] && report_error_and_exit "CONTAINER_ID must be set!"
[ "$BACKEND" = "comfyui" ] && [ -z "$COMFY_MODEL" ] && report_error_and_exit "For comfyui backends, COMFY_MODEL must be set!"
echo "start_server.sh" echo "start_server.sh"
@@ -84,86 +45,51 @@ echo_var MODEL_LOG
# from the run prior to reboot. past logs are saved in $MODEL_LOG.old for debugging only # from the run prior to reboot. past logs are saved in $MODEL_LOG.old for debugging only
if [ -e "$MODEL_LOG" ]; then if [ -e "$MODEL_LOG" ]; then
echo "Rotating model log at $MODEL_LOG to $MODEL_LOG.old" echo "Rotating model log at $MODEL_LOG to $MODEL_LOG.old"
if ! cat "$MODEL_LOG" >> "$MODEL_LOG.old"; then cat "$MODEL_LOG" >> "$MODEL_LOG.old"
report_error_and_exit "Failed to rotate model log" : > "$MODEL_LOG"
fi
if ! : > "$MODEL_LOG"; then
report_error_and_exit "Failed to truncate model log"
fi
fi fi
# Populate /etc/environment with quoted values # Populate /etc/environment with quoted values
if ! grep -q "VAST" /etc/environment; then if ! grep -q "VAST" /etc/environment; then
if ! env -0 | grep -zEv "^(HOME=|SHLVL=)|CONDA" | while IFS= read -r -d '' line; do env -0 | grep -zEv "^(HOME=|SHLVL=)|CONDA" | while IFS= read -r -d '' line; do
name=${line%%=*} name=${line%%=*}
value=${line#*=} value=${line#*=}
printf '%s="%s"\n' "$name" "$value" printf '%s="%s"\n' "$name" "$value"
done > /etc/environment; then done > /etc/environment
echo "WARNING: Failed to populate /etc/environment, continuing anyway"
fi
fi fi
if [ ! -d "$ENV_PATH" ] if [ ! -d "$ENV_PATH" ]
then then
echo "setting up venv" echo "setting up venv"
if ! which uv; then if ! which uv; then
if ! curl -LsSf https://astral.sh/uv/install.sh | sh; then curl -LsSf https://astral.sh/uv/install.sh | sh
report_error_and_exit "Failed to install uv package manager" source ~/.local/bin/env
fi
if [[ -f ~/.local/bin/env ]]; then
if ! source ~/.local/bin/env; then
report_error_and_exit "Failed to source uv environment"
fi
else
echo "WARNING: ~/.local/bin/env not found after uv installation"
fi
fi fi
# Fork testing # Fork testing
if [[ ! -d $SERVER_DIR ]]; then [[ ! -d $SERVER_DIR ]] && git clone "${PYWORKER_REPO:-https://github.com/vast-ai/pyworker}" "$SERVER_DIR"
if ! git clone "${PYWORKER_REPO:-https://github.com/vast-ai/pyworker}" "$SERVER_DIR"; then
report_error_and_exit "Failed to clone pyworker repository"
fi
fi
if [[ -n ${PYWORKER_REF:-} ]]; then if [[ -n ${PYWORKER_REF:-} ]]; then
if ! (cd "$SERVER_DIR" && git checkout "$PYWORKER_REF"); then (cd "$SERVER_DIR" && git checkout "$PYWORKER_REF")
report_error_and_exit "Failed to checkout pyworker reference: $PYWORKER_REF"
fi
fi fi
if ! uv venv --python-preference only-managed "$ENV_PATH" -p 3.10; then uv venv --python-preference only-managed "$ENV_PATH" -p 3.10
report_error_and_exit "Failed to create virtual environment" source "$ENV_PATH/bin/activate"
fi
if ! source "$ENV_PATH/bin/activate"; then
report_error_and_exit "Failed to activate virtual environment"
fi
if ! uv pip install -r "${SERVER_DIR}/requirements.txt"; then uv pip install -r "${SERVER_DIR}/requirements.txt"
report_error_and_exit "Failed to install Python requirements"
fi
if ! touch ~/.no_auto_tmux; then touch ~/.no_auto_tmux
report_error_and_exit "Failed to create ~/.no_auto_tmux"
fi
else else
if [[ -f ~/.local/bin/env ]]; then [[ -f ~/.local/bin/env ]] && source ~/.local/bin/env
if ! source ~/.local/bin/env; then source "$WORKSPACE_DIR/worker-env/bin/activate"
report_error_and_exit "Failed to source uv environment"
fi
fi
if ! source "$WORKSPACE_DIR/worker-env/bin/activate"; then
report_error_and_exit "Failed to activate existing virtual environment"
fi
echo "environment activated" echo "environment activated"
echo "venv: $VIRTUAL_ENV" echo "venv: $VIRTUAL_ENV"
fi fi
[ ! -d "$SERVER_DIR/workers/$BACKEND" ] && report_error_and_exit "$BACKEND not supported!" [ ! -d "$SERVER_DIR/workers/$BACKEND" ] && echo "$BACKEND not supported!" && exit 1
if [ "$USE_SSL" = true ]; then if [ "$USE_SSL" = true ]; then
if ! cat << EOF > /etc/openssl-san.cnf cat << EOF > /etc/openssl-san.cnf
[req] [req]
default_bits = 2048 default_bits = 2048
distinguished_name = req_distinguished_name distinguished_name = req_distinguished_name
@@ -183,25 +109,18 @@ if [ "$USE_SSL" = true ]; then
[alt_names] [alt_names]
IP.1 = 0.0.0.0 IP.1 = 0.0.0.0
EOF EOF
then
report_error_and_exit "Failed to write OpenSSL config"
fi
if ! openssl req -newkey rsa:2048 -subj "/C=US/ST=CA/CN=pyworker.vast.ai/" \ openssl req -newkey rsa:2048 -subj "/C=US/ST=CA/CN=pyworker.vast.ai/" \
-nodes \ -nodes \
-sha256 \ -sha256 \
-keyout /etc/instance.key \ -keyout /etc/instance.key \
-out /etc/instance.csr \ -out /etc/instance.csr \
-config /etc/openssl-san.cnf; then -config /etc/openssl-san.cnf
report_error_and_exit "Failed to generate SSL certificate request"
fi
if ! curl --header 'Content-Type: application/octet-stream' \ curl --header 'Content-Type: application/octet-stream' \
--data-binary @/etc/instance.csr \ --data-binary @//etc/instance.csr \
-X \ -X \
POST "https://console.vast.ai/api/v0/sign_cert/?instance_id=$CONTAINER_ID" > /etc/instance.crt; then POST "https://console.vast.ai/api/v0/sign_cert/?instance_id=$CONTAINER_ID" > /etc/instance.crt;
report_error_and_exit "Failed to sign SSL certificate"
fi
fi fi
@@ -209,9 +128,7 @@ fi
export REPORT_ADDR WORKER_PORT USE_SSL UNSECURED export REPORT_ADDR WORKER_PORT USE_SSL UNSECURED
if ! cd "$SERVER_DIR"; then cd "$SERVER_DIR"
report_error_and_exit "Failed to cd into SERVER_DIR: $SERVER_DIR"
fi
echo "launching PyWorker server" echo "launching PyWorker server"
@@ -221,7 +138,37 @@ PY_STATUS=${PIPESTATUS[0]}
set -e set -e
if [ "${PY_STATUS}" -ne 0 ]; then if [ "${PY_STATUS}" -ne 0 ]; then
report_error_and_exit "PyWorker exited with status ${PY_STATUS}" echo "PyWorker exited with status ${PY_STATUS}; notifying autoscaler..."
ERROR_MSG="PyWorker exited: code ${PY_STATUS}"
MTOKEN="${MASTER_TOKEN:-}"
VERSION="${PYWORKER_VERSION:-0}"
IFS=',' read -r -a REPORT_ADDRS <<< "${REPORT_ADDR}"
for addr in "${REPORT_ADDRS[@]}"; do
curl -sS -X POST -H 'Content-Type: application/json' \
-d "$(cat <<JSON
{
"id": ${CONTAINER_ID:-0},
"mtoken": "${MTOKEN}",
"version": "${VERSION}",
"loadtime": 0,
"new_load": 0,
"cur_load": 0,
"rej_load": 0,
"max_perf": 0,
"cur_perf": 0,
"error_msg": "${ERROR_MSG}",
"num_requests_working": 0,
"num_requests_recieved": 0,
"additional_disk_usage": 0,
"working_request_idxs": [],
"cur_capacity": 0,
"max_capacity": 0,
"url": "${URL}"
}
JSON
)" "${addr%/}/worker_status/" || true
done
fi fi
echo "launching PyWorker server done" echo "launching PyWorker server done"