Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9c6ab78503 | |||
| 45e0c7d9ca | |||
| a4339bd3f1 | |||
| 2b26e5e20c | |||
| d3727d4fd7 |
+1
-1
@@ -417,7 +417,7 @@ class Backend:
|
|||||||
|
|
||||||
async def tail_log():
|
async def tail_log():
|
||||||
log.debug(f"tailing file: {self.model_log_file}")
|
log.debug(f"tailing file: {self.model_log_file}")
|
||||||
async with await open_file(self.model_log_file, encoding='utf-8', errors='ignore'):
|
async with await open_file(self.model_log_file, encoding='utf-8', errors='ignore') as f:
|
||||||
while True:
|
while True:
|
||||||
line = await f.readline()
|
line = await f.readline()
|
||||||
if line:
|
if line:
|
||||||
|
|||||||
+1
-1
@@ -8,4 +8,4 @@ Requests~=2.32
|
|||||||
transformers~=4.52
|
transformers~=4.52
|
||||||
utils==1.0.*
|
utils==1.0.*
|
||||||
hf_transfer>=0.1.9
|
hf_transfer>=0.1.9
|
||||||
vastai-sdk>=0.2.0g
|
vastai-sdk>=0.2.0
|
||||||
+8
-4
@@ -41,6 +41,14 @@ echo_var DEBUG_LOG
|
|||||||
echo_var PYWORKER_LOG
|
echo_var PYWORKER_LOG
|
||||||
echo_var MODEL_LOG
|
echo_var MODEL_LOG
|
||||||
|
|
||||||
|
# if instance is rebooted, we want to clear out the log file so pyworker doesn't read lines
|
||||||
|
# from the run prior to reboot. past logs are saved in $MODEL_LOG.old for debugging only
|
||||||
|
if [ -e "$MODEL_LOG" ]; then
|
||||||
|
echo "Rotating model log at $MODEL_LOG to $MODEL_LOG.old"
|
||||||
|
cat "$MODEL_LOG" >> "$MODEL_LOG.old"
|
||||||
|
: > "$MODEL_LOG"
|
||||||
|
fi
|
||||||
|
|
||||||
# Populate /etc/environment with quoted values
|
# Populate /etc/environment with quoted values
|
||||||
if ! grep -q "VAST" /etc/environment; then
|
if ! grep -q "VAST" /etc/environment; then
|
||||||
env -0 | grep -zEv "^(HOME=|SHLVL=)|CONDA" | while IFS= read -r -d '' line; do
|
env -0 | grep -zEv "^(HOME=|SHLVL=)|CONDA" | while IFS= read -r -d '' line; do
|
||||||
@@ -124,9 +132,5 @@ cd "$SERVER_DIR"
|
|||||||
|
|
||||||
echo "launching PyWorker server"
|
echo "launching PyWorker server"
|
||||||
|
|
||||||
# if instance is rebooted, we want to clear out the log file so pyworker doesn't read lines
|
|
||||||
# from the run prior to reboot. past logs are saved in $MODEL_LOG.old for debugging only
|
|
||||||
[ -e "$MODEL_LOG" ] && cat "$MODEL_LOG" >> "$MODEL_LOG.old" && : > "$MODEL_LOG"
|
|
||||||
|
|
||||||
(python3 -m "workers.$BACKEND.server" |& tee -a "$PYWORKER_LOG") &
|
(python3 -m "workers.$BACKEND.server" |& tee -a "$PYWORKER_LOG") &
|
||||||
echo "launching PyWorker server done"
|
echo "launching PyWorker server done"
|
||||||
|
|||||||
Reference in New Issue
Block a user