From 381a39f201efc981fde6c161c052452ac03852e1 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Thu, 7 May 2026 11:54:20 +0100 Subject: [PATCH] Add well-known fallback path for benchmark.json Read /opt/comfyui-api-wrapper/workflows/pyworker_benchmark.json when neither misc/benchmark.json nor $BENCHMARK_JSON_PATH yields a usable file. The vast.ai ComfyUI base image's convert-workflows.sh maintains that path as a symlink to the first provisioned workflow, so on that image the operator does not need to set BENCHMARK_JSON_PATH at all. A set-but-broken $BENCHMARK_JSON_PATH now warns and falls through to the well-known path instead of dropping straight to the SD1.5 fallback, so a typo in the env var doesn't mask an otherwise-working benchmark. Co-Authored-By: Claude Opus 4.7 (1M context) --- workers/comfyui-json/README.md | 12 +++++++----- workers/comfyui-json/worker.py | 34 +++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/workers/comfyui-json/README.md b/workers/comfyui-json/README.md index b63d5ea..c498c9b 100644 --- a/workers/comfyui-json/README.md +++ b/workers/comfyui-json/README.md @@ -106,11 +106,13 @@ Images will be saved locally AND uploaded to `s3://{bucket}/comfyui/{filename}`. You can provide a custom ComfyUI workflow for benchmarking. This allows you to test performance using your preferred models and workflow complexity. -**Ways to provide the benchmark file:** -- **Fork this repository** and commit your workflow to `workers/comfyui-json/misc/benchmark.json`. -- **Write the file during provisioning** to a path *outside* the pyworker tree (e.g. `/workspace/benchmark.json`) and export `BENCHMARK_JSON_PATH` so the worker can find it. The pyworker repo is cloned by `start_server.sh` *after* provisioning runs, so provisioning cannot write into `misc/` directly — the destination would be clobbered, or the clone would fail. +**Ways to provide the benchmark file** (in resolution order — first match wins): -If both are present, the in-tree `misc/benchmark.json` wins; `BENCHMARK_JSON_PATH` is consulted only when no in-tree file exists. If the env var is set but points at a missing or unreadable file, the worker logs a warning and falls back to the default benchmark. +1. **Fork this repository** and commit your workflow to `workers/comfyui-json/misc/benchmark.json`. +2. **Write the file during provisioning** to a path *outside* the pyworker tree (e.g. `/workspace/benchmark.json`) and export `BENCHMARK_JSON_PATH` so the worker can find it. The pyworker repo is cloned by `start_server.sh` *after* provisioning runs, so provisioning cannot write into `misc/` directly — the destination would be clobbered, or the clone would fail. +3. **Run on the vast.ai ComfyUI base image.** Its `convert-workflows.sh` maintains `/opt/comfyui-api-wrapper/workflows/pyworker_benchmark.json` as a symlink to the first provisioned workflow; the worker reads this automatically when neither of the above is set. No env var required. + +If `BENCHMARK_JSON_PATH` is set but points at a missing or unreadable file, the worker logs a warning and falls through to the next tier rather than going straight to the SD1.5 fallback. An example workflow is provided at `workers/comfyui-json/misc/benchmark.json.example`. To ensure varied generations, use the placeholder `__RANDOM_INT__` in place of static seed values — it will be replaced with a random integer for each benchmark run. @@ -122,7 +124,7 @@ The default benchmark uses Stable Diffusion v1.5 with ComfyUI's standard text-to | Environment Variable | Default Value | Description | | -------------------- | ------------- | ----------- | -| BENCHMARK_JSON_PATH | (unset) | Path to a custom workflow file outside the pyworker tree. Used only if `misc/benchmark.json` is absent. | +| BENCHMARK_JSON_PATH | (unset) | Path to a custom workflow file outside the pyworker tree. Used if `misc/benchmark.json` is absent. Falls through to `/opt/comfyui-api-wrapper/workflows/pyworker_benchmark.json` if set but missing. | | BENCHMARK_TEST_WIDTH | 512 | Fallback benchmark: image width (pixels) | | BENCHMARK_TEST_HEIGHT | 512 | Fallback benchmark: image height (pixels) | | BENCHMARK_TEST_STEPS | 20 | Fallback benchmark: number of denoising steps | diff --git a/workers/comfyui-json/worker.py b/workers/comfyui-json/worker.py index 944fe0f..e6dcee3 100644 --- a/workers/comfyui-json/worker.py +++ b/workers/comfyui-json/worker.py @@ -10,7 +10,12 @@ Each worker runs a benchmark on warm-up. The payload is selected as follows: runs before pyworker is cloned, so it cannot write into ``misc/``, but it can drop the workflow elsewhere (e.g. ``/workspace/``) and export this env var. - 3. Otherwise an SD1.5 Text2Image fallback runs, parameterised by the + 3. Else, if the well-known path + ``/opt/comfyui-api-wrapper/workflows/pyworker_benchmark.json`` exists, + it is used. The vast.ai ComfyUI base image's ``convert-workflows.sh`` + maintains this as a symlink to the first provisioned workflow, so on + that image no env var is needed. + 4. Otherwise an SD1.5 Text2Image fallback runs, parameterised by the ``BENCHMARK_TEST_{WIDTH,HEIGHT,STEPS}`` env vars and a random prompt from ``misc/test_prompts.txt``. @@ -55,26 +60,33 @@ MISC_DIR = Path(__file__).parent / "misc" BENCHMARK_FILE = MISC_DIR / "benchmark.json" TEST_PROMPTS = MISC_DIR / "test_prompts.txt" +# Well-known location maintained by the vast.ai ComfyUI base image. +# convert-workflows.sh symlinks this to the first provisioned workflow, +# letting the base image work out-of-the-box without any env var. +WELLKNOWN_BENCHMARK = Path("/opt/comfyui-api-wrapper/workflows/pyworker_benchmark.json") + log = logging.getLogger(__name__) def _resolve_benchmark_path() -> Path | None: """Return the path to the custom benchmark workflow, or None if absent. - See module docstring for the precedence rule. ``$BENCHMARK_JSON_PATH`` - is logged as a warning when set but missing, so a misconfigured - provisioning script doesn't silently degrade to the fallback benchmark. + See module docstring for the precedence rule. A set-but-broken + ``$BENCHMARK_JSON_PATH`` logs a warning then falls through to the + well-known path, so a typo in the env var doesn't silently mask a + provisioned benchmark sitting at the standard location. """ if BENCHMARK_FILE.exists(): return BENCHMARK_FILE env_path = os.getenv("BENCHMARK_JSON_PATH") - if not env_path: - return None - path = Path(env_path) - if not path.exists(): - log.warning("BENCHMARK_JSON_PATH=%s does not exist; falling back to default benchmark", path) - return None - return path + if env_path: + path = Path(env_path) + if path.exists(): + return path + log.warning("BENCHMARK_JSON_PATH=%s does not exist; trying fallbacks", path) + if WELLKNOWN_BENCHMARK.exists(): + return WELLKNOWN_BENCHMARK + return None def _custom_workflow_payload() -> dict | None: