From cadac033e16655ad86b55e8f520c3e945c168653 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Sun, 5 Oct 2025 22:53:22 +0100 Subject: [PATCH 1/7] Enables use of custom workflow for benchmarking Retains existing method is misc/benchmark.json is nopt present --- workers/comfyui-json/README.md | 18 ++- workers/comfyui-json/data_types.py | 28 ++++- .../comfyui-json/misc/benchmark.json.example | 107 ++++++++++++++++++ 3 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 workers/comfyui-json/misc/benchmark.json.example diff --git a/workers/comfyui-json/README.md b/workers/comfyui-json/README.md index bb07145..7aa1ba3 100644 --- a/workers/comfyui-json/README.md +++ b/workers/comfyui-json/README.md @@ -12,9 +12,21 @@ A docker image is provided but you may use any if the above requirements are met ## Benchmarking -A simple image generation benchmark runs when each worker initializes to validate GPU performance and identify underperforming machines. +### Custom Benchmark Workflows -The benchmark uses Stable Diffusion v1.5 with ComfyUI's default text-to-image workflow. Configure the benchmark complexity and duration using these variables: +You can provide a custom ComfyUI workflow for benchmarking by creating `workers/comfyui-json/misc/benchmark.json`. This allows you to test performance using your preferred models and workflow complexity. + +**Ways to provide the benchmark file:** +- Fork this repository and add your `benchmark.json` file +- Write the file during worker provisioning (onstart script or setup phase) + +An example file is provided in the repository. To ensure varied generations, use the placeholder `__RANDOM_INT__` in place of static seed values - it will be replaced with a random integer for each benchmark run. + +### Default Benchmark (Fallback) + +If `benchmark.json` is not available, a simple image generation benchmark runs when each worker initializes. This validates GPU performance and helps identify underperforming machines. + +The default benchmark uses Stable Diffusion v1.5 with ComfyUI's standard text-to-image workflow. Configure it using these environment variables: | Environment Variable | Default Value | Description | | -------------------- | ------------- | ----------- | @@ -24,7 +36,7 @@ The benchmark uses Stable Diffusion v1.5 with ComfyUI's default text-to-image wo Each benchmark run uses a random prompt from `misc/test_prompts.txt` and a random seed to ensure consistent GPU load patterns. -### Calibrating Benchmark Duration +#### Calibrating Fallback Benchmark Duration To screen for underperforming hardware, set `BENCHMARK_TEST_STEPS` to match your expected production workflow duration. This allows you to identify machines that won't meet performance requirements. diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index fd8c6e5..c7952fe 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -8,10 +8,6 @@ from math import ceil from lib.data_types import ApiPayload, JsonDataException - -with open("workers/comfyui/misc/test_prompts.txt", "r") as f: - test_prompts = f.readlines() - def count_workload() -> float: # Always 100.0 where there is a single instance of ComfyUI handling requests # Results will indicate % or a job completed per second. Avoids sub 0.1 sec performance indication @@ -24,9 +20,31 @@ class ComfyWorkflowData(ApiPayload): @classmethod def for_test(cls): """ - Use the variables available to simulate workflows of the required running time + If the user has provided a benchmark workflow we can use it here to properly gauge performance. + Otherwise, use the variables available to simulate workflows of the required running time Example: SD1.5, simple image gen 10000 steps, 512px x 512px will run for approximately 9 minutes @ ~18 it/s (RTX 4090) """ + # Try to load benchmark.json + benchmark_file = Path("workers/comfyui-json/misc/benchmark.json") + + if benchmark_file.exists(): + try: + with open(benchmark_file, "r") as f: + benchmark_workflow = json.load(f) + return cls( + input={ + "request_id": f"test-{random.randint(1000, 99999)}", + "workflow_json": benchmark_workflow + } + ) + except (json.JSONDecodeError, IOError): + # JSON is malformed or file can't be read, fall through to default + pass + + # Fallback: read prompts and construct payload + with open("workers/comfyui-json/misc/test_prompts.txt", "r") as f: + test_prompts = f.readlines() + test_prompt = random.choice(test_prompts).rstrip() return cls( input={ diff --git a/workers/comfyui-json/misc/benchmark.json.example b/workers/comfyui-json/misc/benchmark.json.example new file mode 100644 index 0000000..3e20040 --- /dev/null +++ b/workers/comfyui-json/misc/benchmark.json.example @@ -0,0 +1,107 @@ +{ + "3": { + "inputs": { + "seed": "__RANDOM_INT__", + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly-fp16.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "5": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "text, watermark", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} \ No newline at end of file From e9b6a14a5e244065f7ce9fed6a7190602c5f131e Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Sun, 5 Oct 2025 22:59:19 +0100 Subject: [PATCH 2/7] Import Path --- workers/comfyui-json/data_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index c7952fe..9239ae5 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -5,6 +5,7 @@ import dataclasses from typing import Dict, Any from functools import cache from math import ceil +from pathlib import Path from lib.data_types import ApiPayload, JsonDataException From a86d4bcf9c4577aac902539a055c6f0c30504111 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Sun, 5 Oct 2025 23:05:33 +0100 Subject: [PATCH 3/7] Import json --- workers/comfyui-json/data_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index 9239ae5..d4cb7c6 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -6,6 +6,7 @@ from typing import Dict, Any from functools import cache from math import ceil from pathlib import Path +import json from lib.data_types import ApiPayload, JsonDataException From 3786cf978dd4f3fec3c53da494cce6c175ff015c Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Sun, 5 Oct 2025 23:14:59 +0100 Subject: [PATCH 4/7] Add awareness of errors thrown by the provisioning script --- workers/comfyui-json/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/comfyui-json/server.py b/workers/comfyui-json/server.py index 44b10fc..dcce49c 100644 --- a/workers/comfyui-json/server.py +++ b/workers/comfyui-json/server.py @@ -19,6 +19,7 @@ MODEL_SERVER_START_LOG_MSG = "To see the GUI go to: " MODEL_SERVER_ERROR_LOG_MSGS = [ "MetadataIncompleteBuffer", # This error is emitted when the downloaded model is corrupted "Value not in list: ", # This error is emitted when the model file is not there at all + "[ERROR] Provisioning Script failed", # Error inserted by provisioning script if models/nodes fail to download ] From 92a04bd7afc0b9d7bfb86ddb90d0b77641df53d4 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Thu, 23 Oct 2025 13:41:03 +0100 Subject: [PATCH 5/7] No silent fail if benchmark file is missing --- workers/comfyui-json/data_types.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index d4cb7c6..578a48b 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -7,9 +7,12 @@ from functools import cache from math import ceil from pathlib import Path import json +import logging from lib.data_types import ApiPayload, JsonDataException +log = logging.getLogger(__file__) + def count_workload() -> float: # Always 100.0 where there is a single instance of ComfyUI handling requests # Results will indicate % or a job completed per second. Avoids sub 0.1 sec performance indication @@ -40,8 +43,7 @@ class ComfyWorkflowData(ApiPayload): } ) except (json.JSONDecodeError, IOError): - # JSON is malformed or file can't be read, fall through to default - pass + log.info(f"{benchmark_file} not found. Using fallback method") # Fallback: read prompts and construct payload with open("workers/comfyui-json/misc/test_prompts.txt", "r") as f: From d51a338e8ff4af09e63ef6f617d5867855b52498 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Thu, 23 Oct 2025 16:41:02 +0100 Subject: [PATCH 6/7] log when benchmark file not used --- workers/comfyui-json/data_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index 578a48b..6d46f43 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -43,9 +43,10 @@ class ComfyWorkflowData(ApiPayload): } ) except (json.JSONDecodeError, IOError): - log.info(f"{benchmark_file} not found. Using fallback method") + log.error(f"Failed to benchmark using {benchmark_file}") # Fallback: read prompts and construct payload + log.info("Using fallback method for benchmarking") with open("workers/comfyui-json/misc/test_prompts.txt", "r") as f: test_prompts = f.readlines() From f4f7080df1f9e798b8ce7a2c43065cb745e70b05 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Thu, 23 Oct 2025 17:00:28 +0100 Subject: [PATCH 7/7] Re-add comment --- workers/comfyui-json/data_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index 6d46f43..1af1f8b 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -43,6 +43,7 @@ class ComfyWorkflowData(ApiPayload): } ) except (json.JSONDecodeError, IOError): + # JSON is malformed or file can't be read, fall through to default log.error(f"Failed to benchmark using {benchmark_file}") # Fallback: read prompts and construct payload