diff --git a/workers/comfyui-json/README.md b/workers/comfyui-json/README.md index bb07145..7aa1ba3 100644 --- a/workers/comfyui-json/README.md +++ b/workers/comfyui-json/README.md @@ -12,9 +12,21 @@ A docker image is provided but you may use any if the above requirements are met ## Benchmarking -A simple image generation benchmark runs when each worker initializes to validate GPU performance and identify underperforming machines. +### Custom Benchmark Workflows -The benchmark uses Stable Diffusion v1.5 with ComfyUI's default text-to-image workflow. Configure the benchmark complexity and duration using these variables: +You can provide a custom ComfyUI workflow for benchmarking by creating `workers/comfyui-json/misc/benchmark.json`. This allows you to test performance using your preferred models and workflow complexity. + +**Ways to provide the benchmark file:** +- Fork this repository and add your `benchmark.json` file +- Write the file during worker provisioning (onstart script or setup phase) + +An example file is provided in the repository. To ensure varied generations, use the placeholder `__RANDOM_INT__` in place of static seed values - it will be replaced with a random integer for each benchmark run. + +### Default Benchmark (Fallback) + +If `benchmark.json` is not available, a simple image generation benchmark runs when each worker initializes. This validates GPU performance and helps identify underperforming machines. + +The default benchmark uses Stable Diffusion v1.5 with ComfyUI's standard text-to-image workflow. Configure it using these environment variables: | Environment Variable | Default Value | Description | | -------------------- | ------------- | ----------- | @@ -24,7 +36,7 @@ The benchmark uses Stable Diffusion v1.5 with ComfyUI's default text-to-image wo Each benchmark run uses a random prompt from `misc/test_prompts.txt` and a random seed to ensure consistent GPU load patterns. -### Calibrating Benchmark Duration +#### Calibrating Fallback Benchmark Duration To screen for underperforming hardware, set `BENCHMARK_TEST_STEPS` to match your expected production workflow duration. This allows you to identify machines that won't meet performance requirements. diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index fd8c6e5..1af1f8b 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -5,12 +5,13 @@ import dataclasses from typing import Dict, Any from functools import cache from math import ceil +from pathlib import Path +import json +import logging from lib.data_types import ApiPayload, JsonDataException - -with open("workers/comfyui/misc/test_prompts.txt", "r") as f: - test_prompts = f.readlines() +log = logging.getLogger(__file__) def count_workload() -> float: # Always 100.0 where there is a single instance of ComfyUI handling requests @@ -24,9 +25,32 @@ class ComfyWorkflowData(ApiPayload): @classmethod def for_test(cls): """ - Use the variables available to simulate workflows of the required running time + If the user has provided a benchmark workflow we can use it here to properly gauge performance. + Otherwise, use the variables available to simulate workflows of the required running time Example: SD1.5, simple image gen 10000 steps, 512px x 512px will run for approximately 9 minutes @ ~18 it/s (RTX 4090) """ + # Try to load benchmark.json + benchmark_file = Path("workers/comfyui-json/misc/benchmark.json") + + if benchmark_file.exists(): + try: + with open(benchmark_file, "r") as f: + benchmark_workflow = json.load(f) + return cls( + input={ + "request_id": f"test-{random.randint(1000, 99999)}", + "workflow_json": benchmark_workflow + } + ) + except (json.JSONDecodeError, IOError): + # JSON is malformed or file can't be read, fall through to default + log.error(f"Failed to benchmark using {benchmark_file}") + + # Fallback: read prompts and construct payload + log.info("Using fallback method for benchmarking") + with open("workers/comfyui-json/misc/test_prompts.txt", "r") as f: + test_prompts = f.readlines() + test_prompt = random.choice(test_prompts).rstrip() return cls( input={ diff --git a/workers/comfyui-json/misc/benchmark.json.example b/workers/comfyui-json/misc/benchmark.json.example new file mode 100644 index 0000000..3e20040 --- /dev/null +++ b/workers/comfyui-json/misc/benchmark.json.example @@ -0,0 +1,107 @@ +{ + "3": { + "inputs": { + "seed": "__RANDOM_INT__", + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly-fp16.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "5": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "text, watermark", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} \ No newline at end of file diff --git a/workers/comfyui-json/server.py b/workers/comfyui-json/server.py index cd970a3..ed4e578 100644 --- a/workers/comfyui-json/server.py +++ b/workers/comfyui-json/server.py @@ -19,6 +19,7 @@ MODEL_SERVER_START_LOG_MSG = "To see the GUI go to: " MODEL_SERVER_ERROR_LOG_MSGS = [ "MetadataIncompleteBuffer", # This error is emitted when the downloaded model is corrupted "Value not in list: ", # This error is emitted when the model file is not there at all + "[ERROR] Provisioning Script failed", # Error inserted by provisioning script if models/nodes fail to download ]