From b03645d1453d89ff69ae8f3fd2c55766b1e731db Mon Sep 17 00:00:00 2001 From: Abiola Akinnubi Date: Tue, 28 Oct 2025 19:22:47 +0000 Subject: [PATCH] Added model type environment variable so we can actually attempt to benchmark with the right payload. --- start_server.sh | 1 + workers/comfyui-json/data_types.py | 21 +- .../comfyui-json/misc/benchmark_audio.json | 118 ++++++++++ .../comfyui-json/misc/benchmark_image.json | 107 +++++++++ .../comfyui-json/misc/benchmark_video.json | 216 ++++++++++++++++++ 5 files changed, 460 insertions(+), 3 deletions(-) create mode 100644 workers/comfyui-json/misc/benchmark_audio.json create mode 100644 workers/comfyui-json/misc/benchmark_image.json create mode 100644 workers/comfyui-json/misc/benchmark_video.json diff --git a/start_server.sh b/start_server.sh index edc16a4..45cf29b 100755 --- a/start_server.sh +++ b/start_server.sh @@ -12,6 +12,7 @@ PYWORKER_LOG="$WORKSPACE_DIR/pyworker.log" REPORT_ADDR="${REPORT_ADDR:-https://run.vast.ai}" USE_SSL="${USE_SSL:-true}" WORKER_PORT="${WORKER_PORT:-3000}" +MODEL_TYPE="${MODEL_TYPE:-image}" mkdir -p "$WORKSPACE_DIR" cd "$WORKSPACE_DIR" diff --git a/workers/comfyui-json/data_types.py b/workers/comfyui-json/data_types.py index 1af1f8b..3e8efad 100644 --- a/workers/comfyui-json/data_types.py +++ b/workers/comfyui-json/data_types.py @@ -1,3 +1,5 @@ +import dataclasses +from enum import Enum import os import sys import random @@ -13,6 +15,12 @@ from lib.data_types import ApiPayload, JsonDataException log = logging.getLogger(__file__) +class ModelType(Enum): + image = "image" + audio = "audio" + video = "video" + + def count_workload() -> float: # Always 100.0 where there is a single instance of ComfyUI handling requests # Results will indicate % or a job completed per second. Avoids sub 0.1 sec performance indication @@ -21,6 +29,11 @@ def count_workload() -> float: @dataclasses.dataclass class ComfyWorkflowData(ApiPayload): input: dict + model_type: ModelType = dataclasses.field( + default_factory=lambda: ModelType( + os.environ.get("MODEL_TYPE", "image").lower() + ) + ) @classmethod def for_test(cls): @@ -30,15 +43,17 @@ class ComfyWorkflowData(ApiPayload): Example: SD1.5, simple image gen 10000 steps, 512px x 512px will run for approximately 9 minutes @ ~18 it/s (RTX 4090) """ # Try to load benchmark.json - benchmark_file = Path("workers/comfyui-json/misc/benchmark.json") - + #Note: We should cross check with Rob if the audio sample benchmark file is correct + model_type = ModelType(os.environ.get("MODEL_TYPE", "image").lower()) + benchmark_file = Path(f"workers/comfyui-json/misc/benchmark_{model_type.value}.json") if benchmark_file.exists(): try: with open(benchmark_file, "r") as f: benchmark_workflow = json.load(f) + log.info(f"using benchmark json file for {model_type.value}") return cls( input={ - "request_id": f"test-{random.randint(1000, 99999)}", + "request_id": f"{model_type.value}-{random.randint(1000, 99999)}", "workflow_json": benchmark_workflow } ) diff --git a/workers/comfyui-json/misc/benchmark_audio.json b/workers/comfyui-json/misc/benchmark_audio.json new file mode 100644 index 0000000..2cabf5e --- /dev/null +++ b/workers/comfyui-json/misc/benchmark_audio.json @@ -0,0 +1,118 @@ +{ + "3": { + "inputs": { + "seed": 98942092715729, + "steps": 50, + "cfg": 4.98, + "sampler_name": "dpmpp_3m_sde_gpu", + "scheduler": "exponential", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "11", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "stable-audio-open-1.0.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "6": { + "inputs": { + "text": "heaven church electronic dance music", + "clip": [ + "10", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "", + "clip": [ + "10", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "10": { + "inputs": { + "clip_name": "t5-base.safetensors", + "type": "stable_audio", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "11": { + "inputs": { + "seconds": 47.6, + "batch_size": 1 + }, + "class_type": "EmptyLatentAudio", + "_meta": { + "title": "EmptyLatentAudio" + } + }, + "12": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecodeAudio", + "_meta": { + "title": "VAEDecodeAudio" + } + }, + "13": { + "inputs": { + "filename_prefix": "audio/ComfyUI", + "audioUI": "", + "audio": [ + "12", + 0 + ] + }, + "class_type": "SaveAudio", + "_meta": { + "title": "SaveAudio" + } + } + } \ No newline at end of file diff --git a/workers/comfyui-json/misc/benchmark_image.json b/workers/comfyui-json/misc/benchmark_image.json new file mode 100644 index 0000000..0ff36ff --- /dev/null +++ b/workers/comfyui-json/misc/benchmark_image.json @@ -0,0 +1,107 @@ +{ + "3": { + "inputs": { + "seed": 588445435278533, + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly-fp16.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "5": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "text, watermark", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } + } \ No newline at end of file diff --git a/workers/comfyui-json/misc/benchmark_video.json b/workers/comfyui-json/misc/benchmark_video.json new file mode 100644 index 0000000..4b4d2d8 --- /dev/null +++ b/workers/comfyui-json/misc/benchmark_video.json @@ -0,0 +1,216 @@ +{ + "90": { + "inputs": { + "clip_name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "type": "wan", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "91": { + "inputs": { + "text": "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走，裸露，NSFW", + "clip": [ + "90", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Negative Prompt)" + } + }, + "92": { + "inputs": { + "vae_name": "wan_2.1_vae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "93": { + "inputs": { + "shift": 8.000000000000002, + "model": [ + "101", + 0 + ] + }, + "class_type": "ModelSamplingSD3", + "_meta": { + "title": "ModelSamplingSD3" + } + }, + "94": { + "inputs": { + "shift": 8, + "model": [ + "102", + 0 + ] + }, + "class_type": "ModelSamplingSD3", + "_meta": { + "title": "ModelSamplingSD3" + } + }, + "95": { + "inputs": { + "add_noise": "disable", + "noise_seed": 0, + "steps": 20, + "cfg": 3.5, + "sampler_name": "euler", + "scheduler": "simple", + "start_at_step": 10, + "end_at_step": 10000, + "return_with_leftover_noise": "disable", + "model": [ + "94", + 0 + ], + "positive": [ + "99", + 0 + ], + "negative": [ + "91", + 0 + ], + "latent_image": [ + "96", + 0 + ] + }, + "class_type": "KSamplerAdvanced", + "_meta": { + "title": "KSampler (Advanced)" + } + }, + "96": { + "inputs": { + "add_noise": "enable", + "noise_seed": "__RANDOM_INT__", + "steps": 20, + "cfg": 3.5, + "sampler_name": "euler", + "scheduler": "simple", + "start_at_step": 0, + "end_at_step": 10, + "return_with_leftover_noise": "enable", + "model": [ + "93", + 0 + ], + "positive": [ + "99", + 0 + ], + "negative": [ + "91", + 0 + ], + "latent_image": [ + "104", + 0 + ] + }, + "class_type": "KSamplerAdvanced", + "_meta": { + "title": "KSampler (Advanced)" + } + }, + "97": { + "inputs": { + "samples": [ + "95", + 0 + ], + "vae": [ + "92", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "98": { + "inputs": { + "filename_prefix": "video/ComfyUI", + "format": "auto", + "codec": "auto", + "video": [ + "100", + 0 + ] + }, + "class_type": "SaveVideo", + "_meta": { + "title": "Save Video" + } + }, + "99": { + "inputs": { + "text": "Beautiful young European woman with honey blonde hair gracefully turning her head back over shoulder, gentle smile, bright eyes looking at camera. Hair flowing in slow motion as she turns. Soft natural lighting, clean background, cinematic portrait.", + "clip": [ + "90", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Positive Prompt)" + } + }, + "100": { + "inputs": { + "fps": 16, + "images": [ + "97", + 0 + ] + }, + "class_type": "CreateVideo", + "_meta": { + "title": "Create Video" + } + }, + "101": { + "inputs": { + "unet_name": "wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "102": { + "inputs": { + "unet_name": "wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "104": { + "inputs": { + "width": 640, + "height": 640, + "length": 81, + "batch_size": 1 + }, + "class_type": "EmptyHunyuanLatentVideo", + "_meta": { + "title": "EmptyHunyuanLatentVideo" + } + } +} \ No newline at end of file