From 16e05f5815c19bafbcd397f7f5232c762b8aa89d Mon Sep 17 00:00:00 2001 From: "kyj@bowong.ai" Date: Fri, 28 Mar 2025 15:11:26 +0800 Subject: [PATCH] =?UTF-8?q?ADD=20=E5=A2=9E=E5=8A=A0=E8=AE=A4=E8=AF=81=20PE?= =?UTF-8?q?RF=20=E8=B0=83=E6=95=B4=E9=83=A8=E7=BD=B2=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server_with_s3.py | 71 +++++++++---- server_with_s3_auth.py | 223 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 20 deletions(-) create mode 100644 server_with_s3_auth.py diff --git a/server_with_s3.py b/server_with_s3.py index f1d40be..4c63a43 100644 --- a/server_with_s3.py +++ b/server_with_s3.py @@ -3,6 +3,7 @@ import os import shutil import subprocess import sys +import traceback import uuid from pathlib import Path from typing import Dict @@ -52,16 +53,14 @@ image = ( .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy") .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials") .run_commands("comfy node install https://github.com/melMass/comfy_mtb") + .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") + .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") .run_commands( "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model" ).run_commands( "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints" - ).run_commands( - "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && mkdir -p /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && rm -rf /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT" ).run_commands( "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints" ).run_commands( @@ -92,13 +91,17 @@ output_dir = "/root/comfy/ComfyUI/output" @app.cls( allow_concurrent_inputs=1, # allow 10 concurrent API calls max_containers=200, + min_containers=0, + buffer_containers=0, scaledown_window=120, # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times timeout=900, - gpu="L4", - cpu=8, + gpu=["L4", "T4"], + cpu=(2,16), # memory=(32768, 32768), # (内存预留量, 内存使用上限) - memory=32768, + memory=(20480,40960), + retries=1, + enable_memory_snapshot=True, secrets=[secret], volumes={ "/root/comfy/ComfyUI/models": vol, @@ -121,15 +124,17 @@ class ComfyUI: def launch_comfy_background(self): # starts the ComfyUI server in the background exactly once when the first input is received self.session_id = str(uuid.uuid4()) - cmd = "echo client_uuid: {}&&rm -rf /root/comfy/ComfyUI/user&&mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}&&ln -s /root/comfy/ComfyUI/output_s3/logs/{} /root/comfy/ComfyUI/user".format(self.session_id,self.session_id, self.session_id) - subprocess.run(cmd, shell=True, check=True) - cmd = "comfy launch --background" + cmd = "echo client_uuid: {}&&rm -rf /root/comfy/ComfyUI/user&&mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}&&ln -s /root/comfy/ComfyUI/output_s3/logs/{} /root/comfy/ComfyUI/user && comfy launch --background".format(self.session_id,self.session_id, self.session_id) subprocess.run(cmd, shell=True, check=True) @modal.method() def infer(self, workflow_json: str = ""): + self.poll_server_health() + self.prompt_uuid = str(uuid.uuid4()) # runs the comfy run --workflow command as a subprocess workflow = json.loads(workflow_json) + print("Workflow JSON:") + print(json.dumps(workflow, indent=4, ensure_ascii=False)) for node in workflow.values(): if node.get("class_type") == "ComfyUIDeployExternalText": if node.get("inputs").get("input_id") == "file_path": @@ -138,10 +143,10 @@ class ComfyUI: os.makedirs(os.path.dirname(file_to_move.replace("input_s3", "input")), exist_ok=True) shutil.copy(file_to_move, file_to_move.replace("input_s3", "input")) node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", "input") - with open(f"/root/{self.session_id}.json", "w", encoding="utf-8") as f: + with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as f: f.write(json.dumps(workflow, ensure_ascii=False)) - cmd = f"comfy run --workflow /root/{self.session_id}.json --wait --timeout 900 --verbose" - subprocess.run(cmd, shell=True, check=True) + cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 890 --verbose" + subprocess.run(cmd, shell=True, check=True, timeout=895) # looks up the name of the output image file based on the workflow file_prefix = [ @@ -165,16 +170,42 @@ class ComfyUI: @modal.fastapi_endpoint(method="POST") def api(self, item: Dict): from fastapi import Response + try: + # save this updated workflow to a new file + new_workflow_file = item["prompt"] + # run inference on the currently running container + fname = self.infer.local(new_workflow_file) + j = {"file_name": fname} + print("json", j) + return Response(content=json.dumps(j), media_type="application/json") + except Exception as e: + raise RuntimeError(str(e)) + finally: + print("Purge Garbage By Restart Comfy") + cmd = "comfy stop" + subprocess.run(cmd, shell=True, check=True) + cmd = "comfy launch --background" + subprocess.run(cmd, shell=True, check=True) - # save this updated workflow to a new file - new_workflow_file = item["prompt"] - # run inference on the currently running container - fname = self.infer.local(new_workflow_file) - j = {"file_name": fname} - print("json", j) + def poll_server_health(self): + import socket + import urllib - return Response(content=json.dumps(j), media_type="application/json") + try: + # dummy request to check if the server is healthy + req = urllib.request.Request("http://127.0.0.1:8188/system_stats") + urllib.request.urlopen(req, timeout=5) + print("ComfyUI server is healthy") + except (socket.timeout, urllib.error.URLError) as e: + # if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container + print(f"Server health check failed: {str(e)} restarting ComfyUI...") + try: + cmd = "comfy stop && comfy launch --background" + subprocess.run(cmd, shell=True, check=True) + except: + modal.experimental.stop_fetching_inputs() + raise Exception("ComfyUI server is not healthy, restart failed, stopping container") # @modal.web_endpoint(method="POST", label="tk") # def tk_api(self, item: Dict): diff --git a/server_with_s3_auth.py b/server_with_s3_auth.py new file mode 100644 index 0000000..b8192a6 --- /dev/null +++ b/server_with_s3_auth.py @@ -0,0 +1,223 @@ +import json +import os +import shutil +import subprocess +import sys +import traceback +import uuid +from pathlib import Path +from typing import Dict + +import modal +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + +image = ( # build up a Modal Image to run ComfyUI, step by step + modal.Image.debian_slim( # start from basic Linux with Python + python_version="3.10" + ) + .apt_install("git") # install git to clone ComfyUI + .apt_install("gcc") + .apt_install("libportaudio2") + .pip_install("fastapi[standard]==0.115.4") # install web dependencies + .pip_install("comfy-cli==1.3.5") # install comfy-cli + .pip_install("cos-python-sdk-v5") + .pip_install("sqlalchemy") + .pip_install("ultralytics") + .pip_install("tencentcloud-sdk-python") + .pip_install("pymysql") + .pip_install("Pillow") + .pip_install("ffmpy") + .pip_install("opencv-python") + .pip_install("av") + .pip_install("imageio") + .pip_install("loguru") + .pip_install("conformer==0.3.2",extra_options="--no-dependencies") + .pip_install("einops>0.6.1",extra_options="--no-dependencies") + .pip_install("openai-whisper") + .run_commands( # use comfy-cli to install ComfyUI and its dependencies + "comfy --skip-prompt install --nvidia --version 0.3.10" + ) +) + +image = ( + image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") + .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") + .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") + .run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools") + .run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts") + .run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy") + .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git") + .run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui") + .run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui") + .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy") + .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials") + .run_commands("comfy node install https://github.com/melMass/comfy_mtb") + .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") + .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") + .run_commands( + "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model" + ).run_commands( + "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints" + ).run_commands( + "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints" + ).run_commands( + "mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai" + ).run_commands( + "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M" + ).run_commands( + "rm -rf /root/comfy/ComfyUI/models" + ).run_commands( + "apt update && apt install -y ffmpeg && ffmpeg -version" + ).add_local_file("config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True + ).add_local_file("config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True + ).workdir("/root/comfy") + + # Add .run_commands(...) calls for any other custom nodes you want to download +) + +app = modal.App(name="highlight-comfyui-s3", image=image) +vol = modal.Volume.from_name("comfyui-model", create_if_missing=True) +bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=True) +bucket_input = str(bucket_dict.get("INPUT")) +bucket_output = str(bucket_dict.get("OUTPUT")) +secret = modal.Secret.from_name("aws-s3-secret") +# completed workflows write output images to this directory +output_dir = "/root/comfy/ComfyUI/output" +auth_scheme = HTTPBearer() + + +@app.cls( + allow_concurrent_inputs=1, # allow 10 concurrent API calls + max_containers=200, + min_containers=0, + buffer_containers=0, + scaledown_window=120, + # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times + timeout=900, + gpu=["L4", "T4"], + cpu=(2,16), + # memory=(32768, 32768), # (内存预留量, 内存使用上限) + memory=(20480,40960), + retries=1, + enable_memory_snapshot=True, + secrets=[secret, modal.Secret.from_name("web_auth_token")], + volumes={ + "/root/comfy/ComfyUI/models": vol, + "/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount( + bucket_name=bucket_input, + # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"], + secret=secret, + key_prefix="/" + ), + "/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount( + bucket_name=bucket_output, + # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"], + secret=secret, + key_prefix="/" + ), + }, +) +class ComfyUI: + @modal.enter() + def launch_comfy_background(self): + # starts the ComfyUI server in the background exactly once when the first input is received + self.session_id = str(uuid.uuid4()) + cmd = "echo client_uuid: {}&&rm -rf /root/comfy/ComfyUI/user&&mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}&&ln -s /root/comfy/ComfyUI/output_s3/logs/{} /root/comfy/ComfyUI/user && comfy launch --background".format(self.session_id,self.session_id, self.session_id) + subprocess.run(cmd, shell=True, check=True) + + @modal.method() + def infer(self, workflow_json: str = ""): + self.poll_server_health() + self.prompt_uuid = str(uuid.uuid4()) + # runs the comfy run --workflow command as a subprocess + workflow = json.loads(workflow_json) + print("Workflow JSON:") + print(json.dumps(workflow, indent=4, ensure_ascii=False)) + for node in workflow.values(): + if node.get("class_type") == "ComfyUIDeployExternalText": + if node.get("inputs").get("input_id") == "file_path": + file_to_move = node.get("inputs").get("default_value") + if file_to_move: + os.makedirs(os.path.dirname(file_to_move.replace("input_s3", "input")), exist_ok=True) + shutil.copy(file_to_move, file_to_move.replace("input_s3", "input")) + node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", "input") + with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as f: + f.write(json.dumps(workflow, ensure_ascii=False)) + cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 890 --verbose" + subprocess.run(cmd, shell=True, check=True, timeout=895) + + # looks up the name of the output image file based on the workflow + file_prefix = [ + node.get("inputs") + for node in workflow.values() + if node.get("class_type") == "VHS_VideoCombine" + ][0]["filename_prefix"] + + # returns the image as bytes + file_list = os.listdir(output_dir) + # 获取按照文件时间创建排序的列表，默认是按时间升序 + new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)), + reverse=True) + # print("file_list", new_file_list) + for f in new_file_list: + if f.startswith(file_prefix): + os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), exist_ok=True) + shutil.copy(os.path.join(output_dir, f), os.path.join(output_dir.replace("output", "output_s3"), f)) + return f + + @modal.fastapi_endpoint(method="POST") + def api(self, item: Dict, token: HTTPAuthorizationCredentials = Depends(auth_scheme)): + if token.credentials != os.environ["AUTH_TOKEN"]: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + from fastapi import Response + try: + # save this updated workflow to a new file + new_workflow_file = item["prompt"] + # run inference on the currently running container + fname = self.infer.local(new_workflow_file) + j = {"file_name": fname} + print("json", j) + return Response(content=json.dumps(j), media_type="application/json") + except Exception as e: + raise RuntimeError(str(e)) + finally: + print("Purge Garbage By Restart Comfy") + cmd = "comfy stop" + subprocess.run(cmd, shell=True, check=True) + cmd = "comfy launch --background" + subprocess.run(cmd, shell=True, check=True) + + + def poll_server_health(self): + import socket + import urllib + + try: + # dummy request to check if the server is healthy + req = urllib.request.Request("http://127.0.0.1:8188/system_stats") + urllib.request.urlopen(req, timeout=5) + print("ComfyUI server is healthy") + except (socket.timeout, urllib.error.URLError) as e: + # if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container + print(f"Server health check failed: {str(e)} restarting ComfyUI...") + try: + cmd = "comfy stop && comfy launch --background" + subprocess.run(cmd, shell=True, check=True) + except: + modal.experimental.stop_fetching_inputs() + raise Exception("ComfyUI server is not healthy, restart failed, stopping container") + + # @modal.web_endpoint(method="POST", label="tk") + # def tk_api(self, item: Dict): + # pass + # +