diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/deploy.py b/src/deploy.py new file mode 100644 index 0000000..9aaaef4 --- /dev/null +++ b/src/deploy.py @@ -0,0 +1,4 @@ +import modal.cli.run + +if __name__ == '__main__': + modal.cli.run.deploy(app_ref='.\\server_cluster\\app.py', name="server-bundle-deploy", tag="initialDeploy", env="main", use_module_mode=False) \ No newline at end of file diff --git a/heygem.py b/src/heygem.py similarity index 100% rename from heygem.py rename to src/heygem.py diff --git a/src/server_cluster/ComfyUI_Auth/__init__.py b/src/server_cluster/ComfyUI_Auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/ComfyUI_Auth/web/__init__.py b/src/server_cluster/ComfyUI_Auth/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server_with_s3_auth.py b/src/server_cluster/ComfyUI_Auth/web/worker.py similarity index 78% rename from server_with_s3_auth.py rename to src/server_cluster/ComfyUI_Auth/web/worker.py index 6133a1a..30bb18d 100644 --- a/server_with_s3_auth.py +++ b/src/server_cluster/ComfyUI_Auth/web/worker.py @@ -1,17 +1,7 @@ # ComfyUI模板--Base Auth -import json -import os -import shutil -import subprocess -import uuid -from typing import Dict - -import loguru import modal -from fastapi import Depends, HTTPException, status -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -image = ( # build up a Modal Image to run ComfyUI, step by step +comfyui_auth_image = ( # build up a Modal Image to run ComfyUI, step by step modal.Image.debian_slim( # start from basic Linux with Python python_version="3.10" ) @@ -22,9 +12,11 @@ image = ( # build up a Modal Image to run ComfyUI, step by step .apt_install("software-properties-common") .apt_install("wget") .run_commands("add-apt-repository -y contrib") - .run_commands("wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb") + .run_commands( + "wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb") .apt_install("cuda-toolkit") - .add_local_file("whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True) + .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", + copy=True) .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl") # install modified-comfy-cli .pip_install("cos-python-sdk-v5") .pip_install("sqlalchemy") @@ -37,16 +29,16 @@ image = ( # build up a Modal Image to run ComfyUI, step by step .pip_install("av") .pip_install("imageio") .pip_install("loguru") - .pip_install("conformer==0.3.2",extra_options="--no-dependencies") - .pip_install("einops>0.6.1",extra_options="--no-dependencies") + .pip_install("conformer==0.3.2", extra_options="--no-dependencies") + .pip_install("einops>0.6.1", extra_options="--no-dependencies") .pip_install("openai-whisper") .run_commands( # use comfy-cli to install ComfyUI and its dependencies "comfy --skip-prompt install --nvidia --version 0.3.10" ) ) -image = ( - image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") +comfyui_auth_image = ( + comfyui_auth_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") @@ -78,36 +70,46 @@ image = ( "rm -rf /root/comfy/ComfyUI/models" ).run_commands( "apt update && apt install -y ffmpeg && ffmpeg -version" - ).add_local_file("config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True - ).add_local_file("config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True - ).workdir("/root/comfy") - - # Add .run_commands(...) calls for any other custom nodes you want to download + ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", + copy=True + ).add_local_file("../config/config.py", + "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", + copy=True + ).workdir("/root/comfy") ) -app = modal.App(name="highlight-comfyui-s3", image=image) +comfyui_auth_app = modal.App(name="ComfyUI-Auth", image=comfyui_auth_image) +comfyui_auth_app.set_description("ComfyUI Auth Server") + +with comfyui_auth_image.imports(): + import json + import os + import shutil + import subprocess + import uuid + from typing import Dict + import loguru + from fastapi import Depends, HTTPException, status + from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + vol = modal.Volume.from_name("comfyui-model", create_if_missing=True) bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False) bucket_input = str(bucket_dict.get("INPUT")) bucket_output = str(bucket_dict.get("OUTPUT")) secret = modal.Secret.from_name("aws-s3-secret") -# completed workflows write output images to this directory output_dir = "/root/comfy/ComfyUI/output" auth_scheme = HTTPBearer() -@app.cls( - allow_concurrent_inputs=1, # allow 10 concurrent API calls +@comfyui_auth_app.cls( max_containers=200, min_containers=0, buffer_containers=0, scaledown_window=120, - # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times timeout=1200, gpu=["L4", "T4"], - cpu=(2,16), - # memory=(32768, 32768), # (内存预留量, 内存使用上限) - memory=(20480,131072), + cpu=(2, 16), + memory=(20480, 131072), enable_memory_snapshot=False, secrets=[secret, modal.Secret.from_name("web_auth_token")], volumes={ @@ -126,19 +128,19 @@ auth_scheme = HTTPBearer() ), }, ) -class ComfyUI: +@modal.concurrent(max_inputs=1) +class ComfyUIAuth: @modal.enter() def launch_comfy_background(self): - # starts the ComfyUI server in the background exactly once when the first input is received self.session_id = str(uuid.uuid4()) - cmd = "echo client_uuid: {}&& mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}/ &&comfy launch --background".format(self.session_id,self.session_id, self.session_id) + cmd = "echo client_uuid: {} && comfy launch --background".format( + self.session_id, self.session_id, self.session_id) subprocess.run(cmd, shell=True, check=True) @modal.method() def infer(self, workflow_json: str = ""): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - # runs the comfy run --workflow command as a subprocess workflow = json.loads(workflow_json) print("Workflow JSON:") print(json.dumps(workflow, indent=4, ensure_ascii=False)) @@ -156,14 +158,17 @@ class ComfyUI: # S3 Fallback import boto3 import yaml - with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",encoding="utf-8",mode="r+") as config: + with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", + encoding="utf-8", mode="r+") as config: yaml_config = yaml.load(config, Loader=yaml.FullLoader) awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"], - aws_secret_access_key=yaml_config["aws_access_key"]) - awss3.meta.client.download_file(bucket_input, file_to_move.split("input_s3/")[1], file_to_move.replace("input_s3", "input")) + aws_secret_access_key=yaml_config["aws_access_key"]) + awss3.meta.client.download_file(bucket_input, file_to_move.split("input_s3/")[1], + file_to_move.replace("input_s3", "input")) except: raise Exception("Failed to download file from S3 manually") - node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", "input") + node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", + "input") with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as fi: fi.write(json.dumps(workflow, ensure_ascii=False)) cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose" @@ -176,28 +181,29 @@ class ComfyUI: subprocess.run(cmd, shell=True, check=True, timeout=1195) - # returns the image as bytes file_list = os.listdir(output_dir) # 获取按照文件时间创建排序的列表,默认是按时间升序 new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)), reverse=True) - # print("file_list", new_file_list) for f in new_file_list: if f.startswith(self.file_prefix): - os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), exist_ok=True) + os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), + exist_ok=True) try: - shutil.copy(os.path.join(output_dir, f), os.path.join(output_dir.replace("output", "output_s3"), f)) + shutil.copy(os.path.join(output_dir, f), + os.path.join(output_dir.replace("output", "output_s3"), f)) except: try: print("Try move file to S3 manually") # S3 Fallback import boto3 import yaml - with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", encoding="utf-8",mode="r+") as config: + with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", + encoding="utf-8", mode="r+") as config: yaml_config = yaml.load(config, Loader=yaml.FullLoader) awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"], - aws_secret_access_key=yaml_config["aws_access_key"]) + aws_secret_access_key=yaml_config["aws_access_key"]) awss3.meta.client.upload_file(os.path.join(output_dir, f), bucket_output, f) except: raise Exception("Failed to move file to S3 manually") @@ -218,11 +224,11 @@ class ComfyUI: fname = self.infer.local(new_workflow_file) if fname is None: raise RuntimeError("Output File not found") - j = {"status":"success", "file_name": fname} + j = {"status": "success", "file_name": fname} loguru.logger.success(j) return j except Exception as e: - j = {"status":"fail", "msg": str(e)} + j = {"status": "fail", "msg": str(e)} loguru.logger.error(j) return j finally: @@ -235,14 +241,19 @@ class ComfyUI: print("Summary Logs") try: with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f: - log_text = f"\n-----------{self.file_prefix}------------\n"+f.read()+"\n" + log_text = f"\n-----------{self.file_prefix}------------\n" + f.read() + "\n" + if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"): + os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True) if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"): - with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", encoding="utf-8") as f: + with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", + encoding="utf-8") as f: log_text = f.read() + log_text - with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "w", encoding="utf-8") as f: + with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "w", + encoding="utf-8") as f: f.write(log_text) if os.path.exists(f"/root/comfy/ComfyUI/user/ffmpeg.txt"): - shutil.copy("/root/comfy/ComfyUI/user/ffmpeg.txt", f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/ffmpeg.txt") + shutil.copy("/root/comfy/ComfyUI/user/ffmpeg.txt", + f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/ffmpeg.txt") except Exception as e: print(f"Summary Logs Failed: {e}") cmd = "comfy launch --background" @@ -277,5 +288,3 @@ class ComfyUI: except: modal.experimental.stop_fetching_inputs() raise Exception("ComfyUI server is not healthy, restart failed, stopping container") - - diff --git a/src/server_cluster/ComfyUI_Auth_HeyGem/__init__.py b/src/server_cluster/ComfyUI_Auth_HeyGem/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/ComfyUI_Auth_HeyGem/web/__init__.py b/src/server_cluster/ComfyUI_Auth_HeyGem/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server_with_s3_auth_heygem.py b/src/server_cluster/ComfyUI_Auth_HeyGem/web/worker.py similarity index 88% rename from server_with_s3_auth_heygem.py rename to src/server_cluster/ComfyUI_Auth_HeyGem/web/worker.py index a108bf2..b2f27a4 100644 --- a/server_with_s3_auth_heygem.py +++ b/src/server_cluster/ComfyUI_Auth_HeyGem/web/worker.py @@ -1,17 +1,7 @@ # ComfyUI模板--Base Auth Heygem -import json -import os -import shutil -import subprocess -import uuid -from typing import Dict - -import loguru import modal -from fastapi import Depends, HTTPException, status -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -image = ( # build up a Modal Image to run ComfyUI, step by step +comfyui_auth_heygem_image = ( # build up a Modal Image to run ComfyUI, step by step modal.Image.debian_slim( # start from basic Linux with Python python_version="3.10" ) @@ -39,8 +29,8 @@ image = ( # build up a Modal Image to run ComfyUI, step by step ) ) -image = ( - image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") +comfyui_auth_heygem_image = ( + comfyui_auth_heygem_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") @@ -54,10 +44,10 @@ image = ( .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy") .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials") .run_commands("comfy node install https://github.com/melMass/comfy_mtb") - .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") - .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") - .run_commands("echo 04111 && comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") - .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") .run_commands( "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model" ).run_commands( @@ -72,13 +62,13 @@ image = ( "rm -rf /root/comfy/ComfyUI/models" ).run_commands( "apt update && apt install -y ffmpeg && ffmpeg -version" - ).add_local_file("config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True - ).add_local_file("config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True + ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True + ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True ) # 添加Python3.8 HeyGem .run_commands("apt update && apt install -y curl build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev lzma liblzma-dev tk-dev libffi-dev") .run_commands("curl -O https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz&&tar -xf Python-3.8.12.tar.xz") .run_commands("cd Python-3.8.12 && ./configure --enable-optimizations && make -j 10 && make altinstall") - .add_local_file("whl/heygem-1.0-py3-none-any.whl","/root/comfy/heygem-1.0-py3-none-any.whl", copy=True) + .add_local_file("../whl/heygem-1.0-py3-none-any.whl","/root/comfy/heygem-1.0-py3-none-any.whl", copy=True) .shell(["/bin/bash", "-c"]) .run_commands("python3.8 -m pip install /root/comfy/heygem-1.0-py3-none-any.whl") .env({"LD_LIBRARY_PATH":"/usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib"}) @@ -86,32 +76,38 @@ image = ( .run_commands("python3.8 -m pip install https://github.com/pydata/numexpr/releases/download/v2.8.6/numexpr-2.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") .add_local_file("heygem.py", "/root/comfy/heygem.py", copy=True) .workdir("/root/comfy") - - # Add .run_commands(...) calls for any other custom nodes you want to download ) -app = modal.App(name="highlight-comfyui-s3", image=image) +comfyui_auth_heygem_app = modal.App(name="ComfyUI-Auth-HeyGem", image=comfyui_auth_heygem_image) +comfyui_auth_heygem_app.set_description("ComfyUI Auth HeyGem Server") + +with comfyui_auth_heygem_image.imports(): + import json + import os + import shutil + import subprocess + import uuid + from typing import Dict + import loguru + from fastapi import Depends, HTTPException, status + from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + vol = modal.Volume.from_name("comfyui-model", create_if_missing=True) bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False) bucket_input = str(bucket_dict.get("INPUT")) bucket_output = str(bucket_dict.get("OUTPUT")) secret = modal.Secret.from_name("aws-s3-secret") -# completed workflows write output images to this directory output_dir = "/root/comfy/ComfyUI/output" auth_scheme = HTTPBearer() - -@app.cls( - allow_concurrent_inputs=1, # allow 10 concurrent API calls +@comfyui_auth_heygem_app.cls( max_containers=200, min_containers=0, buffer_containers=0, scaledown_window=120, - # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times timeout=1200, gpu=["L4", "T4"], cpu=(2,16), - # memory=(32768, 32768), # (内存预留量, 内存使用上限) memory=(20480,81920), enable_memory_snapshot=False, secrets=[secret, modal.Secret.from_name("web_auth_token")], @@ -131,10 +127,10 @@ auth_scheme = HTTPBearer() ), }, ) -class ComfyUI: +@modal.concurrent(max_inputs=1) +class ComfyUIAuthHeyGem: @modal.enter() def launch_comfy_background(self): - # starts the ComfyUI server in the background exactly once when the first input is received self.session_id = str(uuid.uuid4()) cmd = ("echo client_uuid: {}" " && rm -rf /root/comfy/ComfyUI/user" @@ -150,7 +146,6 @@ class ComfyUI: def infer(self, workflow_json: str = ""): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - # runs the comfy run --workflow command as a subprocess workflow = json.loads(workflow_json) print("Workflow JSON:") print(json.dumps(workflow, indent=4, ensure_ascii=False)) @@ -249,8 +244,6 @@ class ComfyUI: raise Exception("Workflow does not contain VHS_VideoCombine/HeygemF2F node, cannot find output file") - - @modal.fastapi_endpoint(method="POST") def api(self, item: Dict, token: HTTPAuthorizationCredentials = Depends(auth_scheme)): if token.credentials != os.environ["AUTH_TOKEN"]: @@ -313,8 +306,3 @@ class ComfyUI: modal.experimental.stop_fetching_inputs() raise Exception("ComfyUI server is not healthy, restart failed, stopping container") - # @modal.web_endpoint(method="POST", label="tk") - # def tk_api(self, item: Dict): - # pass - # - diff --git a/src/server_cluster/ComfyUI_Auth_LatentSync1_5/__init__.py b/src/server_cluster/ComfyUI_Auth_LatentSync1_5/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/__init__.py b/src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server_with_s3_auth_latentsync1_5.py b/src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/worker.py similarity index 89% rename from server_with_s3_auth_latentsync1_5.py rename to src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/worker.py index 69bd99d..d55dd0d 100644 --- a/server_with_s3_auth_latentsync1_5.py +++ b/src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/worker.py @@ -1,19 +1,7 @@ # ComfyUI模板--Base Auth -import glob -import json -import os -import shutil -import subprocess -import time -import uuid -from typing import Dict - -import loguru import modal -from fastapi import Depends, HTTPException, status -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -image = ( # build up a Modal Image to run ComfyUI, step by step +comfyui_auth_latentsync_1_5_image = ( # build up a Modal Image to run ComfyUI, step by step modal.Image.debian_slim( # start from basic Linux with Python python_version="3.10" ) @@ -26,7 +14,7 @@ image = ( # build up a Modal Image to run ComfyUI, step by step .run_commands("add-apt-repository -y contrib") .run_commands("wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb") .apt_install("cuda-toolkit") - .add_local_file("whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True) + .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True) .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl") # install modified-comfy-cli .pip_install("cos-python-sdk-v5") .pip_install("sqlalchemy") @@ -47,8 +35,8 @@ image = ( # build up a Modal Image to run ComfyUI, step by step ) ) -image = ( - image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") +comfyui_auth_latentsync_1_5_image = ( + comfyui_auth_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") @@ -80,35 +68,43 @@ image = ( "rm -rf /root/comfy/ComfyUI/models" ).run_commands( "apt update && apt install -y ffmpeg && ffmpeg -version" - ).add_local_file("config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True - ).add_local_file("config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True + ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True + ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True ).workdir("/root/comfy") - - # Add .run_commands(...) calls for any other custom nodes you want to download ) -app = modal.App(name="highlight-comfyui-s3", image=image) +comfyui_auth_latentsync_1_5_app = modal.App(name="ComfyUI-Auth-LatentSync1_5", image=comfyui_auth_latentsync_1_5_image) +comfyui_auth_latentsync_1_5_app.set_description("ComfyUI Auth LatentSync1_5 Server") + + +with comfyui_auth_latentsync_1_5_image.imports(): + import json + import os + import shutil + import subprocess + import time + import uuid + from typing import Dict + import loguru + from fastapi import Depends, HTTPException, status + from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + vol = modal.Volume.from_name("comfyui-model", create_if_missing=True) bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False) bucket_input = str(bucket_dict.get("INPUT")) bucket_output = str(bucket_dict.get("OUTPUT")) secret = modal.Secret.from_name("aws-s3-secret") -# completed workflows write output images to this directory output_dir = "/root/comfy/ComfyUI/output" auth_scheme = HTTPBearer() - -@app.cls( - allow_concurrent_inputs=1, # allow 10 concurrent API calls +@comfyui_auth_latentsync_1_5_app.cls( max_containers=200, min_containers=0, buffer_containers=0, scaledown_window=120, - # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times timeout=900, gpu=["L4"], cpu=(2,16), - # memory=(32768, 32768), # (内存预留量, 内存使用上限) memory=(32768,131072), enable_memory_snapshot=False, secrets=[secret, modal.Secret.from_name("web_auth_token")], @@ -128,19 +124,18 @@ auth_scheme = HTTPBearer() ), }, ) -class ComfyUI: +@modal.concurrent(max_inputs=1) +class ComfyUIAuthLatentSync15: @modal.enter() def launch_comfy_background(self): - # starts the ComfyUI server in the background exactly once when the first input is received self.session_id = str(uuid.uuid4()) - cmd = "echo client_uuid: {}&& mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}/ &&comfy launch --background".format(self.session_id,self.session_id, self.session_id) + cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id) subprocess.run(cmd, shell=True, check=True) @modal.method() def infer(self, workflow_json: str = ""): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - # runs the comfy run --workflow command as a subprocess workflow = json.loads(workflow_json) print("Workflow JSON:") print(json.dumps(workflow, indent=4, ensure_ascii=False)) @@ -239,6 +234,8 @@ class ComfyUI: try: with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f: log_text = f"\n-----------{self.file_prefix}------------\n"+f.read()+"\n" + if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"): + os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True) if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"): with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", encoding="utf-8") as f: log_text = f.read() + log_text @@ -280,5 +277,3 @@ class ComfyUI: except: modal.experimental.stop_fetching_inputs() raise Exception("ComfyUI server is not healthy, restart failed, stopping container") - - diff --git a/src/server_cluster/ComfyUI_Base/__init__.py b/src/server_cluster/ComfyUI_Base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/ComfyUI_Base/web/__init__.py b/src/server_cluster/ComfyUI_Base/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server_with_s3.py b/src/server_cluster/ComfyUI_Base/web/worker.py similarity index 89% rename from server_with_s3.py rename to src/server_cluster/ComfyUI_Base/web/worker.py index 704e799..aaabeb1 100644 --- a/server_with_s3.py +++ b/src/server_cluster/ComfyUI_Base/web/worker.py @@ -1,15 +1,7 @@ # ComfyUI模板--Base 停止维护!! -import json -import os -import shutil -import subprocess -import uuid -from typing import Dict - -import loguru import modal -image = ( # build up a Modal Image to run ComfyUI, step by step +comfyui_base_image = ( # build up a Modal Image to run ComfyUI, step by step modal.Image.debian_slim( # start from basic Linux with Python python_version="3.10" ) @@ -17,12 +9,12 @@ image = ( # build up a Modal Image to run ComfyUI, step by step .apt_install("gcc") .apt_install("libportaudio2") .pip_install("fastapi[standard]==0.115.4") # install web dependencies - .apt_install("software-properties-common", force_build=True) + .apt_install("software-properties-common") .apt_install("wget") .run_commands("add-apt-repository -y contrib") .run_commands("wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb") .apt_install("cuda-toolkit") - .add_local_file("whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True) + .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True) .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl") # install modified-comfy-cli .pip_install("cos-python-sdk-v5") .pip_install("sqlalchemy") @@ -43,8 +35,8 @@ image = ( # build up a Modal Image to run ComfyUI, step by step ) ) -image = ( - image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") +comfyui_base_image = ( + comfyui_base_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") @@ -76,34 +68,38 @@ image = ( "rm -rf /root/comfy/ComfyUI/models" ).run_commands( "apt update && apt install -y ffmpeg && ffmpeg -version" - ).add_local_file("config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True - ).add_local_file("config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True + ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True + ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True ).workdir("/root/comfy") - - # Add .run_commands(...) calls for any other custom nodes you want to download ) -app = modal.App(name="highlight-comfyui-s3", image=image) +comfyui_base_app = modal.App(name="ComfyUI-Base", image=comfyui_base_image) +comfyui_base_app.set_description("ComfyUI Base Server") + +with comfyui_base_image.imports(): + import json + import os + import shutil + import subprocess + import uuid + from typing import Dict + import loguru + vol = modal.Volume.from_name("comfyui-model", create_if_missing=True) bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False) bucket_input = str(bucket_dict.get("INPUT")) bucket_output = str(bucket_dict.get("OUTPUT")) secret = modal.Secret.from_name("aws-s3-secret") -# completed workflows write output images to this directory output_dir = "/root/comfy/ComfyUI/output" - -@app.cls( - allow_concurrent_inputs=1, # allow 10 concurrent API calls +@comfyui_base_app.cls( max_containers=200, min_containers=0, buffer_containers=0, scaledown_window=120, - # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times timeout=1200, gpu=["L4", "T4"], cpu=(2,16), - # memory=(32768, 32768), # (内存预留量, 内存使用上限) memory=(20480,65536), enable_memory_snapshot=False, secrets=[secret], @@ -123,19 +119,18 @@ output_dir = "/root/comfy/ComfyUI/output" ), }, ) -class ComfyUI: +@modal.concurrent(max_inputs=1) +class ComfyUIBase: @modal.enter() def launch_comfy_background(self): - # starts the ComfyUI server in the background exactly once when the first input is received self.session_id = str(uuid.uuid4()) - cmd = "echo client_uuid: {}&& mkdir -p /root/comfy/ComfyUI/output_s3/logs/{} && comfy launch --background".format(self.session_id,self.session_id, self.session_id) + cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id) subprocess.run(cmd, shell=True, check=True) @modal.method() def infer(self, workflow_json: str = ""): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - # runs the comfy run --workflow command as a subprocess workflow = json.loads(workflow_json) print("Workflow JSON:") print(json.dumps(workflow, indent=4, ensure_ascii=False)) @@ -228,6 +223,8 @@ class ComfyUI: try: with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f: log_text = f"\n-----------{self.file_prefix}------------\n"+f.read()+"\n" + if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"): + os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True) if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"): with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", encoding="utf-8") as f: log_text = f.read() + log_text @@ -270,4 +267,3 @@ class ComfyUI: except: modal.experimental.stop_fetching_inputs() raise Exception("ComfyUI server is not healthy, restart failed, stopping container") - diff --git a/src/server_cluster/HeyGem_Base/__init__.py b/src/server_cluster/HeyGem_Base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/HeyGem_Base/web/__init__.py b/src/server_cluster/HeyGem_Base/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server_pure_heygem.py b/src/server_cluster/HeyGem_Base/web/worker.py similarity index 92% rename from server_pure_heygem.py rename to src/server_cluster/HeyGem_Base/web/worker.py index e2f8bef..987d587 100644 --- a/server_pure_heygem.py +++ b/src/server_cluster/HeyGem_Base/web/worker.py @@ -1,21 +1,7 @@ # HeyGem模板 -import json -import os -import shutil -import socket -import subprocess -import time -import traceback -import uuid -from typing import Any, Optional, Union - -import httpx -import loguru import modal -import requests -from fastapi import Depends, HTTPException, status, UploadFile -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -image = ( + +heygem_base_image = ( modal.Image.debian_slim( # start from basic Linux with Python python_version="3.10" ).apt_install("git") @@ -29,30 +15,51 @@ image = ( .run_commands("apt update && apt install -y curl build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev lzma liblzma-dev tk-dev libffi-dev") .run_commands("curl -O https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz&&tar -xf Python-3.8.12.tar.xz") .run_commands("cd Python-3.8.12 && ./configure --enable-optimizations && make -j 10 && make altinstall") - .add_local_file("whl/heygem-1.0-py3-none-any.whl", "/root/heygem-1.0-py3-none-any.whl", copy=True) + .add_local_file("../whl/heygem-1.0-py3-none-any.whl", "/root/heygem-1.0-py3-none-any.whl", copy=True) .shell(["/bin/bash", "-c"]) .run_commands("python3.8 -m pip install /root/heygem-1.0-py3-none-any.whl") .env({"LD_LIBRARY_PATH":"/usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib"}) .run_commands("ln -s /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so.11.2 /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so") .run_commands("python3.8 -m pip install https://github.com/pydata/numexpr/releases/download/v2.8.6/numexpr-2.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") .add_local_file("heygem.py", "/root/heygem.py", copy=True) - .add_local_file("config/config.yaml", "/root/config.yaml", copy=True) - .workdir("/root").pip_install("boto3").pip_install("PyYAML").pip_install("requests") + .add_local_file("../config/config.yaml", "/root/config.yaml", copy=True) + .pip_install("boto3").pip_install("PyYAML").pip_install("requests").workdir("/root") ) -app = modal.App(name="heygem", image=image) +heygem_base_app = modal.App(name="HeyGem-Base", image=heygem_base_image) +heygem_base_app.set_description("HeyGem Base Server") + + +with heygem_base_image.imports(): + import json + import os + import shutil + import socket + import subprocess + import time + import traceback + import uuid + from typing import Optional, Union + import httpx + import loguru + import requests + from fastapi import Depends, HTTPException, status, UploadFile + from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + secret = modal.Secret.from_name("aws-s3-secret") auth_scheme = HTTPBearer() bucket_output = "bw-heygem-output" -@app.cls( - allow_concurrent_inputs=1, # required for UI startup process which runs several API calls concurrently +@heygem_base_app.cls( max_containers=25, # limit interactive session to 1 container + min_containers=0, + buffer_containers=0, + scaledown_window=240, + timeout=2160, gpu="L40S", # good starter GPU for inference cpu=(2,32), memory=(6144, 32768), - timeout=2160, - scaledown_window=240, + enable_memory_snapshot=False, secrets=[secret, modal.Secret.from_name("web_auth_token")], volumes={ "/code/data/final": modal.CloudBucketMount( @@ -69,7 +76,8 @@ bucket_output = "bw-heygem-output" ) }, # mounts our cached models ) -class HeyGem: +@modal.concurrent(max_inputs=1) +class HeyGemBase: @modal.enter() def start(self): def check_port_in_use(port, host='127.0.0.1'): @@ -286,13 +294,7 @@ class HeyGem: awss3.meta.client.upload_file(file_path, bucket_output, file_path.split(os.path.sep)[-1]) except: return {"status": "fail", "msg": "Failed to move file to S3 manually: " + str(e)} - # resp = FileResponse(path=file_path, filename=Path(file_path).name, status_code=200, headers={ - # "Content-Type": "application/octet-stream", - # "Content-Disposition": f"attachment; filename={Path(file_path).name}", - # }) return {"status": "success", "msg":f"{file_path.split(os.path.sep)[-1]}"} except Exception as e: traceback.print_exc() return {"status": "fail", "msg": "Inference module failed: "+str(e)} - - diff --git a/src/server_cluster/__init__.py b/src/server_cluster/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server_cluster/app.py b/src/server_cluster/app.py new file mode 100644 index 0000000..3bbe6ff --- /dev/null +++ b/src/server_cluster/app.py @@ -0,0 +1,14 @@ +import modal + +from ComfyUI_Base.web.worker import comfyui_base_app +from ComfyUI_Auth.web.worker import comfyui_auth_app +from ComfyUI_Auth_HeyGem.web.worker import comfyui_auth_heygem_app +from ComfyUI_Auth_LatentSync1_5.web.worker import comfyui_auth_latentsync_1_5_app +from HeyGem_Base.web.worker import heygem_base_app + +app = modal.App(name="Server-Bundle") +app.include(comfyui_base_app) +app.include(comfyui_auth_app) +app.include(comfyui_auth_heygem_app) +app.include(comfyui_auth_latentsync_1_5_app) +app.include(heygem_base_app) \ No newline at end of file