538 lines
31 KiB
Python
538 lines
31 KiB
Python
# ComfyUI模板--Base Auth LatentSync1.5
|
||
import modal
|
||
from dotenv import dotenv_values
|
||
|
||
comfyui_latentsync_1_5_image = (
|
||
modal.Image.debian_slim(
|
||
python_version="3.10"
|
||
)
|
||
.pip_install_from_pyproject("pyproject_comfyui.toml")
|
||
.apt_install("wget", "software-properties-common")
|
||
.run_commands("add-apt-repository -y contrib "
|
||
"&& wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb")
|
||
.apt_install("git", "gcc", "libportaudio2", "cuda-toolkit", "ffmpeg")
|
||
.add_local_file("whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True)
|
||
.pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl")
|
||
.run_commands("comfy --skip-prompt install --nvidia --version 0.3.10")
|
||
.env(dotenv_values(".runtime.env"))
|
||
.workdir("/root/comfy")
|
||
)
|
||
|
||
comfyui_latentsync_1_5_image = (
|
||
comfyui_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
|
||
.run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
|
||
.run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
|
||
.run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools")
|
||
.run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts")
|
||
.run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy")
|
||
.run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git")
|
||
.run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui")
|
||
.run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui")
|
||
.run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
|
||
.run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
|
||
.run_commands("comfy node install https://github.com/melMass/comfy_mtb")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
|
||
.run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/Comfy-LatentSync-Next-Node.git")
|
||
.run_commands(
|
||
"mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
|
||
).run_commands(
|
||
"mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints"
|
||
).run_commands(
|
||
"mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints"
|
||
).run_commands(
|
||
"mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai"
|
||
).run_commands(
|
||
"mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M"
|
||
).run_commands(
|
||
"rm -rf /root/comfy/ComfyUI/models"
|
||
)
|
||
.add_local_file("config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True)
|
||
.add_local_file("config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True)
|
||
.add_local_file("src/BowongModalFunctions/template/template_v2.json", "/root/src/BowongModalFunctions/template/template_v2.json", copy=True)
|
||
.add_local_python_source('cluster')
|
||
.add_local_python_source('BowongModalFunctions')
|
||
)
|
||
|
||
app = modal.App(name="ComfyUI v2", image=comfyui_latentsync_1_5_image, include_source=False)
|
||
app.set_description("ComfyUI v2 Server")
|
||
|
||
|
||
with comfyui_latentsync_1_5_image.imports():
|
||
import json
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import time
|
||
import uuid
|
||
from typing import Dict, Tuple, Any, Optional
|
||
import loguru
|
||
import sentry_sdk
|
||
import psutil
|
||
from BowongModalFunctions.config import WorkerConfig
|
||
from modal import current_function_call_id
|
||
from sentry_sdk.integrations.loguru import LoguruIntegration
|
||
from BowongModalFunctions.models.web_model import SentryTransactionInfo
|
||
|
||
config = WorkerConfig()
|
||
vol = modal.Volume.from_name("comfyui-model", create_if_missing=True, environment_name=config.modal_environment)
|
||
secret = modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment)
|
||
output_dir = "/root/comfy/ComfyUI/output"
|
||
|
||
sentry_sdk.init(
|
||
dsn="https://7e3e37e770420e289b3972e1b1b9a794@sentry.bowongai.com/11",
|
||
send_default_pii=True,
|
||
attach_stacktrace=True,
|
||
traces_sample_rate=1.0, # 采样率设为1.0表示捕获所有事务
|
||
profiles_sample_rate=1.0,
|
||
add_full_stack=True,
|
||
shutdown_timeout=2,
|
||
integrations=[LoguruIntegration()],
|
||
environment=config.modal_environment,
|
||
)
|
||
|
||
@sentry_sdk.trace
|
||
def capture_hardware_info() -> Tuple[Any, int, Any]:
|
||
cpu_freq = psutil.cpu_freq()
|
||
cpu_count = psutil.cpu_count()
|
||
mem = psutil.virtual_memory()
|
||
|
||
return (cpu_freq, cpu_count, mem)
|
||
|
||
@app.cls(
|
||
max_containers=200,
|
||
min_containers=0,
|
||
buffer_containers=0,
|
||
scaledown_window=120,
|
||
timeout=1200,
|
||
gpu=["L4"],
|
||
cpu=(2,64),
|
||
memory=(20480,131072),
|
||
enable_memory_snapshot=False,
|
||
secrets=[secret],
|
||
volumes={
|
||
"/root/comfy/ComfyUI/models": vol,
|
||
"/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount(
|
||
bucket_name=config.S3_bucket_name,
|
||
secret=secret,
|
||
key_prefix=config.comfyui_s3_input+"/"
|
||
),
|
||
"/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount(
|
||
bucket_name=config.S3_bucket_name,
|
||
secret=secret,
|
||
key_prefix=config.comfyui_s3_output+"/"
|
||
),
|
||
},
|
||
)
|
||
@modal.concurrent(max_inputs=1)
|
||
class ComfyUILatentSync15:
|
||
@modal.enter()
|
||
def launch_comfy_background(self):
|
||
self.session_id = str(uuid.uuid4())
|
||
self.file_prefix = None
|
||
cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id)
|
||
subprocess.run(cmd, shell=True, check=True)
|
||
|
||
@modal.method()
|
||
def infer(self, workflow: Dict):
|
||
self.poll_server_health()
|
||
self.prompt_uuid = str(uuid.uuid4())
|
||
print("Workflow JSON:")
|
||
print(json.dumps(workflow, indent=4, ensure_ascii=False))
|
||
for node in workflow.values():
|
||
if node.get("class_type") == "ComfyUIDeployExternalText":
|
||
if node.get("inputs").get("input_id") == "file_path":
|
||
file_to_move = node.get("inputs").get("default_value")
|
||
if file_to_move:
|
||
os.makedirs(os.path.dirname(file_to_move.replace("input_s3", "input")), exist_ok=True)
|
||
try:
|
||
shutil.copy(file_to_move, file_to_move.replace("input_s3", "input"))
|
||
except:
|
||
try:
|
||
print("Try download file to S3 manually")
|
||
# S3 Fallback
|
||
import boto3
|
||
import yaml
|
||
with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",encoding="utf-8",mode="r+") as myconfig:
|
||
yaml_config = yaml.load(myconfig, Loader=yaml.FullLoader)
|
||
awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
|
||
aws_secret_access_key=yaml_config["aws_access_key"])
|
||
awss3.meta.client.download_file(config.S3_bucket_name, config.comfyui_s3_input + "/" +
|
||
file_to_move.split("input_s3/")[1],
|
||
file_to_move.replace("input_s3", "input"))
|
||
except:
|
||
raise Exception("Failed to download file from S3 manually")
|
||
node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", "input")
|
||
with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as fi:
|
||
fi.write(json.dumps(workflow, ensure_ascii=False))
|
||
cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose"
|
||
|
||
self.file_prefix = [
|
||
node.get("inputs")
|
||
for node in workflow.values()
|
||
if node.get("class_type") == "VHS_VideoCombine"
|
||
][0]["filename_prefix"]
|
||
|
||
subprocess.run(cmd, shell=True, check=True, timeout=1195)
|
||
|
||
file_list = os.listdir(output_dir)
|
||
# 获取按照文件时间创建排序的列表，默认是按时间升序
|
||
new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)),
|
||
reverse=True)
|
||
for f in new_file_list:
|
||
if f.startswith(self.file_prefix):
|
||
os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), exist_ok=True)
|
||
try:
|
||
shutil.copy(os.path.join(output_dir, f), os.path.join(output_dir.replace("output", "output_s3"), f))
|
||
except:
|
||
try:
|
||
print("Try move file to S3 manually")
|
||
# S3 Fallback
|
||
import boto3
|
||
import yaml
|
||
with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", encoding="utf-8",mode="r+") as myconfig:
|
||
yaml_config = yaml.load(myconfig, Loader=yaml.FullLoader)
|
||
awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
|
||
aws_secret_access_key=yaml_config["aws_access_key"])
|
||
awss3.meta.client.upload_file(os.path.join(output_dir, f), config.S3_bucket_name, config.comfyui_s3_output+"/"+f)
|
||
except:
|
||
raise Exception("Failed to move file to S3 manually")
|
||
return f
|
||
|
||
|
||
@modal.method()
|
||
def api(self, video_path: str, start_time: str = "00:00:03.600", filename_prefix=str(uuid.uuid4()),
|
||
tts_text1: str = "测试",
|
||
tts_text2: str = "", tts_text3: str = "", tts_text4: str = "", anchor_id: str = "dawan",
|
||
speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None):
|
||
# 导入必要模块
|
||
import datetime
|
||
import tempfile
|
||
import traceback
|
||
|
||
# 生成当前函数调用的唯一ID
|
||
function_call_id = current_function_call_id()
|
||
print(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}")
|
||
|
||
# 使用临时文件
|
||
temp_log_path = None
|
||
error_reported = False # 添加标志跟踪是否已报告错误
|
||
|
||
# 创建主Sentry事务
|
||
if sentry_trace and sentry_trace.x_trace_id and sentry_trace.x_baggage:
|
||
transaction = sentry_sdk.continue_trace(environ_or_headers={"sentry-trace": sentry_trace.x_trace_id,
|
||
"baggage": sentry_trace.x_baggage, })
|
||
else:
|
||
transaction = sentry_sdk.start_transaction(op="video.processing", name=f"API调用: {video_path}")
|
||
sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id, x_baggage=transaction.containing_transaction.get_baggage().serialize())
|
||
with transaction:
|
||
# 设置全局Sentry上下文
|
||
with sentry_sdk.configure_scope() as scope:
|
||
# 添加更丰富的上下文数据
|
||
scope.set_tag("session_id", self.session_id)
|
||
scope.set_tag("function_call_id", function_call_id)
|
||
scope.set_tag("video_path", video_path)
|
||
scope.set_context("api_parameters", {
|
||
"current_function_call_id": function_call_id,
|
||
"session_id": self.session_id,
|
||
"video_path": video_path,
|
||
"start_time": start_time,
|
||
"filename_prefix": filename_prefix,
|
||
"tts_text1": tts_text1,
|
||
"tts_text2": tts_text2,
|
||
"tts_text3": tts_text3,
|
||
"tts_text4": tts_text4,
|
||
"anchor_id": anchor_id,
|
||
"speed": speed
|
||
})
|
||
|
||
try:
|
||
# 使用Span跟踪日志文件创建
|
||
with sentry_sdk.start_span(op="db.file", description="创建日志文件") as span:
|
||
# 创建临时日志文件
|
||
temp_log_file = tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt')
|
||
temp_log_path = temp_log_file.name
|
||
print(f"创建临时日志文件: {temp_log_path}")
|
||
|
||
span.set_data("log_path", temp_log_path)
|
||
span.set_tag("file_operation", "create")
|
||
|
||
# 写入初始日志内容
|
||
temp_log_file.write(f"任务开始时间: {datetime.datetime.now()}\n")
|
||
temp_log_file.write(f"会话ID: {self.session_id}\n")
|
||
temp_log_file.write(f"调用ID: {function_call_id}\n")
|
||
temp_log_file.write(f"文件前缀: {filename_prefix}\n")
|
||
temp_log_file.write(f"参数: video_path={video_path}, start_time={start_time}\n")
|
||
temp_log_file.flush()
|
||
temp_log_file.close()
|
||
|
||
# 使用Span跟踪模板处理
|
||
with sentry_sdk.start_span(op="template.json", description="处理工作流模板") as span:
|
||
# 正常业务逻辑
|
||
with open("/root/src/BowongModalFunctions/template/template_v2.json", "r",
|
||
encoding="utf-8") as template:
|
||
workflow_file = json.loads(template.read())
|
||
|
||
# 使用子span跟踪模板修改
|
||
with sentry_sdk.start_span(op="template.modify", description="修改模板参数") as modify_span:
|
||
modify_span.set_data("original_keys", len(workflow_file))
|
||
|
||
workflow_file["211"]["inputs"]["default_value"] = tts_text1
|
||
workflow_file["399"]["inputs"]["default_value"] = tts_text2
|
||
workflow_file["331"]["inputs"]["default_value"] = tts_text3
|
||
workflow_file["406"]["inputs"]["default_value"] = tts_text4
|
||
workflow_file["424"]["inputs"][
|
||
"default_value"] = "/root/comfy/ComfyUI/input_s3/" + video_path
|
||
workflow_file["225"]["inputs"]["filename_prefix"] = filename_prefix
|
||
workflow_file["330"]["inputs"]["default_value"] = start_time
|
||
workflow_file["515"]["inputs"]["anchor_id"] = anchor_id
|
||
workflow_file["515"]["inputs"]["speed"] = speed
|
||
|
||
modify_span.set_data("modified_nodes",
|
||
["211", "399", "331", "406", "424", "225", "330", "515"])
|
||
|
||
span.set_data("workflow_size", len(workflow_file))
|
||
span.set_tag("workflow_type", "video_processing")
|
||
|
||
try:
|
||
# 记录执行的关键步骤作为breadcrumbs
|
||
sentry_sdk.add_breadcrumb(
|
||
category="process",
|
||
message="开始调用infer方法",
|
||
level="info"
|
||
)
|
||
|
||
# 使用Span跟踪推理过程
|
||
with sentry_sdk.start_span(op="ai.inference", description="执行模型推理") as infer_span:
|
||
infer_span.set_data("workflow_size", len(workflow_file))
|
||
infer_start_time = datetime.datetime.now()
|
||
|
||
# 创建预处理子span
|
||
with sentry_sdk.start_span(op="inference.prepare", description="准备推理环境") as prep_span:
|
||
prep_span.set_data("timestamp", datetime.datetime.now().isoformat())
|
||
prep_span.set_tag("model_type", "video_generation")
|
||
|
||
# 调用推理方法
|
||
fname = self.infer.local(workflow_file)
|
||
|
||
# 创建后处理子span
|
||
with sentry_sdk.start_span(op="inference.postprocess",
|
||
description="处理推理结果") as post_span:
|
||
post_span.set_data("output_file", fname)
|
||
post_span.set_data("timestamp", datetime.datetime.now().isoformat())
|
||
|
||
infer_end_time = datetime.datetime.now()
|
||
duration_seconds = (infer_end_time - infer_start_time).total_seconds()
|
||
|
||
infer_span.set_data("start_time", infer_start_time.isoformat())
|
||
infer_span.set_data("end_time", infer_end_time.isoformat())
|
||
infer_span.set_data("duration_seconds", duration_seconds)
|
||
infer_span.set_data("output_file", fname)
|
||
infer_span.set_tag("success", "true" if fname else "false")
|
||
|
||
sentry_sdk.add_breadcrumb(
|
||
category="process",
|
||
message="infer方法调用完成",
|
||
level="info"
|
||
)
|
||
|
||
if fname is None:
|
||
with sentry_sdk.start_span(op="error.handling", description="处理文件缺失错误"):
|
||
raise RuntimeError("Output File not found")
|
||
|
||
# 使用Span跟踪返回结果
|
||
with sentry_sdk.start_span(op="result.preparation", description="准备返回结果"):
|
||
j = {"status": "success", "file_name": fname}
|
||
loguru.logger.success(j)
|
||
|
||
# 设置事务状态为成功
|
||
transaction.set_status("ok")
|
||
|
||
return j, sentry_trace
|
||
|
||
except Exception as infer_error:
|
||
# 使用原始异常对象
|
||
sentry_sdk.add_breadcrumb(
|
||
category="error",
|
||
message=f"infer方法调用失败: {str(infer_error)}",
|
||
level="error"
|
||
)
|
||
with sentry_sdk.start_span(op="error.handling", description="处理推理错误") as error_span:
|
||
error_span.set_data("error_type", type(infer_error).__name__)
|
||
error_span.set_data("error_message", str(infer_error))
|
||
error_span.set_tag("error", "true")
|
||
raise infer_error
|
||
|
||
except Exception as e:
|
||
# 异常处理 - 使用捕获的异常对象e
|
||
j = {"status": "fail", "msg": str(e)}
|
||
|
||
try:
|
||
# 使用Span跟踪错误记录过程
|
||
with sentry_sdk.start_span(op="error.logging", description="记录错误详情") as error_span:
|
||
# 更新日志文件
|
||
with open(temp_log_path, "a", encoding="utf-8") as f:
|
||
# 记录异常详情
|
||
f.write(f"\n====== 异常详情 ======\n")
|
||
f.write(f"异常类型: {type(e).__name__}\n")
|
||
f.write(f"异常信息: {str(e)}\n")
|
||
f.write(f"发生时间: {datetime.datetime.now()}\n")
|
||
f.write(f"会话ID: {self.session_id}\n")
|
||
f.write(f"调用ID: {function_call_id}\n")
|
||
|
||
# 记录完整堆栈
|
||
f.write("\n====== 堆栈跟踪 ======\n")
|
||
f.write(traceback.format_exc())
|
||
|
||
# 尝试添加ComfyUI日志
|
||
with sentry_sdk.start_span(op="db.file.read",
|
||
description="读取ComfyUI日志") as log_span:
|
||
if os.path.exists("/root/comfy/ComfyUI/user/comfyui.log"):
|
||
try:
|
||
with open("/root/comfy/ComfyUI/user/comfyui.log", "r",
|
||
encoding="utf-8") as cf:
|
||
f.write("\n====== ComfyUI日志 ======\n")
|
||
f.write(cf.read())
|
||
log_span.set_tag("success", "true")
|
||
except Exception as log_read_error:
|
||
f.write(f"\n无法读取ComfyUI日志: {str(log_read_error)}\n")
|
||
log_span.set_data("error", str(log_read_error))
|
||
log_span.set_tag("success", "false")
|
||
else:
|
||
log_span.set_tag("file_exists", "false")
|
||
|
||
error_span.set_data("log_path", temp_log_path)
|
||
error_span.set_tag("error_type", type(e).__name__)
|
||
|
||
# 仅当尚未报告错误时才添加附件和报告异常
|
||
if not error_reported:
|
||
with sentry_sdk.configure_scope() as scope:
|
||
with sentry_sdk.start_span(op="error.report",
|
||
description="向Sentry报告错误") as report_span:
|
||
scope.add_attachment(
|
||
path=temp_log_path,
|
||
filename=f"full_{function_call_id}.txt",
|
||
add_to_transactions=True
|
||
)
|
||
|
||
# 添加更多异常上下文
|
||
scope.set_context("error_details", {
|
||
"error_type": type(e).__name__,
|
||
"error_message": str(e),
|
||
"time_of_error": datetime.datetime.now().isoformat()
|
||
})
|
||
|
||
# 设置事务状态为错误
|
||
transaction.set_status("internal_error")
|
||
|
||
# 使用异常对象e
|
||
sentry_sdk.capture_exception(e)
|
||
error_reported = True
|
||
print("已向Sentry报告异常")
|
||
report_span.set_tag("reported", "true")
|
||
|
||
except Exception as log_error:
|
||
print(f"处理日志出错: {log_error}")
|
||
# 设置事务状态为错误
|
||
transaction.set_status("internal_error")
|
||
|
||
# 仅当尚未报告错误时才报告原始异常
|
||
if not error_reported:
|
||
sentry_sdk.capture_exception(e)
|
||
print("已向Sentry报告原始异常（日志处理失败）")
|
||
|
||
return j, sentry_trace
|
||
|
||
finally:
|
||
# 使用Span跟踪清理过程
|
||
with sentry_sdk.start_span(op="system.cleanup", description="清理资源") as cleanup_span:
|
||
# 清理临时文件和重启ComfyUI
|
||
try:
|
||
# 尝试复制日志到S3
|
||
if temp_log_path and os.path.exists(temp_log_path):
|
||
with sentry_sdk.start_span(op="db.file.copy", description="复制日志到S3") as copy_span:
|
||
log_dir = f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"
|
||
try:
|
||
os.makedirs(log_dir, exist_ok=True)
|
||
shutil.copy(temp_log_path, f"{log_dir}/full.txt")
|
||
print(f"已复制日志到: {log_dir}/full.txt")
|
||
copy_span.set_data("source", temp_log_path)
|
||
copy_span.set_data("destination", f"{log_dir}/full.txt")
|
||
copy_span.set_tag("success", "true")
|
||
except Exception as copy_error:
|
||
print(f"复制日志到S3失败: {copy_error}")
|
||
copy_span.set_data("error", str(copy_error))
|
||
copy_span.set_tag("success", "false")
|
||
|
||
# 清理临时文件
|
||
with sentry_sdk.start_span(op="db.file.delete", description="删除临时文件") as del_span:
|
||
try:
|
||
os.unlink(temp_log_path)
|
||
print(f"已清理临时文件: {temp_log_path}")
|
||
del_span.set_data("file", temp_log_path)
|
||
del_span.set_tag("success", "true")
|
||
except Exception as unlink_error:
|
||
print(f"清理临时文件失败: {unlink_error}")
|
||
del_span.set_data("error", str(unlink_error))
|
||
del_span.set_tag("success", "false")
|
||
except Exception as cleanup_error:
|
||
print(f"清理过程出错: {cleanup_error}")
|
||
cleanup_span.set_data("error", str(cleanup_error))
|
||
cleanup_span.set_tag("success", "false")
|
||
|
||
print("清理资源并重启ComfyUI")
|
||
with sentry_sdk.start_span(op="system.restart", description="重启ComfyUI") as restart_span:
|
||
try:
|
||
# 停止ComfyUI
|
||
with sentry_sdk.start_span(op="system.stop", description="停止ComfyUI") as stop_span:
|
||
cmd = "comfy stop"
|
||
subprocess.run(cmd, shell=True, check=True)
|
||
time.sleep(1)
|
||
stop_span.set_tag("success", "true")
|
||
except Exception as stop_error:
|
||
print(f"停止ComfyUI失败: {stop_error}")
|
||
restart_span.set_data("stop_error", str(stop_error))
|
||
restart_span.set_tag("stop_success", "false")
|
||
|
||
try:
|
||
# 重启ComfyUI
|
||
with sentry_sdk.start_span(op="system.start", description="启动ComfyUI") as start_span:
|
||
cmd = "comfy launch --background"
|
||
subprocess.run(cmd, shell=True, check=True)
|
||
start_span.set_tag("success", "true")
|
||
except Exception as start_error:
|
||
print(f"启动ComfyUI失败: {start_error}")
|
||
restart_span.set_data("start_error", str(start_error))
|
||
restart_span.set_tag("start_success", "false")
|
||
modal.experimental.stop_fetching_inputs()
|
||
|
||
|
||
def poll_server_health(self):
|
||
import socket
|
||
import urllib
|
||
|
||
try:
|
||
# dummy request to check if the server is healthy
|
||
req = urllib.request.Request("http://127.0.0.1:8188/system_stats")
|
||
urllib.request.urlopen(req, timeout=5)
|
||
print("ComfyUI server is healthy")
|
||
except (socket.timeout, urllib.error.URLError) as e:
|
||
# if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container
|
||
print(f"Server health check failed: {str(e)} restarting ComfyUI...")
|
||
try:
|
||
cmd = "comfy stop"
|
||
try:
|
||
subprocess.run(cmd, shell=True, check=True)
|
||
except:
|
||
pass
|
||
cmd = "comfy launch --background"
|
||
try:
|
||
subprocess.run(cmd, shell=True, check=True)
|
||
except:
|
||
raise Exception("Failed to launch ComfyUI")
|
||
except:
|
||
modal.experimental.stop_fetching_inputs()
|
||
raise Exception("ComfyUI server is not healthy, restart failed, stopping container") |