From b8fd86a677a412598301e5b0c9ee165c0b417f86 Mon Sep 17 00:00:00 2001 From: gexianmeng Date: Mon, 19 May 2025 15:04:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9sentry=5Fsdk=E7=9A=84?= =?UTF-8?q?=E5=BC=83=E7=94=A8=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cluster/comfyui_v1.py | 154 ++++++++++------------ src/cluster/comfyui_v2.py | 270 ++++++++++++++++---------------------- 2 files changed, 186 insertions(+), 238 deletions(-) diff --git a/src/cluster/comfyui_v1.py b/src/cluster/comfyui_v1.py index 469db92..d5952a5 100644 --- a/src/cluster/comfyui_v1.py +++ b/src/cluster/comfyui_v1.py @@ -33,7 +33,7 @@ comfyui_image = ( .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials") .run_commands("comfy node install https://github.com/melMass/comfy_mtb") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") + .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git", force_build=True) .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git") .run_commands( @@ -149,19 +149,26 @@ with comfyui_image.imports(): @modal.method() async def api(self, video_path: str, start_time: str = "00:00:03.600", filename_prefix=str(uuid.uuid4()), - tts_text1: str = "测试", - tts_text2: str = "", tts_text3: str = "", tts_text4: str = "", anchor_id: str = "dawan", - speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None, webhook:WebhookNotify = None): + tts_text1: str = "测试", tts_text2: str = "", tts_text3: str = "", tts_text4: str = "", + anchor_id: str = "dawan", speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None, + webhook: WebhookNotify = None): + + # 导入必要模块 + import datetime + import tempfile + import traceback + import sys + from loguru import logger # 生成当前函数调用的唯一ID function_call_id = current_function_call_id() - print(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}") + logger.info(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}") @SentryUtils.webhook_handler(webhook=webhook, func_id=function_call_id) async def infer(workflow: Dict): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - loguru.logger.info(f"Workflow JSON: \n{json.dumps(workflow, indent=4, ensure_ascii=False)}") + logger.info(f"Workflow JSON: \n{json.dumps(workflow, indent=4, ensure_ascii=False)}") for node in workflow.values(): if node.get("class_type") == "ComfyUIDeployExternalText": if node.get("inputs").get("input_id") == "file_path": @@ -172,7 +179,7 @@ with comfyui_image.imports(): shutil.copy(file_to_move, file_to_move.replace("input_s3", "input")) except: try: - print("Try download file from S3 manually") + logger.warning("尝试手动从S3下载文件") # S3 Fallback import boto3 import yaml @@ -196,15 +203,15 @@ with comfyui_image.imports(): fi.write(json.dumps(workflow, ensure_ascii=False)) cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose" - self.file_prefix = [ - node.get("inputs") - for node in workflow.values() - if node.get("class_type") == "VHS_VideoCombine" - ][0]["filename_prefix"] + self.file_prefix = \ + [node.get("inputs") for node in workflow.values() if node.get("class_type") == "VHS_VideoCombine"][ + 0][ + "filename_prefix"] subprocess.run(cmd, shell=True, check=True, timeout=1195) - # 获取按照文件时间创建排序的列表，默认是按时间升序 + file_list = os.listdir(output_dir) + # 获取按照文件时间创建排序的列表，默认是按时间升序 new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)), reverse=True) for f in new_file_list: @@ -216,7 +223,7 @@ with comfyui_image.imports(): os.path.join(output_dir.replace("output", "output_s3"), f)) except: try: - print("Try move file to S3 manually") + logger.warning("尝试手动将文件移动到S3") # S3 Fallback import boto3 import yaml @@ -240,31 +247,23 @@ with comfyui_image.imports(): # 创建主Sentry事务 if sentry_trace and sentry_trace.x_trace_id and sentry_trace.x_baggage: - transaction = sentry_sdk.continue_trace(environ_or_headers={"sentry-trace": sentry_trace.x_trace_id, - "baggage": sentry_trace.x_baggage, }) + transaction = sentry_sdk.continue_trace( + environ_or_headers={"sentry-trace": sentry_trace.x_trace_id, "baggage": sentry_trace.x_baggage, }) else: transaction = sentry_sdk.start_transaction(op="video.processing", name=f"API调用: {video_path}") - sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id, x_baggage=transaction.containing_transaction.get_baggage().serialize()) + sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id, + x_baggage=transaction.containing_transaction.get_baggage().serialize()) with transaction: # 设置全局Sentry上下文 - with sentry_sdk.configure_scope() as scope: - # 添加更丰富的上下文数据 - scope.set_tag("session_id", self.session_id) - scope.set_tag("function_call_id", function_call_id) - scope.set_tag("video_path", video_path) - scope.set_context("api_parameters", { - "current_function_call_id": function_call_id, - "session_id": self.session_id, - "video_path": video_path, - "start_time": start_time, - "filename_prefix": filename_prefix, - "tts_text1": tts_text1, - "tts_text2": tts_text2, - "tts_text3": tts_text3, - "tts_text4": tts_text4, - "anchor_id": anchor_id, - "speed": speed - }) + sentry_sdk.set_tag("session_id", self.session_id) + sentry_sdk.set_tag("function_call_id", function_call_id) + sentry_sdk.set_tag("video_path", video_path) + sentry_sdk.set_context("api_parameters", + {"current_function_call_id": function_call_id, "session_id": self.session_id, + "video_path": video_path, "start_time": start_time, + "filename_prefix": filename_prefix, "tts_text1": tts_text1, + "tts_text2": tts_text2, "tts_text3": tts_text3, "tts_text4": tts_text4, + "anchor_id": anchor_id, "speed": speed}) try: # 使用Span跟踪日志文件创建 @@ -272,7 +271,7 @@ with comfyui_image.imports(): # 创建临时日志文件 temp_log_file = tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt') temp_log_path = temp_log_file.name - print(f"创建临时日志文件: {temp_log_path}") + logger.debug(f"创建临时日志文件: {temp_log_path}") span.set_data("log_path", temp_log_path) span.set_tag("file_operation", "create") @@ -316,11 +315,7 @@ with comfyui_image.imports(): try: # 记录执行的关键步骤作为breadcrumbs - sentry_sdk.add_breadcrumb( - category="process", - message="开始调用infer方法", - level="info" - ) + sentry_sdk.add_breadcrumb(category="process", message="开始调用infer方法", level="info") # 使用Span跟踪推理过程 with sentry_sdk.start_span(op="ai.inference", description="执行模型推理") as infer_span: @@ -350,11 +345,7 @@ with comfyui_image.imports(): infer_span.set_data("output_file", fname) infer_span.set_tag("success", "true" if fname else "false") - sentry_sdk.add_breadcrumb( - category="process", - message="infer方法调用完成", - level="info" - ) + sentry_sdk.add_breadcrumb(category="process", message="infer方法调用完成", level="info") if fname is None: with sentry_sdk.start_span(op="error.handling", description="处理文件缺失错误"): @@ -363,7 +354,7 @@ with comfyui_image.imports(): # 使用Span跟踪返回结果 with sentry_sdk.start_span(op="result.preparation", description="准备返回结果"): j = {"status": "success", "file_name": fname} - loguru.logger.success(j) + logger.success(f"处理成功: {j}") # 设置事务状态为成功 transaction.set_status("ok") @@ -372,20 +363,19 @@ with comfyui_image.imports(): except Exception as infer_error: # 使用原始异常对象 - sentry_sdk.add_breadcrumb( - category="error", - message=f"infer方法调用失败: {str(infer_error)}", - level="error" - ) + sentry_sdk.add_breadcrumb(category="error", message=f"infer方法调用失败: {str(infer_error)}", + level="error") with sentry_sdk.start_span(op="error.handling", description="处理推理错误") as error_span: error_span.set_data("error_type", type(infer_error).__name__) error_span.set_data("error_message", str(infer_error)) error_span.set_tag("error", "true") + logger.error(f"推理失败: {str(infer_error)}") raise infer_error except Exception as e: # 异常处理 - 使用捕获的异常对象e j = {"status": "fail", "msg": str(e)} + logger.error(f"处理异常: {str(e)}") try: # 使用Span跟踪错误记录过程 @@ -418,48 +408,46 @@ with comfyui_image.imports(): f.write(f"\n无法读取ComfyUI日志: {str(log_read_error)}\n") log_span.set_data("error", str(log_read_error)) log_span.set_tag("success", "false") + logger.warning(f"无法读取ComfyUI日志: {str(log_read_error)}") else: log_span.set_tag("file_exists", "false") + logger.warning("ComfyUI日志文件不存在") error_span.set_data("log_path", temp_log_path) error_span.set_tag("error_type", type(e).__name__) # 仅当尚未报告错误时才添加附件和报告异常 if not error_reported: - with sentry_sdk.configure_scope() as scope: - with sentry_sdk.start_span(op="error.report", - description="向Sentry报告错误") as report_span: - scope.add_attachment( - path=temp_log_path, - filename=f"full_{function_call_id}.txt", - add_to_transactions=True - ) + with sentry_sdk.start_span(op="error.report", + description="向Sentry报告错误") as report_span: + # 使用新的API替换configure_scope + # 添加附件 + sentry_sdk.add_attachment(path=temp_log_path, filename=f"full_{function_call_id}.txt", + add_to_transactions=True) - # 添加更多异常上下文 - scope.set_context("error_details", { - "error_type": type(e).__name__, - "error_message": str(e), - "time_of_error": datetime.datetime.now().isoformat() - }) + # 添加更多异常上下文 + sentry_sdk.set_context("error_details", + {"error_type": type(e).__name__, "error_message": str(e), + "time_of_error": datetime.datetime.now().isoformat()}) - # 设置事务状态为错误 - transaction.set_status("internal_error") + # 设置事务状态为错误 + transaction.set_status("internal_error") - # 使用异常对象e - sentry_sdk.capture_exception(e) - error_reported = True - print("已向Sentry报告异常") - report_span.set_tag("reported", "true") + # 使用异常对象e + sentry_sdk.capture_exception(e) + error_reported = True + logger.info("已向Sentry报告异常") + report_span.set_tag("reported", "true") except Exception as log_error: - print(f"处理日志出错: {log_error}") + logger.error(f"处理日志出错: {log_error}") # 设置事务状态为错误 transaction.set_status("internal_error") # 仅当尚未报告错误时才报告原始异常 if not error_reported: sentry_sdk.capture_exception(e) - print("已向Sentry报告原始异常（日志处理失败）") + logger.info("已向Sentry报告原始异常（日志处理失败）") return j, sentry_trace @@ -475,12 +463,12 @@ with comfyui_image.imports(): try: os.makedirs(log_dir, exist_ok=True) shutil.copy(temp_log_path, f"{log_dir}/full.txt") - print(f"已复制日志到: {log_dir}/full.txt") + logger.info(f"已复制日志到: {log_dir}/full.txt") copy_span.set_data("source", temp_log_path) copy_span.set_data("destination", f"{log_dir}/full.txt") copy_span.set_tag("success", "true") except Exception as copy_error: - print(f"复制日志到S3失败: {copy_error}") + logger.error(f"复制日志到S3失败: {copy_error}") copy_span.set_data("error", str(copy_error)) copy_span.set_tag("success", "false") @@ -488,19 +476,19 @@ with comfyui_image.imports(): with sentry_sdk.start_span(op="db.file.delete", description="删除临时文件") as del_span: try: os.unlink(temp_log_path) - print(f"已清理临时文件: {temp_log_path}") + logger.debug(f"已清理临时文件: {temp_log_path}") del_span.set_data("file", temp_log_path) del_span.set_tag("success", "true") except Exception as unlink_error: - print(f"清理临时文件失败: {unlink_error}") + logger.error(f"清理临时文件失败: {unlink_error}") del_span.set_data("error", str(unlink_error)) del_span.set_tag("success", "false") except Exception as cleanup_error: - print(f"清理过程出错: {cleanup_error}") + logger.error(f"清理过程出错: {cleanup_error}") cleanup_span.set_data("error", str(cleanup_error)) cleanup_span.set_tag("success", "false") - print("清理资源并重启ComfyUI") + logger.info("清理资源并重启ComfyUI") with sentry_sdk.start_span(op="system.restart", description="重启ComfyUI") as restart_span: try: # 停止ComfyUI @@ -510,7 +498,7 @@ with comfyui_image.imports(): time.sleep(1) stop_span.set_tag("success", "true") except Exception as stop_error: - print(f"停止ComfyUI失败: {stop_error}") + logger.error(f"停止ComfyUI失败: {stop_error}") restart_span.set_data("stop_error", str(stop_error)) restart_span.set_tag("stop_success", "false") @@ -521,7 +509,7 @@ with comfyui_image.imports(): subprocess.run(cmd, shell=True, check=True) start_span.set_tag("success", "true") except Exception as start_error: - print(f"启动ComfyUI失败: {start_error}") + logger.error(f"启动ComfyUI失败: {start_error}") restart_span.set_data("start_error", str(start_error)) restart_span.set_tag("start_success", "false") modal.experimental.stop_fetching_inputs() diff --git a/src/cluster/comfyui_v2.py b/src/cluster/comfyui_v2.py index 89415c1..c67f1a8 100644 --- a/src/cluster/comfyui_v2.py +++ b/src/cluster/comfyui_v2.py @@ -3,61 +3,46 @@ import modal from dotenv import dotenv_values comfyui_latentsync_1_5_image = ( - modal.Image.debian_slim( - python_version="3.10" - ) - .pip_install_from_pyproject("../pyproject_comfyui.toml") - .apt_install("wget", "software-properties-common") - .run_commands("add-apt-repository -y contrib " - "&& wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb") - .apt_install("git", "gcc", "libportaudio2", "cuda-toolkit", "ffmpeg") - .pip_install("comfy_cli==0.0.0", index_url="https://packages-1747622887395:0ee15474ccd7b27b57ca63a9306327678e6c2631@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple") - .run_commands("comfy --skip-prompt install --nvidia --version 0.3.10") - .env(dotenv_values("../.runtime.env")) - .workdir("/root/comfy") -) + modal.Image.debian_slim(python_version="3.10").pip_install_from_pyproject("../pyproject_comfyui.toml").apt_install( + "wget", "software-properties-common").run_commands("add-apt-repository -y contrib " + "&& wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb").apt_install( + "git", "gcc", "libportaudio2", "cuda-toolkit", "ffmpeg").pip_install("comfy_cli==0.0.0", + index_url="https://packages-1747622887395:0ee15474ccd7b27b57ca63a9306327678e6c2631@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple").run_commands( + "comfy --skip-prompt install --nvidia --version 0.3.10").env(dotenv_values("../.runtime.env")).workdir( + "/root/comfy")) comfyui_latentsync_1_5_image = ( - comfyui_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals") - .run_commands("comfy node install https://github.com/evanspearman/ComfyMath") - .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git") - .run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools") - .run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts") - .run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy") - .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git") - .run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui") - .run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui") - .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy") - .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials") - .run_commands("comfy node install https://github.com/melMass/comfy_mtb") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git") - .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/Comfy-LatentSync-Next-Node.git") - .run_commands( - "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model" - ).run_commands( - "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints" - ).run_commands( - "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints" - ).run_commands( - "mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai" - ).run_commands( - "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M" - ).run_commands( - "rm -rf /root/comfy/ComfyUI/models" - ) - .add_local_file("BowongModalFunctions/template/template_v2.json", "/root/src/BowongModalFunctions/template/template_v2.json", copy=True) - .add_local_python_source('cluster') - .add_local_python_source('BowongModalFunctions') -) + comfyui_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals").run_commands( + "comfy node install https://github.com/evanspearman/ComfyMath").run_commands( + "comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved").run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git").run_commands( + "comfy node install https://github.com/crystian/ComfyUI-Crystools").run_commands( + "comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts").run_commands( + "comfy node install https://github.com/BennyKok/comfyui-deploy").run_commands( + "comfy node install https://github.com/yolain/ComfyUI-Easy-Use").run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git").run_commands( + "comfy node install https://github.com/jags111/efficiency-nodes-comfyui").run_commands( + "comfy node install https://github.com/WASasquatch/was-node-suite-comfyui").run_commands( + "comfy node install https://github.com/rgthree/rgthree-comfy").run_commands( + "comfy node install https://github.com/cubiq/ComfyUI_essentials").run_commands( + "comfy node install https://github.com/melMass/comfy_mtb").run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git").run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git", force_build=True).run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git").run_commands( + "comfy node install https://e.coding.net/g-ldyi2063/dev/Comfy-LatentSync-Next-Node.git").run_commands( + "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model").run_commands( + "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints").run_commands( + "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints").run_commands( + "mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai").run_commands( + "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M").run_commands( + "rm -rf /root/comfy/ComfyUI/models").add_local_file("BowongModalFunctions/template/template_v2.json", + "/root/src/BowongModalFunctions/template/template_v2.json", + copy=True).add_local_python_source( + 'cluster').add_local_python_source('BowongModalFunctions')) app = modal.App(name="ComfyUI v2", image=comfyui_latentsync_1_5_image, include_source=False) app.set_description("ComfyUI v2 Server") - with comfyui_latentsync_1_5_image.imports(): import json import os @@ -66,12 +51,9 @@ with comfyui_latentsync_1_5_image.imports(): import time import uuid from typing import Dict, Tuple, Any, Optional - import loguru import sentry_sdk import psutil - import datetime - import tempfile - import traceback + from loguru import logger from BowongModalFunctions.config import WorkerConfig from modal import current_function_call_id from sentry_sdk.integrations.loguru import LoguruIntegration @@ -104,6 +86,7 @@ with comfyui_latentsync_1_5_image.imports(): return (cpu_freq, cpu_count, mem) + @app.cls( max_containers=200, min_containers=0, @@ -111,48 +94,47 @@ with comfyui_latentsync_1_5_image.imports(): scaledown_window=120, timeout=1200, gpu=["L4"], - cpu=(2,64), - memory=(20480,131072), + cpu=(2, 64), + memory=(20480, 131072), enable_memory_snapshot=False, secrets=[secret, custom_secret], - volumes={ - "/root/comfy/ComfyUI/models": vol, - "/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount( - bucket_name=config.S3_bucket_name, - secret=secret, - key_prefix=config.comfyui_s3_input+"/" - ), - "/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount( - bucket_name=config.S3_bucket_name, - secret=secret, - key_prefix=config.comfyui_s3_output+"/" - ), - }, - ) + volumes={"/root/comfy/ComfyUI/models": vol, + "/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount(bucket_name=config.S3_bucket_name, secret=secret, + key_prefix=config.comfyui_s3_input + "/"), + "/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount(bucket_name=config.S3_bucket_name, secret=secret, + key_prefix=config.comfyui_s3_output + "/"), }, ) @modal.concurrent(max_inputs=1) class ComfyUILatentSync15: @modal.enter() def launch_comfy_background(self): self.session_id = str(uuid.uuid4()) self.file_prefix = None - cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id) + cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id, self.session_id, + self.session_id) subprocess.run(cmd, shell=True, check=True) @modal.method() async def api(self, video_path: str, start_time: str = "00:00:03.600", filename_prefix=str(uuid.uuid4()), - tts_text1: str = "测试", - tts_text2: str = "", tts_text3: str = "", tts_text4: str = "", anchor_id: str = "dawan", - speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None, webhook:WebhookNotify = None): + tts_text1: str = "测试", tts_text2: str = "", tts_text3: str = "", tts_text4: str = "", + anchor_id: str = "dawan", speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None, + webhook: WebhookNotify = None): + + # 导入必要模块 + import datetime + import tempfile + import traceback + import sys + from loguru import logger # 生成当前函数调用的唯一ID function_call_id = current_function_call_id() - print(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}") + logger.info(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}") @SentryUtils.webhook_handler(webhook=webhook, func_id=function_call_id) async def infer(workflow: Dict): self.poll_server_health() self.prompt_uuid = str(uuid.uuid4()) - loguru.logger.info(f"Workflow JSON: \n{json.dumps(workflow, indent=4, ensure_ascii=False)}") + logger.info(f"Workflow JSON: \n{json.dumps(workflow, indent=4, ensure_ascii=False)}") for node in workflow.values(): if node.get("class_type") == "ComfyUIDeployExternalText": if node.get("inputs").get("input_id") == "file_path": @@ -163,7 +145,7 @@ with comfyui_latentsync_1_5_image.imports(): shutil.copy(file_to_move, file_to_move.replace("input_s3", "input")) except: try: - print("Try download file to S3 manually") + logger.warning("尝试手动从S3下载文件") # S3 Fallback import boto3 import yaml @@ -187,11 +169,9 @@ with comfyui_latentsync_1_5_image.imports(): fi.write(json.dumps(workflow, ensure_ascii=False)) cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose" - self.file_prefix = [ - node.get("inputs") - for node in workflow.values() - if node.get("class_type") == "VHS_VideoCombine" - ][0]["filename_prefix"] + self.file_prefix = \ + [node.get("inputs") for node in workflow.values() if node.get("class_type") == "VHS_VideoCombine"][0][ + "filename_prefix"] subprocess.run(cmd, shell=True, check=True, timeout=1195) @@ -208,7 +188,7 @@ with comfyui_latentsync_1_5_image.imports(): os.path.join(output_dir.replace("output", "output_s3"), f)) except: try: - print("Try move file to S3 manually") + logger.warning("尝试手动将文件移动到S3") # S3 Fallback import boto3 import yaml @@ -232,31 +212,23 @@ with comfyui_latentsync_1_5_image.imports(): # 创建主Sentry事务 if sentry_trace and sentry_trace.x_trace_id and sentry_trace.x_baggage: - transaction = sentry_sdk.continue_trace(environ_or_headers={"sentry-trace": sentry_trace.x_trace_id, - "baggage": sentry_trace.x_baggage, }) + transaction = sentry_sdk.continue_trace( + environ_or_headers={"sentry-trace": sentry_trace.x_trace_id, "baggage": sentry_trace.x_baggage, }) else: transaction = sentry_sdk.start_transaction(op="video.processing", name=f"API调用: {video_path}") - sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id, x_baggage=transaction.containing_transaction.get_baggage().serialize()) + sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id, + x_baggage=transaction.containing_transaction.get_baggage().serialize()) with transaction: # 设置全局Sentry上下文 - with sentry_sdk.configure_scope() as scope: - # 添加更丰富的上下文数据 - scope.set_tag("session_id", self.session_id) - scope.set_tag("function_call_id", function_call_id) - scope.set_tag("video_path", video_path) - scope.set_context("api_parameters", { - "current_function_call_id": function_call_id, - "session_id": self.session_id, - "video_path": video_path, - "start_time": start_time, - "filename_prefix": filename_prefix, - "tts_text1": tts_text1, - "tts_text2": tts_text2, - "tts_text3": tts_text3, - "tts_text4": tts_text4, - "anchor_id": anchor_id, - "speed": speed - }) + sentry_sdk.set_tag("session_id", self.session_id) + sentry_sdk.set_tag("function_call_id", function_call_id) + sentry_sdk.set_tag("video_path", video_path) + sentry_sdk.set_context("api_parameters", + {"current_function_call_id": function_call_id, "session_id": self.session_id, + "video_path": video_path, "start_time": start_time, + "filename_prefix": filename_prefix, "tts_text1": tts_text1, + "tts_text2": tts_text2, "tts_text3": tts_text3, "tts_text4": tts_text4, + "anchor_id": anchor_id, "speed": speed}) try: # 使用Span跟踪日志文件创建 @@ -264,7 +236,7 @@ with comfyui_latentsync_1_5_image.imports(): # 创建临时日志文件 temp_log_file = tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt') temp_log_path = temp_log_file.name - print(f"创建临时日志文件: {temp_log_path}") + logger.debug(f"创建临时日志文件: {temp_log_path}") span.set_data("log_path", temp_log_path) span.set_tag("file_operation", "create") @@ -308,11 +280,7 @@ with comfyui_latentsync_1_5_image.imports(): try: # 记录执行的关键步骤作为breadcrumbs - sentry_sdk.add_breadcrumb( - category="process", - message="开始调用infer方法", - level="info" - ) + sentry_sdk.add_breadcrumb(category="process", message="开始调用infer方法", level="info") # 使用Span跟踪推理过程 with sentry_sdk.start_span(op="ai.inference", description="执行模型推理") as infer_span: @@ -342,11 +310,7 @@ with comfyui_latentsync_1_5_image.imports(): infer_span.set_data("output_file", fname) infer_span.set_tag("success", "true" if fname else "false") - sentry_sdk.add_breadcrumb( - category="process", - message="infer方法调用完成", - level="info" - ) + sentry_sdk.add_breadcrumb(category="process", message="infer方法调用完成", level="info") if fname is None: with sentry_sdk.start_span(op="error.handling", description="处理文件缺失错误"): @@ -355,7 +319,7 @@ with comfyui_latentsync_1_5_image.imports(): # 使用Span跟踪返回结果 with sentry_sdk.start_span(op="result.preparation", description="准备返回结果"): j = {"status": "success", "file_name": fname} - loguru.logger.success(j) + logger.success(f"处理成功: {j}") # 设置事务状态为成功 transaction.set_status("ok") @@ -364,20 +328,19 @@ with comfyui_latentsync_1_5_image.imports(): except Exception as infer_error: # 使用原始异常对象 - sentry_sdk.add_breadcrumb( - category="error", - message=f"infer方法调用失败: {str(infer_error)}", - level="error" - ) + sentry_sdk.add_breadcrumb(category="error", message=f"infer方法调用失败: {str(infer_error)}", + level="error") with sentry_sdk.start_span(op="error.handling", description="处理推理错误") as error_span: error_span.set_data("error_type", type(infer_error).__name__) error_span.set_data("error_message", str(infer_error)) error_span.set_tag("error", "true") + logger.error(f"推理失败: {str(infer_error)}") raise infer_error except Exception as e: # 异常处理 - 使用捕获的异常对象e j = {"status": "fail", "msg": str(e)} + logger.error(f"处理异常: {str(e)}") try: # 使用Span跟踪错误记录过程 @@ -410,48 +373,46 @@ with comfyui_latentsync_1_5_image.imports(): f.write(f"\n无法读取ComfyUI日志: {str(log_read_error)}\n") log_span.set_data("error", str(log_read_error)) log_span.set_tag("success", "false") + logger.warning(f"无法读取ComfyUI日志: {str(log_read_error)}") else: log_span.set_tag("file_exists", "false") + logger.warning("ComfyUI日志文件不存在") error_span.set_data("log_path", temp_log_path) error_span.set_tag("error_type", type(e).__name__) # 仅当尚未报告错误时才添加附件和报告异常 if not error_reported: - with sentry_sdk.configure_scope() as scope: - with sentry_sdk.start_span(op="error.report", - description="向Sentry报告错误") as report_span: - scope.add_attachment( - path=temp_log_path, - filename=f"full_{function_call_id}.txt", - add_to_transactions=True - ) + with sentry_sdk.start_span(op="error.report", + description="向Sentry报告错误") as report_span: + # 使用新的API替换configure_scope + # 添加附件 + sentry_sdk.add_attachment(path=temp_log_path, filename=f"full_{function_call_id}.txt", + add_to_transactions=True) - # 添加更多异常上下文 - scope.set_context("error_details", { - "error_type": type(e).__name__, - "error_message": str(e), - "time_of_error": datetime.datetime.now().isoformat() - }) + # 添加更多异常上下文 + sentry_sdk.set_context("error_details", + {"error_type": type(e).__name__, "error_message": str(e), + "time_of_error": datetime.datetime.now().isoformat()}) - # 设置事务状态为错误 - transaction.set_status("internal_error") + # 设置事务状态为错误 + transaction.set_status("internal_error") - # 使用异常对象e - sentry_sdk.capture_exception(e) - error_reported = True - print("已向Sentry报告异常") - report_span.set_tag("reported", "true") + # 使用异常对象e + sentry_sdk.capture_exception(e) + error_reported = True + logger.info("已向Sentry报告异常") + report_span.set_tag("reported", "true") except Exception as log_error: - print(f"处理日志出错: {log_error}") + logger.error(f"处理日志出错: {log_error}") # 设置事务状态为错误 transaction.set_status("internal_error") # 仅当尚未报告错误时才报告原始异常 if not error_reported: sentry_sdk.capture_exception(e) - print("已向Sentry报告原始异常（日志处理失败）") + logger.info("已向Sentry报告原始异常（日志处理失败）") return j, sentry_trace @@ -467,12 +428,12 @@ with comfyui_latentsync_1_5_image.imports(): try: os.makedirs(log_dir, exist_ok=True) shutil.copy(temp_log_path, f"{log_dir}/full.txt") - print(f"已复制日志到: {log_dir}/full.txt") + logger.info(f"已复制日志到: {log_dir}/full.txt") copy_span.set_data("source", temp_log_path) copy_span.set_data("destination", f"{log_dir}/full.txt") copy_span.set_tag("success", "true") except Exception as copy_error: - print(f"复制日志到S3失败: {copy_error}") + logger.error(f"复制日志到S3失败: {copy_error}") copy_span.set_data("error", str(copy_error)) copy_span.set_tag("success", "false") @@ -480,19 +441,19 @@ with comfyui_latentsync_1_5_image.imports(): with sentry_sdk.start_span(op="db.file.delete", description="删除临时文件") as del_span: try: os.unlink(temp_log_path) - print(f"已清理临时文件: {temp_log_path}") + logger.debug(f"已清理临时文件: {temp_log_path}") del_span.set_data("file", temp_log_path) del_span.set_tag("success", "true") except Exception as unlink_error: - print(f"清理临时文件失败: {unlink_error}") + logger.error(f"清理临时文件失败: {unlink_error}") del_span.set_data("error", str(unlink_error)) del_span.set_tag("success", "false") except Exception as cleanup_error: - print(f"清理过程出错: {cleanup_error}") + logger.error(f"清理过程出错: {cleanup_error}") cleanup_span.set_data("error", str(cleanup_error)) cleanup_span.set_tag("success", "false") - print("清理资源并重启ComfyUI") + logger.info("清理资源并重启ComfyUI") with sentry_sdk.start_span(op="system.restart", description="重启ComfyUI") as restart_span: try: # 停止ComfyUI @@ -502,7 +463,7 @@ with comfyui_latentsync_1_5_image.imports(): time.sleep(1) stop_span.set_tag("success", "true") except Exception as stop_error: - print(f"停止ComfyUI失败: {stop_error}") + logger.error(f"停止ComfyUI失败: {stop_error}") restart_span.set_data("stop_error", str(stop_error)) restart_span.set_tag("stop_success", "false") @@ -513,12 +474,11 @@ with comfyui_latentsync_1_5_image.imports(): subprocess.run(cmd, shell=True, check=True) start_span.set_tag("success", "true") except Exception as start_error: - print(f"启动ComfyUI失败: {start_error}") + logger.error(f"启动ComfyUI失败: {start_error}") restart_span.set_data("start_error", str(start_error)) restart_span.set_tag("start_success", "false") modal.experimental.stop_fetching_inputs() - def poll_server_health(self): import socket import urllib @@ -527,10 +487,10 @@ with comfyui_latentsync_1_5_image.imports(): # dummy request to check if the server is healthy req = urllib.request.Request("http://127.0.0.1:8188/system_stats") urllib.request.urlopen(req, timeout=5) - print("ComfyUI server is healthy") + logger.info("ComfyUI server is healthy") except (socket.timeout, urllib.error.URLError) as e: # if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container - print(f"Server health check failed: {str(e)} restarting ComfyUI...") + logger.error(f"Server health check failed: {str(e)} restarting ComfyUI...") try: cmd = "comfy stop" try: @@ -544,4 +504,4 @@ with comfyui_latentsync_1_5_image.imports(): raise Exception("Failed to launch ComfyUI") except: modal.experimental.stop_fetching_inputs() - raise Exception("ComfyUI server is not healthy, restart failed, stopping container") \ No newline at end of file + raise Exception("ComfyUI server is not healthy, restart failed, stopping container")