# ComfyUI模板--Base Auth LatentSync1.5
import modal
from dotenv import dotenv_values

comfyui_latentsync_1_5_image = (
    modal.Image.debian_slim(python_version="3.10")
    .pip_install_from_pyproject("../pyproject_comfyui.toml")
    .apt_install("wget", "software-properties-common")
    .run_commands("add-apt-repository -y contrib "
                   "&& wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb")
    .apt_install("git", "gcc", "libportaudio2", "cuda-toolkit", "ffmpeg")
    .pip_install("comfy_cli==0.0.0",index_url="https://packages-1747622887395:0ee15474ccd7b27b57ca63a9306327678e6c2631@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple")
    .run_commands("comfy --skip-prompt install --nvidia --version 0.3.10")
    .env(dotenv_values("../.runtime.env"))
    .workdir("/root/comfy"))

comfyui_latentsync_1_5_image = (
    comfyui_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
    .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
    .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
    .run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools")
    .run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts")
    .run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy")
    .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git")
    .run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui")
    .run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui")
    .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
    .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
    .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/Comfy-LatentSync-Next-Node.git")
    .run_commands("mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model")
    .run_commands("mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints")
    .run_commands("mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints")
    .run_commands("mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai")
    .run_commands("mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M")
    .run_commands("rm -rf /root/comfy/ComfyUI/models")
    .add_local_file("BowongModalFunctions/template/template_v2.json","/root/src/BowongModalFunctions/template/template_v2.json",copy=True)
    .add_local_python_source('cluster')
    .add_local_python_source('BowongModalFunctions'))

app = modal.App(name="ComfyUI v2", image=comfyui_latentsync_1_5_image, include_source=False)
app.set_description("ComfyUI v2 Server")

with comfyui_latentsync_1_5_image.imports():
    import json
    import os
    import shutil
    import subprocess
    import uuid
    from typing import Dict, Tuple, Any, Optional
    import sentry_sdk
    import psutil
    from loguru import logger
    from BowongModalFunctions.config import WorkerConfig
    from modal import current_function_call_id
    from sentry_sdk.integrations.loguru import LoguruIntegration
    from BowongModalFunctions.models.web_model import SentryTransactionInfo, WebhookNotify
    from BowongModalFunctions.utils.SentryUtils import SentryUtils

    config = WorkerConfig()
    vol = modal.Volume.from_name("comfyui-model", create_if_missing=True, environment_name=config.modal_environment)
    secret = modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment)
    custom_secret = modal.Secret.from_name("comfyui-custom-secret", environment_name=config.modal_environment)
    output_dir = "/root/comfy/ComfyUI/output"

    sentry_sdk.init(
        dsn="https://7e3e37e770420e289b3972e1b1b9a794@sentry.bowongai.com/11",
        send_default_pii=True,
        attach_stacktrace=True,
        traces_sample_rate=1.0,  # 采样率设为1.0表示捕获所有事务
        profiles_sample_rate=1.0,
        add_full_stack=True,
        shutdown_timeout=2,
        integrations=[LoguruIntegration()],
        environment=config.modal_environment,
    )

    @sentry_sdk.trace
    def capture_hardware_info() -> Tuple[Any, int, Any]:
        cpu_freq = psutil.cpu_freq()
        cpu_count = psutil.cpu_count()
        mem = psutil.virtual_memory()

        return (cpu_freq, cpu_count, mem)


    @app.cls(
        max_containers=200,
        min_containers=0,
        buffer_containers=0,
        scaledown_window=120,
        timeout=1200,
        gpu=["L4"],
        cpu=(2, 64),
        memory=(20480, 131072),
        enable_memory_snapshot=False,
        secrets=[secret, custom_secret],
        volumes={"/root/comfy/ComfyUI/models": vol,
                 "/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount(bucket_name=config.S3_bucket_name, secret=secret,
                key_prefix=config.comfyui_s3_input + "/"),
                 "/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount(bucket_name=config.S3_bucket_name, secret=secret,
                key_prefix=config.comfyui_s3_output + "/"), }, )
    @modal.concurrent(max_inputs=1)
    class ComfyUILatentSync15:
        @modal.enter()
        def launch_comfy_background(self):
            self.session_id = str(uuid.uuid4())
            self.file_prefix = None
            cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id, self.session_id,
                                                                             self.session_id)
            subprocess.run(cmd, shell=True, check=True)

        @modal.method()
        async def api(self, video_path: str, start_time: str = "00:00:03.600", filename_prefix=str(uuid.uuid4()),
                      tts_text1: str = "测试", tts_text2: str = "", tts_text3: str = "", tts_text4: str = "",
                      anchor_id: str = "dawan", speed: float = 1, sentry_trace: Optional[SentryTransactionInfo] = None,
                      webhook: WebhookNotify = None):

            # 导入必要模块
            import datetime
            import tempfile
            import traceback
            from loguru import logger

            # 生成当前函数调用的唯一ID
            function_call_id = current_function_call_id()
            logger.info(f"开始处理任务，会话ID: {self.session_id}, 调用ID: {function_call_id}")

            @SentryUtils.webhook_handler(webhook=webhook, func_id=function_call_id)
            async def infer(workflow: Dict) -> str | None:
                self.poll_server_health()
                self.prompt_uuid = str(uuid.uuid4())
                logger.info(f"Workflow JSON: \n{json.dumps(workflow, indent=4, ensure_ascii=False)}")
                for node in workflow.values():
                    if node.get("class_type") == "ComfyUIDeployExternalText":
                        if node.get("inputs").get("input_id") == "file_path":
                            file_to_move = node.get("inputs").get("default_value")
                            if file_to_move:
                                os.makedirs(os.path.dirname(file_to_move.replace("input_s3", "input")), exist_ok=True)
                                try:
                                    shutil.copy(file_to_move, file_to_move.replace("input_s3", "input"))
                                except:
                                    try:
                                        logger.warning("尝试手动从S3下载文件")
                                        # S3 Fallback
                                        import boto3
                                        import yaml
                                        if "aws_key_id" in list(os.environ.keys()):
                                            yaml_config = os.environ
                                        else:
                                            with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",
                                                      encoding="utf-8", mode="r+") as myconfig:
                                                yaml_config = yaml.load(myconfig, Loader=yaml.FullLoader)
                                        awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
                                                               aws_secret_access_key=yaml_config["aws_access_key"])
                                        awss3.meta.client.download_file(config.S3_bucket_name,
                                                                        config.comfyui_s3_input + "/" +
                                                                        file_to_move.split("input_s3/")[1],
                                                                        file_to_move.replace("input_s3", "input"))
                                    except:
                                        raise Exception("Failed to download file from S3 manually")
                                node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3",
                                                                                                          "input")
                with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as fi:
                    fi.write(json.dumps(workflow, ensure_ascii=False))
                cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose"

                self.file_prefix = \
                    [node.get("inputs") for node in workflow.values() if node.get("class_type") == "VHS_VideoCombine"][0]["filename_prefix"]

                subprocess.run(cmd, shell=True, check=True, timeout=1195)

                file_list = os.listdir(output_dir)
                # 获取按照文件时间创建排序的列表，默认是按时间升序
                new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)),
                                       reverse=True)
                for f in new_file_list:
                    if f.startswith(self.file_prefix):
                        os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)),
                                    exist_ok=True)
                        try:
                            shutil.copy(os.path.join(output_dir, f),
                                        os.path.join(output_dir.replace("output", "output_s3"), f))
                        except:
                            try:
                                logger.warning("尝试手动将文件移动到S3")
                                # S3 Fallback
                                import boto3
                                import yaml
                                if "aws_key_id" in list(os.environ.keys()):
                                    yaml_config = os.environ
                                else:
                                    with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",
                                              encoding="utf-8", mode="r+") as myconfig:
                                        yaml_config = yaml.load(myconfig, Loader=yaml.FullLoader)
                                awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
                                                       aws_secret_access_key=yaml_config["aws_access_key"])
                                awss3.meta.client.upload_file(os.path.join(output_dir, f), config.S3_bucket_name,
                                                              config.comfyui_s3_output + "/" + f)
                            except:
                                raise Exception("Failed to move file to S3 manually")
                        return "s3://" + "/".join([config.S3_region, config.S3_bucket_name, config.comfyui_s3_output, f])

            # 使用临时文件
            temp_log_path = None
            error_reported = False  # 添加标志跟踪是否已报告错误

            # 创建主Sentry事务
            if sentry_trace and sentry_trace.x_trace_id and sentry_trace.x_baggage:
                transaction = sentry_sdk.continue_trace(environ_or_headers={"sentry-trace": sentry_trace.x_trace_id, "baggage": sentry_trace.x_baggage, })
            else:
                transaction = sentry_sdk.start_transaction(op="video.processing", name=f"API调用: {video_path}")
            sentry_trace = SentryTransactionInfo(x_trace_id=transaction.trace_id,
                                                 x_baggage=transaction.containing_transaction.get_baggage().serialize())
            with transaction:
                # 设置全局Sentry上下文
                sentry_sdk.set_tag("session_id", self.session_id)
                sentry_sdk.set_tag("function_call_id", function_call_id)
                sentry_sdk.set_tag("video_path", video_path)
                sentry_sdk.set_context("api_parameters",
                                       {"current_function_call_id": function_call_id, "session_id": self.session_id,
                                        "video_path": video_path, "start_time": start_time,
                                        "filename_prefix": filename_prefix, "tts_text1": tts_text1,
                                        "tts_text2": tts_text2, "tts_text3": tts_text3, "tts_text4": tts_text4,
                                        "anchor_id": anchor_id, "speed": speed})

                try:
                    # 使用Span跟踪日志文件创建
                    with sentry_sdk.start_span(op="db.file", description="创建日志文件") as span:
                        # 创建临时日志文件
                        temp_log_file = tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt')
                        temp_log_path = temp_log_file.name
                        logger.debug(f"创建临时日志文件: {temp_log_path}")

                        span.set_data("log_path", temp_log_path)
                        span.set_tag("file_operation", "create")

                        # 写入初始日志内容
                        temp_log_file.write(f"任务开始时间: {datetime.datetime.now()}\n")
                        temp_log_file.write(f"会话ID: {self.session_id}\n")
                        temp_log_file.write(f"调用ID: {function_call_id}\n")
                        temp_log_file.write(f"文件前缀: {filename_prefix}\n")
                        temp_log_file.write(f"参数: video_path={video_path}, start_time={start_time}\n")
                        temp_log_file.flush()
                        temp_log_file.close()

                        # 设置状态为成功
                        span.set_status("ok")

                    # 使用Span跟踪模板处理
                    with sentry_sdk.start_span(op="template.json", description="处理工作流模板") as span:
                        # 正常业务逻辑
                        with open("/root/src/BowongModalFunctions/template/template_v2.json", "r",
                                  encoding="utf-8") as template:
                            workflow_file = json.loads(template.read())

                        # 使用子span跟踪模板修改
                        with sentry_sdk.start_span(op="template.modify", description="修改模板参数") as modify_span:
                            modify_span.set_data("original_keys", len(workflow_file))

                            workflow_file["211"]["inputs"]["default_value"] = tts_text1
                            workflow_file["399"]["inputs"]["default_value"] = tts_text2
                            workflow_file["331"]["inputs"]["default_value"] = tts_text3
                            workflow_file["406"]["inputs"]["default_value"] = tts_text4
                            workflow_file["424"]["inputs"]["default_value"] = "/root/comfy/ComfyUI/input_s3/" + video_path
                            workflow_file["225"]["inputs"]["filename_prefix"] = filename_prefix
                            workflow_file["330"]["inputs"]["default_value"] = start_time
                            workflow_file["515"]["inputs"]["anchor_id"] = anchor_id
                            workflow_file["515"]["inputs"]["speed"] = speed

                            modify_span.set_data("modified_nodes",
                                                 ["211", "399", "331", "406", "424", "225", "330", "515"])
                            # 设置状态为成功
                            modify_span.set_status("ok")

                        span.set_data("workflow_size", len(workflow_file))
                        span.set_tag("workflow_type", "video_processing")
                        # 设置状态为成功
                        span.set_status("ok")

                    try:
                        # 记录执行的关键步骤作为breadcrumbs
                        sentry_sdk.add_breadcrumb(category="process", message="开始调用infer方法", level="info")

                        # 使用Span跟踪推理过程
                        with sentry_sdk.start_span(op="ai.inference", description="执行模型推理") as infer_span:
                            infer_span.set_data("workflow_size", len(workflow_file))
                            infer_start_time = datetime.datetime.now()

                            # 创建预处理子span
                            with sentry_sdk.start_span(op="inference.prepare", description="准备推理环境") as prep_span:
                                prep_span.set_data("timestamp", datetime.datetime.now().isoformat())
                                prep_span.set_tag("model_type", "video_generation")
                                # 设置状态为成功
                                prep_span.set_status("ok")

                            # 调用推理方法
                            fname = await infer(workflow_file)

                            # 创建后处理子span
                            with sentry_sdk.start_span(op="inference.postprocess",
                                                       description="处理推理结果") as post_span:
                                post_span.set_data("output_file", fname)
                                post_span.set_data("timestamp", datetime.datetime.now().isoformat())
                                # 设置状态为成功
                                post_span.set_status("ok")

                            infer_end_time = datetime.datetime.now()
                            duration_seconds = (infer_end_time - infer_start_time).total_seconds()

                            infer_span.set_data("start_time", infer_start_time.isoformat())
                            infer_span.set_data("end_time", infer_end_time.isoformat())
                            infer_span.set_data("duration_seconds", duration_seconds)
                            infer_span.set_data("output_file", fname)
                            infer_span.set_tag("success", "true" if fname else "false")
                            # 设置状态为成功
                            infer_span.set_status("ok")

                        sentry_sdk.add_breadcrumb(category="process", message="infer方法调用完成", level="info")

                        if fname is None:
                            with sentry_sdk.start_span(op="error.handling",
                                                       description="处理文件缺失错误") as error_span:
                                error_span.set_status("internal_error")  # 设置状态为错误
                                raise RuntimeError("Output File not found")

                        # 使用Span跟踪返回结果
                        with sentry_sdk.start_span(op="result.preparation", description="准备返回结果") as result_span:
                            j = {"status": "success", "file_name": fname}
                            logger.success(f"处理成功: {j}")

                            # 设置span状态为成功
                            result_span.set_status("ok")

                            # 设置事务状态为成功
                            transaction.set_status("ok")

                            return j, sentry_trace

                    except Exception as infer_error:
                        # 使用原始异常对象
                        sentry_sdk.add_breadcrumb(category="error", message=f"infer方法调用失败: {str(infer_error)}",
                                                  level="error")
                        with sentry_sdk.start_span(op="error.handling", description="处理推理错误") as error_span:
                            error_span.set_data("error_type", type(infer_error).__name__)
                            error_span.set_data("error_message", str(infer_error))
                            error_span.set_tag("error", "true")
                            # 设置状态为错误
                            error_span.set_status("internal_error")
                            logger.error(f"推理失败: {str(infer_error)}")
                            raise infer_error

                except Exception as e:
                    # 异常处理 - 使用捕获的异常对象e
                    j = {"status": "failed", "msg": str(e)}
                    logger.error(f"处理异常: {str(e)}")

                    try:
                        # 使用Span跟踪错误记录过程
                        with sentry_sdk.start_span(op="error.logging", description="记录错误详情") as error_span:
                            # 更新日志文件
                            with open(temp_log_path, "a", encoding="utf-8") as f:
                                # 记录异常详情
                                f.write(f"\n====== 异常详情 ======\n")
                                f.write(f"异常类型: {type(e).__name__}\n")
                                f.write(f"异常信息: {str(e)}\n")
                                f.write(f"发生时间: {datetime.datetime.now()}\n")
                                f.write(f"会话ID: {self.session_id}\n")
                                f.write(f"调用ID: {function_call_id}\n")

                                # 记录完整堆栈
                                f.write("\n====== 堆栈跟踪 ======\n")
                                f.write(traceback.format_exc())

                                # 尝试添加ComfyUI日志
                                with sentry_sdk.start_span(op="db.file.read",
                                                           description="读取ComfyUI日志") as log_span:
                                    if os.path.exists("/root/comfy/ComfyUI/user/comfyui.log"):
                                        try:
                                            with open("/root/comfy/ComfyUI/user/comfyui.log", "r",
                                                      encoding="utf-8") as cf:
                                                f.write("\n====== ComfyUI日志 ======\n")
                                                f.write(cf.read())
                                            log_span.set_tag("success", "true")
                                            log_span.set_status("ok")  # 设置状态为成功
                                        except Exception as log_read_error:
                                            f.write(f"\n无法读取ComfyUI日志: {str(log_read_error)}\n")
                                            log_span.set_data("error", str(log_read_error))
                                            log_span.set_tag("success", "false")
                                            # 设置状态为错误
                                            log_span.set_status("internal_error")
                                            logger.warning(f"无法读取ComfyUI日志: {str(log_read_error)}")
                                    else:
                                        log_span.set_tag("file_exists", "false")
                                        # 设置状态为通知
                                        log_span.set_status("unavailable")
                                        logger.warning("ComfyUI日志文件不存在")

                            error_span.set_data("log_path", temp_log_path)
                            error_span.set_tag("error_type", type(e).__name__)
                            # 设置状态为错误
                            error_span.set_status("internal_error")

                        # 仅当尚未报告错误时才添加附件和报告异常
                        if not error_reported:
                            with sentry_sdk.start_span(op="error.report",
                                                       description="向Sentry报告错误") as report_span:
                                # 使用新的API替换configure_scope
                                # 添加附件
                                sentry_sdk.add_attachment(path=temp_log_path, filename=f"full_{function_call_id}.txt",
                                                          add_to_transactions=True)

                                # 添加更多异常上下文
                                sentry_sdk.set_context("error_details",
                                                       {"error_type": type(e).__name__, "error_message": str(e),
                                                        "time_of_error": datetime.datetime.now().isoformat()})

                                # 设置事务状态为错误
                                transaction.set_status("internal_error")
                                # 设置span状态为错误
                                report_span.set_status("internal_error")

                                # 使用异常对象e
                                sentry_sdk.capture_exception(e)
                                error_reported = True
                                logger.info("已向Sentry报告异常")
                                report_span.set_tag("reported", "true")

                    except Exception as log_error:
                        logger.error(f"处理日志出错: {log_error}")
                        # 设置事务状态为错误
                        transaction.set_status("internal_error")

                        # 仅当尚未报告错误时才报告原始异常
                        if not error_reported:
                            sentry_sdk.capture_exception(e)
                            logger.info("已向Sentry报告原始异常（日志处理失败）")

                    return j, sentry_trace

                finally:
                    # 使用Span跟踪清理过程
                    with sentry_sdk.start_span(op="system.cleanup", description="清理资源") as cleanup_span:
                        cleanup_successful = True  # 跟踪清理是否成功的标志

                        # 清理临时文件和重启ComfyUI
                        try:
                            # 尝试复制日志到S3
                            if temp_log_path and os.path.exists(temp_log_path):
                                with sentry_sdk.start_span(op="db.file.copy", description="复制日志到S3") as copy_span:
                                    log_dir = f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"
                                    try:
                                        os.makedirs(log_dir, exist_ok=True)
                                        shutil.copy(temp_log_path, f"{log_dir}/full.txt")
                                        logger.info(f"已复制日志到: {log_dir}/full.txt")
                                        copy_span.set_data("source", temp_log_path)
                                        copy_span.set_data("destination", f"{log_dir}/full.txt")
                                        copy_span.set_tag("success", "true")
                                        # 设置状态为成功
                                        copy_span.set_status("ok")
                                    except Exception as copy_error:
                                        logger.error(f"复制日志到S3失败: {copy_error}")
                                        copy_span.set_data("error", str(copy_error))
                                        copy_span.set_tag("success", "false")
                                        # 设置状态为错误
                                        copy_span.set_status("internal_error")
                                        cleanup_successful = False

                                # 清理临时文件
                                with sentry_sdk.start_span(op="db.file.delete", description="删除临时文件") as del_span:
                                    try:
                                        os.unlink(temp_log_path)
                                        logger.debug(f"已清理临时文件: {temp_log_path}")
                                        del_span.set_data("file", temp_log_path)
                                        del_span.set_tag("success", "true")
                                        # 设置状态为成功
                                        del_span.set_status("ok")
                                    except Exception as unlink_error:
                                        logger.error(f"清理临时文件失败: {unlink_error}")
                                        del_span.set_data("error", str(unlink_error))
                                        del_span.set_tag("success", "false")
                                        # 设置状态为错误
                                        del_span.set_status("internal_error")
                                        cleanup_successful = False
                        except Exception as cleanup_error:
                            logger.error(f"清理过程出错: {cleanup_error}")
                            cleanup_span.set_data("error", str(cleanup_error))
                            cleanup_span.set_tag("success", "false")
                            cleanup_successful = False

                        # 根据清理和重启结果设置总体清理状态
                        if cleanup_successful:
                            cleanup_span.set_status("ok")
                        else:
                            cleanup_span.set_status("internal_error")


        def poll_server_health(self):
            import socket
            import urllib

            try:
                # dummy request to check if the server is healthy
                req = urllib.request.Request("http://127.0.0.1:8188/system_stats")
                urllib.request.urlopen(req, timeout=5)
                logger.info("ComfyUI server is healthy")
            except (socket.timeout, urllib.error.URLError) as e:
                # if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container
                logger.error(f"Server health check failed: {str(e)} restarting ComfyUI...")
                try:
                    cmd = "comfy stop"
                    try:
                        subprocess.run(cmd, shell=True, check=True)
                    except:
                        pass
                    cmd = "comfy launch --background"
                    try:
                        subprocess.run(cmd, shell=True, check=True)
                    except:
                        raise Exception("Failed to launch ComfyUI")
                except:
                    modal.experimental.stop_fetching_inputs()
                    raise Exception("ComfyUI server is not healthy, restart failed, stopping container")
