llm节点支持链接分析

.gitignore

@@ -1,4 +1,5 @@
 model/*.pth
 model/*.pt
 !model/.gitkeep
-.yaml
+.yaml
+venv

ext/video_agent_deploy.py

@@ -17,7 +17,7 @@ image = (
     .apt_install("git", "gcc", "libportaudio2", "ffmpeg")
     .pip_install("comfy_cli")
     .run_commands(
-        "comfy --skip-prompt install --fast-deps --nvidia --version 0.3.40"
+        "comfy --skip-prompt install --fast-deps --nvidia --version 0.3.55"
     )
     .pip_install_from_pyproject(os.path.join(os.path.dirname(__file__), "pyproject.toml"))
     .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use.git")
@@ -75,37 +75,3 @@ def ui_1():
 @modal.web_server(8000, startup_timeout=120)
 def ui_2():
     subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True)
-
-# @app.function(
-#     min_containers=0,
-#     buffer_containers=0,
-#     max_containers=1,
-#     scaledown_window=600,
-#     secrets=[custom_secret],
-#     volumes={
-#         "/models": vol
-#     }
-# )
-# @modal.concurrent(
-#     max_inputs=10
-# )
-# @modal.web_server(8000, startup_timeout=120)
-# def ui_3():
-#     subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True)
-#
-# @app.function(
-#     min_containers=0,
-#     buffer_containers=0,
-#     max_containers=1,
-#     scaledown_window=600,
-#     secrets=[custom_secret],
-#     volumes={
-#         "/models": vol
-#     }
-# )
-# @modal.concurrent(
-#     max_inputs=10
-# )
-# @modal.web_server(8000, startup_timeout=120)
-# def ui_4():
-#     subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True)

nodes/fetch_task_result.py

@@ -73,7 +73,7 @@ class FetchTaskResult:
                     raw_response_str = json.dumps(data_, indent=2, ensure_ascii=False)
 
                     print(f"处理完成: {len(image_tensors)} 个图像, {len(video_urls)} 个视频URL。")
-                    return (final_images, final_urls, raw_response_str)
+                    return final_images, final_urls, raw_response_str
 
                 print(f"任务未完成。API返回状态: {api_status}。将在 {interval} 秒后重试...")
                 time.sleep(interval)

nodes/save_node.py

@@ -4,7 +4,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from urllib.parse import urlparse
 
-# 导入 ComfyUI 的路径管理器
 import folder_paths
 import requests
 import torch

nodes/union_llm_node.py

@@ -9,18 +9,39 @@ import requests
 import base64
 import mimetypes
 import torch
+import httpx
 import numpy as np
 from PIL import Image
 import folder_paths
 
-tensor_to_file_map = {}
+try:
+    import scipy.io.wavfile as wavfile
+except ImportError:
+    print("------------------------------------------------------------------------------------")
+    print("Scipy 库未安装, 请运行: pip install scipy")
+    print("------------------------------------------------------------------------------------")
+
+
+def handler_google_analytics(prompt: str, model_id: str, media_file_path: str, base_url: str, timeout: int):
+    headers = {'accept': 'application/json'}
+    files = {'prompt': (None, prompt), 'model_id': (None, model_id)}
+    if media_file_path and os.path.exists(media_file_path):
+        files['img_file'] = (os.path.basename(media_file_path), open(media_file_path, 'rb'),
+                             mimetypes.guess_type(media_file_path)[0] or 'application/octet-stream')
+    if media_file_path.startswith("gs:"):
+        files['img_url'] = (None, media_file_path)
+    try:
+        response = requests.post(f'{base_url}/api/llm/google/analysis', headers=headers, files=files,
+                                 timeout=timeout)
+        response.raise_for_status()
+        resp_json = response.json()
+        result = resp_json.get('data') if resp_json else None
+        return result or f"API返回成功，但没有有效的 'data' 内容。 响应: {response.text}"
+    except requests.RequestException as e:
+        return f"Error calling Gemini API: {str(e)}"
 
 
 class LLMUionNode:
-    """
-    一个聚合LLM节点。最终修复版，根据用户指正，彻底重构了执行逻辑，
-    确保代码的清晰、正确和稳定。
-    """
     MODELS = ['gemini-2.5-flash', 'gemini-2.5-pro', "gpt-4o-1120", "gpt-4.1"]
 
     ENVIRONMENTS = ["prod", "dev", "test"]
@@ -35,11 +56,13 @@ class LLMUionNode:
         return {
             "required": {
                 "model_name": (s.MODELS,),
-                "prompt": ("STRING", { "multiline": True, "default": "详细描述这个视频" }),
+                "prompt": ("STRING", {"multiline": True, "default": "", "placeholder": "请输入提示词"}),
             },
             "optional": {
-                "video_input": ("*",),
+                "video": ("*",),
                 "image": ("IMAGE",),
+                "audio": ("AUDIO",),
+                "url": ("STRING", {"multiline": True, "default": "", "placeholder": "【可选】输入要分析的链接"}),
                 "environment": (s.ENVIRONMENTS,),
                 "timeout": ("INT", {"default": 300, "min": 10, "max": 1200}),
             }
@@ -51,7 +74,8 @@ class LLMUionNode:
     CATEGORY = "不忘科技-自定义节点🚩/LLM"
 
     def tensor_to_pil(self, tensor):
-        if tensor is None: return None
+        if tensor is None:
+            return None
         image_np = tensor[0].cpu().numpy()
         image_np = (image_np * 255).astype(np.uint8)
         return Image.fromarray(image_np)
@@ -63,20 +87,39 @@ class LLMUionNode:
         pil_image.save(filepath, 'PNG')
         return filepath
 
-    # --- API 处理函数无需改变, 它们接收文件路径 ---
-    def handler_google_analytics(self, prompt: str, model_id: str, media_file_path: str, base_url: str, timeout: int):
-        headers = {'accept': 'application/json'}
-        files = {'prompt': (None, prompt), 'model_id': (None, model_id)}
-        if media_file_path and os.path.exists(media_file_path):
-            files['img_file'] = (os.path.basename(media_file_path), open(media_file_path, 'rb'), mimetypes.guess_type(media_file_path)[0] or 'application/octet-stream')
-        try:
-            response = requests.post(f'{base_url}/api/llm/google/analysis', headers=headers, files=files, timeout=timeout)
-            response.raise_for_status()
-            resp_json = response.json()
-            result = resp_json.get('data') if resp_json else None
-            return result or f"API返回成功，但没有有效的 'data' 内容。 响应: {response.text}"
-        except requests.RequestException as e:
-            return f"Error calling Gemini API: {str(e)}"
+    def save_link_file(self, link_url: str, is_google: bool = False):
+        def download_file(url):
+            suffix = url.rsplit('.', 1)[-1]
+            response = httpx.get(url, timeout=120)
+            output_dir = folder_paths.get_temp_directory()
+            (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_image",
+                                                                                             output_dir)
+            filepath = os.path.join(full_output_folder, f"{filename}_{counter:05}.{suffix}")
+            with open(filepath, 'wb') as f:
+                f.write(response.content)
+            return filepath
+
+        link_url = link_url.strip()
+        if is_google and link_url.startswith("gs:"):
+            return link_url
+        else:
+            return download_file(link_url)
+
+    def save_audio_tensor_to_temp(self, waveform_tensor, sample_rate):
+        if 'wavfile' not in globals():
+            raise ImportError("Scipy 库未安装。请在您的 ComfyUI 环境中运行 'pip install scipy' 来启用此功能。")
+        waveform_np = waveform_tensor.cpu().numpy()
+        if waveform_np.ndim == 3:
+            waveform_np = waveform_np[0]
+
+        waveform_np = waveform_np.T
+        waveform_int16 = np.int16(waveform_np * 32767)
+        output_dir = folder_paths.get_temp_directory()
+        (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_audio", output_dir)
+        filepath = os.path.join(full_output_folder, f"{filename}_{counter:05}.wav")
+        wavfile.write(filepath, sample_rate, waveform_int16)
+        print(f"音频张量已使用 Scipy 保存到临时文件: {filepath}")
+        return filepath
 
     def handler_other_llm(self, model_name: str, prompt: str, media_path: str, timeout: int):
         messages_content = [{"type": "text", "text": prompt}]
@@ -89,10 +132,14 @@ class LLMUionNode:
                 messages_content.append({"type": "image_url", "image_url": {"url": data_url}})
             except Exception as e:
                 return f"Error encoding media file: {str(e)}"
-        
-        json_payload = {"model": model_name, "messages": [{"role": "user", "content": messages_content}], "temperature": 0.7, "max_tokens": 4096}
+
+        json_payload = {"model": model_name, "messages": [{"role": "user", "content": messages_content}],
+                        "temperature": 0.7, "max_tokens": 4096}
         try:
-            resp = requests.post("https://gateway.bowong.cc/chat/completions", headers={"Content-Type": "application/json", "Authorization": "Bearer auth-bowong7777"}, json=json_payload, timeout=timeout)
+            resp = requests.post("https://gateway.bowong.cc/chat/completions",
+                                 headers={"Content-Type": "application/json",
+                                          "Authorization": "Bearer auth-bowong7777"}, json=json_payload,
+                                 timeout=timeout)
             resp.raise_for_status()
             resp_json = resp.json()
             if 'choices' in resp_json and resp_json['choices']:
@@ -101,68 +148,93 @@ class LLMUionNode:
                 return f'Call LLM failed: {resp_json.get("error", {}).get("message", "LLM API returned no choices.")}'
         except requests.RequestException as e:
             return f"Error calling other LLM API: {str(e)}"
-    
-    def execute(self, model_name: str, prompt: str, environment: str = "prod", 
-                video_input: object = None, image: torch.Tensor = None, timeout=300):
-        
+
+    def execute(self, model_name: str, prompt: str, environment: str = "prod",
+                video: object = None, image: torch.Tensor = None, audio: object = None,
+                url: str = None,
+                timeout=300):
         base_url = self.ENV_URLS.get(environment, self.ENV_URLS["prod"])
         media_path = None
-        
-        # --- **最终的、唯一的、正确的修复逻辑** ---
-        
-        # 优先级 1: 处理 video_input
-        if video_input is not None:
-            unwrapped_input = video_input[0] if isinstance(video_input, (list, tuple)) and video_input else video_input
-            
-            # 检查是否是支持 save_to() 的视频对象
+
+        if video is not None:
+            if 'gemini' not in model_name:
+                raise ValueError(f'{model_name}暂不支持视频分析,\n请使用gemini-2.5-flash或者gemini-2.5-pro')
+            print('多模态处理视频输入...')
+            unwrapped_input = video[0] if isinstance(video, (list, tuple)) and video else video
             if hasattr(unwrapped_input, 'save_to'):
                 try:
                     output_dir = folder_paths.get_temp_directory()
-                    (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_video", output_dir)
+                    (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_video",
+                                                                                                     output_dir)
                     temp_video_path = os.path.join(full_output_folder, f"{filename}_{counter:05}.mp4")
-
-                    print(f"检测到视频对象，使用 save_to() 保存到: {temp_video_path}")
                     unwrapped_input.save_to(temp_video_path)
-                    
                     if os.path.exists(temp_video_path):
                         media_path = temp_video_path
                     else:
                         return (f"错误: 调用 save_to() 后文件未成功创建。",)
                 except Exception as e:
                     return (f"调用 save_to() 时出错: {e}",)
-            
-            # 兼容处理字符串输入的情况
             elif isinstance(unwrapped_input, str):
-                filename = unwrapped_input
-                print(f"检测到字符串输入，作为文件名处理: '{filename}'")
+                full_path = folder_paths.get_full_path("input", unwrapped_input)
+                if full_path and os.path.exists(full_path):
+                    media_path = full_path
+                else:
+                    return (f"错误: 无法在 'input' 文件夹中找到文件 '{unwrapped_input}'。",)
+
+        elif image is not None:
+            print('多模态处理图片输出...')
+            pil_image = self.tensor_to_pil(image)
+            media_path = self.save_pil_to_temp(pil_image)
+
+        elif audio is not None:
+            if 'gemini' not in model_name:
+                raise ValueError(f'{model_name}暂不支持音频分析,\n请使用gemini-2.5-flash或者gemini-2.5-pro')
+            print("多模态处理音频输入...")
+            audio_info = audio[0] if isinstance(audio, (list, tuple)) and audio else audio
+
+            if isinstance(audio_info, dict) and 'filename' in audio_info:
+                filename = audio_info['filename']
+                print(f"从音频对象中找到 'filename': '{filename}'")
                 full_path = folder_paths.get_full_path("input", filename)
                 if full_path and os.path.exists(full_path):
                     media_path = full_path
                 else:
                     return (f"错误: 无法在 'input' 文件夹中找到文件 '{filename}'。",)
 
-        # 优先级 2: 如果没有处理 video_input，再处理 image
-        elif image is not None:
-            print("检测到图像输入, 正在保存为临时文件...")
-            pil_image = self.tensor_to_pil(image)
-            media_path = self.save_pil_to_temp(pil_image)
-        
-        # 优先级 3: 纯文本模式
-        else:
-             print("未提供媒体文件, 以纯文本模式运行。")
-        
-        if media_path:
-             print(f"成功解析媒体文件路径: {media_path}")
+            elif isinstance(audio_info, dict) and 'waveform' in audio_info and 'sample_rate' in audio_info:
+                print("从音频对象中找到 'waveform' 数据，正在使用 Scipy 保存为临时文件...")
+                try:
+                    media_path = self.save_audio_tensor_to_temp(audio_info['waveform'], audio_info['sample_rate'])
+                except Exception as e:
+                    return (f"错误: 保存音频张量时出错: {e}",)
 
-        # 分发到 API handlers
+            elif isinstance(audio_info, str):
+                print(f"检测到音频输入为字符串，作为文件名处理: '{audio_info}'")
+                full_path = folder_paths.get_full_path("input", audio_info)
+                if full_path and os.path.exists(full_path):
+                    media_path = full_path
+                else:
+                    return (f"错误: 无法在 'input' 文件夹中找到文件 '{audio_info}'。",)
+
+            else:
+                return (f"错误: 不支持的音频输入格式或结构。收到类型: {type(audio_info)}",)
+        elif url is not None:
+            url = url.strip()
+            model_name = model_name.strip()
+            is_google = model_name.startswith('gemini')
+            media_path = self.save_link_file(link_url=url, is_google=is_google)
+        else:
+            print("纯文本运行llm")
+        if media_path:
+            print(f"成功解析媒体文件路径: {media_path}")
         model_name = model_name.strip()
         if model_name.startswith('gemini'):
-            result = self.handler_google_analytics(prompt, model_name, media_path, base_url=base_url, timeout=timeout)
+            result = handler_google_analytics(prompt, model_name, media_path, base_url=base_url, timeout=timeout)
         else:
             result = self.handler_other_llm(model_name, prompt, media_path, timeout=timeout)
 
         return (result,)
 
-
-# NODE_CLASS_MAPPINGS = { "LLMUionNode": LLMUionNode }
-# NODE_DISPLAY_NAME_MAPPINGS = { "LLMUionNode": "聚合LLM节点(视频/图像)" }
+# 节点映射
+# NODE_CLASS_MAPPINGS = {"LLMUionNode": LLMUionNode}
+# NODE_DISPLAY_NAME_MAPPINGS = {"LLMUionNode": "聚合LLM节点(视频/图像/音频)"}

nodes/video_agent.py

@@ -118,7 +118,7 @@ class VideoSubmitNode:
         return {
             "required": {
                 "model_name_display": (cls.MODEL_DATA["full_display_list"],),
-                "prompt": ("STRING", {"multiline": True, "default": ""}),
+                "prompt": ("STRING", {"multiline": True, "default": "","placeholder": "请输入提示词"}),
                 "aspect_ratio": ("STRING", {"multiline": False, "default": "9:16"}),
                 "duration": ("STRING", {"multiline": False, "default": "5"}),
                 "resolution": ("STRING", {"multiline": False, "default": "720p"}),
@@ -153,7 +153,7 @@ class VideoSubmitNode:
         file_name = f'{time.time_ns()}.png'
         mime_type = 'image/png'
         files = {'file': (file_name, buffer, mime_type)}
-        response = requests.post(f'{base_url}/api/file/upload/s3', headers={'accept': 'application/json'}, files=files);
+        response = requests.post(f'{base_url}/api/file/upload/s3', headers={'accept': 'application/json'}, files=files)
         response.raise_for_status()
         resp_json = response.json()
         if resp_json.get('status'):
@@ -209,7 +209,8 @@ class VideoSubmitNode:
         try:
             base_url, tech_name = self._get_base_url_and_tech_name(environment, model_name_display)
             model_config = self.MODEL_DATA["configs"].get(tech_name)
-            if not model_config: raise ValueError(f"无法找到模型 '{tech_name}' 的配置。")
+            if not model_config:
+                raise ValueError(f"无法找到模型 '{tech_name}' 的配置。")
 
             is_frame_model = tech_name.startswith('frame/')
 

nodes/video_preview.py

@@ -54,7 +54,8 @@ class VideoDownloaderNode:
                 try:
                     parsed_url = urllib.parse.urlparse(url)
                     filename = os.path.basename(parsed_url.path)
-                    if not filename: raise ValueError
+                    if not filename:
+                        raise ValueError
                 except (ValueError, AttributeError):
                     content_type = response.headers.get('content-type')
                     ext = '.mp4'

requirements.txt

@@ -16,4 +16,5 @@ retry
 pyYAML
 boto3
 Jinja2
-httpx
+httpx
+scipy