diff --git a/.gitignore b/.gitignore index bc0e83b..48b6d39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ model/*.pth model/*.pt !model/.gitkeep -.yaml \ No newline at end of file +.yaml +venv \ No newline at end of file diff --git a/ext/video_agent_deploy.py b/ext/video_agent_deploy.py index ec2fba0..44809c9 100644 --- a/ext/video_agent_deploy.py +++ b/ext/video_agent_deploy.py @@ -17,7 +17,7 @@ image = ( .apt_install("git", "gcc", "libportaudio2", "ffmpeg") .pip_install("comfy_cli") .run_commands( - "comfy --skip-prompt install --fast-deps --nvidia --version 0.3.40" + "comfy --skip-prompt install --fast-deps --nvidia --version 0.3.55" ) .pip_install_from_pyproject(os.path.join(os.path.dirname(__file__), "pyproject.toml")) .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use.git") @@ -75,37 +75,3 @@ def ui_1(): @modal.web_server(8000, startup_timeout=120) def ui_2(): subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True) - -# @app.function( -# min_containers=0, -# buffer_containers=0, -# max_containers=1, -# scaledown_window=600, -# secrets=[custom_secret], -# volumes={ -# "/models": vol -# } -# ) -# @modal.concurrent( -# max_inputs=10 -# ) -# @modal.web_server(8000, startup_timeout=120) -# def ui_3(): -# subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True) -# -# @app.function( -# min_containers=0, -# buffer_containers=0, -# max_containers=1, -# scaledown_window=600, -# secrets=[custom_secret], -# volumes={ -# "/models": vol -# } -# ) -# @modal.concurrent( -# max_inputs=10 -# ) -# @modal.web_server(8000, startup_timeout=120) -# def ui_4(): -# subprocess.Popen("comfy launch -- --cpu --listen 0.0.0.0 --port 8000", shell=True) \ No newline at end of file diff --git a/nodes/fetch_task_result.py b/nodes/fetch_task_result.py index 2e9d298..efb9ea4 100644 --- a/nodes/fetch_task_result.py +++ b/nodes/fetch_task_result.py @@ -73,7 +73,7 @@ class FetchTaskResult: raw_response_str = json.dumps(data_, indent=2, ensure_ascii=False) print(f"处理完成: {len(image_tensors)} 个图像, {len(video_urls)} 个视频URL。") - return (final_images, final_urls, raw_response_str) + return final_images, final_urls, raw_response_str print(f"任务未完成。API返回状态: {api_status}。将在 {interval} 秒后重试...") time.sleep(interval) diff --git a/nodes/save_node.py b/nodes/save_node.py index cc70fc9..40f2365 100644 --- a/nodes/save_node.py +++ b/nodes/save_node.py @@ -4,7 +4,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from urllib.parse import urlparse -# 导入 ComfyUI 的路径管理器 import folder_paths import requests import torch diff --git a/nodes/union_llm_node.py b/nodes/union_llm_node.py index 45dbfd4..8a48188 100644 --- a/nodes/union_llm_node.py +++ b/nodes/union_llm_node.py @@ -9,18 +9,39 @@ import requests import base64 import mimetypes import torch +import httpx import numpy as np from PIL import Image import folder_paths -tensor_to_file_map = {} +try: + import scipy.io.wavfile as wavfile +except ImportError: + print("------------------------------------------------------------------------------------") + print("Scipy 库未安装, 请运行: pip install scipy") + print("------------------------------------------------------------------------------------") + + +def handler_google_analytics(prompt: str, model_id: str, media_file_path: str, base_url: str, timeout: int): + headers = {'accept': 'application/json'} + files = {'prompt': (None, prompt), 'model_id': (None, model_id)} + if media_file_path and os.path.exists(media_file_path): + files['img_file'] = (os.path.basename(media_file_path), open(media_file_path, 'rb'), + mimetypes.guess_type(media_file_path)[0] or 'application/octet-stream') + if media_file_path.startswith("gs:"): + files['img_url'] = (None, media_file_path) + try: + response = requests.post(f'{base_url}/api/llm/google/analysis', headers=headers, files=files, + timeout=timeout) + response.raise_for_status() + resp_json = response.json() + result = resp_json.get('data') if resp_json else None + return result or f"API返回成功，但没有有效的 'data' 内容。 响应: {response.text}" + except requests.RequestException as e: + return f"Error calling Gemini API: {str(e)}" class LLMUionNode: - """ - 一个聚合LLM节点。最终修复版，根据用户指正，彻底重构了执行逻辑， - 确保代码的清晰、正确和稳定。 - """ MODELS = ['gemini-2.5-flash', 'gemini-2.5-pro', "gpt-4o-1120", "gpt-4.1"] ENVIRONMENTS = ["prod", "dev", "test"] @@ -35,11 +56,13 @@ class LLMUionNode: return { "required": { "model_name": (s.MODELS,), - "prompt": ("STRING", { "multiline": True, "default": "详细描述这个视频" }), + "prompt": ("STRING", {"multiline": True, "default": "", "placeholder": "请输入提示词"}), }, "optional": { - "video_input": ("*",), + "video": ("*",), "image": ("IMAGE",), + "audio": ("AUDIO",), + "url": ("STRING", {"multiline": True, "default": "", "placeholder": "【可选】输入要分析的链接"}), "environment": (s.ENVIRONMENTS,), "timeout": ("INT", {"default": 300, "min": 10, "max": 1200}), } @@ -51,7 +74,8 @@ class LLMUionNode: CATEGORY = "不忘科技-自定义节点🚩/LLM" def tensor_to_pil(self, tensor): - if tensor is None: return None + if tensor is None: + return None image_np = tensor[0].cpu().numpy() image_np = (image_np * 255).astype(np.uint8) return Image.fromarray(image_np) @@ -63,20 +87,39 @@ class LLMUionNode: pil_image.save(filepath, 'PNG') return filepath - # --- API 处理函数无需改变, 它们接收文件路径 --- - def handler_google_analytics(self, prompt: str, model_id: str, media_file_path: str, base_url: str, timeout: int): - headers = {'accept': 'application/json'} - files = {'prompt': (None, prompt), 'model_id': (None, model_id)} - if media_file_path and os.path.exists(media_file_path): - files['img_file'] = (os.path.basename(media_file_path), open(media_file_path, 'rb'), mimetypes.guess_type(media_file_path)[0] or 'application/octet-stream') - try: - response = requests.post(f'{base_url}/api/llm/google/analysis', headers=headers, files=files, timeout=timeout) - response.raise_for_status() - resp_json = response.json() - result = resp_json.get('data') if resp_json else None - return result or f"API返回成功，但没有有效的 'data' 内容。 响应: {response.text}" - except requests.RequestException as e: - return f"Error calling Gemini API: {str(e)}" + def save_link_file(self, link_url: str, is_google: bool = False): + def download_file(url): + suffix = url.rsplit('.', 1)[-1] + response = httpx.get(url, timeout=120) + output_dir = folder_paths.get_temp_directory() + (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_image", + output_dir) + filepath = os.path.join(full_output_folder, f"{filename}_{counter:05}.{suffix}") + with open(filepath, 'wb') as f: + f.write(response.content) + return filepath + + link_url = link_url.strip() + if is_google and link_url.startswith("gs:"): + return link_url + else: + return download_file(link_url) + + def save_audio_tensor_to_temp(self, waveform_tensor, sample_rate): + if 'wavfile' not in globals(): + raise ImportError("Scipy 库未安装。请在您的 ComfyUI 环境中运行 'pip install scipy' 来启用此功能。") + waveform_np = waveform_tensor.cpu().numpy() + if waveform_np.ndim == 3: + waveform_np = waveform_np[0] + + waveform_np = waveform_np.T + waveform_int16 = np.int16(waveform_np * 32767) + output_dir = folder_paths.get_temp_directory() + (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_audio", output_dir) + filepath = os.path.join(full_output_folder, f"{filename}_{counter:05}.wav") + wavfile.write(filepath, sample_rate, waveform_int16) + print(f"音频张量已使用 Scipy 保存到临时文件: {filepath}") + return filepath def handler_other_llm(self, model_name: str, prompt: str, media_path: str, timeout: int): messages_content = [{"type": "text", "text": prompt}] @@ -89,10 +132,14 @@ class LLMUionNode: messages_content.append({"type": "image_url", "image_url": {"url": data_url}}) except Exception as e: return f"Error encoding media file: {str(e)}" - - json_payload = {"model": model_name, "messages": [{"role": "user", "content": messages_content}], "temperature": 0.7, "max_tokens": 4096} + + json_payload = {"model": model_name, "messages": [{"role": "user", "content": messages_content}], + "temperature": 0.7, "max_tokens": 4096} try: - resp = requests.post("https://gateway.bowong.cc/chat/completions", headers={"Content-Type": "application/json", "Authorization": "Bearer auth-bowong7777"}, json=json_payload, timeout=timeout) + resp = requests.post("https://gateway.bowong.cc/chat/completions", + headers={"Content-Type": "application/json", + "Authorization": "Bearer auth-bowong7777"}, json=json_payload, + timeout=timeout) resp.raise_for_status() resp_json = resp.json() if 'choices' in resp_json and resp_json['choices']: @@ -101,68 +148,93 @@ class LLMUionNode: return f'Call LLM failed: {resp_json.get("error", {}).get("message", "LLM API returned no choices.")}' except requests.RequestException as e: return f"Error calling other LLM API: {str(e)}" - - def execute(self, model_name: str, prompt: str, environment: str = "prod", - video_input: object = None, image: torch.Tensor = None, timeout=300): - + + def execute(self, model_name: str, prompt: str, environment: str = "prod", + video: object = None, image: torch.Tensor = None, audio: object = None, + url: str = None, + timeout=300): base_url = self.ENV_URLS.get(environment, self.ENV_URLS["prod"]) media_path = None - - # --- **最终的、唯一的、正确的修复逻辑** --- - - # 优先级 1: 处理 video_input - if video_input is not None: - unwrapped_input = video_input[0] if isinstance(video_input, (list, tuple)) and video_input else video_input - - # 检查是否是支持 save_to() 的视频对象 + + if video is not None: + if 'gemini' not in model_name: + raise ValueError(f'{model_name}暂不支持视频分析,\n请使用gemini-2.5-flash或者gemini-2.5-pro') + print('多模态处理视频输入...') + unwrapped_input = video[0] if isinstance(video, (list, tuple)) and video else video if hasattr(unwrapped_input, 'save_to'): try: output_dir = folder_paths.get_temp_directory() - (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_video", output_dir) + (full_output_folder, filename, counter, _, _) = folder_paths.get_save_image_path("llm_temp_video", + output_dir) temp_video_path = os.path.join(full_output_folder, f"{filename}_{counter:05}.mp4") - - print(f"检测到视频对象，使用 save_to() 保存到: {temp_video_path}") unwrapped_input.save_to(temp_video_path) - if os.path.exists(temp_video_path): media_path = temp_video_path else: return (f"错误: 调用 save_to() 后文件未成功创建。",) except Exception as e: return (f"调用 save_to() 时出错: {e}",) - - # 兼容处理字符串输入的情况 elif isinstance(unwrapped_input, str): - filename = unwrapped_input - print(f"检测到字符串输入，作为文件名处理: '{filename}'") + full_path = folder_paths.get_full_path("input", unwrapped_input) + if full_path and os.path.exists(full_path): + media_path = full_path + else: + return (f"错误: 无法在 'input' 文件夹中找到文件 '{unwrapped_input}'。",) + + elif image is not None: + print('多模态处理图片输出...') + pil_image = self.tensor_to_pil(image) + media_path = self.save_pil_to_temp(pil_image) + + elif audio is not None: + if 'gemini' not in model_name: + raise ValueError(f'{model_name}暂不支持音频分析,\n请使用gemini-2.5-flash或者gemini-2.5-pro') + print("多模态处理音频输入...") + audio_info = audio[0] if isinstance(audio, (list, tuple)) and audio else audio + + if isinstance(audio_info, dict) and 'filename' in audio_info: + filename = audio_info['filename'] + print(f"从音频对象中找到 'filename': '{filename}'") full_path = folder_paths.get_full_path("input", filename) if full_path and os.path.exists(full_path): media_path = full_path else: return (f"错误: 无法在 'input' 文件夹中找到文件 '{filename}'。",) - # 优先级 2: 如果没有处理 video_input，再处理 image - elif image is not None: - print("检测到图像输入, 正在保存为临时文件...") - pil_image = self.tensor_to_pil(image) - media_path = self.save_pil_to_temp(pil_image) - - # 优先级 3: 纯文本模式 - else: - print("未提供媒体文件, 以纯文本模式运行。") - - if media_path: - print(f"成功解析媒体文件路径: {media_path}") + elif isinstance(audio_info, dict) and 'waveform' in audio_info and 'sample_rate' in audio_info: + print("从音频对象中找到 'waveform' 数据，正在使用 Scipy 保存为临时文件...") + try: + media_path = self.save_audio_tensor_to_temp(audio_info['waveform'], audio_info['sample_rate']) + except Exception as e: + return (f"错误: 保存音频张量时出错: {e}",) - # 分发到 API handlers + elif isinstance(audio_info, str): + print(f"检测到音频输入为字符串，作为文件名处理: '{audio_info}'") + full_path = folder_paths.get_full_path("input", audio_info) + if full_path and os.path.exists(full_path): + media_path = full_path + else: + return (f"错误: 无法在 'input' 文件夹中找到文件 '{audio_info}'。",) + + else: + return (f"错误: 不支持的音频输入格式或结构。收到类型: {type(audio_info)}",) + elif url is not None: + url = url.strip() + model_name = model_name.strip() + is_google = model_name.startswith('gemini') + media_path = self.save_link_file(link_url=url, is_google=is_google) + else: + print("纯文本运行llm") + if media_path: + print(f"成功解析媒体文件路径: {media_path}") model_name = model_name.strip() if model_name.startswith('gemini'): - result = self.handler_google_analytics(prompt, model_name, media_path, base_url=base_url, timeout=timeout) + result = handler_google_analytics(prompt, model_name, media_path, base_url=base_url, timeout=timeout) else: result = self.handler_other_llm(model_name, prompt, media_path, timeout=timeout) return (result,) - -# NODE_CLASS_MAPPINGS = { "LLMUionNode": LLMUionNode } -# NODE_DISPLAY_NAME_MAPPINGS = { "LLMUionNode": "聚合LLM节点(视频/图像)" } \ No newline at end of file +# 节点映射 +# NODE_CLASS_MAPPINGS = {"LLMUionNode": LLMUionNode} +# NODE_DISPLAY_NAME_MAPPINGS = {"LLMUionNode": "聚合LLM节点(视频/图像/音频)"} diff --git a/nodes/video_agent.py b/nodes/video_agent.py index f168571..99090e9 100644 --- a/nodes/video_agent.py +++ b/nodes/video_agent.py @@ -118,7 +118,7 @@ class VideoSubmitNode: return { "required": { "model_name_display": (cls.MODEL_DATA["full_display_list"],), - "prompt": ("STRING", {"multiline": True, "default": ""}), + "prompt": ("STRING", {"multiline": True, "default": "","placeholder": "请输入提示词"}), "aspect_ratio": ("STRING", {"multiline": False, "default": "9:16"}), "duration": ("STRING", {"multiline": False, "default": "5"}), "resolution": ("STRING", {"multiline": False, "default": "720p"}), @@ -153,7 +153,7 @@ class VideoSubmitNode: file_name = f'{time.time_ns()}.png' mime_type = 'image/png' files = {'file': (file_name, buffer, mime_type)} - response = requests.post(f'{base_url}/api/file/upload/s3', headers={'accept': 'application/json'}, files=files); + response = requests.post(f'{base_url}/api/file/upload/s3', headers={'accept': 'application/json'}, files=files) response.raise_for_status() resp_json = response.json() if resp_json.get('status'): @@ -209,7 +209,8 @@ class VideoSubmitNode: try: base_url, tech_name = self._get_base_url_and_tech_name(environment, model_name_display) model_config = self.MODEL_DATA["configs"].get(tech_name) - if not model_config: raise ValueError(f"无法找到模型 '{tech_name}' 的配置。") + if not model_config: + raise ValueError(f"无法找到模型 '{tech_name}' 的配置。") is_frame_model = tech_name.startswith('frame/') diff --git a/nodes/video_preview.py b/nodes/video_preview.py index 57e68a3..536d45d 100644 --- a/nodes/video_preview.py +++ b/nodes/video_preview.py @@ -54,7 +54,8 @@ class VideoDownloaderNode: try: parsed_url = urllib.parse.urlparse(url) filename = os.path.basename(parsed_url.path) - if not filename: raise ValueError + if not filename: + raise ValueError except (ValueError, AttributeError): content_type = response.headers.get('content-type') ext = '.mp4' diff --git a/requirements.txt b/requirements.txt index f5bff63..4190206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ retry pyYAML boto3 Jinja2 -httpx \ No newline at end of file +httpx +scipy \ No newline at end of file