import re import uuid import modal from ..video import downloader_image, app, config with downloader_image.imports(): import os, httpx, json, time, requests from typing import List from loguru import logger from modal import current_function_call_id from ffmpy import FFmpeg, FFprobe from fastapi import HTTPException from httpx import Timeout from starlette import status import asyncio import random import subprocess from BowongModalFunctions.utils.SentryUtils import SentryUtils from BowongModalFunctions.models.media_model import MediaSource from BowongModalFunctions.models.web_model import SentryTransactionInfo, WebhookNotify from BowongModalFunctions.models.ffmpeg_worker_model import FFMpegSliceSegment from BowongModalFunctions.utils.TimeUtils import TimeDelta, merge_product_data from BowongModalFunctions.utils.VideoUtils import FFMPEGSliceOptions, VideoUtils @app.function(cpu=(0.5, 64), timeout=1800, max_containers=config.video_downloader_concurrency, secrets=[modal.Secret.from_name("gemini-prompt")], volumes={ config.S3_mount_dir: modal.CloudBucketMount( bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment), ), }, region="us", cloud='gcp' ) @modal.concurrent(max_inputs=1) async def video_hls_slice_inference(media: MediaSource, google_api_key: str, product_grid_list: List[str], product_list: List[any], start_time: str, end_time: str, options: FFMPEGSliceOptions, sentry_trace: SentryTransactionInfo, webhook: WebhookNotify = None, retry_time: int = 3, scale:float = 0.9 ): logger.info( f"region {os.environ.get('MODAL_REGION', 'unknown')}, provider {os.environ.get('MODAL_CLOUD_PROVIDER', 'unknown')}") # Gemini可用区 gemini_region = [ #"us-central1", "us-east1", "us-east5", "us-west1", "us-south1", "europe-central2", "europe-north1", "europe-west1", "europe-west4", "europe-west8", ] logger.info(f"\nmedia || {media.urn}," f"\nstart_time || {start_time}, " f"\nend_time || {end_time}, " f"\nproduct_grid_list || {json.dumps(product_grid_list, ensure_ascii=False, indent=2)}, " f"\nproduct_list || {json.dumps(product_list, ensure_ascii=False, indent=2)}" f"\noptions || {options.model_dump_json()}, " f"\nscale || {scale}") # 动态Prompt if "IMAGE_PRODUCT_IDENTIFICATION_PROMPT" in os.environ: IMAGE_PRODUCT_IDENTIFICATION_PROMPT = os.environ["IMAGE_PRODUCT_IDENTIFICATION_PROMPT"] else: IMAGE_PRODUCT_IDENTIFICATION_PROMPT = """ 你是专业的商品识别专家。我上传了商品图片网格,需要你识别图片中的商品并与商品列表进行匹配。 **输入材料**: - 🖼️ **商品图片网格**:包含多个黑色边框区域,每个区域内有商品图片+商品名称文字 - 📋 **商品列表**:标准商品名称参考清单 **核心任务**: 1. **扫描黑色边框区域**:从左上角开始,按行扫描每个黑色边框区域 2. **提取文字信息**:精确提取每个区域内的所有文字信息 3. **与商品列表匹配**:将图片文字与商品列表进行高相似度匹配 4. **提取商品图片特征**:从商品图片提取详细可识别特征,包括颜色、图案、纹理、材质、版型、款式等 **严格约束**: - 🚫 只识别有黑色边框包围的商品区域 - 🚫 每个商品必须有清晰可见的文字标注 - 🚫 不得推测或添加图片中不存在的商品 - ✅ 输出商品数量不得超过图片中的黑色边框区域数量 **商品列表**: {PRODUCT_LIST} 请按以下JSON格式输出: [ {{ "image_order": 1, "image_name": "图片上显示的原始文字", "matched_product": "匹配到的标准商品名称或null", "match_confidence": 95, "visual_features": {{ "color": "商品详细颜色配色等有辨识度的色彩特征", "pattern": "商品详细图案纹理等有辨识度的材质特征", "style": "商品详细款式版型等有辨识度的风格特征" }} }} ] """ if "VIDEO_TIMELINE_ANALYSIS_PROMPT" in os.environ: VIDEO_TIMELINE_ANALYSIS_PROMPT = os.environ["VIDEO_TIMELINE_ANALYSIS_PROMPT"] else: VIDEO_TIMELINE_ANALYSIS_PROMPT = """ 基于已识别的商品清单,分析视频中每个商品的出现时间段。 **输入材料**: - 📹 **视频**:直播带货片段 - 📋 **已识别商品清单**:第一阶段确认的商品列表 **分析任务**: 1. **时间段识别**:找出每个商品在视频中的展示时间 2. **内容类型分类**:判断每个时间段的具体内容类型 3. **时间段合并**:合并连续或相近的时间段 **内容类型**: - (穿着本品+介绍本品) - 主播穿着该商品且正在介绍该商品本身 - (穿着本品+他品) - 主播穿着该商品但正在介绍其他商品 - (穿着本品+无关) - 主播穿着该商品但在做无关商品的事情 **时间格式标准**: - 必须严格使用格式:HH:MM:SS.mmm - HH:MM:SS.mmm - HH是小时(00-23),MM是分钟(00-59),SS是秒(00-59),mmm是毫秒(000-999) - 示例:00:05:23.500 表示0小时5分钟23.5秒,不是5小时23分钟 - 常见错误:不要将05:23:500解释为5小时23分钟,应该是0小时5分钟23.5秒 **已识别商品清单(product-商品名 feature-商品特征 feature.color-商品详细颜色配色等色彩特征 feature.pattern-商品详细图案纹理等材质特征 feature.style-商品详细款式版型等风格特征)**: {IDENTIFIED_PRODUCTS} 请按以下JSON格式输出: [ {{ "product": "商品名称", "timeline": [ "00:01:15.200 - 00:02:30.800 (穿着本品+介绍本品)", "00:05:10.100 - 00:06:25.600 (穿着本品+他品)" ] }} ] """ def upload(file_path): with open(file_path, "rb") as video: video = video.read() content_length = len(video) content_type = f"video/{file_path.split('.')[-1]}" filename = ".".join([str(uuid.uuid4()), file_path.split(".")[-1]]) if content_type not in ['video/mp4', 'video/mpeg', 'video/mov', 'video/avi', 'video/x-flv', 'video/mpg', 'video/webm', 'video/wmv', 'video/3gpp']: raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE) logger.info( f"Uploading name = {filename}, size = {content_length}, type = {content_type} to google file") with httpx.Client(timeout=1800) as client: upload_response = client.post( url=f"https://storage.googleapis.com/upload/storage/v1/b/dy-media-storage/o?uploadType=media&name=videos%2F{filename}", content=video, headers={ "Authorization": f"Bearer {google_api_key}", "Content-Type": content_type }) upload_response.raise_for_status() upload_url = f"gs://dy-media-storage/videos/{filename}" return upload_url, upload_response.status_code def parse_stage1_result(result_text): """解析第一阶段结果,提取已识别的商品列表""" try: # 清理结果文本,提取JSON部分 clean_result = result_text.strip() if clean_result.startswith('```') and clean_result.endswith('```'): clean_result = clean_result[3:-3].strip() if clean_result.startswith('json'): clean_result = clean_result[4:].strip() # 解析JSON stage1_data = json.loads(clean_result) identified_products = [] if isinstance(stage1_data, list): for item in stage1_data: if isinstance(item, dict) and 'matched_product' in item: matched_product = item['matched_product'] if matched_product and matched_product != "null": identified_products.append({"product": matched_product, "feature": item["visual_features"]}) logger.info(f"一阶段解析结果: {len(identified_products)} 个有效商品") return identified_products except Exception as e: logger.exception(f"一阶段结果解析失败 {str(e)}") raise e @SentryUtils.webhook_handler(webhook, current_function_call_id()) @SentryUtils.sentry_tracker(sentry_trace.x_trace_id, sentry_trace.x_baggage, op="inference_gemini", name="Gemini推理", fn_id=current_function_call_id()) async def _handler(media: MediaSource, google_api_key: str, product_grid_list: List[str], product_list: List[any], start_time: str, end_time: str, options: FFMPEGSliceOptions, sentry_trace: SentryTransactionInfo, retry_time: int = 3, scale:float = 0.9): video_gemini_uri = None try: # 1、首先获取全量商品列表 logger.info("1、获取直播间商品列表") if len(product_list) == 0: logger.error("商品列表为空, 退出推理") raise Exception("商品列表为空, 退出推理") product_title_list = [] if isinstance(product_list[0], dict): for product in product_list: product_title_list.append(product["title"]) else: product_title_list = product_list # 2、切20分钟的条 logger.info("2、开始截取指定视频") # 计算缩放尺寸 metadata = VideoUtils.ffprobe_media_metadata(media.path) width = int(metadata.streams[0].width * scale) height = int(metadata.streams[0].height * scale) if width % 2 == 1: width = width + 1 if height % 2 == 1: height = height + 1 options.width = width options.height = height slice_fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_slice_media", environment_name=config.modal_environment) slice_result, sentry_trace = await slice_fn.remote.aio(media, [FFMpegSliceSegment( start=TimeDelta.from_format_string(start_time), end=TimeDelta.from_format_string(end_time))], options, sentry_trace) video = MediaSource.from_str(slice_result[0].urn) logger.success("截取完成") video_path = os.path.join(config.S3_mount_dir, video.path) # 3、上传到gemini logger.info("3、视频文件开始上传到Gemini") video_gemini, code = upload(video_path) if code == 200: video_gemini_uri = video_gemini else: logger.error("视频文件上传Gemini失败") raise Exception("视频文件上传Gemini失败") async def inference_api_first_stage(): try: logger.info("🎈开始一阶段推理") product_list_str = "" for i, product in enumerate(product_title_list): product_list_str += f"{i}. {product}\n" image_parts = [] for i in product_grid_list: image_parts.append( { "file_data": { "mime_type": "image/jpeg", "file_uri": f"{i}" } } ) image_parts.append( { "text": IMAGE_PRODUCT_IDENTIFICATION_PROMPT.format(PRODUCT_LIST=product_list_str) } ) json_data = { "safetySettings": [ {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_CIVIC_INTEGRITY", "threshold": "BLOCK_NONE"} ], "contents": [ { "role": "user", "parts": image_parts } ], "generationConfig": { "temperature": 0.01, "top_p": 0.7 } } resp = requests.post( f"https://gateway.ai.cloudflare.com/v1/67720b647ff2b55cf37ba3ef9e677083/bowong-dev/google-vertex-ai/v1/projects/gen-lang-client-0413414134/locations/{random.choice(gemini_region)}/publishers/google/models/gemini-2.5-flash:generateContent", headers={ "Content-Type": "application/json", "Authorization": f"Bearer {google_api_key}" }, json=json_data, timeout=900 ) except Exception as e: logger.exception(f"😭Gemini一阶段推理请求失败: {e}") raise e if resp.status_code == 200: target_json = resp.json() reason = target_json["candidates"][0]["finishReason"] if "usageMetadata" in target_json: logger.info("😊一阶段用量信息"+json.dumps(target_json["usageMetadata"], indent=4)) if reason == "STOP": result_text: str = target_json["candidates"][0]["content"]["parts"][0]["text"] # 解析识别结果 identified_products = parse_stage1_result(result_text) return identified_products else: logger.error(f"😭Gemini一阶段推理失败, Reason {reason}") return None else: logger.error(f"😭Gemini一阶段推理失败, 状态码{resp.status_code}") if resp.status_code == 429: logger.warning("🥵请求负载过高, 随机更换地区重试") return None async def inference_api_second_stage(identified_products): try: logger.info("🎈开始二阶段推理") # 构建已识别商品清单字符串 identified_products_str = "" for i, product in enumerate(identified_products): identified_products_str += f"{i}. {json.dumps(product,ensure_ascii=False)}\n" image_parts = [] image_parts.append( { "file_data": { "mime_type": "video/mp4", "file_uri": f"{video_gemini_uri}" } }) image_parts.append( { "text": VIDEO_TIMELINE_ANALYSIS_PROMPT.format(IDENTIFIED_PRODUCTS=identified_products_str) } ) json_data = { "safetySettings": [ {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, {"category": "HARM_CATEGORY_CIVIC_INTEGRITY", "threshold": "BLOCK_NONE"} ], "contents": [ { "role": "user", "parts": image_parts } ], "generationConfig": { "temperature": 0.01, "top_p": 0.7 } } resp = requests.post( f"https://gateway.ai.cloudflare.com/v1/67720b647ff2b55cf37ba3ef9e677083/bowong-dev/google-vertex-ai/v1/projects/gen-lang-client-0413414134/locations/{random.choice(gemini_region)}/publishers/google/models/gemini-2.5-flash:generateContent", headers={ "Content-Type": "application/json", "Authorization": f"Bearer {google_api_key}" }, json=json_data, timeout=900 ) except Exception as e: logger.exception(f"😭Gemini二阶段推理请求失败: {e}") raise e if resp.status_code == 200: target_json = resp.json() reason = target_json["candidates"][0]["finishReason"] if "usageMetadata" in target_json: logger.info("😊二阶段用量信息"+json.dumps(target_json["usageMetadata"], indent=4)) if reason == "STOP": result_text: str = target_json["candidates"][0]["content"]["parts"][0]["text"] if len(result_text)>10: parts = result_text.split("```")[-2].replace("```", "").replace("json\n", "").replace("\n", "").replace("\\", "") parts = json.loads(parts) logger.info(f"👌合并前 {json.dumps(parts, indent=4, ensure_ascii=False)}") # 合并产品和时间线 parts = merge_product_data(parts, start_time, end_time, merge_diff=5) for part in parts: part["product"] = re.sub(r'^\x20*\d+\.\x20*', '', part["product"]) return parts else: return [] else: logger.error(f"😭Gemini二阶段推理失败, Reason {reason}") return None else: logger.error(f"😭Gemini二阶段推理失败, 状态码{resp.status_code}") if resp.status_code == 429: logger.warning("🥵请求负载过高, 随机更换地区重试") return None logger.info("4、发起Gemini推理") # 一阶段推理 product_info = None first_retry_time = retry_time while product_info is None and first_retry_time > 0: product_info = await inference_api_first_stage() first_retry_time -= 1 if product_info is None: raise Exception("一阶段推理失败") else: logger.info(f"一阶段推理完成JSON \n{json.dumps(product_info, indent=4, ensure_ascii=False)}") # 二阶段推理 product_timeline_info = None second_retry_time = retry_time while product_timeline_info is None and second_retry_time > 0: product_timeline_info = await inference_api_second_stage(product_info) second_retry_time -= 1 if product_timeline_info is None: raise Exception("二阶段推理失败") else: logger.info(f"二阶段推理完成JSON \n{json.dumps(product_timeline_info, indent=4, ensure_ascii=False)}") return product_timeline_info, sentry_trace except Exception as e: logger.exception(f"推理失败, {e}") raise Exception(f"推理失败, {e}") return await _handler(media, google_api_key, product_grid_list, product_list, start_time, end_time, options, sentry_trace, retry_time, scale)