diff --git a/.env b/.env deleted file mode 100644 index 9ac7c27..0000000 --- a/.env +++ /dev/null @@ -1,13 +0,0 @@ -R2_API_TOKEN=_AmmE60blmX2wLIu6cuBWDhOuNpHOE4xQH6n5fYH -R2_ACCESS_KEY_ID=79cde91614e6e43eeb6ee9d63836b64e -R2_SECRET_ACCESS_KEY=7a712d6026e9f96fbde2b2efc2411f3dc5e4f9bed6e7d190e0f437a7aa77f9a0 - -VOD_SECRET_ID=AKIDsrihIyjZOBsjimt8TsN8yvv1AMh5dB44 -VOD_SECRET_KEY=CPZcxdk6W39Jd4cGY95wvupoyMd0YFqW - -AWS_ACCESS_KEY_ID=AKIAYRH5NGRSWHN2L4M6 -AWS_SECRET_ACCESS_KEY=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 - -CF_KV_API_TOKEN=hAiXy3jB-Lt5q-XiIV588lyvyH1c8FJaG3oieqeT -CF_KV_NAMESPACE_ID=f24d396e0daa418e89a1d7074b435c24 -CF_ACCOUNT_ID=67720b647ff2b55cf37ba3ef9e677083 diff --git a/.gitignore b/.gitignore index e69de29..18ba43b 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +*/**/.pypirc \ No newline at end of file diff --git a/.idea/modalDeploy.iml b/.idea/modalDeploy.iml index 851df58..62d1b49 100644 --- a/.idea/modalDeploy.iml +++ b/.idea/modalDeploy.iml @@ -2,6 +2,7 @@ + diff --git a/.idea/runConfigurations/modal_deploy.xml b/.idea/runConfigurations/modal_deploy.xml new file mode 100644 index 0000000..efb908e --- /dev/null +++ b/.idea/runConfigurations/modal_deploy.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/README.md b/README.md index c7fa7fa..cbb4a96 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,32 @@ +# Modal + +## 部署 + ```bash modal deploy -e dev ./src/cluster/app.py +``` + +# Pip + +## 构建 + +确保当前路径在`bowong_modal_functions/pyproject.toml`同级下运行以下命令 + +```bash +uv build +``` + +## 推送到coding私有制品库 + +依据[制品库](https://g-ldyi2063.coding.net/p/dev/artifacts/23892503/pypi/packages) 创建本地使用的.pypirc配置文件 +其中包含访问密钥，严禁提交到代码库 + +```bash +py -m twine upload --config-file=.pypirc --repository=coding-pypi dist/* +``` + +## 安装 + +```bash + uv add bowong-modal-functions --index-url=https://<用户名>:<密钥>@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple ``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ab88689..63eb6c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,50 @@ [project] -name = "modaldeploy" -version = "0.2.0" -description = "管理Modal worker的工程" -readme = "README.md" +name = "BowongModalFunctions" +version = "0.1.1-pre" +authors = [ + { name = "Yudi Xiao", email = "xyd@bowong.ai" }, +] +description = "Bowong modal云函数以及对于的FastAPI接口" +readme = "src/BowongModalFunctions/readme.md" requires-python = ">=3.11" dependencies = [ - "backoff>=2.2.1", - "boto3>=1.37.37", - "cos-python-sdk-v5>=1.9.36", - "crcmod>=1.7", "fastapi[standard]>=0.115.12", - "httpx>=0.28.1", + "backoff>=2.2.1", + "crcmod>=1.7", "loguru>=0.7.3", - "modal>=0.74.57", - "noisereduce>=3.0.3", - "pedalboard==0.9.2", - "psutil>=7.0.0", + "httpx>=0.28.1", "pydantic>=2.11.3", "pydantic-settings>=2.9.1", - "pyloudnorm>=0.1.1", - "python-ffmpeg>=2.0.12", - "requests>=2.32.3", - "scalar-fastapi>=1.0.3", - "sentry-sdk[loguru]>=2.26.1", + "webvtt-py>=0.5.1", "soundfile>=0.13.1", + "pyloudnorm>=0.1.1", + "noisereduce>=3.0.3", + "pedalboard==0.9.2", + "python-ffmpeg>=2.0.12", + "sentry-sdk[loguru]>=2.26.1", "tencentcloud-sdk-python-common>=3.0.1363", "tencentcloud-sdk-python-vod>=3.0.1363", - "tqdm>=4.67.1", - "webvtt-py>=0.5.1", + "cos-python-sdk-v5>=1.9.36", + "boto3>=1.37.37", + "psutil>=7.0.0", + "twine>=6.1.0", + "scalar-fastapi>=1.0.3", + "modal>=0.76.3", ] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "Development Status :: 2 - Pre-Alpha" +] +license = "MIT" +license-files = ["LICEN[CS]E*"] + +[build-system] +requires = ["hatchling >= 1.26"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/BowongModalFunctions"] +sources = ["src"] +only-packages = true +require-runtime-dependencies = true diff --git a/src/BowongModalFunctions/__init__.py b/src/BowongModalFunctions/__init__.py new file mode 100644 index 0000000..5745e00 --- /dev/null +++ b/src/BowongModalFunctions/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.1-pre" \ No newline at end of file diff --git a/src/BowongModalFunctions/api.py b/src/BowongModalFunctions/api.py new file mode 100644 index 0000000..ef44a84 --- /dev/null +++ b/src/BowongModalFunctions/api.py @@ -0,0 +1,516 @@ +import os +import modal +import httpx +import asyncio +from typing import List, Annotated, Tuple, Any, Optional +from modal import current_function_call_id +from modal.call_graph import InputStatus +from loguru import logger +from fastapi import FastAPI, Response, Depends, Header +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi.responses import JSONResponse, RedirectResponse +from fastapi.exceptions import HTTPException +from starlette import status +from scalar_fastapi import get_scalar_api_reference +import sentry_sdk +from sentry_sdk.integrations.loguru import LoguruIntegration, LoggingLevels +from sentry_sdk.integrations.fastapi import FastApiIntegration +from .utils.KVCache import KVCache +from .utils.SentryUtils import SentryUtils +from .models.media_model import (MediaSource, + MediaSources, + MediaCacheStatus, + CacheResult, + DownloadResult) +from .models.web_model import (TaskStatus, ErrorCode, + SentryTransactionInfo, + SentryTransactionHeader, + ModalTaskResponse, + FFMPEGSliceRequest, + FFMPEGConcatRequest, + FFMPEGExtractAudioRequest, + FFMPEGCornerMirrorRequest, + FFMPEGSliceTaskStatusResponse, + FFMPEGConcatTaskStatusResponse, + FFMPEGExtractAudioTaskStatusResponse, + FFMPEGCornerMirrorTaskStatusResponse) +from .config import WorkerConfig + +bearer_scheme = HTTPBearer() + +config = WorkerConfig( + app_name="bowong-ai-video", + video_downloader_concurrency=10, + ffmpeg_worker_concurrency=10, + modal_kv_name="media-cache", + s3_region='ap-northeast-2', + s3_bucket_name="modal-media-cache", + environment="dev" +) + +web_app = FastAPI(title="Modal worker API", + summary="Modal Worker的API, 包括缓存视频, 发起生产任务等", + servers=[ + {'url': 'https://bowongai-dev--bowong-ai-video-fastapi-webapp.modal.run', + 'description': 'modal dev环境测试服务'}, + {'url': 'https://modal-dev.bowong.cc', + 'description': 'modal dev环境测试服务'}, + ]) + + +async def verify_token(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + token = credentials.credentials + # 在这里实现你的token验证逻辑 + if not is_valid_token(token): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + return token + + +def is_valid_token(token: str) -> bool: + # 这里实现具体的token验证逻辑 + # 例如：验证JWT token，检查数据库中的token等 + return token == "bowong7777" + + +sentry_sdk.init(dsn="https://dab7b7ae652216282c89f029a76bb10a@sentry.bowongai.com/2", + send_default_pii=True, + traces_sample_rate=1.0, + profiles_sample_rate=1.0, + add_full_stack=True, + environment=config.environment, + integrations=[ + LoguruIntegration(level=LoggingLevels.INFO.value, event_level=LoggingLevels.ERROR.value), + FastApiIntegration() + ] + ) +modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment) +cf_account_id = os.environ.get("CF_ACCOUNT_ID") +cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN") +cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID") + +sentry_header_schema = { + "x-trace-id": { + "description": "Sentry Transaction ID", + "schema": { + "type": "string", + } + }, + "x-baggage": { + "description": "Sentry Transaction baggage", + "schema": { + "type": "string", + } + } +} + + +@sentry_sdk.trace +def batch_update_cloudflare_kv(caches: List[MediaSource]): + with httpx.Client() as client: + try: + response = client.put( + f"https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/storage/kv/namespaces/{cf_kv_namespace_id}/bulk", + headers={"Authorization": f"Bearer {cf_kv_api_token}"}, + json=[ + { + "based64": False, + "key": cache.urn, + "value": cache.model_dump_json(), + } + for cache in caches + ] + ) + response.raise_for_status() + except httpx.RequestError as e: + logger.error(f"An error occurred while put kv to cloudflare") + raise e + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error occurred while get kv from cloudflare {str(e)}") + raise e + except Exception as e: + logger.error(f"An unexpected error occurred: {str(e)}") + raise e + + +@sentry_sdk.trace +def batch_remove_cloudflare_kv(keys: List[str]): + with httpx.Client() as client: + try: + response = client.post( + f"https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/storage/kv/namespaces/{cf_kv_namespace_id}/bulk/delete", + headers={"Authorization": f"Bearer {cf_kv_api_token}"}, + json=keys + ) + response.raise_for_status() + except httpx.RequestError as e: + logger.error(f"An error occurred while put kv to cloudflare") + raise e + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error occurred while get kv from cloudflare {str(e)}") + raise e + except Exception as e: + logger.error(f"An unexpected error occurred: {str(e)}") + raise e + + +@sentry_sdk.trace +async def get_modal_task_status(task_id: str) -> Tuple[ + TaskStatus, Optional[int], Optional[str], Optional[Any], Optional[SentryTransactionInfo]]: + """ + + :param task_id: + :return: (TaskStaus, errorCode, errorReason, results, sentryTransactionInfo) + """ + try: + fn_task = modal.FunctionCall.from_id(task_id) + call_graph = fn_task.get_call_graph() + root = call_graph[0] + if len(root.children) == 0: + return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None + task = root.children[0] + if not task.function_call_id == task_id: + return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None + + logger.info(task) + logger.info(f"Task function call status: {task.status}") + match task.status: + case InputStatus.PENDING: + return TaskStatus.running, None, None, None, None + case InputStatus.SUCCESS: + try: + results, sentry_trace = fn_task.get(timeout=2) + return TaskStatus.success, ErrorCode.SUCCESS.value, None, results, sentry_trace + except modal.exception.OutputExpiredError: + return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "EXPIRED", None, None + except TimeoutError: + return TaskStatus.running, None, None, None, None + except Exception as e: + logger.exception(e) + return TaskStatus.failed, ErrorCode.SYSTEM_ERROR.value, 'FAILURE', None, None + case _: + error = 'FAILURE' + match task.status: + case InputStatus.INIT_FAILURE: + error = 'INIT_FAILURE' + case InputStatus.TERMINATED: + error = 'TERMINATED' + case InputStatus.TIMEOUT: + error = 'TIMEOUT' + return TaskStatus.failed, ErrorCode.SYSTEM_ERROR.value, error, None, None + except Exception as e: + logger.exception(e) + return TaskStatus.failed, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None + + +@web_app.get("/scalar", include_in_schema=False) +async def scalar(): + return get_scalar_api_reference(openapi_url=web_app.openapi_schema, title="Modal worker web endpoint") + + +@web_app.post("/cache", + tags=["缓存"], + summary="缓存视频文件", + description="异步缓存视频文件到S3存储桶和Modal Dict(KV)", + dependencies=[Depends(verify_token)]) +@sentry_sdk.trace +async def cache(medias: MediaSources) -> CacheResult: + fn_id = current_function_call_id() + caches: MediaSources + sentry_trace = SentryTransactionInfo(x_trace_id=sentry_sdk.get_traceparent(), + x_baggage=sentry_sdk.get_baggage()) + + @SentryUtils.sentry_tracker(name="同步视频缓存", op="cache.get", fn_id=fn_id, + sentry_trace_id=None, sentry_baggage=None) + async def cache_handler(media: MediaSource): + cache_span = sentry_sdk.get_current_span() + cache_span.set_data("runner_id", fn_id) + cache_span.set_data("cache.key", [media.urn]) + cached_media = modal_kv_cache.get_cache(media.urn) + cache_hit: bool = False + if not cached_media: + # start new download task + with cache_span.start_child(name="视频缓存任务入队", + op="queue.publish") as queue_publish_span: + fn = modal.Function.from_name(config.app_name, 'cache_submit') + fn_task = fn.spawn(media, sentry_trace) + queue_publish_span.set_data("cache.key", media.urn) + queue_publish_span.set_data("messaging.message.id", fn_task.object_id) + queue_publish_span.set_data("messaging.destination.name", + "video-downloader.cache_submit") + queue_publish_span.set_data("messaging.message.body.size", 0) + # video_cache = MediaCache(status=MediaCacheStatus.downloading, + # downloader_id=fn_task.object_id) + media.status = MediaCacheStatus.downloading + media.downloader_id = fn_task.object_id + # video_cache_status_json = video_cache.model_dump_json() + modal_kv_cache.set_cache(media) + else: + media = cached_media + # video_cache = MediaCache.model_validate_json(video_cache_status_json) + match media.status: + case MediaCacheStatus.ready: + cache_hit = True + case MediaCacheStatus.downloading: # 下载任务已经在进行 + cache_hit = True + case _: + # start new download task + with cache_span.start_child(name="视频缓存任务入队", + op="queue.publish") as queue_publish_span: + fn = modal.Function.from_name(config.app_name, 'cache_submit') + fn_task = fn.spawn(media, sentry_trace) + queue_publish_span.set_data("cache.key", media.urn) + queue_publish_span.set_data("messaging.message.id", fn_task.object_id) + queue_publish_span.set_data("messaging.destination.name", + "video-downloader.cache_submit") + queue_publish_span.set_data("messaging.message.body.size", 0) + media.status = MediaCacheStatus.downloading + media.downloader_id = fn_task.object_id + # video_cache = MediaCache(status=MediaCacheStatus.downloading, + # downloader_id=fn_task.object_id) + # video_cache_status_json = video_cache.model_dump_json() + modal_kv_cache.set_cache(media) + cache_hit = False + + # caches[media.urn] = video_cache + # logger.info(f"Media cache hit ? {cache_hit}") + cache_span.set_data("cache.hit", cache_hit) + return media + + async with asyncio.TaskGroup() as group: + tasks = [group.create_task(cache_handler(media)) for media in medias.inputs] + + cache_task_result = [task.result() for task in tasks] + + batch_update_cloudflare_kv(cache_task_result) + return CacheResult(caches={media.urn: media for media in cache_task_result}) + + +@web_app.post("/cache/download", + tags=["缓存"], + summary="批量获取下载地址", + description="获取已缓存的视频下载地址", + dependencies=[Depends(verify_token)]) +@sentry_sdk.trace +async def download_caches(medias: MediaSources) -> DownloadResult: + cdn_endpoint = config.cdn_endpoint + urls = [] + for media in medias.inputs: + urls.append(f"{cdn_endpoint}/{media.get_cdn_url()}") + return DownloadResult(urls=urls) + + +@web_app.get("/cache/download", + tags=["缓存"], + summary="下载已缓存的视频", + description="通过CDN下载已缓存的视频文件") +@sentry_sdk.trace +async def download_cache(media: str) -> RedirectResponse: + cdn_endpoint = config.cdn_endpoint + media = MediaSource.from_str(media) + return RedirectResponse(url=f"{cdn_endpoint}/{media.get_cdn_url()}", status_code=status.HTTP_302_FOUND) + + +@web_app.delete("/cache/kv", + tags=["缓存"], + summary="清除KV记录", + description="清除当前环境下KV缓存过的所有数据(S3存储桶内的文件会保留)", + dependencies=[Depends(verify_token)]) +async def purge_kv_all(): + parent = sentry_sdk.get_current_span() + span = parent.start_child(name="清除缓存KV", op="purge.flush") + modal_kv_cache.clear() + span.set_data("cache.success", True) + span.finish() + return JSONResponse(content={"success": True}) + + +@web_app.post("/cache/kv", + tags=["缓存"], + summary="删除对应的KV记录", + description="删除请求中对应的视频缓存记录", + dependencies=[Depends(verify_token)]) +async def purge_kv(medias: MediaSources): + try: + for media in medias.inputs: + modal_kv_cache.pop(media.urn) + keys = [media.urn for media in medias.inputs] + batch_remove_cloudflare_kv(keys) + return JSONResponse(content={"success": True, "keys": keys}) + except Exception as e: + return JSONResponse(content={"success": False, "error": str(e)}) + + +@web_app.post("/cache/media", + tags=["缓存"], + summary="清除指定的所有缓存", + description="清除指定的所有缓存(包括KV记录和S3存储文件)", + dependencies=[Depends(verify_token)]) +async def purge_media(medias: MediaSources): + fn_id = current_function_call_id() + fn = modal.Function.from_name(config.app_name, "cache_delete", environment_name=config.environment) + + @SentryUtils.sentry_tracker(name="清除媒体源缓存", op="cache.purge", fn_id=fn_id, + sentry_trace_id=None, sentry_baggage=None) + async def purge_handle(media: MediaSource): + cache_media = modal_kv_cache.pop(media.urn) + if cache_media: + deleted_cache: MediaSource = await fn.remote.aio(cache_media) + return deleted_cache.urn + return None + + async with asyncio.TaskGroup() as group: + tasks = [group.create_task(purge_handle(media)) for media in medias.inputs] + + keys = [task.result() for task in tasks] + batch_remove_cloudflare_kv(keys) + return JSONResponse(content={"success": True, "keys": keys}) + + +@web_app.post("/ffmpeg/slice", + tags=["发起任务"], + summary="发起切割任务", + description="依据打点信息切出多个片段", + dependencies=[Depends(verify_token)]) +async def slice_media(request: FFMPEGSliceRequest, + headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: + fn = modal.Function.from_name(config.app_name, "ffmpeg_slice_media", environment_name=config.environment) + sentry_trace = None + if headers.x_trace_id and headers.x_baggage: + sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) + fn_call = fn.spawn(request.media, request.markers, sentry_trace) + return ModalTaskResponse(success=True, taskId=fn_call.object_id) + + +@web_app.get("/ffmpeg/slice/{task_id}", + tags=["查询任务"], + summary="查询切割任务状态/结果", + description="根据任务Id查询运行状态", + responses={ + status.HTTP_200_OK: { + "description": "", + "headers": sentry_header_schema + }, + }, + dependencies=[Depends(verify_token)]) +async def slice_media(task_id: str, response: Response) -> FFMPEGSliceTaskStatusResponse: + task_status, code, reason, results, transaction = await get_modal_task_status(task_id) + if transaction: + response.headers["x-trace-id"] = transaction.x_trace_id + response.headers["x-baggage"] = transaction.x_baggage + return FFMPEGSliceTaskStatusResponse(taskId=task_id, status=task_status, code=code, error=reason, + result=results) + + +@web_app.post("/ffmpeg/concat", + tags=["发起任务"], + summary="发起合并任务", + description="依据AI分析的结果发起合并任务", + dependencies=[Depends(verify_token)]) +async def concat_media(body: FFMPEGConcatRequest, + headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: + medias = body.medias + fn = modal.Function.from_name(config.app_name, "ffmpeg_concat_medias", + environment_name=config.environment) + sentry_trace = None + if headers.x_trace_id and headers.x_baggage: + sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) + fn_call = fn.spawn(medias, sentry_trace) + return ModalTaskResponse(success=True, taskId=fn_call.object_id) + + +@web_app.get("/ffmpeg/concat/{task_id}", + tags=["查询任务"], + summary="获取合并任务结果", + description="获取合并任务的处理状态和结果", + responses={ + status.HTTP_200_OK: { + "description": "", + "headers": sentry_header_schema + }, + }, + dependencies=[Depends(verify_token)]) +async def concat_media_status(task_id: str, response: Response) -> FFMPEGConcatTaskStatusResponse: + task_status, code, reason, results, transaction = await get_modal_task_status(task_id) + if transaction: + response.headers["x-trace-id"] = transaction.x_trace_id + response.headers["x-baggage"] = transaction.x_baggage + return FFMPEGConcatTaskStatusResponse(taskId=task_id, status=task_status, + code=code, error=reason, + result=results) + + +@web_app.post("/ffmpeg/extract-audio", + tags=["发起任务"], + summary="发起音频提取任务", + description="提取视频文件的音频", + dependencies=[Depends(verify_token)] + ) +async def extract_audio(body: FFMPEGExtractAudioRequest, + headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: + media = body.media + fn = modal.Function.from_name(config.app_name, "ffmpeg_extract_audio", + environment_name=config.environment) + sentry_trace = None + if headers.x_trace_id and headers.x_baggage: + sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) + fn_call = fn.spawn(media, sentry_trace) + return ModalTaskResponse(success=True, taskId=fn_call.object_id) + + +@web_app.get("/ffmpeg/extract-audio/{task_id}", + tags=["查询任务"], + summary="查询音频提取任务状态", + description="查询音频提取任务状态", + responses={ + status.HTTP_200_OK: { + "description": "", + "headers": sentry_header_schema + }, + }, + dependencies=[Depends(verify_token)]) +async def extract_audio_status(task_id: str, response: Response) -> FFMPEGExtractAudioTaskStatusResponse: + task_status, code, reason, results, transaction = await get_modal_task_status(task_id) + if transaction: + response.headers["x-trace-id"] = transaction.x_trace_id + response.headers["x-baggage"] = transaction.x_baggage + + return FFMPEGExtractAudioTaskStatusResponse(taskId=task_id, status=task_status, code=code, + error=reason, result=results) + + +@web_app.post("/ffmpeg/corner-mirror", tags=["发起任务"], summary="发起镜像小窗去重任务", + description="发起镜像小窗去重任务", dependencies=[Depends(verify_token)]) +async def corner_mirror(body: FFMPEGCornerMirrorRequest, + headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: + media = body.media + fn = modal.Function.from_name(config.app_name, "ffmpeg_corner_mirror", + environment_name=config.environment) + sentry_trace = None + if headers.x_trace_id and headers.x_baggage: + sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) + fn_call = fn.spawn(media=media, sentry_trace=sentry_trace) + return ModalTaskResponse(success=True, taskId=fn_call.object_id) + + +@web_app.get("/ffmpeg/corner-mirror/{task_id}", + tags=["查询任务"], + summary="查询镜像小窗去重任务状态", + description="", + responses={ + status.HTTP_200_OK: { + "description": "", + "headers": sentry_header_schema + }, + }, + dependencies=[Depends(verify_token)]) +async def corner_mirror_status(task_id: str, response: Response) -> FFMPEGCornerMirrorTaskStatusResponse: + task_status, code, reason, result, transaction = await get_modal_task_status(task_id) + if transaction: + response.headers["x-trace-id"] = transaction.x_trace_id + response.headers["x-baggage"] = transaction.x_baggage + return FFMPEGCornerMirrorTaskStatusResponse(taskId=task_id, status=task_status, code=code, + error=reason, result=result) diff --git a/src/BowongModalFunctions/config.py b/src/BowongModalFunctions/config.py new file mode 100644 index 0000000..85ad579 --- /dev/null +++ b/src/BowongModalFunctions/config.py @@ -0,0 +1,18 @@ +from typing import Optional, Any +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +class WorkerConfig(BaseSettings): + app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称") + video_downloader_concurrency: Optional[int] = Field(default=10, description="处理缓存任务的并行数") + ffmpeg_worker_concurrency: Optional[int] = Field(default=10, description="处理视频合成任务的并行数") + + modal_kv_name: Optional[str] = Field(default='media-cache', description="Modal视频缓存KV库") + s3_region: Optional[str] = Field(default='ap-northeast-2', description="S3挂载桶的地域") + s3_bucket_name: Optional[str] = Field(default='modal-media-cache', description="集群挂载的S3存储桶") + s3_mount_dir: str = Field(default='/mntS3', description="集群S3存储桶挂载在本地的根目录") + cdn_endpoint: Optional[str] = Field(default="https://d2nj71io21vkj2.cloudfront.net", + description="集群挂载S3存储桶的对应AWS Cloudfront CDN") + environment: Optional[str] = Field(default="dev", description="Modal worker运行环境") + + modal_config: Any = SettingsConfigDict() diff --git a/src/__init__.py b/src/BowongModalFunctions/models/__init__.py similarity index 100% rename from src/__init__.py rename to src/BowongModalFunctions/models/__init__.py diff --git a/src/cluster/ffmpeg_worker/model.py b/src/BowongModalFunctions/models/ffmpeg_worker_model.py similarity index 96% rename from src/cluster/ffmpeg_worker/model.py rename to src/BowongModalFunctions/models/ffmpeg_worker_model.py index 7c4963a..2b8e784 100644 --- a/src/cluster/ffmpeg_worker/model.py +++ b/src/BowongModalFunctions/models/ffmpeg_worker_model.py @@ -1,9 +1,7 @@ from typing import Union, Any - from pydantic import BaseModel, Field, computed_field, field_validator from pydantic.json_schema import JsonSchemaValue - -from src.cluster.ffmpeg_worker.Utils.TimeUtils import TimeDelta +from ..utils.TimeUtils import TimeDelta class FFMpegSliceSegment(BaseModel): diff --git a/src/cluster/video_downloader/model.py b/src/BowongModalFunctions/models/media_model.py similarity index 94% rename from src/cluster/video_downloader/model.py rename to src/BowongModalFunctions/models/media_model.py index 60005c8..d415dbe 100644 --- a/src/cluster/video_downloader/model.py +++ b/src/BowongModalFunctions/models/media_model.py @@ -3,10 +3,13 @@ from datetime import datetime from enum import Enum from typing import List, Union, Optional, Any, Dict from urllib.parse import urlparse -from pydantic import BaseModel, Field, field_validator, ValidationError, field_serializer, SerializationInfo, \ - computed_field +from pydantic import (BaseModel, Field, field_validator, ValidationError, + field_serializer, SerializationInfo, computed_field) from pydantic.json_schema import JsonSchemaValue -from src.cluster.config import config + +# todo: 临时解决方案，需要想个办法获取本地挂载点配置 +s3_region = 'ap-northeast-2' +s3_bucket_name = 'modal-media-cache' class MediaProtocol(str, Enum): @@ -37,8 +40,6 @@ class MediaSource(BaseModel): downloader_id: Optional[str] = Field(description="正在处理下载的Downloader ID", default=None) progress: int = Field(description="缓存进度", default=0) - # cache_filepath: Optional[str] = Field(description="s3挂载路径下的缓存相对路径", default=None) - @classmethod def from_str(cls, media_url: str) -> 'MediaSource': if media_url.startswith('http://') or media_url.startswith('https://'): @@ -101,11 +102,11 @@ class MediaSource(BaseModel): @computed_field(description="s3挂载路径下的缓存相对路径") @property - def cache_filepath(self) -> str: + def cache_filepath(self, local_mount: bool = False) -> str: match self.protocol: case MediaProtocol.s3: # 本地挂载缓存 - if self.protocol == MediaProtocol.s3 and self.endpoint == config.s3_region and self.bucket == config.s3_bucket_name: + if self.protocol == MediaProtocol.s3 and self.endpoint == s3_region and self.bucket == s3_bucket_name: return f"{self.path}" else: return f"{self.protocol.value}/{self.endpoint}/{self.bucket}/{self.path}" diff --git a/src/cluster/web/model.py b/src/BowongModalFunctions/models/web_model.py similarity index 95% rename from src/cluster/web/model.py rename to src/BowongModalFunctions/models/web_model.py index 8e9a055..d4e3c78 100644 --- a/src/cluster/web/model.py +++ b/src/BowongModalFunctions/models/web_model.py @@ -1,8 +1,8 @@ from enum import Enum -from typing import List, Union, Any, Optional +from typing import List, Union, Optional from pydantic import BaseModel, Field, field_validator, ConfigDict, HttpUrl -from src.cluster.ffmpeg_worker.model import FFMpegSliceSegment -from src.cluster.video_downloader.model import MediaSource, MediaSources +from .ffmpeg_worker_model import FFMpegSliceSegment +from .media_model import MediaSource, MediaSources class SentryTransactionHeader(BaseModel): diff --git a/src/BowongModalFunctions/readme.md b/src/BowongModalFunctions/readme.md new file mode 100644 index 0000000..397e3b8 --- /dev/null +++ b/src/BowongModalFunctions/readme.md @@ -0,0 +1 @@ +Modal 上使用的 函数包 \ No newline at end of file diff --git a/src/cluster/web/Utils/KVCache.py b/src/BowongModalFunctions/utils/KVCache.py similarity index 88% rename from src/cluster/web/Utils/KVCache.py rename to src/BowongModalFunctions/utils/KVCache.py index 43d6569..aec9754 100644 --- a/src/cluster/web/Utils/KVCache.py +++ b/src/BowongModalFunctions/utils/KVCache.py @@ -2,7 +2,7 @@ from typing import Optional import modal -from src.cluster.video_downloader.model import MediaSource +from ..models.media_model import MediaSource class KVCache: @@ -30,4 +30,4 @@ class KVCache: if raise_exception: raise KeyError("URN错误，资源不存在") return None - return MediaSource.model_validate_json(cache_json) \ No newline at end of file + return MediaSource.model_validate_json(cache_json) diff --git a/src/cluster/ffmpeg_worker/Utils/PathUtils.py b/src/BowongModalFunctions/utils/PathUtils.py similarity index 100% rename from src/cluster/ffmpeg_worker/Utils/PathUtils.py rename to src/BowongModalFunctions/utils/PathUtils.py diff --git a/src/cluster/ffmpeg_worker/Utils/SentryUtils.py b/src/BowongModalFunctions/utils/SentryUtils.py similarity index 100% rename from src/cluster/ffmpeg_worker/Utils/SentryUtils.py rename to src/BowongModalFunctions/utils/SentryUtils.py diff --git a/src/cluster/ffmpeg_worker/Utils/TimeUtils.py b/src/BowongModalFunctions/utils/TimeUtils.py similarity index 100% rename from src/cluster/ffmpeg_worker/Utils/TimeUtils.py rename to src/BowongModalFunctions/utils/TimeUtils.py diff --git a/src/cluster/ffmpeg_worker/Utils/VideoUtils.py b/src/BowongModalFunctions/utils/VideoUtils.py similarity index 62% rename from src/cluster/ffmpeg_worker/Utils/VideoUtils.py rename to src/BowongModalFunctions/utils/VideoUtils.py index 7df0350..1a605e6 100644 --- a/src/cluster/ffmpeg_worker/Utils/VideoUtils.py +++ b/src/BowongModalFunctions/utils/VideoUtils.py @@ -19,6 +19,7 @@ from pedalboard import ( Distortion, ) from pedalboard.io import AudioFile +from loguru import logger from src.cluster.ffmpeg_worker.Utils.PathUtils import FileUtils @@ -179,3 +180,104 @@ class VideoUtils: subtype="PCM_16", ) return processed_audio_path + + @staticmethod + def ffmpeg_init_async(use_ffprobe: bool = False) -> AsyncFFmpeg: + if use_ffprobe: + ffmpeg_cmd = AsyncFFmpeg('ffprobe') + else: + ffmpeg_cmd = AsyncFFmpeg().option('y').option('hide_banner') + + @ffmpeg_cmd.on("start") + def on_start(arguments: list[str]): + try: + filter_index = arguments.index("-filter_complex") + filter_content = arguments[filter_index + 1] + arguments[filter_index + 1] = f'"{filter_content}"' + args = " ".join(arguments) + logger.info(f"FFmpeg command:{args}") + arguments[filter_index + 1] = filter_content + except ValueError: + args = " ".join(arguments) + logger.info(f"FFmpeg command:{args}") + + @ffmpeg_cmd.on("progress") + def on_progress(progress): + logger.info(f"处理进度: {progress}") + + @ffmpeg_cmd.on("completed") + def on_completed(): + logger.info(f"FFMpeg task completed.") + + @ffmpeg_cmd.on("stderr") + def on_stderr(line: str): + if line.startswith('Error'): + logger.error(line) + raise RuntimeError(line) + else: + logger.warning(line) + + return ffmpeg_cmd + + @staticmethod + def ffmpeg_init(use_ffprobe: bool = False) -> FFmpeg: + if use_ffprobe: + ffmpeg_cmd = FFmpeg('ffprobe') + else: + ffmpeg_cmd = FFmpeg().option('y').option('hide_banner') + + @ffmpeg_cmd.on("start") + def on_start(arguments: list[str]): + try: + filter_index = arguments.index("-filter_complex") + filter_content = arguments[filter_index + 1] + arguments[filter_index + 1] = f'"{filter_content}"' + args = " ".join(arguments) + logger.info(f"FFmpeg command:{args}") + arguments[filter_index + 1] = filter_content + except ValueError: + args = " ".join(arguments) + logger.info(f"FFmpeg command:{args}") + + @ffmpeg_cmd.on("progress") + def on_progress(progress): + logger.info(f"处理进度: {progress}") + + @ffmpeg_cmd.on("completed") + def on_completed(): + logger.info(f"FFMpeg task completed.") + + @ffmpeg_cmd.on("stderr") + def on_stderr(line: str): + if line.startswith('Error'): + logger.error(line) + raise RuntimeError(line) + else: + logger.warning(line) + + return ffmpeg_cmd + + @staticmethod + async def ffmpeg_extract_audio_async(media_path: str, output_path: str): + import json, os + + ffprobe_cmd = VideoUtils.ffmpeg_init_async(use_ffprobe=True) + ffprobe_cmd.input(media_path, + v="quiet", + print_format="json", + select_streams="a", + show_entries="stream=codec_type") + audio_check_bytes = await ffprobe_cmd.execute() + audio_check = json.loads(audio_check_bytes) + logger.info(audio_check) + if len(audio_check['streams']) == 0: + raise RuntimeError(f"Media has no audio streams.") + # output_path = f"{output_path_prefix}/extract_audio/outputs/{fn_id}/output.wav" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + ffmpeg_cmd = VideoUtils.ffmpeg_init_async() + ffmpeg_cmd.input(media_path).output(output_path, + map="0:a", + acodec="pcm_s16le", + ar=44100, + ac=1) + await ffmpeg_cmd.execute() diff --git a/src/cluster/ffmpeg_worker/Utils/__init__.py b/src/BowongModalFunctions/utils/__init__.py similarity index 100% rename from src/cluster/ffmpeg_worker/Utils/__init__.py rename to src/BowongModalFunctions/utils/__init__.py diff --git a/src/cluster/app.py b/src/cluster/app.py index 94e2ae7..91e4366 100644 --- a/src/cluster/app.py +++ b/src/cluster/app.py @@ -1,14 +1,24 @@ import modal -from config import config -from video_downloader.worker import worker_app -from web.worker import fastapi_app -from ffmpeg_worker.worker import app as ffmpeg_app +from config import WorkerConfig +from video import media_app +from web import fastapi_app +from ffmpeg_app import ffmpeg_app -app = modal.App(config.app_name, +config = WorkerConfig( + app_name="bowong-ai-video-test", + video_downloader_concurrency=10, + ffmpeg_worker_concurrency=10, + modal_kv_name="media-cache", + s3_region='ap-northeast-2', + s3_bucket_name="modal-media-cache", + environment="dev" +) + +app = modal.App('bowong-modal-ai-video-preview', include_source=False, secrets=[modal.Secret.from_name("cf-kv-secret", environment_name=config.environment)]) app.include(fastapi_app) -app.include(worker_app) +app.include(media_app) app.include(ffmpeg_app) diff --git a/src/cluster/config.py b/src/cluster/config.py index 6859381..e9d5825 100644 --- a/src/cluster/config.py +++ b/src/cluster/config.py @@ -4,7 +4,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class WorkerConfig(BaseSettings): - app_name: str = Field(default='video-downloader', description="Modal App集群名称") + app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称") video_downloader_concurrency: Optional[int] = Field(default=10, description="处理缓存任务的并行数") ffmpeg_worker_concurrency: Optional[int] = Field(default=10, description="处理视频合成任务的并行数") @@ -17,14 +17,3 @@ class WorkerConfig(BaseSettings): environment: Optional[str] = Field(default="dev", description="Modal worker运行环境") modal_config: Any = SettingsConfigDict() - - -config = WorkerConfig( - app_name="bowong-ai-video", - video_downloader_concurrency=10, - ffmpeg_worker_concurrency=10, - modal_kv_name="media-cache", - s3_region='ap-northeast-2', - s3_bucket_name="modal-media-cache", - environment="dev" -) diff --git a/src/deploy.py b/src/cluster/deploy.py similarity index 92% rename from src/deploy.py rename to src/cluster/deploy.py index aa255e5..a22462b 100644 --- a/src/deploy.py +++ b/src/cluster/deploy.py @@ -1,5 +1,5 @@ import modal.cli.run -from cluster.config import config +from .config import config if __name__ == '__main__': # modal.cli.run.deploy(app_ref='src.cluster.app', env=config.environment, use_module_mode=True) diff --git a/src/cluster/ffmpeg_worker/worker.py b/src/cluster/ffmpeg_app.py similarity index 82% rename from src/cluster/ffmpeg_worker/worker.py rename to src/cluster/ffmpeg_app.py index 01bef9a..ee4145e 100644 --- a/src/cluster/ffmpeg_worker/worker.py +++ b/src/cluster/ffmpeg_app.py @@ -1,32 +1,42 @@ import modal -from modal import current_function_call_id ffmpeg_worker_image = ( modal.Image.debian_slim(python_version="3.11") .apt_install('ffmpeg') - .pip_install('sentry-sdk', 'loguru', 'pydantic', 'pydantic_settings', 'httpx', 'python-ffmpeg', 'psutil', 'backoff', - 'pedalboard==0.9.2', 'soundfile', 'pyloudnorm', 'noisereduce') - .add_local_python_source("src.cluster.config", copy=True) - .add_local_python_source("src.cluster.ffmpeg_worker", copy=True) - .add_local_python_source("src.cluster.video_downloader.model", copy=True) + .pip_install( + 'BowongModalFunctions==0.1.1-pre', + extra_index_url="https://packages-1747117713372:a8865382010036f1e8ad5ca8d0cdab520b78a8ff@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple", + ) ) -app = modal.App(image=ffmpeg_worker_image, ) +ffmpeg_app = modal.App(image=ffmpeg_worker_image, include_source=False) with ffmpeg_worker_image.imports(): + import shutil, psutil, os, backoff, json, sentry_sdk from typing import List, Optional, Tuple, Dict, Any from loguru import logger + from modal import current_function_call_id from ffmpeg.asyncio import FFmpeg - from src.cluster.config import config - from src.cluster.ffmpeg_worker.Utils.SentryUtils import SentryUtils - from src.cluster.ffmpeg_worker.Utils.PathUtils import FileUtils - from src.cluster.ffmpeg_worker.Utils.VideoUtils import VideoUtils - from src.cluster.ffmpeg_worker.model import FFMpegSliceSegment - from src.cluster.video_downloader.model import (MediaSources, - MediaSource, - MediaProtocol) - from src.cluster.web.model import SentryTransactionInfo - import shutil, psutil, os, backoff, json, sentry_sdk + from BowongModalFunctions.utils.SentryUtils import SentryUtils + from BowongModalFunctions.utils.PathUtils import FileUtils + from BowongModalFunctions.utils.VideoUtils import VideoUtils + from BowongModalFunctions.models.ffmpeg_worker_model import FFMpegSliceSegment + from BowongModalFunctions.models.media_model import MediaSources, MediaSource, MediaProtocol + from BowongModalFunctions.models.web_model import SentryTransactionInfo + from BowongModalFunctions.config import WorkerConfig + + config = WorkerConfig( + app_name="bowong-ai-video-test", + video_downloader_concurrency=10, + ffmpeg_worker_concurrency=10, + modal_kv_name="media-cache", + s3_region='ap-northeast-2', + s3_bucket_name="modal-media-cache", + environment="dev" + ) + + s3_mount = config.s3_mount_dir + output_path_prefix = "/mnt/outputs" sentry_sdk.init( dsn="https://75cca970bfcc3d45d24361e7c0f1833c@sentry.bowongai.com/4", @@ -37,12 +47,6 @@ with ffmpeg_worker_image.imports(): environment=config.environment, ) - modal_kv = modal.Dict.from_name(config.modal_kv_name, - environment_name=config.environment, - create_if_missing=True) - s3_mount = config.s3_mount_dir - output_path_prefix = "/mnt/outputs" - @sentry_sdk.trace def capture_hardware_info() -> Tuple[Any, int, Any]: @@ -53,8 +57,11 @@ with ffmpeg_worker_image.imports(): return (cpu_freq, cpu_count, mem) - def ffmpeg_init(): - ffmpeg_cmd = FFmpeg().option('y').option('hide_banner') + def ffmpeg_init(use_ffprobe: bool = False) -> FFmpeg: + if use_ffprobe: + ffmpeg_cmd = FFmpeg('ffprobe') + else: + ffmpeg_cmd = FFmpeg().option('y').option('hide_banner') @ffmpeg_cmd.on("start") def on_start(arguments: list[str]): @@ -102,7 +109,7 @@ with ffmpeg_worker_image.imports(): return s3_outputs - @app.function( + @ffmpeg_app.function( timeout=1800, cloud="aws", # region='ap-northeast', @@ -144,7 +151,7 @@ with ffmpeg_worker_image.imports(): sentry_trace_id=sentry_trace.x_trace_id if sentry_trace else None, sentry_baggage=sentry_trace.x_baggage if sentry_trace else None) async def ffmpeg_process(media_sources: MediaSources, output_filepath: str) -> str: - ffmpeg_cmd = ffmpeg_init() + ffmpeg_cmd = VideoUtils.ffmpeg_init_async() input_videos = [] for media_source in media_sources.inputs: cache_filepath = f"{s3_mount}/{media_source.cache_filepath}" @@ -208,7 +215,7 @@ with ffmpeg_worker_image.imports(): return s3_output, sentry_trace - @app.function( + @ffmpeg_app.function( timeout=900, cloud="aws", max_containers=config.ffmpeg_worker_concurrency, @@ -236,7 +243,7 @@ with ffmpeg_worker_image.imports(): cache_filepath = f"{s3_mount}/{media_source.cache_filepath}" logger.info(cache_filepath) outputs: List[str] = [] - ffmpeg_cmd = ffmpeg_init() + ffmpeg_cmd = VideoUtils.ffmpeg_init_async() ffmpeg_cmd.input(cache_filepath) filter_complex: List[str] = [] for index, marker in enumerate(media_markers): @@ -280,7 +287,7 @@ with ffmpeg_worker_image.imports(): hls_m3u8_url = media_source.path logger.info(hls_m3u8_url) outputs: List[str] = [] - ffmpeg_cmd = ffmpeg_init() + ffmpeg_cmd = VideoUtils.ffmpeg_init_async() ffmpeg_cmd.input(hls_m3u8_url, protocol_whitelist="file,http,https,tcp,tls", reconnect="1", # 自动重连 @@ -331,7 +338,7 @@ with ffmpeg_worker_image.imports(): return outputs, sentry_trace - @app.function( + @ffmpeg_app.function( timeout=600, cloud="aws", max_containers=config.ffmpeg_worker_concurrency, @@ -356,56 +363,8 @@ with ffmpeg_worker_image.imports(): sentry_baggage=sentry_trace.x_baggage if sentry_trace else None) async def ffmpeg_process(media: MediaSource, fn_id: str) -> str: cache_filepath = f"{s3_mount}/{media.cache_filepath}" - ffprobe_cmd = FFmpeg('ffprobe').input(cache_filepath, - v="quiet", - print_format="json", - select_streams="a", - show_entries="stream=codec_type") - - @ffprobe_cmd.on("start") - def on_start(arguments: list[str]): - try: - filter_index = arguments.index("-filter_complex") - filter_content = arguments[filter_index + 1] - arguments[filter_index + 1] = f'"{filter_content}"' - args = " ".join(arguments) - logger.info(f"FFmpeg command:{args}") - arguments[filter_index + 1] = filter_content - except ValueError: - args = " ".join(arguments) - logger.info(f"FFmpeg command:{args}") - - @ffprobe_cmd.on("progress") - def on_progress(progress): - logger.info(f"处理进度: {progress}") - - @ffprobe_cmd.on("completed") - def on_completed(): - logger.info(f"FFMpeg task completed.") - - @ffprobe_cmd.on("stderr") - def on_stderr(line: str): - if line.startswith('Error'): - logger.error(line) - raise RuntimeError(line) - else: - logger.warning(line) - - audio_check_bytes = await ffprobe_cmd.execute() - audio_check = json.loads(audio_check_bytes) - logger.info(audio_check) - if len(audio_check['streams']) == 0: - raise RuntimeError(f"Media has no audio streams.") output_path = f"{output_path_prefix}/extract_audio/outputs/{fn_id}/output.wav" - os.makedirs(os.path.dirname(output_path), exist_ok=True) - ffmpeg_cmd = ffmpeg_init() - ffmpeg_cmd.input(cache_filepath).output(output_path, - map="0:a", - acodec="pcm_s16le", - ar=44100, - ac=1) - - await ffmpeg_cmd.execute() + await VideoUtils.ffmpeg_extract_audio_async(cache_filepath, output_path) s3_outputs = local_copy_to_s3([output_path]) return s3_outputs[0] @@ -421,7 +380,7 @@ with ffmpeg_worker_image.imports(): raise NotImplementedError - @app.function( + @ffmpeg_app.function( timeout=600, cloud="aws", max_containers=config.ffmpeg_worker_concurrency, @@ -464,12 +423,9 @@ with ffmpeg_worker_image.imports(): output_path = f"{output_path_prefix}/corner_mirror/outputs/{func_id}/output.mp4" os.makedirs(os.path.dirname(output_path), exist_ok=True) - ffmpeg_cmd = ffmpeg_init() - + ffmpeg_cmd = VideoUtils.ffmpeg_init_async() ffmpeg_cmd.input(media_filepath).output(output_path, - options={ - "filter_complex": ";".join(filter_complex) - }, + options={"filter_complex": ";".join(filter_complex)}, map=["[video]", "0:a"], vcodec="libx264", crf=16, @@ -496,6 +452,11 @@ with ffmpeg_worker_image.imports(): fn_id = current_function_call_id() async def ffmpeg_process(video: MediaSource, bgm: MediaSource, noise_sample: Optional[MediaSource] = None): + local_input_filepath = f"{s3_mount}/{media.cache_filepath}" + audio_filepath = FileUtils.file_path_extend(local_input_filepath, "audio").replace('mp4', 'wav') + output_audio_filepath = f"{s3_mount}/{audio_filepath}" + await VideoUtils.ffmpeg_extract_audio_async(local_input_filepath, output_audio_filepath) + # VideoUtils.noise_reduce() raise NotImplementedError raise NotImplementedError diff --git a/src/cluster/ffmpeg_worker/__init__.py b/src/cluster/ffmpeg_worker/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cluster/video_downloader/worker.py b/src/cluster/video.py similarity index 87% rename from src/cluster/video_downloader/worker.py rename to src/cluster/video.py index 1b7c2b1..bc58db1 100644 --- a/src/cluster/video_downloader/worker.py +++ b/src/cluster/video.py @@ -3,34 +3,46 @@ import modal downloader_image = ( modal.Image .debian_slim(python_version="3.11") - .pip_install('httpx', 'cos-python-sdk-v5', 'loguru', 'pydantic', 'pydantic_settings', 'sentry-sdk[loguru]', - 'tqdm', 'crcmod', 'tencentcloud-sdk-python-common', 'tencentcloud-sdk-python-vod') - .add_local_python_source("src.cluster.config", copy=True) - .add_local_python_source("src.cluster.video_downloader.model", copy=True) - .add_local_python_source("src.cluster.web.model", copy=True) - .add_local_python_source("src.cluster.config", copy=True) + .pip_install( + 'BowongModalFunctions==0.1.1-pre', + extra_index_url="https://packages-1747117713372:a8865382010036f1e8ad5ca8d0cdab520b78a8ff@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple", + ) ) +media_app = modal.App(image=downloader_image, + include_source=False, + secrets=[ + modal.Secret.from_name("cf-kv-secret", environment_name='dev'), + ]) + with downloader_image.imports(): import os, httpx, crcmod + import sentry_sdk + from sentry_sdk.integrations.loguru import LoguruIntegration from tqdm import tqdm from typing import Tuple, Dict, List from loguru import logger - from modal import current_function_call_id - from src.cluster.config import config - from src.cluster.web.Utils.KVCache import KVCache - from src.cluster.video_downloader.model import MediaSource, MediaCacheStatus, MediaProtocol - from src.cluster.web.model import SentryTransactionInfo from datetime import datetime, UTC, timedelta + from modal import current_function_call_id + from tencentcloud.common.credential import Credential from tencentcloud.vod.v20180717.vod_client import VodClient from tencentcloud.vod.v20180717 import models as vod_request_models - import sentry_sdk - from sentry_sdk.integrations.loguru import LoguruIntegration - worker_app = modal.App(image=downloader_image, secrets=[ - modal.Secret.from_name("cf-kv-secret", environment_name=config.environment), - ]) + from BowongModalFunctions.config import WorkerConfig + from BowongModalFunctions.utils.KVCache import KVCache + from BowongModalFunctions.models.media_model import MediaSource, MediaCacheStatus, MediaProtocol + from BowongModalFunctions.models.web_model import SentryTransactionInfo + + config = WorkerConfig( + app_name="bowong-ai-video-test", + video_downloader_concurrency=10, + ffmpeg_worker_concurrency=10, + modal_kv_name="media-cache", + s3_region='ap-northeast-2', + s3_bucket_name="modal-media-cache", + environment="dev" + ) sentry_sdk.init(dsn="https://85632fdcd62f699c2f88af6ca489e9ec@sentry.bowongai.com/3", send_default_pii=True, @@ -98,17 +110,17 @@ with downloader_image.imports(): raise e - @worker_app.function(cpu=1, timeout=1800, - cloud="aws", - # region='ap-northeast', - max_containers=config.video_downloader_concurrency, - volumes={ - "/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, - secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), - ), - }, - secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.environment)]) + @media_app.function(cpu=1, timeout=1800, + cloud="aws", + # region='ap-northeast', + max_containers=config.video_downloader_concurrency, + volumes={ + "/mntS3": modal.CloudBucketMount( + bucket_name=config.s3_bucket_name, + secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), + ), + }, + secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.environment)]) @modal.concurrent(max_inputs=10) async def cache_submit(media: MediaSource, sentry_trace: SentryTransactionInfo) -> MediaSource: def vod_init(): @@ -293,14 +305,14 @@ with downloader_image.imports(): return media - @worker_app.function(cpu=1, timeout=300, - max_containers=config.video_downloader_concurrency, - volumes={ - "/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, - secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), - ), - }) + @media_app.function(cpu=1, timeout=300, + max_containers=config.video_downloader_concurrency, + volumes={ + "/mntS3": modal.CloudBucketMount( + bucket_name=config.s3_bucket_name, + secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), + ), + }) @modal.concurrent(max_inputs=10) async def cache_delete(cache: MediaSource) -> MediaSource: if os.path.exists(cache.cache_filepath): diff --git a/src/cluster/video_downloader/__init__.py b/src/cluster/video_downloader/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cluster/web.py b/src/cluster/web.py new file mode 100644 index 0000000..c2f7950 --- /dev/null +++ b/src/cluster/web.py @@ -0,0 +1,28 @@ +import modal + +fastapi_image = ( + modal.Image + .debian_slim(python_version="3.11") + # .pip_install("fastapi[standard]", "sentry-sdk[fastapi]", + # 'loguru', 'pydantic', 'pydantic_settings', 'scalar-fastapi', 'psutil') + .pip_install( + 'BowongModalFunctions==0.1.1-pre', + extra_index_url="https://packages-1747117713372:a8865382010036f1e8ad5ca8d0cdab520b78a8ff@g-ldyi2063-pypi.pkg.coding.net/dev/packages/simple", + ) +) + +fastapi_app = modal.App(image=fastapi_image, include_source=False) + +with fastapi_image.imports(): + from BowongModalFunctions.api import web_app + + + @fastapi_app.function(scaledown_window=60, + secrets=[ + modal.Secret.from_name("cf-kv-secret", environment_name='dev'), + ]) + @modal.concurrent(max_inputs=100) + # @modal.asgi_app(custom_domains=["modal-dev.bowong.cc"]) + @modal.asgi_app() + def fastapi_webapp(): + return web_app diff --git a/src/cluster/web/Utils/__init__.py b/src/cluster/web/Utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cluster/web/__init__.py b/src/cluster/web/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cluster/web/worker.py b/src/cluster/web/worker.py deleted file mode 100644 index b902ba6..0000000 --- a/src/cluster/web/worker.py +++ /dev/null @@ -1,511 +0,0 @@ -import modal -from src.cluster.config import config - -fastapi_image = ( - modal.Image - .debian_slim(python_version="3.11") - .pip_install("fastapi[standard]", "sentry-sdk[fastapi]", - 'loguru', 'pydantic', 'pydantic_settings', 'scalar-fastapi', 'psutil') - .add_local_python_source("src.cluster.config", copy=True) - .add_local_python_source("src.cluster.web", copy=True) - .add_local_python_source("src.cluster.video_downloader", copy=True) - .add_local_python_source("src.cluster.ffmpeg_worker", copy=True) -) - -fastapi_app = modal.App(image=fastapi_image, ) - - -@fastapi_app.function(scaledown_window=60, - secrets=[ - modal.Secret.from_name("cf-kv-secret", environment_name=config.environment), - ]) -@modal.concurrent(max_inputs=100) -@modal.asgi_app(custom_domains=["modal-dev.bowong.cc"]) -def fastapi_webapp(): - with fastapi_image.imports(): - import os - import httpx - import asyncio - from typing import List, Annotated, Tuple, Any, Optional - from modal import current_function_call_id - from modal.call_graph import InputStatus - from loguru import logger - from fastapi import FastAPI, Response, Depends, Header - from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials - from fastapi.responses import JSONResponse, RedirectResponse - from fastapi.exceptions import HTTPException - from starlette import status - from scalar_fastapi import get_scalar_api_reference - import sentry_sdk - from sentry_sdk.integrations.loguru import LoguruIntegration, LoggingLevels - from sentry_sdk.integrations.fastapi import FastApiIntegration - from src.cluster.web.Utils.KVCache import KVCache - from src.cluster.ffmpeg_worker.Utils.SentryUtils import SentryUtils - from src.cluster.video_downloader.model import (MediaSource, - MediaSources, - MediaCacheStatus, - CacheResult, - DownloadResult) - from src.cluster.web.model import (TaskStatus, ErrorCode, - SentryTransactionInfo, - SentryTransactionHeader, - ModalTaskResponse, - FFMPEGSliceRequest, - FFMPEGConcatRequest, - FFMPEGExtractAudioRequest, - FFMPEGCornerMirrorRequest, - FFMPEGSliceTaskStatusResponse, - FFMPEGConcatTaskStatusResponse, - FFMPEGExtractAudioTaskStatusResponse, - FFMPEGCornerMirrorTaskStatusResponse) - - bearer_scheme = HTTPBearer() - - web_app = FastAPI(title="Modal worker API", - summary="Modal Worker的API, 包括缓存视频, 发起生产任务等", - servers=[ - # {'url': 'https://bowongai-dev--video-downloader-fastapi-webapp.modal.run', - # 'description': 'modal dev环境测试服务'}, - {'url': 'https://modal-dev.bowong.cc', - 'description': 'modal dev环境测试服务'}, - ]) - - async def verify_token(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): - token = credentials.credentials - # 在这里实现你的token验证逻辑 - if not is_valid_token(token): - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authentication credentials", - headers={"WWW-Authenticate": "Bearer"}, - ) - return token - - def is_valid_token(token: str) -> bool: - # 这里实现具体的token验证逻辑 - # 例如：验证JWT token，检查数据库中的token等 - return token == "bowong7777" - - sentry_sdk.init(dsn="https://dab7b7ae652216282c89f029a76bb10a@sentry.bowongai.com/2", - send_default_pii=True, - traces_sample_rate=1.0, - profiles_sample_rate=1.0, - add_full_stack=True, - environment=config.environment, - integrations=[ - LoguruIntegration(level=LoggingLevels.INFO.value, event_level=LoggingLevels.ERROR.value), - FastApiIntegration() - ] - ) - modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment) - cf_account_id = os.environ.get("CF_ACCOUNT_ID") - cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN") - cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID") - - sentry_header_schema = { - "x-trace-id": { - "description": "Sentry Transaction ID", - "schema": { - "type": "string", - } - }, - "x-baggage": { - "description": "Sentry Transaction baggage", - "schema": { - "type": "string", - } - } - } - - @sentry_sdk.trace - def batch_update_cloudflare_kv(caches: List[MediaSource]): - with httpx.Client() as client: - try: - response = client.put( - f"https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/storage/kv/namespaces/{cf_kv_namespace_id}/bulk", - headers={"Authorization": f"Bearer {cf_kv_api_token}"}, - json=[ - { - "based64": False, - "key": cache.urn, - "value": cache.model_dump_json(), - } - for cache in caches - ] - ) - response.raise_for_status() - except httpx.RequestError as e: - logger.error(f"An error occurred while put kv to cloudflare") - raise e - except httpx.HTTPStatusError as e: - logger.error(f"HTTP error occurred while get kv from cloudflare {str(e)}") - raise e - except Exception as e: - logger.error(f"An unexpected error occurred: {str(e)}") - raise e - - @sentry_sdk.trace - def batch_remove_cloudflare_kv(keys: List[str]): - with httpx.Client() as client: - try: - response = client.post( - f"https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/storage/kv/namespaces/{cf_kv_namespace_id}/bulk/delete", - headers={"Authorization": f"Bearer {cf_kv_api_token}"}, - json=keys - ) - response.raise_for_status() - except httpx.RequestError as e: - logger.error(f"An error occurred while put kv to cloudflare") - raise e - except httpx.HTTPStatusError as e: - logger.error(f"HTTP error occurred while get kv from cloudflare {str(e)}") - raise e - except Exception as e: - logger.error(f"An unexpected error occurred: {str(e)}") - raise e - - @sentry_sdk.trace - async def get_modal_task_status(task_id: str) -> Tuple[ - TaskStatus, Optional[int], Optional[str], Optional[Any], Optional[SentryTransactionInfo]]: - """ - - :param task_id: - :return: (TaskStaus, errorCode, errorReason, results, sentryTransactionInfo) - """ - try: - fn_task = modal.FunctionCall.from_id(task_id) - call_graph = fn_task.get_call_graph() - root = call_graph[0] - if len(root.children) == 0: - return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None - task = root.children[0] - if not task.function_call_id == task_id: - return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None - - logger.info(task) - logger.info(f"Task function call status: {task.status}") - match task.status: - case InputStatus.PENDING: - return TaskStatus.running, None, None, None, None - case InputStatus.SUCCESS: - try: - results, sentry_trace = fn_task.get(timeout=2) - return TaskStatus.success, ErrorCode.SUCCESS.value, None, results, sentry_trace - except modal.exception.OutputExpiredError: - return TaskStatus.expired, ErrorCode.NOT_FOUND.value, "EXPIRED", None, None - except TimeoutError: - return TaskStatus.running, None, None, None, None - except Exception as e: - logger.exception(e) - return TaskStatus.failed, ErrorCode.SYSTEM_ERROR.value, 'FAILURE', None, None - case _: - error = 'FAILURE' - match task.status: - case InputStatus.INIT_FAILURE: - error = 'INIT_FAILURE' - case InputStatus.TERMINATED: - error = 'TERMINATED' - case InputStatus.TIMEOUT: - error = 'TIMEOUT' - return TaskStatus.failed, ErrorCode.SYSTEM_ERROR.value, error, None, None - except Exception as e: - logger.exception(e) - return TaskStatus.failed, ErrorCode.NOT_FOUND.value, "NOT_FOUND", None, None - - @web_app.get("/scalar", include_in_schema=False) - async def scalar(): - return get_scalar_api_reference(openapi_url=web_app.openapi_schema, title="Modal worker web endpoint") - - @web_app.post("/cache", - tags=["缓存"], - summary="缓存视频文件", - description="异步缓存视频文件到S3存储桶和Modal Dict(KV)", - dependencies=[Depends(verify_token)]) - @sentry_sdk.trace - async def cache(medias: MediaSources) -> CacheResult: - fn_id = current_function_call_id() - caches: MediaSources - sentry_trace = SentryTransactionInfo(x_trace_id=sentry_sdk.get_traceparent(), - x_baggage=sentry_sdk.get_baggage()) - - @SentryUtils.sentry_tracker(name="同步视频缓存", op="cache.get", fn_id=fn_id, - sentry_trace_id=None, sentry_baggage=None) - async def cache_handler(media: MediaSource): - cache_span = sentry_sdk.get_current_span() - cache_span.set_data("runner_id", fn_id) - cache_span.set_data("cache.key", [media.urn]) - cached_media = modal_kv_cache.get_cache(media.urn) - cache_hit: bool = False - if not cached_media: - # start new download task - with cache_span.start_child(name="视频缓存任务入队", - op="queue.publish") as queue_publish_span: - fn = modal.Function.from_name(config.app_name, 'cache_submit') - fn_task = fn.spawn(media, sentry_trace) - queue_publish_span.set_data("cache.key", media.urn) - queue_publish_span.set_data("messaging.message.id", fn_task.object_id) - queue_publish_span.set_data("messaging.destination.name", - "video-downloader.cache_submit") - queue_publish_span.set_data("messaging.message.body.size", 0) - # video_cache = MediaCache(status=MediaCacheStatus.downloading, - # downloader_id=fn_task.object_id) - media.status = MediaCacheStatus.downloading - media.downloader_id = fn_task.object_id - # video_cache_status_json = video_cache.model_dump_json() - modal_kv_cache.set_cache(media) - else: - media = cached_media - # video_cache = MediaCache.model_validate_json(video_cache_status_json) - match media.status: - case MediaCacheStatus.ready: - cache_hit = True - case MediaCacheStatus.downloading: # 下载任务已经在进行 - cache_hit = True - case _: - # start new download task - with cache_span.start_child(name="视频缓存任务入队", - op="queue.publish") as queue_publish_span: - fn = modal.Function.from_name(config.app_name, 'cache_submit') - fn_task = fn.spawn(media, sentry_trace) - queue_publish_span.set_data("cache.key", media.urn) - queue_publish_span.set_data("messaging.message.id", fn_task.object_id) - queue_publish_span.set_data("messaging.destination.name", - "video-downloader.cache_submit") - queue_publish_span.set_data("messaging.message.body.size", 0) - media.status = MediaCacheStatus.downloading - media.downloader_id = fn_task.object_id - # video_cache = MediaCache(status=MediaCacheStatus.downloading, - # downloader_id=fn_task.object_id) - # video_cache_status_json = video_cache.model_dump_json() - modal_kv_cache.set_cache(media) - cache_hit = False - - # caches[media.urn] = video_cache - # logger.info(f"Media cache hit ? {cache_hit}") - cache_span.set_data("cache.hit", cache_hit) - return media - - async with asyncio.TaskGroup() as group: - tasks = [group.create_task(cache_handler(media)) for media in medias.inputs] - - cache_task_result = [task.result() for task in tasks] - - batch_update_cloudflare_kv(cache_task_result) - return CacheResult(caches={media.urn: media for media in cache_task_result}) - - @web_app.post("/cache/download", - tags=["缓存"], - summary="批量获取下载地址", - description="获取已缓存的视频下载地址", - dependencies=[Depends(verify_token)]) - @sentry_sdk.trace - async def download_caches(medias: MediaSources) -> DownloadResult: - cdn_endpoint = config.cdn_endpoint - urls = [] - for media in medias.inputs: - urls.append(f"{cdn_endpoint}/{media.get_cdn_url()}") - return DownloadResult(urls=urls) - - @web_app.get("/cache/download", - tags=["缓存"], - summary="下载已缓存的视频", - description="通过CDN下载已缓存的视频文件") - @sentry_sdk.trace - async def download_cache(media: str) -> RedirectResponse: - cdn_endpoint = config.cdn_endpoint - media = MediaSource.from_str(media) - return RedirectResponse(url=f"{cdn_endpoint}/{media.get_cdn_url()}", status_code=status.HTTP_302_FOUND) - - @web_app.delete("/cache/kv", - tags=["缓存"], - summary="清除KV记录", - description="清除当前环境下KV缓存过的所有数据(S3存储桶内的文件会保留)", - dependencies=[Depends(verify_token)]) - async def purge_kv_all(): - parent = sentry_sdk.get_current_span() - span = parent.start_child(name="清除缓存KV", op="purge.flush") - modal_kv_cache.clear() - span.set_data("cache.success", True) - span.finish() - return JSONResponse(content={"success": True}) - - @web_app.post("/cache/kv", - tags=["缓存"], - summary="删除对应的KV记录", - description="删除请求中对应的视频缓存记录", - dependencies=[Depends(verify_token)]) - async def purge_kv(medias: MediaSources): - try: - for media in medias.inputs: - modal_kv_cache.pop(media.urn) - keys = [media.urn for media in medias.inputs] - batch_remove_cloudflare_kv(keys) - return JSONResponse(content={"success": True, "keys": keys}) - except Exception as e: - return JSONResponse(content={"success": False, "error": str(e)}) - - @web_app.post("/cache/media", - tags=["缓存"], - summary="清除指定的所有缓存", - description="清除指定的所有缓存(包括KV记录和S3存储文件)", - dependencies=[Depends(verify_token)]) - async def purge_media(medias: MediaSources): - fn_id = current_function_call_id() - fn = modal.Function.from_name(config.app_name, "cache_delete", environment_name=config.environment) - - @SentryUtils.sentry_tracker(name="清除媒体源缓存", op="cache.purge", fn_id=fn_id, - sentry_trace_id=None, sentry_baggage=None) - async def purge_handle(media: MediaSource): - cache_media = modal_kv_cache.pop(media.urn) - if cache_media: - deleted_cache: MediaSource = await fn.remote.aio(cache_media) - return deleted_cache.urn - return None - - async with asyncio.TaskGroup() as group: - tasks = [group.create_task(purge_handle(media)) for media in medias.inputs] - - keys = [task.result() for task in tasks] - batch_remove_cloudflare_kv(keys) - return JSONResponse(content={"success": True, "keys": keys}) - - @web_app.post("/ffmpeg/slice", - tags=["发起任务"], - summary="发起切割任务", - description="依据打点信息切出多个片段", - dependencies=[Depends(verify_token)]) - async def slice_media(request: FFMPEGSliceRequest, - headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - - fn = modal.Function.from_name(config.app_name, "ffmpeg_slice_media", environment_name=config.environment) - sentry_trace = None - if headers.x_trace_id and headers.x_baggage: - sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) - fn_call = fn.spawn(request.media, request.markers, sentry_trace) - return ModalTaskResponse(success=True, taskId=fn_call.object_id) - - @web_app.get("/ffmpeg/slice/{task_id}", - tags=["查询任务"], - summary="查询切割任务状态/结果", - description="根据任务Id查询运行状态", - responses={ - status.HTTP_200_OK: { - "description": "", - "headers": sentry_header_schema - }, - }, - dependencies=[Depends(verify_token)]) - async def slice_media(task_id: str, response: Response) -> FFMPEGSliceTaskStatusResponse: - task_status, code, reason, results, transaction = await get_modal_task_status(task_id) - if transaction: - response.headers["x-trace-id"] = transaction.x_trace_id - response.headers["x-baggage"] = transaction.x_baggage - return FFMPEGSliceTaskStatusResponse(taskId=task_id, status=task_status, code=code, error=reason, - result=results) - - @web_app.post("/ffmpeg/concat", - tags=["发起任务"], - summary="发起合并任务", - description="依据AI分析的结果发起合并任务", - dependencies=[Depends(verify_token)]) - async def concat_media(body: FFMPEGConcatRequest, - headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - medias = body.medias - fn = modal.Function.from_name(config.app_name, "ffmpeg_concat_medias", - environment_name=config.environment) - sentry_trace = None - if headers.x_trace_id and headers.x_baggage: - sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) - fn_call = fn.spawn(medias, sentry_trace) - return ModalTaskResponse(success=True, taskId=fn_call.object_id) - - @web_app.get("/ffmpeg/concat/{task_id}", - tags=["查询任务"], - summary="获取合并任务结果", - description="获取合并任务的处理状态和结果", - responses={ - status.HTTP_200_OK: { - "description": "", - "headers": sentry_header_schema - }, - }, - dependencies=[Depends(verify_token)]) - async def concat_media_status(task_id: str, response: Response) -> FFMPEGConcatTaskStatusResponse: - task_status, code, reason, results, transaction = await get_modal_task_status(task_id) - if transaction: - response.headers["x-trace-id"] = transaction.x_trace_id - response.headers["x-baggage"] = transaction.x_baggage - return FFMPEGConcatTaskStatusResponse(taskId=task_id, status=task_status, - code=code, error=reason, - result=results) - - @web_app.post("/ffmpeg/extract-audio", - tags=["发起任务"], - summary="发起音频提取任务", - description="提取视频文件的音频", - dependencies=[Depends(verify_token)] - ) - async def extract_audio(body: FFMPEGExtractAudioRequest, - headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - media = body.media - fn = modal.Function.from_name(config.app_name, "ffmpeg_extract_audio", - environment_name=config.environment) - sentry_trace = None - if headers.x_trace_id and headers.x_baggage: - sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) - fn_call = fn.spawn(media, sentry_trace) - return ModalTaskResponse(success=True, taskId=fn_call.object_id) - - @web_app.get("/ffmpeg/extract-audio/{task_id}", - tags=["查询任务"], - summary="查询音频提取任务状态", - description="查询音频提取任务状态", - responses={ - status.HTTP_200_OK: { - "description": "", - "headers": sentry_header_schema - }, - }, - dependencies=[Depends(verify_token)]) - async def extract_audio_status(task_id: str, response: Response) -> FFMPEGExtractAudioTaskStatusResponse: - task_status, code, reason, results, transaction = await get_modal_task_status(task_id) - if transaction: - response.headers["x-trace-id"] = transaction.x_trace_id - response.headers["x-baggage"] = transaction.x_baggage - - return FFMPEGExtractAudioTaskStatusResponse(taskId=task_id, status=task_status, code=code, - error=reason, result=results) - - @web_app.post("/ffmpeg/corner-mirror", tags=["发起任务"], summary="发起镜像小窗去重任务", - description="发起镜像小窗去重任务", dependencies=[Depends(verify_token)]) - async def corner_mirror(body: FFMPEGCornerMirrorRequest, - headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - media = body.media - fn = modal.Function.from_name(config.app_name, "ffmpeg_corner_mirror", - environment_name=config.environment) - sentry_trace = None - if headers.x_trace_id and headers.x_baggage: - sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) - fn_call = fn.spawn(media=media, sentry_trace=sentry_trace) - return ModalTaskResponse(success=True, taskId=fn_call.object_id) - - @web_app.get("/ffmpeg/corner-mirror/{task_id}", - tags=["查询任务"], - summary="查询镜像小窗去重任务状态", - description="", - responses={ - status.HTTP_200_OK: { - "description": "", - "headers": sentry_header_schema - }, - }, - dependencies=[Depends(verify_token)]) - async def corner_mirror_status(task_id: str, response: Response) -> FFMPEGCornerMirrorTaskStatusResponse: - task_status, code, reason, result, transaction = await get_modal_task_status(task_id) - if transaction: - response.headers["x-trace-id"] = transaction.x_trace_id - response.headers["x-baggage"] = transaction.x_baggage - return FFMPEGCornerMirrorTaskStatusResponse(taskId=task_id, status=task_status, code=code, - error=reason, result=result) - - return web_app diff --git a/uv.lock b/uv.lock index 9419994..899ecc6 100644 --- a/uv.lock +++ b/uv.lock @@ -129,6 +129,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148 }, ] +[[package]] +name = "backports-tarfile" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181 }, +] + [[package]] name = "boto3" version = "1.38.10" @@ -353,6 +362,39 @@ version = "1.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/6b/b0/e595ce2a2527e169c3bcd6c33d2473c1918e0b7f6826a043ca1245dd4e5b/crcmod-1.7.tar.gz", hash = "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e", size = 89670 } +[[package]] +name = "cryptography" +version = "44.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/d6/1411ab4d6108ab167d06254c5be517681f1e331f90edf1379895bcb87020/cryptography-44.0.3.tar.gz", hash = "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053", size = 711096 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/06/af2cf8d56ef87c77319e9086601bef621bedf40f6f59069e1b6d1ec498c5/cryptography-44.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", size = 3959305 }, + { url = "https://files.pythonhosted.org/packages/ae/01/80de3bec64627207d030f47bf3536889efee8913cd363e78ca9a09b13c8e/cryptography-44.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", size = 4171040 }, + { url = "https://files.pythonhosted.org/packages/bd/48/bb16b7541d207a19d9ae8b541c70037a05e473ddc72ccb1386524d4f023c/cryptography-44.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", size = 3963411 }, + { url = "https://files.pythonhosted.org/packages/42/b2/7d31f2af5591d217d71d37d044ef5412945a8a8e98d5a2a8ae4fd9cd4489/cryptography-44.0.3-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", size = 3689263 }, + { url = "https://files.pythonhosted.org/packages/25/50/c0dfb9d87ae88ccc01aad8eb93e23cfbcea6a6a106a9b63a7b14c1f93c75/cryptography-44.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", size = 4196198 }, + { url = "https://files.pythonhosted.org/packages/66/c9/55c6b8794a74da652690c898cb43906310a3e4e4f6ee0b5f8b3b3e70c441/cryptography-44.0.3-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", size = 3966502 }, + { url = "https://files.pythonhosted.org/packages/b6/f7/7cb5488c682ca59a02a32ec5f975074084db4c983f849d47b7b67cc8697a/cryptography-44.0.3-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", size = 4196173 }, + { url = "https://files.pythonhosted.org/packages/d2/0b/2f789a8403ae089b0b121f8f54f4a3e5228df756e2146efdf4a09a3d5083/cryptography-44.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", size = 4087713 }, + { url = "https://files.pythonhosted.org/packages/1d/aa/330c13655f1af398fc154089295cf259252f0ba5df93b4bc9d9c7d7f843e/cryptography-44.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", size = 4299064 }, + { url = "https://files.pythonhosted.org/packages/b1/f0/7491d44bba8d28b464a5bc8cc709f25a51e3eac54c0a4444cf2473a57c37/cryptography-44.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", size = 3960307 }, + { url = "https://files.pythonhosted.org/packages/f7/c8/e5c5d0e1364d3346a5747cdcd7ecbb23ca87e6dea4f942a44e88be349f06/cryptography-44.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", size = 4170876 }, + { url = "https://files.pythonhosted.org/packages/73/96/025cb26fc351d8c7d3a1c44e20cf9a01e9f7cf740353c9c7a17072e4b264/cryptography-44.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", size = 3964127 }, + { url = "https://files.pythonhosted.org/packages/01/44/eb6522db7d9f84e8833ba3bf63313f8e257729cf3a8917379473fcfd6601/cryptography-44.0.3-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", size = 3689164 }, + { url = "https://files.pythonhosted.org/packages/68/fb/d61a4defd0d6cee20b1b8a1ea8f5e25007e26aeb413ca53835f0cae2bcd1/cryptography-44.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", size = 4198081 }, + { url = "https://files.pythonhosted.org/packages/1b/50/457f6911d36432a8811c3ab8bd5a6090e8d18ce655c22820994913dd06ea/cryptography-44.0.3-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", size = 3967716 }, + { url = "https://files.pythonhosted.org/packages/35/6e/dca39d553075980ccb631955c47b93d87d27f3596da8d48b1ae81463d915/cryptography-44.0.3-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", size = 4197398 }, + { url = "https://files.pythonhosted.org/packages/9b/9d/d1f2fe681eabc682067c66a74addd46c887ebacf39038ba01f8860338d3d/cryptography-44.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", size = 4087900 }, + { url = "https://files.pythonhosted.org/packages/c4/f5/3599e48c5464580b73b236aafb20973b953cd2e7b44c7c2533de1d888446/cryptography-44.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", size = 4301067 }, + { url = "https://files.pythonhosted.org/packages/58/11/0a6bf45d53b9b2290ea3cec30e78b78e6ca29dc101e2e296872a0ffe1335/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", size = 3895216 }, + { url = "https://files.pythonhosted.org/packages/0a/27/b28cdeb7270e957f0077a2c2bfad1b38f72f1f6d699679f97b816ca33642/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", size = 4115044 }, + { url = "https://files.pythonhosted.org/packages/35/b0/ec4082d3793f03cb248881fecefc26015813199b88f33e3e990a43f79835/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", size = 3898034 }, + { url = "https://files.pythonhosted.org/packages/0b/7f/adf62e0b8e8d04d50c9a91282a57628c00c54d4ae75e2b02a223bd1f2613/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", size = 4114449 }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -371,6 +413,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, ] +[[package]] +name = "docutils" +version = "0.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, +] + [[package]] name = "email-validator" version = "2.2.0" @@ -653,6 +704,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007 }, ] +[[package]] +name = "id" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/11/102da08f88412d875fa2f1a9a469ff7ad4c874b0ca6fed0048fe385bdb3d/id-1.5.0.tar.gz", hash = "sha256:292cb8a49eacbbdbce97244f47a97b4c62540169c976552e497fd57df0734c1d", size = 15237 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/cb/18326d2d89ad3b0dd143da971e77afd1e6ca6674f1b1c3df4b6bec6279fc/id-1.5.0-py3-none-any.whl", hash = "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658", size = 13611 }, +] + [[package]] name = "idna" version = "3.10" @@ -662,6 +725,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656 }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777 }, +] + +[[package]] +name = "jaraco-context" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/ad/f3777b81bf0b6e7bc7514a1656d3e637b2e8e15fab2ce3235730b3e7a4e6/jaraco_context-6.0.1.tar.gz", hash = "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", size = 13912 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/db/0c52c4cf5e4bd9f5d7135ec7669a3a767af21b3a308e1ed3674881e52b62/jaraco.context-6.0.1-py3-none-any.whl", hash = "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4", size = 6825 }, +] + +[[package]] +name = "jaraco-functools" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/23/9894b3df5d0a6eb44611c36aec777823fc2e07740dabbd0b810e19594013/jaraco_functools-4.1.0.tar.gz", hash = "sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d", size = 19159 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/4f/24b319316142c44283d7540e76c7b5a6dbd5db623abd86bb7b3491c21018/jaraco.functools-4.1.0-py3-none-any.whl", hash = "sha256:ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649", size = 10187 }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010 }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -692,6 +812,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/d3/13ee227a148af1c693654932b8b0b02ed64af5e1f7406d56b088b57574cd/joblib-1.5.0-py3-none-any.whl", hash = "sha256:206144b320246485b712fc8cc51f017de58225fa8b414a1fe1764a7231aca491", size = 307682 }, ] +[[package]] +name = "keyring" +version = "25.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/09/d904a6e96f76ff214be59e7aa6ef7190008f52a0ab6689760a98de0bf37d/keyring-25.6.0.tar.gz", hash = "sha256:0b39998aa941431eb3d9b0d4b2460bc773b9df6fed7621c2dfb291a7e0187a66", size = 62750 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/32/da7f44bcb1105d3e88a0b74ebdca50c59121d2ddf71c9e34ba47df7f3a56/keyring-25.6.0-py3-none-any.whl", hash = "sha256:552a3f7af126ece7ed5c89753650eec89c7eaae8617d0aa4d9ad2b75111266bd", size = 39085 }, +] + [[package]] name = "kiwisolver" version = "1.4.8" @@ -934,6 +1072,7 @@ dependencies = [ { name = "tencentcloud-sdk-python-common" }, { name = "tencentcloud-sdk-python-vod" }, { name = "tqdm" }, + { name = "twine" }, { name = "webvtt-py" }, ] @@ -961,9 +1100,19 @@ requires-dist = [ { name = "tencentcloud-sdk-python-common", specifier = ">=3.0.1363" }, { name = "tencentcloud-sdk-python-vod", specifier = ">=3.0.1363" }, { name = "tqdm", specifier = ">=4.67.1" }, + { name = "twine", specifier = ">=6.1.0" }, { name = "webvtt-py", specifier = ">=0.5.1" }, ] +[[package]] +name = "more-itertools" +version = "10.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a0/834b0cebabbfc7e311f30b46c8188790a37f89fc8d756660346fe5abfd09/more_itertools-10.7.0.tar.gz", hash = "sha256:9fddd5403be01a94b204faadcff459ec3568cf110265d3c54323e1e866ad29d3", size = 127671 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/9f/7ba6f94fc1e9ac3d2b853fdff3035fb2fa5afbed898c4a72b8a020610594/more_itertools-10.7.0-py3-none-any.whl", hash = "sha256:d43980384673cb07d2f7d2d918c616b30c659c089ee23953f601d6609c67510e", size = 65278 }, +] + [[package]] name = "multidict" version = "6.4.3" @@ -1041,6 +1190,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/10/7d526c8974f017f1e7ca584c71ee62a638e9334d8d33f27d7cdfc9ae79e4/multidict-6.4.3-py3-none-any.whl", hash = "sha256:59fe01ee8e2a1e8ceb3f6dbb216b09c8d9f4ef1c22c4fc825d045a147fa2ebc9", size = 10400 }, ] +[[package]] +name = "nh3" +version = "0.2.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/30/2f81466f250eb7f591d4d193930df661c8c23e9056bdc78e365b646054d8/nh3-0.2.21.tar.gz", hash = "sha256:4990e7ee6a55490dbf00d61a6f476c9a3258e31e711e13713b2ea7d6616f670e", size = 16581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/81/b83775687fcf00e08ade6d4605f0be9c4584cb44c4973d9f27b7456a31c9/nh3-0.2.21-cp313-cp313t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:fcff321bd60c6c5c9cb4ddf2554e22772bb41ebd93ad88171bbbb6f271255286", size = 1297678 }, + { url = "https://files.pythonhosted.org/packages/22/ee/d0ad8fb4b5769f073b2df6807f69a5e57ca9cea504b78809921aef460d20/nh3-0.2.21-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31eedcd7d08b0eae28ba47f43fd33a653b4cdb271d64f1aeda47001618348fde", size = 733774 }, + { url = "https://files.pythonhosted.org/packages/ea/76/b450141e2d384ede43fe53953552f1c6741a499a8c20955ad049555cabc8/nh3-0.2.21-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d426d7be1a2f3d896950fe263332ed1662f6c78525b4520c8e9861f8d7f0d243", size = 760012 }, + { url = "https://files.pythonhosted.org/packages/97/90/1182275db76cd8fbb1f6bf84c770107fafee0cb7da3e66e416bcb9633da2/nh3-0.2.21-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9d67709bc0d7d1f5797b21db26e7a8b3d15d21c9c5f58ccfe48b5328483b685b", size = 923619 }, + { url = "https://files.pythonhosted.org/packages/29/c7/269a7cfbec9693fad8d767c34a755c25ccb8d048fc1dfc7a7d86bc99375c/nh3-0.2.21-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:55823c5ea1f6b267a4fad5de39bc0524d49a47783e1fe094bcf9c537a37df251", size = 1000384 }, + { url = "https://files.pythonhosted.org/packages/68/a9/48479dbf5f49ad93f0badd73fbb48b3d769189f04c6c69b0df261978b009/nh3-0.2.21-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:818f2b6df3763e058efa9e69677b5a92f9bc0acff3295af5ed013da544250d5b", size = 918908 }, + { url = "https://files.pythonhosted.org/packages/d7/da/0279c118f8be2dc306e56819880b19a1cf2379472e3b79fc8eab44e267e3/nh3-0.2.21-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:b3b5c58161e08549904ac4abd450dacd94ff648916f7c376ae4b2c0652b98ff9", size = 909180 }, + { url = "https://files.pythonhosted.org/packages/26/16/93309693f8abcb1088ae143a9c8dbcece9c8f7fb297d492d3918340c41f1/nh3-0.2.21-cp313-cp313t-win32.whl", hash = "sha256:637d4a10c834e1b7d9548592c7aad760611415fcd5bd346f77fd8a064309ae6d", size = 532747 }, + { url = "https://files.pythonhosted.org/packages/a2/3a/96eb26c56cbb733c0b4a6a907fab8408ddf3ead5d1b065830a8f6a9c3557/nh3-0.2.21-cp313-cp313t-win_amd64.whl", hash = "sha256:713d16686596e556b65e7f8c58328c2df63f1a7abe1277d87625dcbbc012ef82", size = 528908 }, + { url = "https://files.pythonhosted.org/packages/ba/1d/b1ef74121fe325a69601270f276021908392081f4953d50b03cbb38b395f/nh3-0.2.21-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:a772dec5b7b7325780922dd904709f0f5f3a79fbf756de5291c01370f6df0967", size = 1316133 }, + { url = "https://files.pythonhosted.org/packages/b8/f2/2c7f79ce6de55b41e7715f7f59b159fd59f6cdb66223c05b42adaee2b645/nh3-0.2.21-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d002b648592bf3033adfd875a48f09b8ecc000abd7f6a8769ed86b6ccc70c759", size = 758328 }, + { url = "https://files.pythonhosted.org/packages/6d/ad/07bd706fcf2b7979c51b83d8b8def28f413b090cf0cb0035ee6b425e9de5/nh3-0.2.21-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2a5174551f95f2836f2ad6a8074560f261cf9740a48437d6151fd2d4d7d617ab", size = 747020 }, + { url = "https://files.pythonhosted.org/packages/75/99/06a6ba0b8a0d79c3d35496f19accc58199a1fb2dce5e711a31be7e2c1426/nh3-0.2.21-cp38-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b8d55ea1fc7ae3633d758a92aafa3505cd3cc5a6e40470c9164d54dff6f96d42", size = 944878 }, + { url = "https://files.pythonhosted.org/packages/79/d4/dc76f5dc50018cdaf161d436449181557373869aacf38a826885192fc587/nh3-0.2.21-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ae319f17cd8960d0612f0f0ddff5a90700fa71926ca800e9028e7851ce44a6f", size = 903460 }, + { url = "https://files.pythonhosted.org/packages/cd/c3/d4f8037b2ab02ebf5a2e8637bd54736ed3d0e6a2869e10341f8d9085f00e/nh3-0.2.21-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ca02ac6f27fc80f9894409eb61de2cb20ef0a23740c7e29f9ec827139fa578", size = 839369 }, + { url = "https://files.pythonhosted.org/packages/11/a9/1cd3c6964ec51daed7b01ca4686a5c793581bf4492cbd7274b3f544c9abe/nh3-0.2.21-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5f77e62aed5c4acad635239ac1290404c7e940c81abe561fd2af011ff59f585", size = 739036 }, + { url = "https://files.pythonhosted.org/packages/fd/04/bfb3ff08d17a8a96325010ae6c53ba41de6248e63cdb1b88ef6369a6cdfc/nh3-0.2.21-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:087ffadfdcd497658c3adc797258ce0f06be8a537786a7217649fc1c0c60c293", size = 768712 }, + { url = "https://files.pythonhosted.org/packages/9e/aa/cfc0bf545d668b97d9adea4f8b4598667d2b21b725d83396c343ad12bba7/nh3-0.2.21-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ac7006c3abd097790e611fe4646ecb19a8d7f2184b882f6093293b8d9b887431", size = 930559 }, + { url = "https://files.pythonhosted.org/packages/78/9d/6f5369a801d3a1b02e6a9a097d56bcc2f6ef98cffebf03c4bb3850d8e0f0/nh3-0.2.21-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:6141caabe00bbddc869665b35fc56a478eb774a8c1dfd6fba9fe1dfdf29e6efa", size = 1008591 }, + { url = "https://files.pythonhosted.org/packages/a6/df/01b05299f68c69e480edff608248313cbb5dbd7595c5e048abe8972a57f9/nh3-0.2.21-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:20979783526641c81d2f5bfa6ca5ccca3d1e4472474b162c6256745fbfe31cd1", size = 925670 }, + { url = "https://files.pythonhosted.org/packages/3d/79/bdba276f58d15386a3387fe8d54e980fb47557c915f5448d8c6ac6f7ea9b/nh3-0.2.21-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a7ea28cd49293749d67e4fcf326c554c83ec912cd09cd94aa7ec3ab1921c8283", size = 917093 }, + { url = "https://files.pythonhosted.org/packages/e7/d8/c6f977a5cd4011c914fb58f5ae573b071d736187ccab31bfb1d539f4af9f/nh3-0.2.21-cp38-abi3-win32.whl", hash = "sha256:6c9c30b8b0d291a7c5ab0967ab200598ba33208f754f2f4920e9343bdd88f79a", size = 537623 }, + { url = "https://files.pythonhosted.org/packages/23/fc/8ce756c032c70ae3dd1d48a3552577a325475af2a2f629604b44f571165c/nh3-0.2.21-cp38-abi3-win_amd64.whl", hash = "sha256:bb0014948f04d7976aabae43fcd4cb7f551f9f8ce785a4c9ef66e6c2590f8629", size = 535283 }, +] + [[package]] name = "noisereduce" version = "3.0.3" @@ -1509,6 +1689,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756 }, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -1544,6 +1733,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, ] +[[package]] +name = "readme-renderer" +version = "44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "nh3" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz", hash = "sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1", size = 32056 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl", hash = "sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151", size = 13310 }, +] + [[package]] name = "requests" version = "2.32.3" @@ -1559,6 +1762,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, ] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481 }, +] + +[[package]] +name = "rfc3986" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326 }, +] + [[package]] name = "rich" version = "14.0.0" @@ -1654,6 +1878,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/06/0a5e5349474e1cbc5757975b21bd4fad0e72ebf138c5592f191646154e06/scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca", size = 40308097 }, ] +[[package]] +name = "secretstorage" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", size = 19739 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", size = 15221 }, +] + [[package]] name = "sentry-sdk" version = "2.27.0" @@ -1800,6 +2037,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, ] +[[package]] +name = "twine" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "id" }, + { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x'" }, + { name = "packaging" }, + { name = "readme-renderer" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "rfc3986" }, + { name = "rich" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/a2/6df94fc5c8e2170d21d7134a565c3a8fb84f9797c1dd65a5976aaf714418/twine-6.1.0.tar.gz", hash = "sha256:be324f6272eff91d07ee93f251edf232fc647935dd585ac003539b42404a8dbd", size = 168404 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/b6/74e927715a285743351233f33ea3c684528a0d374d2e43ff9ce9585b73fe/twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384", size = 40791 }, +] + [[package]] name = "typer" version = "0.15.3" @@ -2112,3 +2369,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/93/f73b61353b2a699d489e782c3f5998b59f974ec3156a2050a52dfd7e8946/yarl-1.20.0-cp313-cp313t-win_amd64.whl", hash = "sha256:53b2da3a6ca0a541c1ae799c349788d480e5144cac47dba0266c7cb6c76151fe", size = 101093 }, { url = "https://files.pythonhosted.org/packages/ea/1f/70c57b3d7278e94ed22d85e09685d3f0a38ebdd8c5c73b65ba4c0d0fe002/yarl-1.20.0-py3-none-any.whl", hash = "sha256:5d0fe6af927a47a230f31e6004621fd0959eaa915fc62acfafa67ff7229a3124", size = 46124 }, ] + +[[package]] +name = "zipp" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630 }, +]