comi-logic/modalDeploy
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FIX AutoDL修复临界区资源竞争问题

Browse Source
This commit is contained in:
kyj@bowong.ai
2025-04-17 15:37:31 +08:00
parent bf4c66fc27
commit 1dfab45d83
3 changed files with 72 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
AutoDL/autodl_scheduling/server.py
View File

@@ -21,13 +21,15 @@ class Server:
self.app = FastAPI()
self.waiting_queue = WaitingQueue()
self.running_pool = RunningPool()
self.instance_pool = InstancePool(max_instance=29)
#账号限制max_instance不能超过30
self.instance_pool = InstancePool(max_instance=5)
self.result_map = ResultMap()
self.executor = ThreadPoolExecutor(max_workers=2)
self.worker_1 = self.executor.submit(self.scaling_worker)
self.worker_2 = self.executor.submit(self.introspect_instance)
self.app.routes.append(APIRoute("/", endpoint=self.submit, methods=['POST']))
self.app.routes.append(APIRoute("/{uid}", endpoint=self.get_result, methods=['GET']))
self.app.routes.append(APIRoute("/", endpoint=self.get_all_result, methods=['GET']))
# async def submit(self, video_url, audio_url, token: HTTPAuthorizationCredentials = Depends(HTTPBearer())):
async def submit(self, video_url, audio_url):
@@ -48,10 +50,14 @@ class Server:
try:
return self.result_map.get(uid)
except:
if uid in self.running_pool.tasks:
return {"status": "running", "msg": ""}
return {"status": "queuing", "msg":""}
async def get_all_result(self):
try:
return self.result_map.map
except:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def introspect_instance(self):
loguru.logger.info("Introspecting worker started || Scaledown Window: %ds" % self.instance_pool.scaledown_window)
while True:
@@ -74,23 +80,27 @@ class Server:
instance.active = True
instance.last_active_time = time.time()
loguru.logger.info("Task Submitted")
self.result_map.set(uid, {"status": "running", "msg": "Task Submitted, Waiting for result"})
else:
loguru.logger.error("Submit Task Failed")
self.result_map.set(uid, {"status":"fail", "msg":"Submit Task Failed"})
#查询结果放到结果缓存里
for uid, task in list(self.running_pool.tasks.items()):
result = self.running_pool.result(uid)
# 任务运行完成或失败
if result:
self.result_map.set(uid, result)
self.result_map.update(uid, result)
self.running_pool.tasks.pop(uid)
if result["status"] == "fail":
#删除失败任务的实例
self.instance_pool.remove_instance(Instance(uuid=task["instance_id"]))
loguru.logger.info("Instance[%s] Removed Due to Task Failure" % task["instance_id"])
else:
#更新成功任务实例的状态
for instance in self.instance_pool.instances:
if instance.uuid == task["instance_id"]:
loguru.logger.info("Instance[%s] Task Finished" % instance.uuid)
instance.active = False
instance.last_active_time = time.time()
time.sleep(0.5)