FIX AutoDL调度修复暖池功能

AutoDL/autodl_scheduling/entity/instance_pool.py

@@ -33,15 +33,16 @@ class InstancePool:
         self.instances:List[Instance] = []
         self.executor = ThreadPoolExecutor(max_workers=os.cpu_count()*2)
         self.threads = []
+        self.intro_threads = []
 
-    def scale_instance(self, target_instance):
+    def scale_instance(self, target_instance, disable_shrink=True):
         if target_instance + self.buffer_instance < self.min_instance:
             return self._scale(self.min_instance)
         if target_instance + self.buffer_instance > self.max_instance:
             return self._scale(self.max_instance)
         if target_instance + self.buffer_instance == len(self.instances):
             return True
-        return self._scale(target_instance + self.buffer_instance)
+        return self._scale(target_instance + self.buffer_instance, disable_shrink)
 
     def remove_instance(self, instance:Instance):
         if instance_operate(instance.uuid, "power_off"):
@@ -73,21 +74,34 @@ class InstancePool:
 
     def introspection(self):
         # 停止超时实例(运行超时和无任务超时)
+        before=len(self.instances)
         instance_copy = copy.deepcopy(self.instances)
+        flag = False
         for instance in instance_copy:
             if instance.active:
                 if (time.time() - instance.last_active_time) > self.timeout:
-                    self.threads.append(self.executor.submit(self.remove_instance, instance=instance))
+                    flag = True
+                    self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
             else:
                 if (time.time() - instance.last_active_time) > self.scaledown_window:
-                    self.threads.append(self.executor.submit(self.remove_instance, instance=instance))
+                    flag = True
+                    self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
+        while len(self.intro_threads) > 0:
+            for t in self.intro_threads:
+                t.result(timeout=self.timeout//2)
+                self.intro_threads.remove(t)
+        after = len(self.instances)
+        if flag:
+            loguru.logger.info("Instance Num Before Introspecting %d After Introspecting %d" % (before, after))
 
-    def _scale(self, target_instance:int):
-        loguru.logger.info("Instance Num Before Scaling %d ; Target %d" % (len(self.instances), target_instance))
+    def _scale(self, target_instance:int, disable_shrink=True):
         self.introspection()
         # 调整实例数量
         instance_copy = copy.deepcopy(self.instances)
         dest = target_instance - len(instance_copy)
+        if (disable_shrink and dest < 0) or dest == 0:
+            return True
+        loguru.logger.info("Instance Num Before Scaling %d ; Target %d" % (len(self.instances), target_instance))
         if dest < 0:
             dest = abs(dest)
             for instance in instance_copy: