FIX AutoDL修复队列空等空指针问题

AutoDL/autodl_scheduling/entity/instance_pool.py

@@ -84,22 +84,25 @@ class InstancePool:
         try:
             before=len(self.instances)
             instance_copy = copy.deepcopy(self.instances)
-            flag = False
+            flag = []
             for instance in instance_copy:
                 if instance.active:
                     if (time.time() - instance.last_active_time) > self.timeout:
-                        flag = True
+                        flag.append(instance.uuid)
                         self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
                 else:
                     if (time.time() - instance.last_active_time) > self.scaledown_window:
-                        flag = True
+                        flag.append(instance.uuid)
                         self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
             while len(self.intro_threads) > 0:
                 for t in self.intro_threads:
                     t.result(timeout=self.timeout//2)
                     self.intro_threads.remove(t)
             after = len(self.instances)
-            if flag:
+            for instance in self.instances:
+                if instance.uuid in flag:
+                    raise Exception("Instance[%s] Remove Failed" % instance.uuid)
+            if len(flag) > 0:
                 loguru.logger.info("Instance Num Before Introspecting %d After Introspecting %d" % (before, after))
         except Exception as e:
             loguru.logger.error("Fail to Introspect Instances: %s" % e)

AutoDL/autodl_scheduling/entity/waiting_queue.py

@@ -11,10 +11,10 @@ class WaitingQueue:
             "video_path": video_path,
             "audio_path": audio_path
         }
-        self.queue.put(data)
+        self.queue.put(data,timeout=10)
 
     def dequeue(self):
-        data = self.queue.get()
+        data = self.queue.get(timeout=10)
         return data["uid"], data["video_path"], data["audio_path"]
 
     def get_size(self):