Просмотр исходного кода

feature:排队模式-多实例串行执行

XQ 4 месяцев назад
Родитель
Сommit
2e87fd4427

+ 3 - 1
config/template.json

@@ -28,7 +28,9 @@
       "combat_screenshot_interval": 1.0,
       "combat_screenshot_interval": 1.0,
       "task_hoarding_duration": 0,
       "task_hoarding_duration": 0,
       "when_task_queue_empty": "goto_main",
       "when_task_queue_empty": "goto_main",
-      "schedule_rule": "Filter"
+      "schedule_rule": "Filter",
+      "queue_mode": false,
+      "queue_idle_threshold": 10
     }
     }
   },
   },
   "restart": {
   "restart": {

+ 318 - 0
module/config/instance_guard.py

@@ -0,0 +1,318 @@
+# This Python file uses the following encoding: utf-8
+# @author handman123
+# github https://github.com/runhey
+"""跨进程实例看守器。
+
+通过共享 JSON 状态文件协调多个进程的实例执行顺序,确保同一时间
+只有一个实例操作模拟器。持有执行权(token)的实例可执行任务,
+其他实例排队等待。
+"""
+import json
+import threading
+from pathlib import Path
+from datetime import datetime
+
+from filelock import FileLock
+from module.logger import logger
+
+
+class _QueueState:
+    """共享状态文件的持久化层。
+
+    负责 JSON 文件的原子读写和持有者心跳过期判断。
+    文件锁(FileLock)保证跨进程写入安全。
+
+    文件格式:
+        {
+            "current": "oas1",               # 当前持有执行权的实例名
+            "timestamp": "2026-07-08 ...",   # 持有者心跳时间戳
+            "queue": ["oas2", "oas3"]        # 等待队列
+        }
+    """
+
+    STATE_FILE = Path.cwd() / "log" / ".queue_state.json"
+    STALE_TIMEOUT = 300  # 心跳超时 5 分钟,超过视为持有者崩溃
+
+    def __init__(self, config_name: str):
+        """Args:
+            config_name: 配置文件名称,如 'oas1'。
+        """
+        self.config_name = config_name
+        self._lock = FileLock(str(self.STATE_FILE) + ".lock", timeout=5)
+
+    def read(self) -> dict:
+        """读取状态文件。
+
+        调用方需持有 _lock。文件不存在或损坏时返回空状态。
+
+        Returns:
+            dict: 包含 current、timestamp、queue 的状态字典。
+        """
+        if not self.STATE_FILE.exists():
+            return {"current": None, "timestamp": None, "queue": []}
+        try:
+            with open(self.STATE_FILE, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            logger.warning("[InstanceGuard] State file corrupted, resetting")
+            return {"current": None, "timestamp": None, "queue": []}
+
+    def write(self, state: dict) -> None:
+        """写入状态文件。调用方需持有 _lock。"""
+        self.STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.STATE_FILE, "w", encoding="utf-8") as f:
+            json.dump(state, f, indent=2, ensure_ascii=False)
+
+    def is_stale(self, state: dict) -> bool:
+        """判断持有者心跳是否过期。
+
+        如果 timestamp 距现在超过 STALE_TIMEOUT 秒,认为持有者崩溃。
+
+        Args:
+            state: 从 read() 获取的状态字典。
+
+        Returns:
+            bool: True 表示心跳过期,可安全接管。
+        """
+        ts_str = state.get("timestamp")
+        if not ts_str:
+            return False
+        try:
+            ts = datetime.fromisoformat(ts_str)
+            elapsed = (datetime.now() - ts).total_seconds()
+            return elapsed > self.STALE_TIMEOUT
+        except (ValueError, TypeError):
+            return True
+
+    def __enter__(self):
+        self._lock.acquire()
+        return self.read()
+
+    def __exit__(self, *args):
+        self._lock.release()
+
+
+class _Heartbeat:
+    """心跳保活线程。
+
+    持有 token 期间定期更新 timestamp,防止被误判崩溃。
+    同时监控 token 是否被其他实例抢占(例如被管理员从状态文件强制接管),
+    检测到丢失时设置 token_lost 标志通知主线程。
+    """
+
+    INTERVAL = 180  # 心跳间隔 3 分钟,小于 STALE_TIMEOUT(5min) 留出缓冲
+
+    def __init__(self, config_name: str, state: _QueueState):
+        self._config_name = config_name
+        self._state = state
+        self._thread: threading.Thread = None
+        self._stop_event: threading.Event = None
+        self._token_lost = threading.Event()
+
+    @property
+    def token_lost(self) -> bool:
+        """token 是否被其他实例抢占。"""
+        return self._token_lost.is_set()
+
+    def start(self) -> None:
+        """启动心跳 daemon 线程。已运行则跳过。"""
+        if self._thread and self._thread.is_alive():
+            return
+        self._token_lost.clear()
+        self._stop_event = threading.Event()
+        self._thread = threading.Thread(target=self._loop, daemon=True)
+        self._thread.start()
+        logger.info(
+            f"[InstanceGuard] Heartbeat started for '{self._config_name}'")
+
+    def stop(self) -> None:
+        """停止心跳线程。"""
+        if self._stop_event:
+            self._stop_event.set()
+        self._thread = None
+        self._stop_event = None
+
+    def _loop(self) -> None:
+        """心跳循环。每 INTERVAL 秒刷新 timestamp。"""
+        while not self._stop_event.wait(timeout=self.INTERVAL):
+            try:
+                with self._state._lock:
+                    state = self._state.read()
+                    if state["current"] == self._config_name:
+                        state["timestamp"] = datetime.now().isoformat()
+                        self._state.write(state)
+                    elif self._config_name not in state.get("queue", []):
+                        # 不在 current 也不在 queue,说明被强制移除了
+                        logger.warning(
+                            f"[InstanceGuard] '{self._config_name}' lost token, "
+                            f"current holder: {state['current']}")
+                        self._token_lost.set()
+                        return
+            except Exception as e:
+                logger.error(f"[InstanceGuard] Heartbeat error: {e}")
+
+
+class InstanceGuard:
+    """跨进程实例看守器。
+
+    通过共享状态文件协调多个实例的执行顺序,确保同一时间只有一个实例
+    操作模拟器。集成心跳保活和崩溃检测。
+
+    用法:
+        guard = InstanceGuard(config_name)
+        if guard.try_acquire():
+            # 执行业务逻辑
+            guard.release()
+    """
+
+    def __init__(self, config_name: str):
+        """Args:
+            config_name: 配置文件名称。
+        """
+        self.config_name = config_name
+        self._state = _QueueState(config_name)
+        self._heartbeat = _Heartbeat(config_name, self._state)
+
+    @property
+    def token_lost(self) -> bool:
+        """token 是否被其他实例抢占。"""
+        return self._heartbeat.token_lost
+
+    # ==================== 执行权操作 ====================
+
+    def try_acquire(self) -> bool:
+        """尝试获取执行权。
+
+        四种情况:
+        1. 自己已是持有者 — 刷新 heartbeat,返回 True
+        2. 持有者心跳过期 — 强制接管,返回 True
+        3. 无人持有 — 自己成为持有者,启动心跳,返回 True
+        4. 他人持有 — 加入等待队列,返回 False
+
+        Returns:
+            True: 已获得执行权。
+            False: 执行权被他人持有,已加入队列。
+        """
+        with self._state._lock:
+            state = self._state.read()
+
+            # 自己已是持有者,刷新心跳
+            if state["current"] == self.config_name:
+                state["timestamp"] = datetime.now().isoformat()
+                self._state.write(state)
+                return True
+
+            # 持有者崩溃,强制接管
+            if state["current"] and self._state.is_stale(state):
+                logger.warning(
+                    f"[InstanceGuard] Holder '{state['current']}' "
+                    f"heartbeat stale, taking over")
+                state["current"] = None
+                state["timestamp"] = None
+
+            # 无人持有,自己成为持有者
+            if state["current"] is None:
+                state["current"] = self.config_name
+                state["timestamp"] = datetime.now().isoformat()
+                if self.config_name in state["queue"]:
+                    state["queue"].remove(self.config_name)
+                self._state.write(state)
+                logger.info(
+                    f"[InstanceGuard] '{self.config_name}' acquired token")
+                self._heartbeat.start()
+                return True
+
+            # 他人持有,加入等待队列
+            if self.config_name not in state["queue"]:
+                state["queue"].append(self.config_name)
+                self._state.write(state)
+                logger.info(
+                    f"[InstanceGuard] '{self.config_name}' waiting, "
+                    f"holder: '{state['current']}', "
+                    f"queue: {state['queue']}")
+            return False
+
+    def release(self) -> None:
+        """释放执行权。
+
+        停止心跳线程。如果等待队列非空,自动移交给下一个实例。
+        如果自己不是持有者,静默返回。
+        """
+        with self._state._lock:
+            state = self._state.read()
+            if state["current"] != self.config_name:
+                return
+
+            self._heartbeat.stop()
+
+            if state["queue"]:
+                next_holder = state["queue"].pop(0)
+                state["current"] = next_holder
+                state["timestamp"] = datetime.now().isoformat()
+                logger.info(
+                    f"[InstanceGuard] '{self.config_name}' released token "
+                    f"→ '{next_holder}'")
+            else:
+                state["current"] = None
+                state["timestamp"] = None
+                logger.info(
+                    f"[InstanceGuard] '{self.config_name}' released token, "
+                    f"queue empty")
+            self._state.write(state)
+
+    def should_release(self, pending_task: list, waiting_task: list,
+                       idle_threshold_minutes: int = 10) -> bool:
+        """判断是否应该在进入空闲前释放执行权。
+
+        条件:
+        1. pending_task 为空(当前无待执行任务)
+        2. waiting_task 为空,或最早的任务距现在超过 idle_threshold_minutes 分钟
+
+        Args:
+            pending_task: 当前到期的任务列表。
+            waiting_task: 未来任务列表,按 next_run 升序。
+            idle_threshold_minutes: 空闲阈值(分钟)。
+
+        Returns:
+            True: 空闲时间足够长,应释放执行权。
+            False: 即将有任务,应继续持有。
+        """
+        if pending_task:
+            return False
+        if not waiting_task:
+            return True
+
+        next_task_time = waiting_task[0].next_run
+        idle_seconds = (next_task_time - datetime.now()).total_seconds()
+        return idle_seconds > idle_threshold_minutes * 60
+
+    def remove_from_queue(self) -> None:
+        """从排队系统中完全移除自己。
+
+        进程停止时调用,防止残留状态导致死锁。
+        如果自己是持有者,将执行权移交给下一个等待者。
+        """
+        with self._state._lock:
+            state = self._state.read()
+
+            if self.config_name in state["queue"]:
+                state["queue"].remove(self.config_name)
+                logger.info(
+                    f"[InstanceGuard] '{self.config_name}' removed from queue")
+
+            if state["current"] == self.config_name:
+                self._heartbeat.stop()
+                if state["queue"]:
+                    next_holder = state["queue"].pop(0)
+                    state["current"] = next_holder
+                    state["timestamp"] = datetime.now().isoformat()
+                    logger.info(
+                        f"[InstanceGuard] '{self.config_name}' removed, "
+                        f"token → '{next_holder}'")
+                else:
+                    state["current"] = None
+                    state["timestamp"] = None
+                self._state.write(state)
+                return
+
+            self._state.write(state)

+ 9 - 1
module/device/method/adb.py

@@ -12,7 +12,7 @@ from module.device.connection import Connection
 from module.device.method.minitouch import smooth_path
 from module.device.method.minitouch import smooth_path
 from module.device.method.utils import (RETRY_TRIES, retry_sleep, remove_prefix, handle_adb_error,
 from module.device.method.utils import (RETRY_TRIES, retry_sleep, remove_prefix, handle_adb_error,
                                         ImageTruncated, PackageNotInstalled)
                                         ImageTruncated, PackageNotInstalled)
-from module.exception import RequestHumanTakeover, ScriptError
+from module.exception import RequestHumanTakeover, ScriptError, EmulatorNotRunningError
 from module.logger import logger
 from module.logger import logger
 
 
 
 
@@ -33,6 +33,14 @@ def retry(func):
             # Can't handle
             # Can't handle
             except RequestHumanTakeover:
             except RequestHumanTakeover:
                 break
                 break
+            # Emulator not running, try to restart it
+            except EmulatorNotRunningError:
+                logger.error('Emulator not running, attempting to restart')
+                if hasattr(self, 'emulator_start'):
+                    self.emulator_start()
+
+                def init():
+                    self.adb_reconnect()
             # When adb server was killed
             # When adb server was killed
             except ConnectionResetError as e:
             except ConnectionResetError as e:
                 logger.error(e)
                 logger.error(e)

+ 1 - 0
module/server/config_manager.py

@@ -26,6 +26,7 @@ class ConfigManager:
             # 如果没有脚本文件 则创建一个
             # 如果没有脚本文件 则创建一个
             ConfigManager.copy(file='oas1', template='template')
             ConfigManager.copy(file='oas1', template='template')
             result.append('oas1')
             result.append('oas1')
+        result.sort()
         return result
         return result
 
 
     @staticmethod
     @staticmethod

+ 11 - 0
module/server/script_process.py

@@ -123,6 +123,11 @@ class ScriptProcess(ScriptWSManager):
 def func(config: str, state_queue: multiprocessing.Queue, log_pipe_in) -> None:
 def func(config: str, state_queue: multiprocessing.Queue, log_pipe_in) -> None:
     def signal_handler(signum, frame):
     def signal_handler(signum, frame):
         logger.info(f'Script {config} received signal {signum}, exiting gracefully')
         logger.info(f'Script {config} received signal {signum}, exiting gracefully')
+        try:
+            from module.config.instance_guard import InstanceGuard
+            InstanceGuard(config).remove_from_queue()
+        except Exception:
+            pass
         log_pipe_in.close()
         log_pipe_in.close()
         state_queue.close()
         state_queue.close()
         sys.exit(0)
         sys.exit(0)
@@ -160,6 +165,12 @@ def func(config: str, state_queue: multiprocessing.Queue, log_pipe_in) -> None:
         logger.exception(f'Run script {config} error')
         logger.exception(f'Run script {config} error')
         logger.error(f'Error: {e}')
         logger.error(f'Error: {e}')
         raise
         raise
+    finally:
+        try:
+            from module.config.instance_guard import InstanceGuard
+            InstanceGuard(config).remove_from_queue()
+        except Exception:
+            pass
 
 
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':

+ 97 - 0
script.py

@@ -28,6 +28,7 @@ from multiprocessing.queues import Queue
 from module.config.utils import convert_to_underscore
 from module.config.utils import convert_to_underscore
 from module.config.config import Config
 from module.config.config import Config
 from module.config.config_model import ConfigModel
 from module.config.config_model import ConfigModel
+from module.config.instance_guard import InstanceGuard
 from module.device.device import Device
 from module.device.device import Device
 from module.device.env import IS_WINDOWS
 from module.device.env import IS_WINDOWS
 from module.base.utils import load_module
 from module.base.utils import load_module
@@ -57,6 +58,8 @@ class Script:
         self.failure_record = {}
         self.failure_record = {}
         # 运行loop的线程
         # 运行loop的线程
         self.loop_thread: Thread = None
         self.loop_thread: Thread = None
+        # 跨进程排队管理器(仅在 queue_mode=True 时初始化)
+        self.instance_guard: InstanceGuard = None
 
 
     @cached_property
     @cached_property
     def config(self) -> "Config":
     def config(self) -> "Config":
@@ -316,10 +319,65 @@ class Script:
             if task.next_run <= now:
             if task.next_run <= now:
                 return task.command
                 return task.command
             # 根据策略执行等待逻辑
             # 根据策略执行等待逻辑
+            if not self._try_acquire_queue_token():
+                del_cached_property(self, "config")
+                continue
             if not self._handle_wait_during_idle(task.next_run):
             if not self._handle_wait_during_idle(task.next_run):
                 # 若等待被打断, 则刷新配置
                 # 若等待被打断, 则刷新配置
                 del_cached_property(self, "config")
                 del_cached_property(self, "config")
 
 
+    def _try_acquire_queue_token(self) -> bool:
+        """
+        尝试获取排队执行权。
+        如果排队模式未启用,返回 True。
+        如果排队模式启用但未能获取到执行权,进入等待循环直到获取成功或配置变更。
+
+        Returns:
+            True: 获取得执行权,可以执行任务
+            False: 等待被配置变更打断,调用方应重新加载配置后重试
+        """
+        # 是否开启排队模式
+        if not self.config.script.optimization.queue_mode:
+            if self.instance_guard:
+                self.instance_guard.remove_from_queue()
+                self.instance_guard = None
+            return True
+
+        # 懒加载instance_guard
+        if self.instance_guard is None:
+            try:
+                self.instance_guard = InstanceGuard(self.config_name)
+                logger.info(f"[Queue] Queue mode enabled for '{self.config_name}'")
+            except Exception:
+                self.instance_guard = None
+                return True
+
+        # 尝试获取执行权
+        if self.instance_guard.try_acquire():
+            return True
+
+        # 执行权获取失败,关闭模拟器并进入等待循环
+        logger.info(f"[Queue] '{self.config_name}' waiting for execution token...")
+        if (self.config.script.optimization.when_task_queue_empty == 'close_game'
+                and not self._emulator_down
+                and 'device' in self.__dict__):
+            try:
+                self.device.emulator_stop()
+                self._emulator_down = True
+                logger.info(f"[Queue] Emulator closed during queue wait")
+            except Exception:
+                pass
+        self.config.start_watching()
+        while True:
+            time.sleep(30)
+
+            if self.config.should_reload():
+                logger.info(f"[Queue] Config changed, re-evaluating")
+                return False
+
+            if self.instance_guard.try_acquire():
+                return True
+
     def _handle_wait_during_idle(self, next_run: datetime) -> bool:
     def _handle_wait_during_idle(self, next_run: datetime) -> bool:
         """
         """
         处理任务空闲期间的行为策略
         处理任务空闲期间的行为策略
@@ -362,6 +420,8 @@ class Script:
 
 
         if self._emulator_down:
         if self._emulator_down:
             logger.info("Wake emulator before next task")
             logger.info("Wake emulator before next task")
+            if not self._try_acquire_queue_token():
+                return False
             self.device = Device(self.config)
             self.device = Device(self.config)
             self._emulator_down = False
             self._emulator_down = False
 
 
@@ -384,6 +444,13 @@ class Script:
             self.device.emulator_stop()
             self.device.emulator_stop()
             self._emulator_down = True
             self._emulator_down = True
 
 
+            if self.instance_guard and self.instance_guard.should_release(
+                pending_task=self.config.pending_task,
+                waiting_task=self.config.waiting_task,
+                idle_threshold_minutes=self.config.script.optimization.queue_idle_threshold
+            ):
+                self.instance_guard.release()
+
             if not self._wait_until_with_emulator_preheat(next_run):
             if not self._wait_until_with_emulator_preheat(next_run):
                 return False
                 return False
 
 
@@ -426,6 +493,13 @@ class Script:
             self.device.emulator_stop()
             self.device.emulator_stop()
             self._emulator_down = True
             self._emulator_down = True
 
 
+            if self.instance_guard and self.instance_guard.should_release(
+                pending_task=self.config.pending_task,
+                waiting_task=self.config.waiting_task,
+                idle_threshold_minutes=self.config.script.optimization.queue_idle_threshold
+            ):
+                self.instance_guard.release()
+
             if not self._wait_until_with_emulator_preheat(next_run):
             if not self._wait_until_with_emulator_preheat(next_run):
                 return False
                 return False
 
 
@@ -435,6 +509,12 @@ class Script:
         logger.info("Goto main page during wait")
         logger.info("Goto main page during wait")
         self.run("GotoMain")
         self.run("GotoMain")
         self.device.release_during_wait()
         self.device.release_during_wait()
+        if self.instance_guard and self.instance_guard.should_release(
+            pending_task=self.config.pending_task,
+            waiting_task=self.config.waiting_task,
+            idle_threshold_minutes=self.config.script.optimization.queue_idle_threshold
+        ):
+            self.instance_guard.release()
         return self.wait_until(next_run)
         return self.wait_until(next_run)
 
 
     def _wait_stay_there(self, next_run: datetime) -> bool:
     def _wait_stay_there(self, next_run: datetime) -> bool:
@@ -466,6 +546,19 @@ class Script:
         if command == 'start' or command == 'goto_main':
         if command == 'start' or command == 'goto_main':
             logger.error(f'Invalid command `{command}`')
             logger.error(f'Invalid command `{command}`')
 
 
+        if not self._try_acquire_queue_token():
+            return False
+
+        if self.instance_guard and self.instance_guard.token_lost:
+            logger.warning(f'Token lost, stopping emulator and rejoining queue')
+            try:
+                self.device.emulator_stop()
+            except Exception:
+                pass
+            self._emulator_down = True
+            self.instance_guard.release()
+            return False
+
         try:
         try:
             self.device.screenshot()
             self.device.screenshot()
             module_name = 'script_task'
             module_name = 'script_task'
@@ -581,6 +674,10 @@ class Script:
                 del_cached_property(self, 'config')
                 del_cached_property(self, 'config')
                 continue
                 continue
 
 
+            if not self._try_acquire_queue_token():
+                del_cached_property(self, 'config')
+                continue
+
             if self._emulator_down:
             if self._emulator_down:
                 self.device = Device(self.config)
                 self.device = Device(self.config)
                 self._emulator_down = False
                 self._emulator_down = False

+ 4 - 0
tasks/Script/config_optimization.py

@@ -32,4 +32,8 @@ class Optimization(BaseModel):
     emulator_startup_lead_time: Time = Field(default=Time(minute=2),
     emulator_startup_lead_time: Time = Field(default=Time(minute=2),
                                             description='emulator_startup_lead_time_help')
                                             description='emulator_startup_lead_time_help')
     schedule_rule: ScheduleRule = Field(default=ScheduleRule.FILTER, description='schedule_rule_help')
     schedule_rule: ScheduleRule = Field(default=ScheduleRule.FILTER, description='schedule_rule_help')
+    # 排队模式:多个实例排队依次执行任务,避免同时执行造成服务器压力
+    queue_mode: bool = Field(default=False, description='queue_mode_help')
+    # 释放执行权的空闲阈值(分钟),仅 queue_mode=True 时生效
+    queue_idle_threshold: int = Field(default=10, description='queue_idle_threshold_help')