config.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327
  1. # This Python file uses the following encoding: utf-8
  2. # @author runhey
  3. # github https://github.com/runhey
  4. import copy
  5. import datetime
  6. import operator
  7. import threading
  8. from datetime import datetime
  9. from module.base.filter import Filter
  10. from module.config.config_updater import ConfigUpdater
  11. from module.config.config_manual import ConfigManual
  12. from module.config.config_watcher import ConfigWatcher
  13. from module.config.config_menu import ConfigMenu
  14. from module.config.config_model import ConfigModel
  15. from module.config.config_state import ConfigState
  16. from module.config.scheduler import TaskScheduler
  17. from module.config.utils import *
  18. from module.exception import RequestHumanTakeover, ScriptError
  19. from module.logger import logger
  20. class Function:
  21. def __init__(self, key: str, data: dict):
  22. """
  23. 输入的是每一个ConfigModel的一个字段对象
  24. :param data:
  25. """
  26. if isinstance(data, dict) is False:
  27. self.enable = False
  28. self.command = "Unknown"
  29. self.next_run = DEFAULT_TIME
  30. return
  31. if data.get("scheduler") is None:
  32. self.enable = False
  33. self.command = "Unknown"
  34. self.next_run = DEFAULT_TIME
  35. return
  36. self.enable: bool = data['scheduler']['enable']
  37. self.command: str = ConfigModel.type(key)
  38. next_run = data['scheduler']['next_run']
  39. if isinstance(next_run, str):
  40. next_run = datetime.strptime(next_run, "%Y-%m-%d %H:%M:%S")
  41. self.next_run: datetime = next_run
  42. priority = data['scheduler']['priority']
  43. if isinstance(priority, str):
  44. priority = int(priority)
  45. self.priority: int = priority
  46. if not isinstance(self.priority, int):
  47. logger.error(f"Invalid priority: {self.priority}")
  48. # self.enable = deep_get(data, keys="Scheduler.Enable", default=False)
  49. # self.command = deep_get(data, keys="Scheduler.Command", default="Unknown")
  50. # self.next_run = deep_get(data, keys="Scheduler.NextRun", default=DEFAULT_TIME)
  51. def __str__(self):
  52. enable = "Enable" if self.enable else "Disable"
  53. return f"{self.command} ({enable}, {self.priority}, {str(self.next_run)})"
  54. __repr__ = __str__
  55. def __eq__(self, other):
  56. if not isinstance(other, Function):
  57. return False
  58. if self.command == other.command and self.next_run == other.next_run:
  59. return True
  60. else:
  61. return False
  62. def name_to_function(name):
  63. """
  64. Args:
  65. name (str):
  66. Returns:
  67. Function:
  68. """
  69. function = Function({})
  70. function.command = name
  71. function.enable = True
  72. return function
  73. class Config(ConfigState, ConfigManual, ConfigWatcher, ConfigMenu):
  74. def __init__(self, config_name: str, task=None) -> None:
  75. """
  76. :param config_name:
  77. :param task:
  78. """
  79. super().__init__(config_name) # 调用 ConfigState 的初始化方法
  80. super(ConfigManual, self).__init__()
  81. super(ConfigWatcher, self).__init__()
  82. super(ConfigMenu, self).__init__()
  83. self.model = ConfigModel(config_name=config_name)
  84. def __getattr__(self, name):
  85. """
  86. 一开始是打算直接继承ConfigModel的,但是pydantic会接管所有的变量
  87. 故而选择持有ConfigModel
  88. :param name:
  89. :return:
  90. """
  91. try:
  92. return getattr(self.model, name)
  93. except AttributeError:
  94. logger.error(f'can not ask this variable {name}')
  95. return None # 或者抛出异常,或者返回其他默认值
  96. def gui_args(self, task: str) -> str:
  97. """
  98. 获取给gui显示的参数
  99. :return:
  100. """
  101. return self.model.gui_args(task=task)
  102. def get_arg(self, task: str, group: str, argument: str):
  103. """
  104. :param task:
  105. :param group:
  106. :param argument:
  107. :return: str/int/float
  108. """
  109. try:
  110. return self.data[task][group][argument]
  111. except:
  112. logger.exception(f'have no arg {task}.{group}.{argument}')
  113. def set_arg(self, task: str, group: str, argument: str, value) -> None:
  114. """
  115. :param task:
  116. :param group:
  117. :param argument:
  118. :param value:
  119. :return:
  120. """
  121. try:
  122. self.data[task][group][argument] = value
  123. except:
  124. logger.exception(f'have no arg {task}.{group}.{argument}')
  125. def save(self) -> None:
  126. """
  127. 保存配置文件
  128. :return:
  129. """
  130. self.model.write_json(self.config_name, self.model.dict())
  131. def update_scheduler(self) -> None:
  132. """
  133. 更新调度器, 设置pending_task and waiting_task
  134. :return:
  135. """
  136. pending_task = []
  137. waiting_task = []
  138. error = []
  139. now = datetime.now()
  140. for key, value in self.model.dict().items():
  141. func = Function(key, value)
  142. if not func.enable:
  143. continue
  144. if not isinstance(func.next_run, datetime):
  145. error.append(func)
  146. elif func.next_run < now:
  147. pending_task.append(func)
  148. else:
  149. waiting_task.append(func)
  150. # f = Filter(regex=r"(.*)", attr=["command"])
  151. # f.load(self.SCHEDULER_PRIORITY)
  152. if pending_task:
  153. pending_task = TaskScheduler.schedule(rule=self.model.script.optimization.schedule_rule,
  154. pending=pending_task)
  155. if waiting_task:
  156. # waiting_task = f.apply(waiting_task)
  157. waiting_task = sorted(waiting_task, key=operator.attrgetter("next_run"))
  158. if error:
  159. pending_task = error + pending_task
  160. self.pending_task = pending_task
  161. self.waiting_task = waiting_task
  162. def get_next(self) -> Function:
  163. """
  164. 获取下一个要执行的任务
  165. :return:
  166. """
  167. self.update_scheduler()
  168. if self.pending_task:
  169. logger.info(f"Pending tasks: {[f.command for f in self.pending_task]}")
  170. task = self.pending_task[0]
  171. logger.attr("Task", task)
  172. return task
  173. # 哪怕是没有任务,也要返回一个任务,这样才能保证调度器正常运行
  174. if self.waiting_task:
  175. logger.info("No task pending")
  176. task = copy.deepcopy(self.waiting_task[0])
  177. # task.next_run = (task.next_run + self.hoarding).replace(microsecond=0)
  178. logger.attr("Task", task)
  179. return task
  180. else:
  181. logger.critical("No task waiting or pending")
  182. logger.critical("Please enable at least one task")
  183. raise RequestHumanTakeover
  184. def task_call(self, task: str=None, force_call=True):
  185. """
  186. 回调任务,这会是在任务结束后调用
  187. :param task: 调用的任务的大写名称
  188. :param force_call:
  189. :return:
  190. """
  191. task = convert_to_underscore(task)
  192. if self.model.deep_get(self.model, keys=f'{task}.scheduler.next_run') is None:
  193. raise ScriptError(f"Task to call: `{task}` does not exist in user config")
  194. task_enable = self.model.deep_get(self.model, keys=f'{task}.scheduler.enable')
  195. if force_call or task_enable:
  196. logger.info(f"Task call: {task}")
  197. next_run = datetime.now().replace(
  198. microsecond=0
  199. )
  200. self.model.deep_set(self.model, keys=f'{task}.scheduler.next_run', value=next_run)
  201. self.save()
  202. return True
  203. else:
  204. logger.info(f"Task call: {task} (skipped because disabled by user)")
  205. return False
  206. def task_delay(self, task: str, start_time: datetime = None,
  207. success: bool=None, server: bool=True, target: timedelta=None) -> None:
  208. """
  209. 设置下次运行时间 当然这个也是可以重写的
  210. :param target: 可以自定义的下次运行时间
  211. :param server: True
  212. :param success: 判断是成功的还是失败的时间间隔
  213. :param task: 任务名称,大驼峰的
  214. :param finish: 是完成任务后的时间为基准还是开始任务的时间为基准
  215. :return:
  216. """
  217. # 任务预处理
  218. if not task:
  219. task = self.task.command
  220. task = convert_to_underscore(task)
  221. task_object = getattr(self.model, task, None)
  222. if not task_object:
  223. logger.warning(f'No task named {task}')
  224. return
  225. scheduler = getattr(task_object, 'scheduler', None)
  226. if not scheduler:
  227. logger.warning(f'No scheduler in {task}')
  228. return
  229. # 任务开始时间
  230. if not start_time:
  231. start_time = datetime.now().replace(microsecond=0)
  232. # 依次判断是否有自定义的下次运行时间
  233. run = []
  234. if success is not None:
  235. interval = (
  236. scheduler.success_interval
  237. if success
  238. else scheduler.failure_interval
  239. )
  240. run.append(start_time + interval)
  241. # if server is not None:
  242. # if server:
  243. # server = scheduler.server_update
  244. # run.append(get_server_next_update(server))
  245. if target is not None:
  246. target = [target] if not isinstance(target, list) else target
  247. target = nearest_future(target)
  248. run.append(target)
  249. next_run = None
  250. # 排序
  251. if not len(run):
  252. raise ScriptError(
  253. "Missing argument in delay_next_run, should set at least one"
  254. )
  255. run = min(run).replace(microsecond=0)
  256. next_run = run
  257. # 将这些连接起来,方便日志输出
  258. kv = dict_to_kv(
  259. {
  260. "success": success,
  261. "server_update": server,
  262. "target": target,
  263. },
  264. allow_none=False,
  265. )
  266. logger.info(f"Delay task `{task}` to {next_run} ({kv})")
  267. # 强制设定下一次的运行时间
  268. if server and scheduler.server_update != time(hour=9):
  269. next_run = parse_tomorrow_server(scheduler.server_update)
  270. # 设置
  271. logger.attr(f'{task}.scheduler.next_run', next_run)
  272. scheduler.next_run = next_run
  273. self.save()
  274. if __name__ == '__main__':
  275. config = Config(config_name='oas1')
  276. config.update_scheduler()
  277. print(config.pending_task)
  278. print(config.waiting_task)
  279. config.update_scheduler()
  280. print(config.pending_task)
  281. # print(config.get_next())