config.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390
  1. # This Python file uses the following encoding: utf-8
  2. # @author runhey
  3. # github https://github.com/runhey
  4. import copy
  5. import datetime
  6. import operator
  7. import threading
  8. import random
  9. from datetime import datetime, timedelta
  10. from cached_property import cached_property
  11. from threading import Lock
  12. from module.base.filter import Filter
  13. from module.config.config_updater import ConfigUpdater
  14. from module.config.config_manual import ConfigManual
  15. from module.config.config_watcher import ConfigWatcher
  16. from module.config.config_menu import ConfigMenu
  17. from module.config.config_model import ConfigModel
  18. from module.config.config_state import ConfigState
  19. from module.config.scheduler import TaskScheduler
  20. from module.config.utils import *
  21. from module.notify.notify import Notifier
  22. from module.exception import RequestHumanTakeover, ScriptError
  23. from module.logger import logger
  24. class Function:
  25. def __init__(self, key: str, data: dict):
  26. """
  27. 输入的是每一个ConfigModel的一个字段对象
  28. :param data:
  29. """
  30. if isinstance(data, dict) is False:
  31. self.enable = False
  32. self.command = "Unknown"
  33. self.next_run = DEFAULT_TIME
  34. return
  35. if data.get("scheduler") is None:
  36. self.enable = False
  37. self.command = "Unknown"
  38. self.next_run = DEFAULT_TIME
  39. return
  40. self.enable: bool = data['scheduler']['enable']
  41. self.command: str = ConfigModel.type(key)
  42. next_run = data['scheduler']['next_run']
  43. if isinstance(next_run, str):
  44. next_run = datetime.strptime(next_run, "%Y-%m-%d %H:%M:%S")
  45. self.next_run: datetime = next_run
  46. priority = data['scheduler']['priority']
  47. if isinstance(priority, str):
  48. priority = int(priority)
  49. self.priority: int = priority
  50. if not isinstance(self.priority, int):
  51. logger.error(f"Invalid priority: {self.priority}")
  52. # self.enable = deep_get(data, keys="Scheduler.Enable", default=False)
  53. # self.command = deep_get(data, keys="Scheduler.Command", default="Unknown")
  54. # self.next_run = deep_get(data, keys="Scheduler.NextRun", default=DEFAULT_TIME)
  55. def __str__(self):
  56. enable = "Enable" if self.enable else "Disable"
  57. return f"{self.command} ({enable}, {self.priority}, {str(self.next_run)})"
  58. __repr__ = __str__
  59. def __eq__(self, other):
  60. if not isinstance(other, Function):
  61. return False
  62. if self.command == other.command and self.next_run == other.next_run:
  63. return True
  64. else:
  65. return False
  66. def name_to_function(name):
  67. """
  68. Args:
  69. name (str):
  70. Returns:
  71. Function:
  72. """
  73. function = Function({})
  74. function.command = name
  75. function.enable = True
  76. return function
  77. class Config(ConfigState, ConfigManual, ConfigWatcher, ConfigMenu):
  78. def __init__(self, config_name: str, task=None) -> None:
  79. """
  80. :param config_name:
  81. :param task:
  82. """
  83. super().__init__(config_name) # 调用 ConfigState 的初始化方法
  84. super(ConfigManual, self).__init__()
  85. super(ConfigWatcher, self).__init__()
  86. super(ConfigMenu, self).__init__()
  87. self.model = ConfigModel(config_name=config_name)
  88. self.scheduler_update_dt = None # 调度器更新时间
  89. def __getattr__(self, name):
  90. """
  91. 一开始是打算直接继承ConfigModel的,但是pydantic会接管所有的变量
  92. 故而选择持有ConfigModel
  93. :param name:
  94. :return:
  95. """
  96. try:
  97. return getattr(self.model, name)
  98. except AttributeError:
  99. # 这个导致 大量的无用log
  100. # logger.error(f'can not ask this variable {name}')
  101. return None # 或者抛出异常,或者返回其他默认值
  102. @cached_property
  103. def lock_config(self) -> Lock:
  104. return Lock()
  105. @cached_property
  106. def notifier(self):
  107. notifier = Notifier(self.model.script.error.notify_config, enable=self.model.script.error.notify_enable)
  108. notifier.config_name = self.config_name.upper()
  109. logger.info(f'Notifier: {notifier.config_name}')
  110. return notifier
  111. def gui_args(self, task: str) -> str:
  112. """
  113. 获取给gui显示的参数
  114. :return:
  115. """
  116. return self.model.gui_args(task=task)
  117. def get_arg(self, task: str, group: str, argument: str):
  118. """
  119. :param task:
  120. :param group:
  121. :param argument:
  122. :return: str/int/float
  123. """
  124. try:
  125. return self.data[task][group][argument]
  126. except:
  127. logger.exception(f'have no arg {task}.{group}.{argument}')
  128. def set_arg(self, task: str, group: str, argument: str, value) -> None:
  129. """
  130. :param task:
  131. :param group:
  132. :param argument:
  133. :param value:
  134. :return:
  135. """
  136. try:
  137. self.data[task][group][argument] = value
  138. except:
  139. logger.exception(f'have no arg {task}.{group}.{argument}')
  140. def reload(self):
  141. self.model = ConfigModel(config_name=self.config_name)
  142. def save(self) -> None:
  143. """
  144. 保存配置文件
  145. :return:
  146. """
  147. self.model.write_json(self.config_name, self.model.dict())
  148. def update_scheduler(self) -> None:
  149. """
  150. 更新调度器, 设置pending_task and waiting_task
  151. :return:
  152. """
  153. pending_task = []
  154. waiting_task = []
  155. error = []
  156. self.scheduler_update_dt = datetime.now()
  157. for key, value in self.model.dict().items():
  158. func = Function(key, value)
  159. if not func.enable:
  160. continue
  161. if not isinstance(func.next_run, datetime):
  162. error.append(func)
  163. elif func.next_run < self.scheduler_update_dt:
  164. pending_task.append(func)
  165. else:
  166. waiting_task.append(func)
  167. # f = Filter(regex=r"(.*)", attr=["command"])
  168. # f.load(self.SCHEDULER_PRIORITY)
  169. if pending_task:
  170. pending_task = TaskScheduler.schedule(rule=self.model.script.optimization.schedule_rule,
  171. pending=pending_task)
  172. # 防止正在运行的任务被新上来的pending队列中的任务给顶替掉
  173. if self.model.running_task and pending_task:
  174. for i, obj in enumerate(pending_task):
  175. if obj.command == self.model.running_task:
  176. pending_task.insert(0, pending_task.pop(i))
  177. logger.info(f'{self.model.running_task} is running')
  178. break
  179. if waiting_task:
  180. # waiting_task = f.apply(waiting_task)
  181. waiting_task = sorted(waiting_task, key=operator.attrgetter("next_run"))
  182. if error:
  183. pending_task = error + pending_task
  184. self.pending_task = pending_task
  185. self.waiting_task = waiting_task
  186. def get_next(self) -> Function:
  187. """
  188. 获取下一个要执行的任务
  189. :return:
  190. """
  191. self.update_scheduler()
  192. if self.pending_task:
  193. logger.info(f"Pending tasks: {[f.command for f in self.pending_task]}")
  194. task = self.pending_task[0]
  195. self.task = task
  196. logger.attr("Task", task)
  197. return task
  198. # 哪怕是没有任务,也要返回一个任务,这样才能保证调度器正常运行
  199. if self.waiting_task:
  200. logger.info("No task pending")
  201. task = copy.deepcopy(self.waiting_task[0])
  202. # task.next_run = (task.next_run + self.hoarding).replace(microsecond=0)
  203. logger.attr("Task", task)
  204. return task
  205. else:
  206. logger.critical("No task waiting or pending")
  207. logger.critical("Please enable at least one task")
  208. raise RequestHumanTakeover
  209. def get_schedule_data(self) -> dict[str, dict]:
  210. """
  211. 获取调度器的数据, 但是你必须使用update_scheduler来更新信息
  212. :return:
  213. """
  214. # 根据调度器更新时间来判断是否有可运行的任务,保证逻辑一致性
  215. scheduler_update_dt = getattr(self, 'scheduler_update_dt', datetime.now())
  216. running = {}
  217. if self.task is not None and self.task.next_run < scheduler_update_dt:
  218. running = {"name": self.task.command, "next_run": str(self.task.next_run)}
  219. pending = []
  220. for p in self.pending_task[1:]:
  221. item = {"name": p.command, "next_run": str(p.next_run)}
  222. pending.append(item)
  223. waiting = []
  224. for w in self.waiting_task:
  225. item = {"name": w.command, "next_run": str(w.next_run)}
  226. waiting.append(item)
  227. data = {"running": running, "pending": pending, "waiting": waiting}
  228. return data
  229. def task_call(self, task: str = None, force_call=True):
  230. """
  231. 回调任务,这会是在任务结束后调用
  232. :param task: 调用的任务的大写名称
  233. :param force_call:
  234. :return:
  235. """
  236. task = convert_to_underscore(task)
  237. if self.model.deep_get(self.model, keys=f'{task}.scheduler.next_run') is None:
  238. raise ScriptError(f"Task to call: `{task}` does not exist in user config")
  239. task_enable = self.model.deep_get(self.model, keys=f'{task}.scheduler.enable')
  240. if force_call or task_enable:
  241. logger.info(f"Task call: {task}")
  242. next_run = datetime.now().replace(
  243. microsecond=0
  244. )
  245. self.model.deep_set(self.model, keys=f'{task}.scheduler.next_run', value=next_run)
  246. self.save()
  247. return True
  248. else:
  249. logger.info(f"Task call: {task} (skipped because disabled by user)")
  250. return False
  251. def task_delay(self, task: str, start_time: datetime = None,
  252. success: bool = None, server: bool = True, target: datetime = None) -> None:
  253. """
  254. 设置下次运行时间 当然这个也是可以重写的
  255. :param target: 可以自定义的下次运行时间
  256. :param server: True
  257. :param success: 判断是成功的还是失败的时间间隔
  258. :param task: 任务名称,大驼峰的
  259. :param finish: 是完成任务后的时间为基准还是开始任务的时间为基准
  260. :return:
  261. """
  262. # 加载配置文件
  263. self.reload()
  264. # 任务预处理
  265. if not task:
  266. task = self.task.command
  267. task = convert_to_underscore(task)
  268. task_object = getattr(self.model, task, None)
  269. if not task_object:
  270. logger.warning(f'No task named {task}')
  271. return
  272. scheduler = getattr(task_object, 'scheduler', None)
  273. if not scheduler:
  274. logger.warning(f'No scheduler in {task}')
  275. return
  276. # 任务开始时间
  277. if not start_time:
  278. start_time = datetime.now().replace(microsecond=0)
  279. # 依次判断是否有自定义的下次运行时间
  280. run = []
  281. if success is not None:
  282. interval = (
  283. scheduler.success_interval
  284. if success
  285. else scheduler.failure_interval
  286. )
  287. if isinstance(interval, str):
  288. interval = timedelta(interval)
  289. run.append(start_time + interval)
  290. # if server is not None:
  291. # if server:
  292. # server = scheduler.server_update
  293. # run.append(get_server_next_update(server))
  294. if target is not None:
  295. target = [target] if not isinstance(target, list) else target
  296. target = nearest_future(target)
  297. run.append(target)
  298. next_run = None
  299. # 排序
  300. if not len(run):
  301. raise ScriptError(
  302. "Missing argument in delay_next_run, should set at least one"
  303. )
  304. run = min(run).replace(microsecond=0)
  305. next_run = run
  306. if server and hasattr(scheduler, 'server_update'):
  307. # 加入随机延迟时间
  308. float_seconds = (scheduler.float_time.hour * 3600 +
  309. scheduler.float_time.minute * 60 +
  310. scheduler.float_time.second)
  311. random_float = random.randint(0, float_seconds)
  312. # 如果有强制运行时间
  313. if scheduler.server_update == time(hour=9):
  314. next_run += timedelta(seconds=random_float)
  315. else:
  316. next_run = parse_tomorrow_server(scheduler.server_update, scheduler.delay_date, random_float)
  317. # 将这些连接起来,方便日志输出
  318. kv = dict_to_kv(
  319. {
  320. "success": success,
  321. "server_update": server,
  322. "target": target,
  323. },
  324. allow_none=False,
  325. )
  326. logger.info(f"Delay task `{task}` to {next_run} ({kv})")
  327. # 保证线程安全的
  328. self.lock_config.acquire()
  329. try:
  330. scheduler.next_run = next_run
  331. self.save()
  332. finally:
  333. self.lock_config.release()
  334. # 设置
  335. logger.attr(f'{task}.scheduler.next_run', next_run)
  336. if __name__ == '__main__':
  337. config = Config(config_name='oas1')
  338. config.notifier.push(title="0000", content="dddddddd")
  339. # print(config.get_next())