| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340 |
- # This Python file uses the following encoding: utf-8
- # @author runhey
- # github https://github.com/runhey
- import copy
- import datetime
- import operator
- import threading
- from datetime import datetime, timedelta
- from cached_property import cached_property
- from module.base.filter import Filter
- from module.config.config_updater import ConfigUpdater
- from module.config.config_manual import ConfigManual
- from module.config.config_watcher import ConfigWatcher
- from module.config.config_menu import ConfigMenu
- from module.config.config_model import ConfigModel
- from module.config.config_state import ConfigState
- from module.config.scheduler import TaskScheduler
- from module.config.utils import *
- from module.notify.notify import Notifier
- from module.exception import RequestHumanTakeover, ScriptError
- from module.logger import logger
- class Function:
- def __init__(self, key: str, data: dict):
- """
- 输入的是每一个ConfigModel的一个字段对象
- :param data:
- """
- if isinstance(data, dict) is False:
- self.enable = False
- self.command = "Unknown"
- self.next_run = DEFAULT_TIME
- return
- if data.get("scheduler") is None:
- self.enable = False
- self.command = "Unknown"
- self.next_run = DEFAULT_TIME
- return
- self.enable: bool = data['scheduler']['enable']
- self.command: str = ConfigModel.type(key)
- next_run = data['scheduler']['next_run']
- if isinstance(next_run, str):
- next_run = datetime.strptime(next_run, "%Y-%m-%d %H:%M:%S")
- self.next_run: datetime = next_run
- priority = data['scheduler']['priority']
- if isinstance(priority, str):
- priority = int(priority)
- self.priority: int = priority
- if not isinstance(self.priority, int):
- logger.error(f"Invalid priority: {self.priority}")
- # self.enable = deep_get(data, keys="Scheduler.Enable", default=False)
- # self.command = deep_get(data, keys="Scheduler.Command", default="Unknown")
- # self.next_run = deep_get(data, keys="Scheduler.NextRun", default=DEFAULT_TIME)
- def __str__(self):
- enable = "Enable" if self.enable else "Disable"
- return f"{self.command} ({enable}, {self.priority}, {str(self.next_run)})"
- __repr__ = __str__
- def __eq__(self, other):
- if not isinstance(other, Function):
- return False
- if self.command == other.command and self.next_run == other.next_run:
- return True
- else:
- return False
- def name_to_function(name):
- """
- Args:
- name (str):
- Returns:
- Function:
- """
- function = Function({})
- function.command = name
- function.enable = True
- return function
- class Config(ConfigState, ConfigManual, ConfigWatcher, ConfigMenu):
- def __init__(self, config_name: str, task=None) -> None:
- """
- :param config_name:
- :param task:
- """
- super().__init__(config_name) # 调用 ConfigState 的初始化方法
- super(ConfigManual, self).__init__()
- super(ConfigWatcher, self).__init__()
- super(ConfigMenu, self).__init__()
- self.model = ConfigModel(config_name=config_name)
- def __getattr__(self, name):
- """
- 一开始是打算直接继承ConfigModel的,但是pydantic会接管所有的变量
- 故而选择持有ConfigModel
- :param name:
- :return:
- """
- try:
- return getattr(self.model, name)
- except AttributeError:
- logger.error(f'can not ask this variable {name}')
- return None # 或者抛出异常,或者返回其他默认值
- def gui_args(self, task: str) -> str:
- """
- 获取给gui显示的参数
- :return:
- """
- return self.model.gui_args(task=task)
- def get_arg(self, task: str, group: str, argument: str):
- """
- :param task:
- :param group:
- :param argument:
- :return: str/int/float
- """
- try:
- return self.data[task][group][argument]
- except:
- logger.exception(f'have no arg {task}.{group}.{argument}')
- def set_arg(self, task: str, group: str, argument: str, value) -> None:
- """
- :param task:
- :param group:
- :param argument:
- :param value:
- :return:
- """
- try:
- self.data[task][group][argument] = value
- except:
- logger.exception(f'have no arg {task}.{group}.{argument}')
- def save(self) -> None:
- """
- 保存配置文件
- :return:
- """
- self.model.write_json(self.config_name, self.model.dict())
- def update_scheduler(self) -> None:
- """
- 更新调度器, 设置pending_task and waiting_task
- :return:
- """
- pending_task = []
- waiting_task = []
- error = []
- now = datetime.now()
- for key, value in self.model.dict().items():
- func = Function(key, value)
- if not func.enable:
- continue
- if not isinstance(func.next_run, datetime):
- error.append(func)
- elif func.next_run < now:
- pending_task.append(func)
- else:
- waiting_task.append(func)
- # f = Filter(regex=r"(.*)", attr=["command"])
- # f.load(self.SCHEDULER_PRIORITY)
- if pending_task:
- pending_task = TaskScheduler.schedule(rule=self.model.script.optimization.schedule_rule,
- pending=pending_task)
- if waiting_task:
- # waiting_task = f.apply(waiting_task)
- waiting_task = sorted(waiting_task, key=operator.attrgetter("next_run"))
- if error:
- pending_task = error + pending_task
- self.pending_task = pending_task
- self.waiting_task = waiting_task
- def get_next(self) -> Function:
- """
- 获取下一个要执行的任务
- :return:
- """
- self.update_scheduler()
- if self.pending_task:
- logger.info(f"Pending tasks: {[f.command for f in self.pending_task]}")
- task = self.pending_task[0]
- logger.attr("Task", task)
- return task
- # 哪怕是没有任务,也要返回一个任务,这样才能保证调度器正常运行
- if self.waiting_task:
- logger.info("No task pending")
- task = copy.deepcopy(self.waiting_task[0])
- # task.next_run = (task.next_run + self.hoarding).replace(microsecond=0)
- logger.attr("Task", task)
- return task
- else:
- logger.critical("No task waiting or pending")
- logger.critical("Please enable at least one task")
- raise RequestHumanTakeover
- def task_call(self, task: str=None, force_call=True):
- """
- 回调任务,这会是在任务结束后调用
- :param task: 调用的任务的大写名称
- :param force_call:
- :return:
- """
- task = convert_to_underscore(task)
- if self.model.deep_get(self.model, keys=f'{task}.scheduler.next_run') is None:
- raise ScriptError(f"Task to call: `{task}` does not exist in user config")
- task_enable = self.model.deep_get(self.model, keys=f'{task}.scheduler.enable')
- if force_call or task_enable:
- logger.info(f"Task call: {task}")
- next_run = datetime.now().replace(
- microsecond=0
- )
- self.model.deep_set(self.model, keys=f'{task}.scheduler.next_run', value=next_run)
- self.save()
- return True
- else:
- logger.info(f"Task call: {task} (skipped because disabled by user)")
- return False
- def task_delay(self, task: str, start_time: datetime = None,
- success: bool=None, server: bool=True, target: timedelta=None) -> None:
- """
- 设置下次运行时间 当然这个也是可以重写的
- :param target: 可以自定义的下次运行时间
- :param server: True
- :param success: 判断是成功的还是失败的时间间隔
- :param task: 任务名称,大驼峰的
- :param finish: 是完成任务后的时间为基准还是开始任务的时间为基准
- :return:
- """
- # 任务预处理
- if not task:
- task = self.task.command
- task = convert_to_underscore(task)
- task_object = getattr(self.model, task, None)
- if not task_object:
- logger.warning(f'No task named {task}')
- return
- scheduler = getattr(task_object, 'scheduler', None)
- if not scheduler:
- logger.warning(f'No scheduler in {task}')
- return
- # 任务开始时间
- if not start_time:
- start_time = datetime.now().replace(microsecond=0)
- # 依次判断是否有自定义的下次运行时间
- run = []
- if success is not None:
- interval = (
- scheduler.success_interval
- if success
- else scheduler.failure_interval
- )
- if isinstance(interval, str):
- interval = timedelta(interval)
- run.append(start_time + interval)
- # if server is not None:
- # if server:
- # server = scheduler.server_update
- # run.append(get_server_next_update(server))
- if target is not None:
- target = [target] if not isinstance(target, list) else target
- target = nearest_future(target)
- run.append(target)
- next_run = None
- # 排序
- if not len(run):
- raise ScriptError(
- "Missing argument in delay_next_run, should set at least one"
- )
- run = min(run).replace(microsecond=0)
- next_run = run
- # 将这些连接起来,方便日志输出
- kv = dict_to_kv(
- {
- "success": success,
- "server_update": server,
- "target": target,
- },
- allow_none=False,
- )
- logger.info(f"Delay task `{task}` to {next_run} ({kv})")
- # 强制设定下一次的运行时间
- if server and scheduler.server_update != time(hour=9):
- next_run = parse_tomorrow_server(scheduler.server_update)
- # 设置
- logger.attr(f'{task}.scheduler.next_run', next_run)
- scheduler.next_run = next_run
- self.save()
- @cached_property
- def notifier(self):
- notifier = Notifier(self.model.script.error.notify_config, enable=self.model.script.error.notify_enable)
- notifier.config_name = self.config_name.upper()
- logger.info(f'Notifier: {notifier.config_name}')
- return notifier
- if __name__ == '__main__':
- config = Config(config_name='oas1')
- config.update_scheduler()
- print(config.pending_task)
- print(config.waiting_task)
- config.update_scheduler()
- print(config.pending_task)
- # print(config.get_next())
|