Просмотр исходного кода

Merge pull request #10 from runhey/dev

Dev
huangrunheng 3 лет назад
Родитель
Сommit
cecb9d70b7

+ 6 - 7
README.md

@@ -42,18 +42,18 @@ oas目前还在开发中,我们期待你的加入,Q群: 465946275
 
 如果单纯了解本项目,你可以下载[最新发布](https://github.com/runhey/OnmyojiAutoScript/releases),会自动安装环境。
 
-具体是:下载->解压->打开'oas.exe', 稍等一两分钟待环境安装好后即可启动
+具体为:下载->解压->打开`oas.exe`, 稍等一两分钟待环境安装好后即可启动
 
 ### 参与开发
 
 1. 打开本项目[主仓库](https://github.com/runhey/OnmyojiAutoScript/tree/master), 点击右上角的 `Fork`,然后点击下面绿色的 `Create fork`
 
-2. 进入自己账号下的 Alas 仓库,并 `clone` 其中的 `dev` 分支到本地
+2. 进入自己账号下的 oas仓库,并 `clone` 其中的 `dev` 分支到本地
 	```
 	git clone url -b dev
 	```
 
-3. 下载[最新环境包](https://github.com/runhey/OnmyojiAutoScript/releases), 解压复制`toolkit`,`deploy`两个文件夹和`consloe.bat`到本地的仓库根目录,`toolkit`里面包含python最小环境、git环境和GUI启动环境。
+3. 下载[最新环境包](https://github.com/runhey/OnmyojiAutoScript/releases), 解压复制`toolkit`,`deploy`两个文件夹和`console.bat`到本地的仓库根目录,`toolkit`里面包含python最小环境、git环境和GUI启动环境。
 
 3. 安装pip库, 安装后你可以执行`gui.py`启动进行测试
 
@@ -96,9 +96,9 @@ oas目前还在开发中,我们期待你的加入,Q群: 465946275
 | ✔自动化部署                                                  |
 | ✔GUI引入工程                                                 |
 | ✔config读写参数字段                                          |
-| 🔨优化window截图+控制                                         |
-| 🔨投屏到OAS                                                   |
-| 🔨操作格式化json生成工具:可以从用户界面简单操作生成维护数据(包括图片匹配,点击操作,滑动操作,ocr识别等等) |
+| ✔优化window截图+控制                                         |
+| ✔投屏到OAS                                                   |
+| 🔨操作格式化json生成工具:可以从用户界面简单操作生成维护数据(包括✔图片匹配,点击操作,长按操作,滑动操作,ocr识别等等) |
 | 📆Atom: 对于上一条数据的接口                                  |
 | 📆实现ocr单独进程                                             |
 | 📆到此我们可以(读参数)、(对游戏截屏控制)、(操控数据)。实现第一个脚本任务如挖土 |
@@ -110,4 +110,3 @@ oas目前还在开发中,我们期待你的加入,Q群: 465946275
 | 📆实现更多的任务 | 如何输出Log到GUI |
 | 📆建立任务调度   | 优化速度         |
 |                 |                  |
-

+ 0 - 33
config/template.json

@@ -1,33 +0,0 @@
-{
-  "script": {
-    "device": {
-      "serial": "127.0.0.1:7555",
-      "handle": "auto",
-      "package_name": "auto",
-      "screenshot_method": "window_background",
-      "control_method": "window_message",
-      "adb_restart": false
-    },
-    "error": {
-      "handle_error": false,
-      "save_error": false,
-      "screenshot_length": 1
-    },
-    "optimization": {
-      "screenshot_interval": 0.3,
-      "combat_screenshot_interval": 1.0,
-      "task_hoarding_duration": 0,
-      "when_task_queue_empty": "goto_home"
-    }
-  },
-  "restart": {
-    "scheduler": {
-      "enable": false,
-      "next_run": "2023-01-01 00:00:00",
-      "command": "Restart",
-      "success_interval": 0,
-      "failure_interval": 0,
-      "server_update": "00:00"
-    }
-  }
-}

+ 4 - 0
gui.py

@@ -6,6 +6,8 @@ from module.gui.utils import check_admin
 from module.gui.context.add import Add
 from module.gui.context.settings import Setting
 from module.gui.context.process_manager import ProcessManager
+from module.gui.register_type.paint_image import PaintImage
+from module.gui.register_type.rule_file import RuleFile
 from module.gui.fluent_app import FluentApp
 
 if __name__ == "__main__":
@@ -22,5 +24,7 @@ if __name__ == "__main__":
     app.set_context_property(setting, 'setting')
     app.set_context_property(add_config, 'add_config')
     app.set_context_property(process_manager, 'process_manager')
+    app.qml_register_type(PaintImage, 'PaintImage')
+    app.qml_register_type(RuleFile, 'RuleFile')
     # 启动一个GUI
     app.run()

+ 2 - 0
module/config/config_menu.py

@@ -16,6 +16,8 @@ class ConfigMenu:
         self.menu["Overview"] = []
         # 脚本设置
         self.menu['Script'] = ['Script', 'General', 'Restart']
+        # 开发工具
+        self.menu["Tools"] = ['Image Rule', 'Ocr Rule', 'Click Rule', 'Long Click Rule', 'Swipe Rule']
         # 日常的任务
         self.menu["Daily Task"] = ['Territorial Demon', 'Coin Challenge']
 

+ 4 - 4
module/device/app_control.py

@@ -12,7 +12,7 @@ class AppControl(Adb, Uiautomator2):
     _app_u2_family = ['uiautomator2', 'minitouch', 'scrcpy']
 
     def app_is_running(self) -> bool:
-        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        method = self.config.script.emulator.control_method
         # if self.is_wsa:
         #     package = self.app_current_wsa()
         if method in AppControl._app_u2_family:
@@ -25,7 +25,7 @@ class AppControl(Adb, Uiautomator2):
         return package == self.package
 
     def app_start(self):
-        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        method = self.config.script.device.screenshot_method
         logger.info(f'App start: {self.package}')
         # if self.config.Emulator_Serial == 'wsa-0':
         #     self.app_start_wsa(display=0)
@@ -35,7 +35,7 @@ class AppControl(Adb, Uiautomator2):
             self.app_start_adb()
 
     def app_stop(self):
-        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        method = self.config.script.device.screenshot_method
         logger.info(f'App stop: {self.package}')
         if method in AppControl._app_u2_family:
             self.app_stop_uiautomator2()
@@ -47,7 +47,7 @@ class AppControl(Adb, Uiautomator2):
         Returns:
             etree._Element: Select elements with `self.hierarchy.xpath('//*[@text="Hermit"]')` for example.
         """
-        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        method = self.config.script.device.screenshot_method
         if method in AppControl._app_u2_family:
             self.hierarchy = self.dump_hierarchy_uiautomator2()
         else:

+ 3 - 3
module/device/connection.py

@@ -104,7 +104,7 @@ class Connection(ConnectionAttr):
 
         # Package
         # self.package = self.config.Emulator_PackageName
-        self.package = self.config.get_arg('Script', 'Device', 'PackageName')
+        self.package = self.config.script.device.package_name
         if self.package == 'auto':
             self.detect_package()
         else:
@@ -576,7 +576,7 @@ class Connection(ConnectionAttr):
            Reboot adb client if no device found, otherwise try reconnecting device.
         """
         # if self.config.Emulator_AdbRestart and len(self.list_device()) == 0:
-        if self.config.get_arg('Script', 'Device', 'AdbRestart') and len(self.list_device()) == 0:
+        if self.config.script.device.adb_restart and len(self.list_device()) == 0:
             # Restart Adb
             self.adb_restart()
             # Connect to device
@@ -737,7 +737,7 @@ class Connection(ConnectionAttr):
                 logger.info(f'{device.serial} ({device.status})')
 
         # Auto device detection
-        if self.config.get_arg('Script', 'Device', 'Serial') == 'auto':
+        if self.config.script.device.serial == 'auto':
         # if self.config.Emulator_Serial == 'auto':
             if available.count == 0:
                 logger.critical('No available device found, auto device detection cannot work, '

+ 3 - 2
module/device/connection_attr.py

@@ -41,7 +41,8 @@ class ConnectionAttr:
         # Remove global proxies, or uiautomator2 will go through it
         count = 0
         d = dict(**os.environ)
-        d.update(self.config.args)
+        #----------------------------------------------------------------------------------下面的是我注释掉的
+        # d.update(self.config.args)
         for _, v in deep_iter(d, depth=3):
             if not isinstance(v, dict):
                 continue
@@ -64,7 +65,7 @@ class ConnectionAttr:
 
         # Parse custom serial
         # self.serial = str(self.config.Emulator_Serial)
-        self.serial = str(self.config.get_arg('Script', 'Device', 'Serial'))
+        self.serial = str(self.config.script.device.serial)
         self.serial_check()
         self.config.DEVICE_OVER_HTTP = self.is_over_http
 

+ 4 - 4
module/device/control.py

@@ -42,7 +42,7 @@ class Control(Minitouch, Scrcpy, Window):
             'Click %s @ %s' % (point2str(x, y), button)
         )
         method = self.click_methods.get(
-            self.config.get_arg("Script", 'Emulator', 'ControlMethod'),
+            self.config.script.emulator.control_method,
             self.click_adb
         )
         method(x, y)
@@ -79,7 +79,7 @@ class Control(Minitouch, Scrcpy, Window):
         logger.info(
             'Click %s @ %s, %s' % (point2str(x, y), button, duration)
         )
-        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        method = self.config.script.emulator.control_method
         if method == 'minitouch':
             self.long_click_minitouch(x, y, duration)
         elif method == 'window_message':
@@ -97,7 +97,7 @@ class Control(Minitouch, Scrcpy, Window):
         self.handle_control_check(name)
         p1, p2 = ensure_int(p1, p2)
         duration = ensure_time(duration)
-        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        method = self.config.script.emulator.control_method
         if method == 'minitouch':
             logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
         elif method == 'window_message':
@@ -168,7 +168,7 @@ class Control(Minitouch, Scrcpy, Window):
         logger.info(
             'Drag %s -> %s' % (point2str(*p1), point2str(*p2))
         )
-        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        method = self.config.script.emulator.control_method
         if method == 'minitouch':
             self.drag_minitouch(p1, p2, point_random=point_random)
         elif method == 'uiautomator2':

+ 11 - 1
module/device/device.py

@@ -1,4 +1,5 @@
 import sys
+import cv2
 from collections import deque
 from datetime import datetime
 
@@ -6,6 +7,7 @@ from module.base.timer import Timer
 from module.config.utils import get_server_next_update
 from module.device.app_control import AppControl
 from module.device.control import Control
+from module.device.method.window import Window
 from module.device.screenshot import Screenshot
 from module.exception import (GameNotRunningError, GameStuckError,
                               GameTooManyClickError, RequestHumanTakeover)
@@ -30,9 +32,10 @@ class Device(Screenshot, Control, AppControl, EmulatorManager):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.screenshot_interval_set()
+        # 我不能接受为什么继承了Window但是不会自动初始化
 
         # Auto-select the fastest screenshot method
-        if not self.config.is_template_config and self.config.get_arg("Script", 'Device', 'ScreenshotMethod') == 'auto':
+        if self.config.script.device.screenshot_method == 'auto':
             self.run_simple_screenshot_benchmark()
 
     def run_simple_screenshot_benchmark(self):
@@ -213,3 +216,10 @@ class Device(Screenshot, Control, AppControl, EmulatorManager):
         super().app_stop()
         self.stuck_record_clear()
         self.click_record_clear()
+
+
+if __name__ == "__main__":
+    device = Device(config="oas1")
+    cv2.imshow("imgSrceen", device.screenshot())  # 显示
+    cv2.waitKey(0)
+    cv2.destroyAllWindows()

+ 7 - 6
module/device/handle.py

@@ -112,15 +112,16 @@ class Handle:
         :param config:
         """
         logger.hr('Handle')
-        if isinstance(config, str):
-            self.config = Config(config, task=None)
-        else:
-            self.config = config
+        if self.config is None:
+            if isinstance(config, str):
+                self.config = Config(config, task=None)
+            else:
+                self.config = config
 
         # 获取根的句柄
         self.root_handle_title = ''
         self.root_handle_num = 0
-        self.root_handle = self.config.get_arg('Script', 'Device', 'Handle')
+        self.root_handle = self.config.script.device.handle
         if self.root_handle == "auto":
             logger.info('handle is auto. oas will find window emulator')
             window_list = Handle.all_windows()
@@ -134,7 +135,7 @@ class Handle:
                 self.root_handle_num = self.root_handle
         elif isinstance(self.root_handle, str):
             logger.info('handle is handle string. oas use it as root handle title')
-            if handle_title2num(self.root_handle) !=0:
+            if handle_title2num(self.root_handle) != 0:
                 self.root_handle_num = handle_title2num(self.root_handle)
                 self.root_handle_title = self.root_handle
         logger.info(f'the root handle title is {self.root_handle_title} and num is {self.root_handle_num}')

+ 13 - 6
module/device/method/window.py

@@ -29,6 +29,10 @@ from module.device.handle import Handle, window_scale_rate
 
 class Window(Handle):
 
+    def __init__(self):
+        logger.info("Window init")
+        super().__init__()
+
     def screenshot_window_background(self):
         """
         后台截屏
@@ -58,13 +62,14 @@ class Window(Handle):
         imgSrceen = frombuffer(signedIntsArray, dtype='uint8')
         imgSrceen.shape = (heightScreen, widthScreen, 4)
         # imgSrceen = cv2.cvtColor(imgSrceen, cv2.COLOR_BGRA2GRAY)
-        imgSrceen = cv2.resize(imgSrceen, (win_size[0], win_size[1]))
+        # imgSrceen = cv2.resize(imgSrceen, (win_size[0], win_size[1]))
+        # 很奇怪的
 
         # 测试显示截图图片
-        cv2.namedWindow('imgSrceen')  # 命名窗口
-        cv2.imshow("imgSrceen", imgSrceen)  # 显示
-        cv2.waitKey(0)
-        cv2.destroyAllWindows()
+        # cv2.namedWindow('imgSrceen')  # 命名窗口
+        # cv2.imshow("imgSrceen", imgSrceen)  # 显示
+        # cv2.waitKey(0)
+        # cv2.destroyAllWindows()
 
         # 内存释放
         DeleteObject(saveBitMap.GetHandle())
@@ -275,5 +280,7 @@ class Window(Handle):
 
 if __name__ == "__main__":
     w = Window(config='oas1')
+    img = w.screenshot_window_background()
+    logger.info(img.shape)
     # w.click_window_message(x=350, y=220)
-    w.swipe_window_message(startPos=[200, 5], endPos=[300, 300])
+    # w.swipe_window_message(startPos=[200, 5], endPos=[300, 300])

+ 21 - 11
module/device/screenshot.py

@@ -26,6 +26,10 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
     _last_save_time = {}
     image: np.ndarray
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        super(Window, self).__init__(*args, **kwargs)
+
     @cached_property
     def screenshot_methods(self):
         return {
@@ -50,7 +54,7 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
 
         for _ in range(2):
             method = self.screenshot_methods.get(
-                self.config.get_arg('Script', 'Device', 'ScreenshotMethod'),
+                self.config.script.device.screenshot_method,
                 self.screenshot_adb  # 第二个参数默认的是screenshot_adb
             )
             self.image = method()
@@ -61,7 +65,7 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
             # self.image = self._handle_orientated_image(self.image)
 
             # if self.config.Error_SaveError:
-            if self.config.get_arg('Script', 'Error', 'SaveError'):
+            if self.config.script.error.save_error:
                 self.screenshot_deque.append({'time': datetime.now(), 'image': self.image})
 
             if self.check_screen_size() and self.check_screen_black():
@@ -100,7 +104,7 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
     @cached_property
     def screenshot_deque(self):
         # return deque(maxlen=int(self.config.Error_ScreenshotLength))
-        return deque(maxlen=int(self.config.get_arg("Script", "Error", "ScreenshotLength")))
+        return deque(maxlen=int(self.config.script.error.screenshot_length))
 
     def save_screenshot(self, genre='items', interval=None, to_base_folder=False):
         """Save a screenshot. Use millisecond timestamp as file name.
@@ -145,17 +149,17 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
                 Or None for Optimization_ScreenshotInterval, 'combat' for Optimization_CombatScreenshotInterval
         """
         if interval is None:
-            origin = self.config.get_arg('Script', 'Optimization', 'ScreenshotInterval')
+            origin = self.config.script.optimization.screenshot_interval
             interval = limit_in(origin, 0.1, 0.3)
             if interval != origin:
                 logger.warning(f'Optimization.ScreenshotInterval {origin} is revised to {interval}')
-                self.config.set_arg('Script', 'Optimization', 'ScreenshotInterval', interval)
+                self.config.script.optimization.screenshot_interval = interval
         elif interval == 'combat':
-            origin = self.config.get_arg('Script', 'Optimization', 'CombatScreenshotInterval')
+            origin = self.config.script.optimization.combat_screenshot_interval
             interval = limit_in(origin, 0.3, 1.0)
             if interval != origin:
                 logger.warning(f'Optimization.CombatScreenshotInterval {origin} is revised to {interval}')
-                self.config.set_arg('Script', 'Optimization', 'CombatScreenshotInterval', interval)
+                self.config.script.optimization.combat_screenshot_interval = interval
         elif isinstance(interval, (int, float)):
             # No limitation for manual set in code
             pass
@@ -165,7 +169,7 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
         # Screenshot interval in scrcpy is meaningless,
         # video stream is received continuously no matter you use it or not.
         # if self.config.Emulator_ScreenshotMethod == 'scrcpy':
-        if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'scrcpy':
+        if self.config.script.device.screenshot_method == 'scrcpy':
             interval = 0.1
 
         if interval != self._screenshot_interval.limit:
@@ -234,7 +238,7 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
             #     logger.warning('Game not running on display 0, will be restarted')
             #     self.app_stop_uiautomator2()
             #     return False
-            if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'uiautomator2':
+            if self.config.script.device.screenshot_method == 'uiautomator2':
                 logger.warning(f'Received pure black screenshots from emulator, color: {color}')
                 logger.warning('Uninstall minicap and retry')
                 self.uninstall_minicap()
@@ -242,10 +246,10 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
                 return False
             else:
                 logger.warning(f'Received pure black screenshots from emulator, color: {color}')
-                logger.warning(f'Screenshot method {self.config.get_arg("Script", "Device", "ScreenshotMethod")}'
+                logger.warning(f'Screenshot method {self.config.script.device.screenshot_method}'
                                f'may not work on emulator `{self.serial}`, or the emulator is not fully started')
                 if self.is_mumu_family:
-                    if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'DroidCast':
+                    if self.config.script.device.screenshot_method == 'DroidCast':
                         self.droidcast_stop()
                     else:
                         logger.warning('If you are using MuMu X, please upgrade to version >= 12.1.5.0')
@@ -254,3 +258,9 @@ class Screenshot(Adb, DroidCast, Scrcpy, Window):
         else:
             self._screen_black_checked = True
             return True
+
+
+if __name__ == "__main__":
+    s = Screenshot(config="oas1")
+    s.screenshot()
+    s.image_show()

+ 41 - 10
module/gui/context/process_manager.py

@@ -4,11 +4,18 @@
 import socket
 import random
 import zerorpc
+import cv2
+import msgpack
+import numpy as np
+import io
 
 from typing import Union, Any
-from PySide6.QtCore import QObject, Slot, Signal
+from cached_property import cached_property
+from PySide6.QtCore import QObject, Slot, Signal, Property
+from PySide6.QtGui import QImage
 
 from module.gui.process.script_process import ScriptProcess
+from module.config.config_menu import ConfigMenu
 from module.gui.context.add import Add
 from module.logger import logger
 
@@ -139,20 +146,15 @@ class ProcessManager(QObject):
             logger.info(f'script {config} is not running')
             return None
 
-    @Slot(str, result="QString")
-    def gui_menu(self, config: str) -> str:
+    @Slot(result="QString")
+    def gui_menu(self) -> str:
         """
         get_gui_menu
         :param config:
         :return:
         """
-        if config in self.clients:
-            logger.info(f'gui get menu of {config}')
-            return self.clients[config].gui_menu()
-        else:
-            logger.info(self.clients)
-            logger.info(f'script {config} is not running')
-            return None
+        menu = ConfigMenu()
+        return menu.gui_menu
 
     @Slot(str, str, result="QString")
     def gui_args(self, config: str, task: str) -> str:
@@ -246,3 +248,32 @@ class ProcessManager(QObject):
         else:
             logger.info(f'script {config} is not running')
             return False
+
+    @Slot(str, result="QImage")
+    def gui_mirror_image(self, config: str) -> QImage:
+        """
+        :param config:
+        :return:
+        """
+        if config in self.clients:
+            logger.info(f'gui get mirror image of {config}')
+            # 接收流对象
+            stream = self.clients[config].gui_mirror_image()
+            # 创建 BytesIO 对象来存储图像数据
+            buffer = io.BytesIO()
+            for data in stream:
+                buffer.write(data)
+            # 将 BytesIO 对象的内容作为字节流解码为 cv2 图像
+            buffer.seek(0)  # 将读取位置重置为起始位置
+            image_data = np.frombuffer(buffer.getvalue(), dtype=np.uint8)
+            image = cv2.imdecode(image_data, cv2.IMREAD_COLOR)
+
+            height, width, _ = image.shape
+            image_qt = QImage(image.data, width, height, QImage.Format_RGB888).rgbSwapped()
+            return image_qt
+
+
+
+        else:
+            logger.info(f'script {config} is not running')
+            return None

+ 14 - 2
module/gui/fluent_app.py

@@ -5,7 +5,7 @@ import sys
 import os
 
 from PySide6.QtGui import QGuiApplication, QIcon
-from PySide6.QtQml import QQmlApplicationEngine
+from PySide6.QtQml import QQmlApplicationEngine, qmlRegisterType
 from PySide6.QtCore import Qt, QObject, QTranslator, QLocale, Slot
 from pathlib import Path
 
@@ -24,7 +24,7 @@ class FluentApp():
     def __init__(self):
         super().__init__()
         # 适配高分辨率、声明、设置Logo、设置软件名字
-        # QGuiApplication.setHighDpiScaleFactorRoundingPolicy(Qt.HighDpiScaleFactorRoundingPolicy.Round)
+        QGuiApplication.setHighDpiScaleFactorRoundingPolicy(Qt.HighDpiScaleFactorRoundingPolicy.Round)
         QGuiApplication.setAttribute(Qt.AA_EnableHighDpiScaling)
         QGuiApplication.setAttribute(Qt.AA_UseHighDpiPixmaps)
         QGuiApplication.setAttribute(Qt.AA_ShareOpenGLContexts)
@@ -58,6 +58,18 @@ class FluentApp():
         """
         FluentApp.engine.rootContext().setContextProperty(name, context)
 
+
+
+    def qml_register_type(self, Class, qml_class: str) -> None:
+        """
+        注册qml类型
+        :param Class:
+        :param qml_class:
+        :return:
+        """
+        qmlRegisterType(Class, "Oas", 1, 0, qml_class)
+
+
 class Translator(QObject):
 
     def __init__(self, engine, app) -> None:

+ 8 - 0
module/gui/qml.qrc

@@ -23,5 +23,13 @@
         <file>qml/Component/Args.qml</file>
         <file>qml/Component/Taskmini.qml</file>
         <file>qml/Component/ModelParse.js</file>
+        <file>qml/Content/ImageRule.qml</file>
+        <file>qml/Content/ClickRule.qml</file>
+        <file>qml/Content/LongClickRule.qml</file>
+        <file>qml/Content/SwipeRule.qml</file>
+        <file>qml/Content/OcrRule.qml</file>
+        <file>qml/Component/MirrorImage.qml</file>
+        <file>qml/Component/ResizableRectangle.qml</file>
+        <file>qml/Component/RuleFile.qml</file>
     </qresource>
 </RCC>

+ 7 - 7
module/gui/qml/Component/Args.qml

@@ -453,15 +453,15 @@ Item {
             console.debug("没有数据")
         }
 
-        var args = JSON.parse(argsData)
-        var values = JSON.parse(valueData)
-        var group = MP.parseGroup(args["properties"])
-        var groups = MP.parseGroups(args["properties"])
+        const args = JSON.parse(argsData)
+        const values = JSON.parse(valueData)
+        const group = MP.parseGroup(args["properties"])
+        const groups = MP.parseGroups(args["properties"])
 
         for(let task in group){
-            var argument = MP.parseArgument(args.definitions, group[task])
+            const argument = MP.parseArgument(args.definitions, group[task])
             const groupLetter = groups[group[task]]
-            var argumentVuale = MP.mergeArgument(argument, values[groupLetter])
+            const argumentVuale = MP.mergeArgument(argument, values[groupLetter])
 //            console.debug(JSON.stringify( argumentVuale ))
 //            console.debug("---------------------------------------------------------------")
             createGroup(groupLetter, group[task], argumentVuale)
@@ -485,7 +485,7 @@ Item {
             return
         }
 
-        var object = group_item.createObject()
+        const object = group_item.createObject(contentScrollable)
         if(object !== null){
             object.groupName = groupName
             object.groupTitle = groupTitle

+ 349 - 0
module/gui/qml/Component/MirrorImage.qml

@@ -0,0 +1,349 @@
+import QtQuick 2.0
+import QtQuick.Controls
+import QtQuick.Dialogs
+import FluentUI
+import Oas
+import "../Global"
+
+Item {
+    id: imageItem
+    implicitWidth: 1280
+    implicitHeight: 720 + 50
+
+    property alias imageWidth: paintImage.width
+    property alias imageHeight: paintImage.height
+
+    property alias roiRed_x: roi_Red.x
+    property alias roiRed_y: roi_Red.y
+    property alias roiRed_width: roi_Red.width
+    property alias roiRed_height: roi_Red.height
+    property alias roiGreen_x: roi_Green.x
+    property alias roiGreen_y: roi_Green.y
+    property alias roiGreen_width: roi_Green.width
+    property alias roiGreen_height: roi_Green.height
+
+    signal roi_red_changed(string roi)
+    signal roi_green_changed(string roi)
+
+    onRoiRed_xChanged: {
+//        roi_red_changed( '${roiRed_x},${roiRed_y},${roiRed_width},${roiRed_height}' )
+        roi_red_changed( roiRed_x + "," +  roiRed_y + "," + roiRed_width + "," + roiRed_height)
+    }
+    onRoiRed_yChanged: {
+        roi_red_changed( roiRed_x + "," +  roiRed_y + "," + roiRed_width + "," + roiRed_height)
+    }
+    onRoiRed_widthChanged: {
+        roi_red_changed( roiRed_x + "," +  roiRed_y + "," + roiRed_width + "," + roiRed_height)
+    }
+    onRoiRed_heightChanged: {
+        roi_red_changed( roiRed_x + "," +  roiRed_y + "," + roiRed_width + "," + roiRed_height)
+    }
+
+
+    onRoiGreen_xChanged: {
+        roi_green_changed( roiGreen_x + "," +  roiGreen_y + "," + roiGreen_width + "," + roiGreen_height )
+    }
+    onRoiGreen_yChanged: {
+        roi_green_changed( roiGreen_x + "," +  roiGreen_y + "," + roiGreen_width + "," + roiGreen_height )
+    }
+    onRoiGreen_widthChanged: {
+        roi_green_changed( roiGreen_x + "," +  roiGreen_y + "," + roiGreen_width + "," + roiGreen_height )
+    }
+    onRoiGreen_heightChanged: {
+        roi_green_changed( roiGreen_x + "," +  roiGreen_y + "," + roiGreen_width + "," + roiGreen_height )
+    }
+
+    enum RoiMode {
+        None,
+        Same,
+        Include
+    }
+    property int roi_mode: MirrorImage.RoiMode.None
+
+    Item{
+
+    anchors{
+        left: parent.left
+        top: parent.top
+        topMargin: 45
+    }
+    width: imageWidth
+    height: imageHeight
+
+    PaintImage{
+        id: paintImage
+        width: 1280
+        height: 720
+        Timer{
+            id: timerImage
+            repeat: true
+            running: if(imageSoucre.currentText === "Device Image"){return true}
+                     else{ return false}
+            interval: 1000
+
+            onTriggered: {
+                paintImage.update_image()
+            }
+        }
+
+        function update_image(){
+            if(imageSoucre.currentText === "Local Image"){
+                const imagePath = localImageName.text
+                if(!imagePath.startsWith("file:///")){
+                    return
+                }
+                paintImage.set_local(imagePath)
+            }
+            else{
+                const i = process_manager.gui_mirror_image(MainEvent.scriptName)
+                paintImage.image = i
+            }
+        }
+        Component.onCompleted:{
+            timerImage.start()
+        }
+    }
+
+
+    Rectangle{
+        id: roi_Green // 默认第二个是 back
+        width: 100
+        height: 100
+        color: "transparent"
+        border.width: 3
+        border.color: "green"
+        ResizableRectangle{
+            resizeTarget: roi_Green//设置调整目标ID
+        }
+    }
+
+    Rectangle{
+        id: roi_Red // 默认第一个是红色的front
+        width: 100
+        height: 100
+        color: "transparent"
+        border.width: 1
+        border.color: "red"
+
+        ResizableRectangle{
+            resizeTarget: roi_Red//设置调整目标ID
+        }
+        onXChanged: {
+            if(roi_mode ===  MirrorImage.RoiMode.Include){
+                if(roi_Red.x < roi_Green.x || roi_Red.x+roi_Red.width > roi_Green.x+roi_Green.width){
+                    roi_Red.x = roi_Green.x + roi_Green.width/2 - roi_Red.width/2
+
+                }
+            }
+            else if(roi_mode ===  MirrorImage.RoiMode.Same){
+                roi_Green.x = roi_Red.x
+            }
+        }
+        onYChanged: {
+            if(roi_mode ===  MirrorImage.RoiMode.Include){
+                if(roi_Red.y < roi_Green.y || roi_Red.y+roi_Red.height > roi_Green.y+roi_Green.height){
+                    roi_Red.y = roi_Green.y + roi_Green.height/2 - roi_Red.height/2
+
+                }
+            }
+            else if(roi_mode ===  MirrorImage.RoiMode.Same){
+                roi_Green.y = roi_Red.y
+            }
+        }
+        onWidthChanged: {
+            if(roi_mode ===  MirrorImage.RoiMode.Include){
+                if(roi_Red.width > roi_Green.width){
+                    roi_Red.width = roi_Green.width - 5
+                }
+            }
+            else if(roi_mode ===  MirrorImage.RoiMode.Same){
+                roi_Green.width = roi_Red.width
+            }
+        }
+        onHeightChanged: {
+            if(roi_mode ===  MirrorImage.RoiMode.Include){
+                if(roi_Red.height > roi_Green.height){
+                    roi_Red.height = roi_Green.height - 5
+                }
+            }
+            else if(roi_mode ===  MirrorImage.RoiMode.Same){
+                roi_Green.height = roi_Red.height
+            }
+        }
+    }
+    }
+
+
+
+
+
+    FluExpander{
+        id: imageExpander
+        headerText: qsTr("Screen Settings")
+        width: parent.width
+        anchors.left: parent.left
+        anchors.top: parent.top
+        contentHeight: paintImage.height
+
+        Column{
+            anchors.fill: parent
+            anchors.leftMargin: 20
+
+        // 设置你的图片源
+        Item{
+            width: imageExpander.width
+            height: 50
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "Image Source"
+            }
+            FComboBox{
+                id: imageSoucre
+                anchors{
+                    left: parent.left
+                    leftMargin: 150
+                    verticalCenter: parent.verticalCenter
+                }
+                width: 200
+                model: ["Device Image", "Local Image"]
+                Component.onCompleted:{
+                    currentIndex = 0
+                }
+            }
+        }
+        // 设置你的本地的图片地址
+        Item{
+            width: imageExpander.width
+            height: 50
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "Local Image Name"
+            }
+
+            FluFilledButton{
+                id: localImageName
+                anchors{
+                    left: parent.left
+                    leftMargin: 150
+                    verticalCenter: parent.verticalCenter
+                }
+                width: 200
+                text:"select your image"
+                clip: true
+//                horizontalAlignment: Text.AlignRight
+//                elide: Text.ElideRight
+                onClicked: {
+                    imageFileDialog.open()
+                }
+            }
+
+            FileDialog{
+                   id: imageFileDialog
+                   title: "select your image"
+                   nameFilters: ["Image Files (*.png *.jpg)"]
+                   onAccepted: {
+                       const selectedUrl = imageFileDialog.selectedFile.toString()
+                       localImageName.text = selectedUrl
+                       paintImage.update_image()
+                   }
+               }
+        }
+        // 设置两个框的交互模式
+        Item{
+            width: imageExpander.width
+            height: 50
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "ROI Model"
+            }
+            FComboBox{
+                id: roiModel
+                anchors{
+                    left: parent.left
+                    verticalCenter: parent.verticalCenter
+                    leftMargin: 150
+                }
+                width: 200
+                model: ["None", "Same Size", "Include", ]
+
+                function setMode(){
+                    if(currentText === "None"){
+                        imageItem.roi_mode = MirrorImage.RoiMode.None
+                    }
+                    else if(currentText === "Same Size"){
+                        imageItem.roi_mode = MirrorImage.RoiMode.Same
+                    }
+                    else{
+                        imageItem.roi_mode = MirrorImage.RoiMode.Include
+                    }
+                }
+
+                function updateRoi(){
+                    if(roi_mode ===  MirrorImage.RoiMode.Include){
+                        roi_Red.x = roi_Green.x + 5
+                        roi_Red.y = roi_Green.y + 5
+                        roi_Red.width = roi_Green.width - 5
+                        roi_Red.height = roi_Green.height - 5
+
+                    }
+                    else if(roi_mode ===  MirrorImage.RoiMode.Same){
+                        roi_Red.x = roi_Green.x
+                        roi_Red.y = roi_Green.y
+                        roi_Red.width = roi_Green.width
+                        roi_Red.height = roi_Green.height
+                    }
+                }
+
+                onActivated: {
+                    setMode()
+                    updateRoi()
+                }
+            }
+        }
+        // 设置显示的帧率
+        Item{
+            width: imageExpander.width
+            height: 50
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "Frame Rate"
+            }
+            FluSlider{
+                id: frameRate
+                anchors{
+                    left: parent.left
+                    verticalCenter: parent.verticalCenter
+                    leftMargin: 150
+                }
+                size: 200
+                value: 2
+                maxValue: 10
+                onReleased: {
+                    if(value > 0){
+                        timerImage.interval = 1000 / value
+                    }
+                }
+            }
+        }
+        // 帮助
+        Item{
+            width: imageExpander.width
+            height: 50
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "python+qml 就别想着高帧率了,而且是以截图方式而不是视频流"
+            }
+        }
+
+        }
+    }
+
+    function save_target_image(roi, file){
+        paintImage.save_target_image(roi, file)
+    }
+}

+ 4 - 11
module/gui/qml/Component/ModelParse.js

@@ -62,11 +62,11 @@ function parseArgument(definitions, group){
     if(!(group in definitions)){
         return null
     }
-    var result = []
-    var pro = definitions[group]["properties"]
+    const result = []
+    const pro = definitions[group]["properties"]
     for(const key in pro){
-        var arg = {}
-        var argName = pro[key]
+        const arg = {}
+        const argName = pro[key]
 
         arg["name"] = key
 
@@ -116,13 +116,6 @@ function parseArgument(definitions, group){
  * @returns {Object} - 就是添加了value这个一个值
  */
 function mergeArgument(argument, values){
-//    for(let arg in argument){
-//        for(let vname in values){
-//            if(arg.name === vname){
-//                arg["value"] = values[vname]
-//            }
-//        }
-//    }
     for(let key in argument){
         const name = argument[key].name
         if(name in values){

+ 257 - 0
module/gui/qml/Component/ResizableRectangle.qml

@@ -0,0 +1,257 @@
+import QtQuick 2.0
+import QtQuick.Window 2.15
+
+Item {
+    id:rect
+    property int minimumWidth: 20
+    property int minimumHeight: 20
+    property int mouseRegion: 5
+    property var resizeTarget: rect
+    anchors.fill: resizeTarget
+
+    MouseArea{
+        id: move
+        anchors.fill: parent
+        anchors.margins: mouseRegion
+        cursorShape: Qt.ClosedHandCursor
+        property int xPosition: 0
+        property int yPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+            yPosition = mouse.y
+        }
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            var yOffset = mouse.y-yPosition
+
+            if(rect.resizeTarget.x+xOffset>0 && rect.resizeTarget.width+xOffset+rect.resizeTarget.x < rect.resizeTarget.parent.width){
+                rect.resizeTarget.x = rect.resizeTarget.x+xOffset
+            }
+
+            if(rect.resizeTarget.y+yOffset>0 && rect.resizeTarget.height+yOffset+rect.resizeTarget.y < rect.resizeTarget.parent.height){
+                rect.resizeTarget.y = rect.resizeTarget.y+yOffset
+            }
+        }
+    }
+
+    MouseArea {
+        id:leftX
+        width: mouseRegion
+        anchors.left: parent.left
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.leftMargin: 0
+        anchors.bottomMargin: 0
+        anchors.topMargin: 0
+        cursorShape: Qt.SizeHorCursor
+        property int xPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+        }
+
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            if(rect.resizeTarget.x+xOffset>0 && rect.resizeTarget.width-xOffset>minimumWidth){
+                rect.resizeTarget.x = rect.resizeTarget.x+xOffset
+                rect.resizeTarget.width = rect.resizeTarget.width-xOffset
+            }
+        }
+    }
+
+    MouseArea{
+        id:rightX
+        width: mouseRegion
+        anchors.right: parent.right
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.bottomMargin: 0
+        anchors.topMargin: 0
+        anchors.rightMargin: 0
+        cursorShape: Qt.SizeHorCursor
+        property int xPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+        }
+
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            var xWidth = rect.resizeTarget.width+xOffset
+            var availableWidth = rect.resizeTarget.parent ? rect.resizeTarget.parent.width:Screen.desktopAvailableWidth
+            if(xWidth+rect.resizeTarget.x<availableWidth && xWidth>minimumWidth){
+                rect.resizeTarget.width = xWidth
+            }
+        }
+    }
+
+    MouseArea{
+        id:topY
+        height: mouseRegion
+        anchors.left: parent.left
+        anchors.right: parent.right
+        anchors.top: parent.top
+        anchors.rightMargin: 0
+        anchors.leftMargin: 0
+        anchors.topMargin: 0
+        cursorShape: Qt.SizeVerCursor
+        property int yPosition: 0
+        onPressed: function(mouse){
+            yPosition = mouse.y
+        }
+
+        onPositionChanged: function(mouse){
+            var yOffset = mouse.y-yPosition
+            if(rect.resizeTarget.y+yOffset>0 && rect.resizeTarget.height-yOffset>minimumHeight){
+                rect.resizeTarget.y = rect.resizeTarget.y+yOffset
+                rect.resizeTarget.height = rect.resizeTarget.height-yOffset
+            }
+        }
+    }
+
+    MouseArea{
+        id:bottomY
+        height: mouseRegion
+        anchors.left: parent.left
+        anchors.right: parent.right
+        anchors.bottom: parent.bottom
+        anchors.rightMargin: 0
+        anchors.leftMargin: 0
+        anchors.bottomMargin: 0
+        cursorShape: Qt.SizeVerCursor
+        property int yPosition: 0
+        onPressed: function(mouse){
+            yPosition = mouse.y
+        }
+
+        onPositionChanged: function(mouse){
+            var yOffset = mouse.y-yPosition
+            var yHeight = rect.resizeTarget.height+yOffset
+            var availableHeight = rect.resizeTarget.parent ? rect.resizeTarget.parent.height : Screen.desktopAvailableHeight
+            if(yHeight+rect.resizeTarget.y<availableHeight && yHeight>minimumHeight){
+                rect.resizeTarget.height = yHeight
+            }
+        }
+    }
+
+    MouseArea{
+        width: mouseRegion
+        height: mouseRegion
+        anchors.left: parent.left
+        anchors.top: parent.top
+        anchors.topMargin: 0
+        anchors.leftMargin: 0
+        cursorShape: Qt.SizeFDiagCursor
+        property int xPosition: 0
+        property int yPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+            yPosition = mouse.y
+        }
+
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            if(rect.resizeTarget.x+xOffset>0 && rect.resizeTarget.width-xOffset>minimumWidth){
+                rect.resizeTarget.x = rect.resizeTarget.x+xOffset
+                rect.resizeTarget.width = rect.resizeTarget.width-xOffset
+            }
+            var yOffset = mouse.y-yPosition
+            if(rect.resizeTarget.y+yOffset>0 && rect.resizeTarget.height-yOffset>minimumHeight){
+                rect.resizeTarget.y = rect.resizeTarget.y+yOffset
+                rect.resizeTarget.height = rect.resizeTarget.height-yOffset
+            }
+        }
+    }
+
+    MouseArea{
+        width: mouseRegion
+        height: mouseRegion
+        anchors.right: parent.right
+        anchors.top: parent.top
+        anchors.topMargin: 0
+        anchors.rightMargin: 0
+        cursorShape: Qt.SizeBDiagCursor
+        property int xPosition: 0
+        property int yPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+            yPosition = mouse.y
+        }
+
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            var xWidth = rect.resizeTarget.width+xOffset
+            var availableWidth = rect.resizeTarget.parent ? rect.resizeTarget.parent.width:Screen.desktopAvailableWidth
+            var availableHeight = rect.resizeTarget.parent ? rect.resizeTarget.parent.height : Screen.desktopAvailableHeight
+
+            if(xWidth+rect.resizeTarget.x<availableWidth && xWidth>minimumWidth){
+                rect.resizeTarget.width = xWidth
+            }
+            var yOffset = mouse.y-yPosition
+            if(rect.resizeTarget.y+yOffset>0 && rect.resizeTarget.height-yOffset>minimumHeight){
+                rect.resizeTarget.y = rect.resizeTarget.y+yOffset
+                rect.resizeTarget.height = rect.resizeTarget.height-yOffset
+            }
+        }
+    }
+
+    MouseArea{
+        width: mouseRegion
+        height: mouseRegion
+        anchors.left: parent.left
+        anchors.bottom: parent.bottom
+        anchors.leftMargin: 0
+        anchors.bottomMargin: 0
+        cursorShape: Qt.SizeBDiagCursor
+        property int xPosition: 0
+        property int yPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+            yPosition = mouse.y
+        }
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            var availableWidth = rect.resizeTarget.parent ? rect.resizeTarget.parent.width:Screen.desktopAvailableWidth
+            var availableHeight = rect.resizeTarget.parent ? rect.resizeTarget.parent.height : Screen.desktopAvailableHeight
+            if(rect.resizeTarget.x+xOffset>0 && availableWidth-xOffset>minimumWidth){
+                rect.resizeTarget.x = rect.resizeTarget.x+xOffset
+                rect.resizeTarget.width = rect.resizeTarget.width-xOffset
+            }
+            var yOffset = mouse.y-yPosition
+            var yHeight = rect.resizeTarget.height+yOffset
+            if(yHeight+rect.resizeTarget.y<availableHeight && yHeight>minimumHeight){
+                rect.resizeTarget.height = yHeight
+            }
+        }
+    }
+
+
+    MouseArea{
+        width: mouseRegion
+        height: mouseRegion
+        anchors.right: parent.right
+        anchors.bottom: parent.bottom
+        anchors.rightMargin: 0
+        anchors.bottomMargin: 0
+        cursorShape: Qt.SizeFDiagCursor
+        property int xPosition: 0
+        property int yPosition: 0
+        onPressed: function(mouse){
+            xPosition = mouse.x
+            yPosition = mouse.y
+        }
+        onPositionChanged: function(mouse){
+            var xOffset = mouse.x-xPosition
+            var xWidth = rect.resizeTarget.width+xOffset
+            var availableWidth = rect.resizeTarget.parent ? rect.resizeTarget.parent.width:Screen.desktopAvailableWidth
+            var availableHeight = rect.resizeTarget.parent ? rect.resizeTarget.parent.height : Screen.desktopAvailableHeight
+
+            if(xWidth+rect.resizeTarget.x<availableWidth && xWidth>minimumWidth){
+                rect.resizeTarget.width = xWidth
+            }
+            var yOffset = mouse.y-yPosition
+            var yHeight = rect.resizeTarget.height+yOffset
+            if(yHeight+rect.resizeTarget.y<availableHeight && yHeight>minimumHeight){
+                rect.resizeTarget.height = yHeight
+            }
+        }
+    }
+}

+ 220 - 0
module/gui/qml/Component/RuleFile.qml

@@ -0,0 +1,220 @@
+import QtQuick
+import QtQuick.Controls
+import FluentUI
+import QtQuick.Dialogs
+import Oas
+
+FluArea {
+    id: root
+    property string file: ""  //完整的文件路径带后缀
+    property alias folder: folderButton.text
+    property alias list_model: listModel
+    property alias rule_file: ruleFile
+
+    property var currentItem: listModel.get(0)
+    property var addFunc: function(){}
+    property var editFunc: function(){}
+    property var saveFunc: function(){}
+    ListModel{
+        id: listModel
+    }
+
+    RuleFile{
+        id: ruleFile
+    }
+
+    Column{
+        id: coi
+        anchors{
+            top: parent.top
+//            topMargin: 10
+            bottom: parent.bottom
+            left: parent.left
+            leftMargin: 20
+            right: parent.right
+            rightMargin: 20
+        }
+
+        spacing: 5
+        // 头
+        Item{
+            width: coi.width
+            height: 40
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "Select rule"
+                font: FluTextStyle.BodyStrong
+            }
+        }
+        //路径
+        Item{
+            width: coi.width
+            height: 40
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "Folder"
+                font: FluTextStyle.BodyStrong
+            }
+            FluButton{
+                id: folderButton
+                anchors{
+                    right: parent.right
+                    verticalCenter: parent.verticalCenter
+                }
+                disabled: true
+                text:"Select folder"
+                width: 200
+                onClicked: {
+
+                }
+            }
+        }
+        // 文件名
+        Item{
+            width: coi.width
+            height: 40
+            FluText{
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                text: "File"
+                font: FluTextStyle.BodyStrong
+            }
+            FluButton{
+                id: fileButton
+                anchors{
+                    right: parent.right
+                    verticalCenter: parent.verticalCenter
+                }
+                text:"Select file"
+                width: 200
+                onClicked: {
+                    jsonFileDialog.open()
+                }
+            }
+            FileDialog{
+                   id: jsonFileDialog
+                   title: "select your json file"
+                   nameFilters: ["(*.json)"]
+                   onAccepted: {
+                       const selectedUrl = jsonFileDialog.selectedFile.toString()
+                       if(!selectedUrl.startsWith("file:///")){
+                           console.error("请你选择正确的文件路径")
+                           return
+                       }
+                       const filePath = selectedUrl.substring("file:///".length)
+                       const lastSlashIndex = filePath.lastIndexOf("/"); // 获取最后一个斜杠的索引
+                       const secondLastSlashIndex = filePath.lastIndexOf("/", lastSlashIndex - 1); // 获取倒数第二个斜杠的索引
+
+                       const filename = filePath.substring(lastSlashIndex + 1); // 获取文件名(带后缀)
+                       const directory = filePath.substring(secondLastSlashIndex+1, lastSlashIndex); // 获取文件所在的目录
+
+                       root.file = filePath
+                       folderButton.text = directory
+                       fileButton.text = filename
+                       listModel.clear()
+                       loadFile()
+                   }
+               }
+        }
+        // new
+        Item{
+            width: coi.width
+            height: 40
+            FluFilledButton{
+                id: saveItemButton
+                anchors{
+                    right: parent.right
+                    verticalCenter: parent.verticalCenter
+                }
+                text:"New"
+                disabled: root.file === ""? true : false
+                width: parent.width
+                onClicked: {
+                    listModel.append(addFunc())
+                }
+            }
+        }
+        // 保存 文件
+        Item{
+            width: coi.width
+            height: 40
+            FluFilledButton{
+                id: saveFileButton
+                anchors{
+                    right: parent.right
+                    verticalCenter: parent.verticalCenter
+                }
+                text:"Save file"
+                width: parent.width
+                disabled: root.file === ""? true : false
+                onClicked: {
+                    root.saveFunc()
+                }
+            }
+        }
+        //表格视图
+        Item{
+            width: coi.width
+            height: 400
+            ListView{
+                id: listView
+                width: 200
+                height: parent.height
+                anchors{
+                    right: parent.right
+                }
+                model: listModel
+                delegate: listDelegate
+            }
+            Component{
+                id: listDelegate
+                Item{
+                    width: 200
+                    height: 40
+                FluTextBox{
+                    id: itemName
+                    width: parent.width
+                    anchors{
+                        verticalCenter: parent.verticalCenter
+                    }
+                    placeholderText:"item name"
+                    text: model.itemName
+                    validator: RegularExpressionValidator { regularExpression: /^[a-zA-Z0-9]*$/ }
+                    onActiveFocusChanged: {
+                        if(activeFocus){
+                            currentItem = listModel.get(index)
+                        }
+                    }
+                    onEditingFinished: {
+                        model.itemName = text
+                        root.editFunc(model)
+                    }
+                    onAccepted: {
+
+                    }
+                }
+                }
+            }
+        }
+    }
+
+    // 加载文件,前提是文件的路径是加载好的
+    function loadFile(){
+        const getData = ruleFile.read_file(file)
+        var data = []
+        if (getData === "" || typeof getData === "undefined") {
+            console.log("String is empty or undefined.")
+            // 如果是空的表示这个是一个刚刚建立的rule文件
+            return
+        }
+        console.debug('加载文件的数据', getData)
+        data = JSON.parse(getData)
+        for(let item of data){
+            listModel.append(item)
+        }
+
+    }
+
+}

+ 1 - 0
module/gui/qml/Component/ScriptItem.qml

@@ -23,5 +23,6 @@ FluPaneItem{
 //            // 组件加载失败
 //        }
     navigationView.pushScript(Qt.resolvedUrl("../../qml/Page/ScriptView.qml"), title)
+        MainEvent.scriptName = title
     }
 }

+ 5 - 0
module/gui/qml/Content/ClickRule.qml

@@ -0,0 +1,5 @@
+import QtQuick 2.0
+
+Item {
+
+}

+ 350 - 0
module/gui/qml/Content/ImageRule.qml

@@ -0,0 +1,350 @@
+import QtQuick
+import QtQuick.Controls
+import FluentUI
+import "../Component"
+
+Item {
+
+    property alias aItemName: itemName.text
+    property alias aImageName: imageName.text
+    property alias aRoiFront: roiFront.text
+    property alias aRoiBack: roiBack.text
+    property alias aMethod: method.currentText
+    property alias aThreshold: threshold.text
+    property alias aDescription: description.text
+
+    MirrorImage{
+        id: mirrorImage
+        anchors{
+            left: parent.left
+            top: parent.top
+        }
+        width: 1280
+        height: 720 + 50
+        imageWidth: 1280
+        imageHeight: 720
+
+        onRoi_red_changed: function(roi){
+            if(typeof ruleFile.currentItem === "undefined"){
+                return
+            }
+            ruleFile.currentItem.roiFront = roi
+        }
+
+        onRoi_green_changed: function(roi){
+            if(typeof ruleFile.currentItem === "undefined"){
+                return
+            }
+            ruleFile.currentItem.roiBack = roi
+        }
+    }
+    RuleFile{
+        id: ruleFile
+        anchors{
+            left: mirrorImage.right
+            leftMargin: 10
+            right: parent.right
+            top: parent.top
+        }
+        height: 600
+        addFunc: add
+        editFunc: edit
+        saveFunc: save
+
+        onCurrentItemChanged: {
+            if(currentItem === "" || typeof currentItem === "undefined"){
+                return
+            }
+            var [x, y, width, height] = currentItem.roiFront.split(',')
+            mirrorImage.roiRed_x = parseInt(x)
+            mirrorImage.roiRed_y = parseInt(y)
+            mirrorImage.roiRed_width = parseInt(width)
+            mirrorImage.roiRed_height = parseInt(height)
+            var [x, y, width, height] = currentItem.roiBack.split(',')
+            mirrorImage.roiGreen_x = parseInt(x)
+            mirrorImage.roiGreen_y = parseInt(y)
+            mirrorImage.roiGreen_width = parseInt(width)
+            mirrorImage.roiGreen_height = parseInt(height)
+
+        }
+    }
+
+    FluArea {
+        id: ruleArea
+        anchors{
+            left: mirrorImage.right
+            leftMargin: 10
+            right: parent.right
+            top: ruleFile.bottom
+            topMargin: 12
+            bottom: parent.bottom
+        }
+        Column{
+            id: coi
+            anchors{
+                top: parent.top
+                bottom: parent.bottom
+                left: parent.left
+                leftMargin: 20
+                right: parent.right
+                rightMargin: 20
+            }
+            spacing: 5
+            //头
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Editing rule"
+                    font: FluTextStyle.Subtitle
+                }
+            }
+            //item name 这个参数项的名称
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Item name"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: itemName
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    placeholderText:"单行输入框"
+                    disabled: true
+                    text: (typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.itemName
+                    width: 200
+                }
+            }
+            // 文件名
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Image name"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: imageName
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    disabled: true
+                    text:(typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.imageName
+                    placeholderText:"Save image name"
+                    width: 200
+                }
+            }
+            // roi front
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Roi front"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: roiFront
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    disabled: true
+                    text:(typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.roiFront
+                    placeholderText:"0,0,100,100"
+                    width: 200
+                }
+            }
+            //roi back
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Roi Back"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: roiBack
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    disabled: true
+                    text:(typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.roiBack
+                    placeholderText:"0,0,100,100"
+                    width: 200
+
+
+                }
+            }
+            //找图的方法
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Find method"
+                    font: FluTextStyle.BodyStrong
+                }
+                FComboBox{
+                    id: method
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    width: 200
+                    currentIndex: 0
+                    model: ["Template matching"]
+                    onActivated: {
+
+                    }
+                }
+            }
+            //阈值
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Threshod"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: threshold
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    width: 200
+                    text:(typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.threshold
+                    placeholderText:"0.8"
+                    onEditingFinished: {
+                        if(typeof ruleFile.currentItem === "undefined"){
+                            return
+                        }
+                        ruleFile.currentItem.threshold = parseFloat(text)
+                    }
+                }
+            }
+            //描述 description
+            Item{
+                width: coi.width
+                height: 40
+                FluText{
+                    anchors.left: parent.left
+                    anchors.verticalCenter: parent.verticalCenter
+                    text: "Description"
+                    font: FluTextStyle.BodyStrong
+                }
+                FluTextBox{
+                    id: description
+                    anchors{
+                        right: parent.right
+                        verticalCenter: parent.verticalCenter
+                    }
+                    width: 200
+                    text:(typeof ruleFile.currentItem === "undefined")? "" : ruleFile.currentItem.description
+                    placeholderText:"input your description"
+                    onEditingFinished: {
+                        if(typeof ruleFile.currentItem === "undefined"){
+                            return
+                        }
+                        ruleFile.currentItem.description = text
+                    }
+                }
+            }
+
+        }
+
+    }
+
+    // 帮助
+    FluArea{
+        id: ruleHelp
+        anchors{
+            left: parent.left
+            right: ruleArea.left
+            rightMargin: 10
+            top: mirrorImage.bottom
+            topMargin: 10
+            bottom: parent.bottom
+        }
+        FluText{
+            leftPadding: 12
+            topPadding: 12
+            text: "红色的框(Roi front)表示这一个项要匹配的图片,绿色的框(Roi back)表示匹配的范围。
+在移动边框时右边的roi并不会实时显示,当切换软件的焦点时更新显示。但是具体的数值是更新的,可以放心的保存截图"
+        }
+        FluFilledButton{
+            anchors{
+                right: parent.right
+                rightMargin: 10
+                top: parent.top
+                topMargin: 10
+            }
+
+            text:"Save image"
+            onClicked: {
+                const i = ruleFile.file.lastIndexOf("/")
+                const imagepath = ruleFile.file.substring(0, i)
+                mirrorImage.save_target_image(ruleFile.currentItem.roiFront, imagepath +"/"+ruleFile.currentItem.imageName + ".png")
+                showSuccess("Save target image")
+            }
+        }
+    }
+
+    function add(){
+        const item ={}
+        item["itemName"] = "new"
+        item["imageName"] = "image name"
+        item["roiFront"] = "0,0,100,100"
+        item["roiBack"] = "0,0,100,100"
+        item["method"] = "Template matching"
+        item["threshold"] = 0.8
+        item["description"] = "description"
+        showSuccess("Add new item")
+        return item
+    }
+
+    function edit(model){
+        model.imageName = ruleFile.folder +"_"+ model.itemName
+    }
+
+    function save(){
+        const data = []
+        for(var i=0; i<ruleFile.list_model.count; i++){
+            const item = ruleFile.list_model.get(i)
+            const itemData = {}
+            itemData["itemName"] = item["itemName"]
+            itemData["imageName"] = item["imageName"]
+            itemData["roiFront"] = item["roiFront"]
+            itemData["roiBack"] = item["roiBack"]
+            itemData["method"] = item["method"]
+            itemData["threshold"] = item["threshold"]
+            itemData["description"] = item["description"]
+            data.push(itemData)
+        }
+
+        ruleFile.rule_file.write_file(ruleFile.file, JSON.stringify(data))
+        showSuccess("Save file")
+
+    }
+
+
+}

+ 5 - 0
module/gui/qml/Content/LongClickRule.qml

@@ -0,0 +1,5 @@
+import QtQuick 2.0
+
+Item {
+
+}

+ 5 - 0
module/gui/qml/Content/OcrRule.qml

@@ -0,0 +1,5 @@
+import QtQuick 2.0
+
+Item {
+
+}

+ 5 - 0
module/gui/qml/Content/SwipeRule.qml

@@ -0,0 +1,5 @@
+import QtQuick 2.0
+
+Item {
+
+}

+ 49 - 5
module/gui/qml/Page/ScriptView.qml

@@ -9,15 +9,58 @@ SplitPanel{
     property string configName: ""
 
     onTitleChanged: {
+        // 如果是总览
         if(title === "Overview"){
             showDefalut()
         }
+        //如果是其他的开发的工具
+        else if(title === "Image Rule"){
+            const component = Qt.createComponent("../../qml/Content/ImageRule.qml")
+            if(component.status !== Component.Ready){
+                console.error("image rule component is not ready")
+                return
+            }
+            setLoader(component)
+        }
+        else if(title === "Ocr Rule"){
+            const component = Qt.createComponent("../../qml/Content/OcrRule.qml")
+            if(component.status !== Component.Ready){
+                console.error("ocr rule component is not ready")
+                return
+            }
+            setLoader(component)
+        }
+        else if(title === "Click Rule"){
+            const component = Qt.createComponent("../../qml/Content/ClickRule.qml")
+            if(component.status !== Component.Ready){
+                console.error("click rule component is not ready")
+                return
+            }
+            setLoader(component)
+        }
+        else if(title === "Long Click Rule"){
+            const component = Qt.createComponent("../../qml/Content/LongClickRule.qml")
+            if(component.status !== Component.Ready){
+                console.error("long click rule component is not ready")
+                return
+            }
+            setLoader(component)
+        }
+        else if(title === "Swipe Rule"){
+            const component = Qt.createComponent("../../qml/Content/SwipeRule.qml")
+            if(component.status !== Component.Ready){
+                console.error("swipe rule component is not ready")
+                return
+            }
+            setLoader(component)
+        }
+        //最后才是一般的参数页面
         else{
-            var component = Qt.createComponent("../../qml/Component/Args.qml")
+            const component = Qt.createComponent("../../qml/Component/Args.qml")
             if (component.status === Component.Ready) {
                 setLoader(component)
-                var args = process_manager.gui_args(configName, title)
-                var value = process_manager.gui_task(configName, title)
+                const args = process_manager.gui_args(configName, title)
+                const value = process_manager.gui_task(configName, title)
                 setLoaderData(args, value)
                 setLoaderContext(configName, title)
             }else{
@@ -31,7 +74,8 @@ SplitPanel{
             console.info('configname is emtry')
             return
         }
-        var data = JSON.parse(process_manager.gui_menu(configName))
+        console.debug(process_manager.gui_menu)
+        const data = JSON.parse(process_manager.gui_menu())
         create(data)
     }
 
@@ -42,7 +86,7 @@ SplitPanel{
 //        var data = {'Overview':[], 'Script': ['Script', 'General', 'Restart']}
 
 
-        var component = Qt.createComponent("../../qml/Component/Overview.qml")
+        const component = Qt.createComponent("../../qml/Component/Overview.qml")
         if (component.status === Component.Ready) {
             setDefalut(component)
         } else {

+ 74 - 0
module/gui/register_type/paint_image.py

@@ -0,0 +1,74 @@
+# This Python file uses the following encoding: utf-8
+# @author runhey
+# github https://github.com/runhey
+import cv2
+
+from numpy import float32, int32, uint8, fromfile
+from PySide6.QtCore import QUrl, Property
+from PySide6.QtGui import QImage, QPainter
+from PySide6.QtQuick import QQuickPaintedItem
+from PySide6.QtCore import QObject, Slot, Signal
+
+from module.logger import logger
+
+
+class PaintImage(QQuickPaintedItem):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._image = QImage()
+
+    def paint(self, painter: QPainter):
+        if self._image.isNull():
+            logger.error("image is null")
+            return
+        painter.drawImage(self.boundingRect(), self._image)
+
+    def image(self):
+        return self._image
+
+    def set_image(self, image: QImage):
+        if self._image == image:
+            return None
+        self._image = image
+        # 更新会触发paint事件
+        self.update()
+
+    @Slot(str)
+    def set_local(self, image_name: str):
+        if not image_name.startswith("file:///"):
+            logger.error("image path must start with file:///")
+            return None
+        image_name = image_name.lstrip("file:///")
+
+
+        if self._image.load(image_name):
+            logger.info("load image success")
+            self.update()
+        else:
+            logger.error("load image failed")
+            return None
+
+    image = Property(QImage, fget=image, fset=set_image, notify=logger.info)
+
+    @Slot(str, str)
+    def save_target_image(self, roi: str, file: str) -> None:
+        """
+        保存目标图片
+        :param file: 保存的文件名
+        :param roi: 截图的范围 文本如"0,0,100,100"
+        :return:
+        """
+        if not roi:
+            return
+        if not isinstance(roi, str):
+            logger.error("roi must be str")
+        if not file:
+            return
+        if not isinstance(file, str):
+            logger.error("file must be str")
+            return
+        x, y, width, height = map(int, roi.split(','))
+        roi_image = self._image.copy(x, y, width, height)
+        roi_image.save(file)
+        logger.info(f"save target image {file}success")

+ 65 - 0
module/gui/register_type/rule_file.py

@@ -0,0 +1,65 @@
+# This Python file uses the following encoding: utf-8
+# @author runhey
+# github https://github.com/runhey
+import os
+import json
+
+from filelock import FileLock
+from PySide6.QtCore import QObject, Slot
+
+from module.logger import logger
+from module.config.atomicwrites import atomic_write
+
+
+
+class RuleFile(QObject):
+
+    def __init__(self):
+        super().__init__()
+
+    @Slot(str, result="QString")
+    def read_file(self, file: str) -> str:
+        """
+        读取json文件,如果不存在则创建
+        :param file: 包含路径
+        :return:
+        """
+        folder = os.path.dirname(file)
+        if not os.path.exists(folder):
+            os.mkdir(folder)
+
+        if not os.path.exists(file):
+            return ""
+
+        _, ext = os.path.splitext(file)
+        lock = FileLock(f"{file}.lock")
+        with lock:
+            logger.info(f'read: {file}')
+            if ext == '.json':
+                with open(file, mode='r', encoding='utf-8') as f:
+                    return f.read()
+            else:
+                logger.error(f"not support {ext} file")
+                return ""
+
+    @Slot(str, str)
+    def write_file(self, file: str, data: str) -> None:
+        """
+        写入json文件,如果不存在则创建
+        :param file: 包含路径
+        :param data: 写入的数据
+        :return:
+        """
+        folder = os.path.dirname(file)
+        if not os.path.exists(folder):
+            os.mkdir(folder)
+
+        _, ext = os.path.splitext(file)
+        lock = FileLock(f"{file}.lock")
+        with lock:
+            logger.info(f'write: {file}')
+            if ext == '.json':
+                with atomic_write(file, overwrite=True, encoding='utf-8') as f:
+                    f.write(data)
+            else:
+                logger.error(f"not support {ext} file")

+ 1 - 0
module/tasks/CoinChallenge/res/test.json

@@ -0,0 +1 @@
+[{"itemName":"new1","imageName":"res_new1","roiFront":"179,241,70,79","roiBack":"595,238,100,100","method":"Template matching","threshold":0.8,"description":"description"},{"itemName":"new2","imageName":"res_new2","roiFront":"362,243,100,100","roiBack":"803,101,100,100","method":"Template matching","threshold":0.8,"description":"description"},{"itemName":"new3","imageName":"res_new3","roiFront":"568,212,240,220","roiBack":"0,0,100,100","method":"Template matching","threshold":0.9,"description":"desc55"}]

+ 29 - 15
script.py

@@ -3,9 +3,13 @@
 # github https://github.com/runhey
 import zerorpc
 import zmq
+import msgpack
 import random
+import cv2
 from cached_property import cached_property
 from pydantic import BaseModel, ValidationError
+from PySide6.QtGui import QImage
+
 
 from module.config.utils import convert_to_underscore
 from module.logger import logger
@@ -16,6 +20,8 @@ class Script:
         self.server = None
         self.config_name = config_name
 
+
+
     @cached_property
     def config(self) -> "Config":
         try:
@@ -29,18 +35,18 @@ class Script:
             logger.exception(e)
             exit(1)
 
-    # @cached_property
-    # def device(self) -> "Device":
-    #     try:
-    #         from module.device.device import Device
-    #         device = Device(config=self.config)
-    #         return device
-    #     except RequestHumanTakeover:
-    #         logger.critical('Request human takeover')
-    #         exit(1)
-    #     except Exception as e:
-    #         logger.exception(e)
-    #         exit(1)
+    @cached_property
+    def device(self) -> "Device":
+        try:
+            from module.device.device import Device
+            device = Device(config=self.config)
+            return device
+        except RequestHumanTakeover:
+            logger.critical('Request human takeover')
+            exit(1)
+        except Exception as e:
+            logger.exception(e)
+            exit(1)
 
     def init_server(self, port: int) -> int:
         """
@@ -112,9 +118,17 @@ class Script:
             logger.error(e)
             return False
 
+    @zerorpc.stream
+    def gui_mirror_image(self):
+        """
+        获取给gui显示的镜像
+        :return: cv2的对象将 numpy 数组转换为字节串。接下来MsgPack 进行序列化发送方将图像数据转换为字节串
+        """
+        # return msgpack.packb(cv2.imencode('.jpg', self.device.screenshot())[1].tobytes())
+        ret, buffer = cv2.imencode('.jpg', self.device.screenshot())
+        yield buffer.tobytes()
+
+
 
 if __name__ == "__main__":
     script = Script("oas1")
-    print(script.config.model.json())
-    script.gui_set_task("Script", "device", "serial", "175")
-    print(script.config.script.device.serial)