Explorar el Código

feat(ocr) ocr done

runhey hace 3 años
padre
commit
04b27b6d94

+ 22 - 6
module/atom/ocr.py

@@ -2,7 +2,7 @@
 # @author runhey
 # github https://github.com/runhey
 
-
+import numpy as np
 import cv2
 
 from module.ocr.base_ocr import BaseCor, OcrMode, OcrMethod
@@ -11,7 +11,7 @@ from module.logger import logger
 
 
 
-class RuleOcr(Full, Single, Digit, DigitCounter, Duration):
+class RuleOcr(Digit, DigitCounter, Duration, Single, Full):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -22,7 +22,7 @@ class RuleOcr(Full, Single, Digit, DigitCounter, Duration):
             case OcrMode.FULL: return Full.after_process(self, result)
             case OcrMode.SINGLE: return Single.after_process(self, result)
             case OcrMode.DIGIT: return Digit.after_process(self, result)
-            case OcrMode.DIGIT_COUNTER: return DigitCounter.after_process(self, result)
+            case OcrMode.DIGITCOUNTER: return DigitCounter.after_process(self, result)
             case OcrMode.DURATION: return Duration.after_process(self, result)
             case _: return result
 
@@ -30,14 +30,30 @@ class RuleOcr(Full, Single, Digit, DigitCounter, Duration):
 
         match self.mode:
             case OcrMode.FULL: return Full.ocr_full(self, image, keyword)
-            case OcrMode.SINGLE: return Single.ocr_single_line(self, image)
+            case OcrMode.SINGLE: return Single.ocr_single(self, image)
             case OcrMode.DIGIT: return Digit.ocr_digit(self, image)
-            case OcrMode.DIGIT_COUNTER: return DigitCounter.ocr_digit_counter(self, image)
+            case OcrMode.DIGITCOUNTER: return DigitCounter.ocr_digit_counter(self, image)
             case OcrMode.DURATION: return Duration.ocr_duration(self, image)
             case _: return None
 
+    def coord(self) -> tuple:
+        """
+        获取一个区域,随机返回一个坐标
+        :return:
+        """
+        area = None
+        if self.mode == OcrMode.FULL:
+            area = self.area
+        else:
+            area = self.roi
+
+        x, y, w, h = self.area
+        x = np.random.randint(x, x + w)
+        y = np.random.randint(y, y + h)
+        return x, y
+
 
 
 if __name__ == "__main__":
-    o = RuleOcr("ocr", "Single", "Default", (1, 2, 3, 4), (1, 2, 3, 4), "keyword")
+    pass
 

+ 4 - 4
module/gui/qml/Content/OcrRule.qml

@@ -200,13 +200,13 @@ Item {
                         else if(currentIndex === 1){
                             ruleFile.currentItem.mode = "Full"
                         }
-                        else if(currentIndex === 1){
+                        else if(currentIndex === 2){
                             ruleFile.currentItem.mode = "Digit"
                         }
-                        else if(currentIndex === 1){
+                        else if(currentIndex === 3){
                             ruleFile.currentItem.mode = "DigitCounter"
                         }
-                        else if(currentIndex === 1){
+                        else if(currentIndex === 4){
                             ruleFile.currentItem.mode = "Duration"
                         }
                     }
@@ -319,7 +319,7 @@ Item {
 
     function add(){
         const item ={}
-        item["itemName"] = "new-ocr-item"
+        item["itemName"] = "new"
         item["roiFront"] = "0,0,100,100"
         item["roiBack"] = "0,0,100,100"
         item["mode"] = "Single"

+ 43 - 5
module/ocr/base_ocr.py

@@ -3,9 +3,12 @@
 # github https://github.com/runhey
 import time
 import cv2
+import numpy as np
+
 from ppocronnx.predict_system import BoxedResult
 from enum import Enum
 
+
 from module.base.decorator import cached_property
 from module.base.utils import area_pad, crop, float2str
 from module.ocr.ppocr import TextSystem
@@ -18,7 +21,7 @@ class OcrMode(Enum):
     FULL = 1  # str: "Full"
     SINGLE = 2  # str: "Single"
     DIGIT = 3  # str: "Digit"
-    DIGIT_COUNTER = 4  # str: "DigitCounter"
+    DIGITCOUNTER = 4  # str: "DigitCounter"
     DURATION = 5  # str: "Duration"
 
 class OcrMethod(Enum):
@@ -26,7 +29,7 @@ class OcrMethod(Enum):
 
 class BaseCor:
 
-    lang: str = "cn"
+    lang: str = "ch"
     score: float = 0.6  # 阈值默认为0.5
 
     name: str = "ocr"
@@ -86,6 +89,37 @@ class BaseCor:
         """
         return result
 
+    @classmethod
+    def crop(cls, image: np.array, roi: tuple) -> np.array:
+        """
+        截取图片
+        :param roi:
+        :param image:
+        :return:
+        """
+        x, y, w, h = roi
+        return image[y:y + h, x:x + w]
+
+    def ocr_item(self, image):
+        """
+        这个函数区别于ocr_single_line,这个函数不对图片进行裁剪,是什么就是什么的
+        :param image:
+        :return:
+        """
+        # pre process
+        start_time = time.time()
+        image = self.pre_process(image)
+        # ocr
+        result, score = self.model.ocr_single_line(image)
+        if score < self.score:
+            result = ""
+        # after proces
+        result = self.after_process(result)
+        logger.info("ocr result score: %s%s" % (result,score))
+        logger.attr(name='%s %ss' % (self.name, float2str(time.time() - start_time)),
+                    text=f'[{result}]')
+        return result
+
     def ocr_single_line(self, image):
         """
         只支持横方向的单行ocr,不支持竖方向的单行ocr
@@ -95,7 +129,7 @@ class BaseCor:
         """
         # pre process
         start_time = time.time()
-        image = crop(image, self.roi, copy=False)
+        image = self.crop(image, self.roi)
         image = self.pre_process(image)
         # ocr
         result, score = self.model.ocr_single_line(image)
@@ -103,8 +137,9 @@ class BaseCor:
             result = ""
         # after proces
         result = self.after_process(result)
+        logger.info("ocr result score: %s" % score)
         logger.attr(name='%s %ss' % (self.name, float2str(time.time() - start_time)),
-                    text=str(result))
+                    text=f'[{result}]')
         return result
 
     def detect_and_ocr(self, image) -> list[BoxedResult]:
@@ -115,15 +150,18 @@ class BaseCor:
         """
         # pre process
         start_time = time.time()
+        image = self.crop(image, self.roi)
         image = self.pre_process(image)
+
         # ocr
         boxed_results: list[BoxedResult] = self.model.detect_and_ocr(image)
         results = []
         # after proces
         for result in boxed_results:
+            logger.info("ocr result score: %s" % result.score)
             if result.score < self.score:
                 continue
-            boxed_results.ocr_text = self.after_process(boxed_results.ocr_text)
+            result.ocr_text = self.after_process(result.ocr_text)
             results.append(result)
 
         logger.attr(name='%s %ss' % (self.name, float2str(time.time() - start_time)),

+ 0 - 32
module/ocr/ocr_test.py

@@ -1,32 +0,0 @@
-# This Python file uses the following encoding: utf-8
-# @author runhey
-# github https://github.com/runhey
-
-import cv2
-from module.ocr.ppocr import TextSystem
-# from ppocronnx.predict_system import TextSystem
-
-
-
-
-if __name__ == "__main__":
-
-    text_sys = TextSystem()
-
-    # 检测并识别文本
-    img = cv2.imread('D:/test_single1.jpg')
-    if img is None:
-        print("Error: img is None")
-        exit(1)
-
-
-    res = text_sys.detect_and_ocr(img)
-    if res is None:
-        print("Error: res is None")
-        exit(1)
-    for boxed_result in res:
-        print("{}, {:.3f}".format(boxed_result.ocr_text, boxed_result.score))
-        print(boxed_result.box)
-
-    res = text_sys.ocr_single_line(img)
-    print(res)

+ 11 - 7
module/ocr/sub_ocr.py

@@ -29,6 +29,9 @@ class Full(BaseCor):
         :param keyword:
         :return:
         """
+        if keyword is None:
+            keyword = self.keyword
+
         boxed_results = self.detect_and_ocr(image)
         if not boxed_results:
             return 0, 0, 0, 0
@@ -47,10 +50,12 @@ class Full(BaseCor):
                 boxed_results[index].box[2, 1] - boxed_results[index].box[0, 1],     # height
             ) for index in index_list]
             box = merge_area(area_list)
+            self.area = box
             return box
         else:
             box = boxed_results[index_list[0]].box
-            return box[0, 0], box[0, 1], box[1, 0] - box[0, 0], box[2, 1] - box[0, 1]
+            self.area = box[0, 0]+self.roi[0], box[0, 1]+self.roi[1], box[1, 0] - box[0, 0], box[2, 1] - box[0, 1]
+            return self.area
 
 class Single(BaseCor):
     """
@@ -70,11 +75,11 @@ class Single(BaseCor):
             if result != "":
                 return result
 
-            # 如果没有识别到,就尝试逆时针旋转90度再识别一次。这个时候考虑到可能是竖方向的文本
-            image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
-            result = self.ocr_single_line(image)
-            if result != "":
-                return result
+            # 如果没有识别到,这个时候考虑到可能是竖方向的文本, 使用detect_and_ocr来进行识别
+            logger.info(f"OCR {self.name}: No text detected in ROI, try to detect vertically")
+            result = self.detect_and_ocr(image)
+            if result[0].ocr_text != "" and result[0].score > self.score:
+                return result[0].ocr_text
 
             # 如果还是没有识别到。那可能就是真的没有识别到了
             return ""
@@ -84,7 +89,6 @@ class Single(BaseCor):
 class Digit(Single):
 
     def after_process(self, result):
-        print('this is digit after_process')
         result = super().after_process(result)
         result = result.replace('I', '1').replace('D', '0').replace('S', '5')
         result = result.replace('B', '8')

+ 5 - 3
tasks/CoinChallenge/assets.py

@@ -44,11 +44,13 @@ class CoinChallengeAssets:
 	# Ocr-description 
 	S_NEW_SINGLE_H = RuleOcr(roi=(78,19,143,63), area=(314,169,100,100), mode="Single", method="Default", keyword="式神育成", name="new_single_h")
 	# Ocr-description 
-	S_DIGIT = RuleOcr(roi=(408,173,51,43), area=(0,0,100,100), mode="Single", method="Default", keyword="", name="digit")
+	S_DIGIT = RuleOcr(roi=(434,173,25,43), area=(0,0,100,100), mode="Digit", method="Default", keyword="", name="digit")
 	# Ocr-description 
-	S_DIGIT_COUNTER = RuleOcr(roi=(1134,6,100,56), area=(0,0,100,100), mode="Single", method="Default", keyword="", name="digit_counter")
+	S_DIGIT_COUNTER = RuleOcr(roi=(1134,6,100,56), area=(0,0,100,100), mode="DigitCounter", method="Default", keyword="", name="digit_counter")
 	# Ocr-description 
-	S_DURATION = RuleOcr(roi=(923,256,100,46), area=(0,0,100,100), mode="Single", method="Default", keyword="育成_结界卡剩余时间", name="duration")
+	S_DURATION = RuleOcr(roi=(923,256,100,46), area=(0,0,100,100), mode="Duration", method="Default", keyword="育成_结界卡剩余时间", name="duration")
+	# Ocr-description 
+	S_NEW_DUR_00 = RuleOcr(roi=(848,624,76,37), area=(0,0,100,100), mode="Duration", method="Default", keyword="", name="new_dur_00")
 
 
 	# Swipe Rule Assets

+ 13 - 4
tasks/CoinChallenge/res/test_ocr.json

@@ -28,9 +28,9 @@
   },
   {
     "itemName": "digit",
-    "roiFront": "408,173,51,43",
+    "roiFront": "434,173,25,43",
     "roiBack": "0,0,100,100",
-    "mode": "Single",
+    "mode": "Digit",
     "method": "Default",
     "keyword": "",
     "description": "Ocr-description"
@@ -39,7 +39,7 @@
     "itemName": "digit_counter",
     "roiFront": "1134,6,100,56",
     "roiBack": "0,0,100,100",
-    "mode": "Single",
+    "mode": "DigitCounter",
     "method": "Default",
     "keyword": "",
     "description": "Ocr-description"
@@ -48,9 +48,18 @@
     "itemName": "duration",
     "roiFront": "923,256,100,46",
     "roiBack": "0,0,100,100",
-    "mode": "Single",
+    "mode": "Duration",
     "method": "Default",
     "keyword": "育成_结界卡剩余时间",
     "description": "Ocr-description"
+  },
+  {
+    "itemName": "new_dur_00",
+    "roiFront": "848,624,76,37",
+    "roiBack": "0,0,100,100",
+    "mode": "Duration",
+    "method": "Default",
+    "keyword": "",
+    "description": "Ocr-description"
   }
 ]

+ 86 - 0
tasks/base_task.py

@@ -1,6 +1,8 @@
 # This Python file uses the following encoding: utf-8
 # @author runhey
 # github https://github.com/runhey
+import numpy as np
+
 from datetime import datetime
 from typing import Union
 
@@ -8,11 +10,14 @@ from module.atom.image import RuleImage
 from module.atom.click import RuleClick
 from module.atom.long_click import RuleLongClick
 from module.atom.swipe import RuleSwipe
+from module.atom.ocr import RuleOcr
+from module.ocr.base_ocr import OcrMode, OcrMethod
 from module.logger import logger
 from module.base.timer import Timer
 from module.config.config import Config
 from module.device.device import Device
 
+
 class BaseTask:
     config: Config = None
     device: Device = None
@@ -239,3 +244,84 @@ class BaseTask:
         # 执行后,如果有限制时间,则重置限制时间
         if interval:
             self.interval_timer[click.name].reset()
+
+    def ocr(self, target: RuleOcr, interval: float=None):
+        """
+        ocr识别目标  根据不同的target的类型返回不同的数据
+        :param interval:
+        :param target:
+        :return:
+        """
+        if not isinstance(target, RuleOcr):
+            return None
+
+        if interval:
+            if target.name in self.interval_timer:
+                # 如果传入的限制时间不一样,则替换限制新的传入的时间
+                if self.interval_timer[target.name].limit != interval:
+                    self.interval_timer[target.name] = Timer(interval)
+            else:
+                # 如果没有限制时间,则创建限制时间
+                self.interval_timer[target.name] = Timer(interval)
+            # 如果时间还没到达,则不执行
+            if not self.interval_timer[target.name].reached():
+                return None
+
+        result = target.ocr(self.device.image, target.keyword)
+
+        # 执行后,如果有限制时间,则重置限制时间
+        if interval:
+            self.interval_timer[target.name].reset()
+
+        return result
+
+    def ocr_appear(self, target: RuleOcr, interval: float=None) -> bool:
+        """
+        ocr识别目标
+        :param interval:
+        :param target:
+        :return: 如果target有keyword或者是keyword存在,返回是True,否则返回False
+                 但是没有指定keyword,返回的是匹配到的值,具体取决于target的mode
+        """
+        result = self.ocr(target, interval)
+        if not target.keyword or target.keyword == '':
+            return False
+        match target.mode:
+            case OcrMode.FULL:  # 全匹配
+                return result != (0, 0, 0, 0)
+            case OcrMode.SINGLE:
+                return result == target.keyword
+            case OcrMode.DIGIT:
+                return result == int(target.keyword)
+            case OcrMode.DIGITCOUNTER:
+                return result == target.ocr_str_digit_counter(target.keyword)
+            case OcrMode.DURATION:
+                return result == target.parse_time(target.keyword)
+
+    def ocr_appear_click(self,
+                         target: RuleOcr,
+                         action: Union[RuleClick, RuleLongClick]=None,
+                         interval: float=None,
+                         duration: float = None) -> bool:
+        """
+        ocr识别目标,如果目标存在,则触发动作
+        :param target:
+        :param action:
+        :param interval:
+        :param duration:
+        :return:
+        """
+        appear = self.ocr_appear(target, interval)
+
+        if not appear:
+            return False
+
+        if action:
+            x, y = action.coord()
+            self.click(action, interval)
+        else:
+            x, y = target.coord()
+            self.device.click(x=x, y=y, control_name=target.name)
+
+
+