Browse Source

perf(ActivityShikigami): Add ocr image recognition preprocessing and post-processing operations

Gan1Ser 1 year ago
parent
commit
681e1593f2

+ 15 - 5
module/ocr/base_ocr.py

@@ -48,6 +48,7 @@ class BaseCor:
 
     lang: str = "ch"
     score: float = 0.6  # 阈值默认为0.5
+    min_score: float = 0.3  # 宽松阈值,用于挽救数字等结果
 
     name: str = "ocr"
     mode: OcrMode = OcrMode.FULL
@@ -150,7 +151,16 @@ class BaseCor:
         image = self.pre_process(image)
         # ocr
         result, score = self.model.ocr_single_line(image)
-        if score < self.score:
+        contains_digit = any(char.isdigit() for char in result)
+
+        if score >= self.score:
+            pass
+        elif score >= self.min_score and contains_digit and self.mode in [OcrMode.DIGIT, OcrMode.DIGITCOUNTER,
+                                                                          OcrMode.QUANTITY]:
+            logger.warning(
+                f'[{self.name}] Score {score:.2f} is low, but result "{result}" contains a digit. Accepting it.')
+            print(f'能保留')
+        else:
             result = ""
         # after proces
         result = self.after_process(result)
@@ -159,7 +169,7 @@ class BaseCor:
                     text=f'[{result}]')
         return result
 
-    def detect_and_ocr(self, image) -> list[BoxedResult]:
+    def detect_and_ocr(self, image, logDisplay: bool = True) -> list[BoxedResult]:
         """
         注意:这里使用了预处理和后处理
         :param image:
@@ -181,9 +191,9 @@ class BaseCor:
                 continue
             result.ocr_text = self.after_process(result.ocr_text)
             results.append(result)
-
-        logger.attr(name='%s %ss' % (self.name, float2str(time.time() - start_time)),
-                    text=str([result.ocr_text for result in results]))
+            if logDisplay:
+                logger.attr(name='%s %ss' % (self.name, float2str(time.time() - start_time)),
+                            text=str([result.ocr_text for result in results]))
         return results
 
     def match(self, result: str, included: bool=False) -> bool:

+ 20 - 6
module/ocr/sub_ocr.py

@@ -96,9 +96,16 @@ class Digit(Single):
 
     def after_process(self, result):
         result = super().after_process(result)
-        result = result.replace('I', '1').replace('D', '0').replace('S', '5')
-        result = result.replace('B', '8').replace('?', '2').replace('?', '2')
-        result = result.replace('d', '6')
+        replacements = {
+            'I': '1', 'D': '0', 'S': '5',
+            'B': '8', '?': '2', '?': '2',
+            'd': '6', 'o': '0', 'O': '0',
+            '→': '1'
+        }
+
+        for old, new in replacements.items():
+            result = result.replace(old, new)
+
         result = [char for char in result if char.isdigit()]
         result = ''.join(result)
 
@@ -125,9 +132,16 @@ class Digit(Single):
 class DigitCounter(Single):
     def after_process(self, result):
         result = super().after_process(result)
-        result = result.replace('I', '1').replace('D', '0').replace('S', '5')
-        result = result.replace('B', '8').replace('?', '2').replace('?', '2')
-        result = result.replace('d', '6')
+        replacements = {
+            'I': '1', 'D': '0', 'S': '5',
+            'B': '8', '?': '2', '?': '2',
+            'd': '6', 'o': '0', 'O': '0',
+            '→': '1'
+        }
+
+        for old, new in replacements.items():
+            result = result.replace(old, new)
+
         result = [char for char in result if char.isdigit() or char == '/']
         result = ''.join(result)
         return result

BIN
tasks/ActivityShikigami/as/as_red_exit.png


+ 2 - 2
tasks/ActivityShikigami/as/image.json

@@ -128,8 +128,8 @@
   {
     "itemName": "red_exit",
     "imageName": "as_red_exit.png",
-    "roiFront": "1108,88,39,38",
-    "roiBack": "1071,52,110,135",
+    "roiFront": "1162,96,39,38",
+    "roiBack": "1120,49,110,135",
     "method": "Template matching",
     "threshold": 0.8,
     "description": "红色退出"

+ 1 - 1
tasks/ActivityShikigami/assets.py

@@ -73,7 +73,7 @@ class ActivityShikigamiAssets:
 	# 右上跳过按钮 
 	I_SKIP_BUTTON = RuleImage(roi_front=(1154,34,64,30), roi_back=(1154,34,64,30), threshold=0.8, method="Template matching", file="./tasks/ActivityShikigami/as/as_skip_button.png")
 	# 红色退出 
-	I_RED_EXIT = RuleImage(roi_front=(1108,88,39,38), roi_back=(1071,52,110,135), threshold=0.8, method="Template matching", file="./tasks/ActivityShikigami/as/as_red_exit.png")
+	I_RED_EXIT = RuleImage(roi_front=(1162,96,39,38), roi_back=(1120,49,110,135), threshold=0.8, method="Template matching", file="./tasks/ActivityShikigami/as/as_red_exit.png")
 	# description 
 	I_TOGGLE_BUTTON = RuleImage(roi_front=(1193,463,27,25), roi_back=(1169,146,73,403), threshold=0.8, method="Template matching", file="./tasks/ActivityShikigami/as/as_toggle_button.png")
 

+ 70 - 4
tasks/ActivityShikigami/script_task.py

@@ -3,7 +3,7 @@
 # github https://github.com/runhey
 import random
 from datetime import datetime, timedelta, time
-
+import time as time_mode
 from tasks.base_task import BaseTask
 from tasks.Component.GeneralBattle.general_battle import GeneralBattle
 from tasks.AreaBoss.assets import AreaBossAssets
@@ -18,6 +18,9 @@ from module.logger import logger
 from module.exception import TaskEnd
 from module.base.protect import random_sleep
 
+import cv2
+import numpy as np
+from module.atom.ocr import RuleOcr
 
 class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
 
@@ -158,6 +161,7 @@ class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
             if self.ocr_appear_click(self.O_ENTRY_ACTIVITY, interval=1):
                 continue
             if self.appear_then_click(self.I_TOGGLE_BUTTON, interval=3):
+                time_mode.sleep(2)
                 continue
             if self.appear_then_click(self.I_SKIP_BUTTON, interval=1.5):
                 continue
@@ -196,7 +200,8 @@ class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
         """
         self.screenshot()
         if current_ap == ApMode.AP_ACTIVITY:
-            res: int = self.O_REMAIN_AP_ACTIVITY2.ocr_digit(self.device.image)
+            res: int = self.O_REMAIN_AP_ACTIVITY2.ocr_digit(
+                self._prepare_image_for_ocr(self.device.image, asset=self.O_REMAIN_AP_ACTIVITY2))
             if res <= 0:
                 logger.warning(f'Activity ap {res} not enough')
                 return False
@@ -208,7 +213,7 @@ class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
             # return True
 
         elif current_ap == ApMode.AP_GAME:
-            cu: int = self.O_REMAIN_AP.ocr_digit(self.device.image)
+            cu: int = self.O_REMAIN_AP.ocr_digit(self._prepare_image_for_ocr(self.device.image,asset=self.O_REMAIN_AP))
             if cu > 0:
                 logger.warning(f'Game ap {cu} more than 0')
                 return True
@@ -278,7 +283,8 @@ class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
                         break
                     if self.appear_then_click(self.I_WIN, action=self.C_RANDOM_ALL, interval=1.1):
                         continue
-                return True
+                if self.appear(self.I_FIRE):
+                    return True
             # 失败 -> 正常人不会失败
             if self.appear(self.I_FALSE):
                 logger.warning('False battle')
@@ -288,6 +294,66 @@ class ScriptTask(GameUi, BaseActivity, SwitchSoul, ActivityShikigamiAssets):
             if random_click_swipt_enable:
                 self.random_click_swipt()
 
+    def _prepare_image_for_ocr(self, image: np.ndarray, asset: RuleOcr) -> np.ndarray:
+
+        image_copy = image.copy()
+        x, y, w, h = asset.roi
+
+        roi_to_process = image_copy[y:y + h, x:x + w]
+        if len(roi_to_process.shape) == 3:
+            gray_image = cv2.cvtColor(roi_to_process, cv2.COLOR_BGR2GRAY)
+        else:
+            gray_image = roi_to_process
+
+        # 自适应二值化
+        _, binary_norm = cv2.threshold(gray_image, 127, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
+        _, binary_inv = cv2.threshold(gray_image, 127, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+        if cv2.countNonZero(binary_norm) < cv2.countNonZero(binary_inv):
+            binary_correct = binary_norm
+        else:
+            binary_correct = binary_inv
+
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 1))
+        dilated_image = cv2.dilate(binary_correct, kernel, iterations=1)
+
+        # 找轮廓
+        contours, _ = cv2.findContours(dilated_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        processed_roi_content = None
+        if contours:
+            all_points = np.concatenate(contours, axis=0)
+            bx, by, bw, bh = cv2.boundingRect(all_points)
+
+            processed_roi_content = binary_correct[by:by + bh, bx:bx + bw]
+
+        centered_roi = np.full((h, w), 255, dtype=np.uint8)  # 255代表白色
+
+        if processed_roi_content is not None:
+            content_h, content_w = processed_roi_content.shape
+
+            if content_h <= h and content_w <= w:
+                # 计算居中粘贴的位置,放到中间
+                start_y = (h - content_h) // 2
+                start_x = (w - content_w) // 2
+
+                paste_area = centered_roi[start_y:start_y + content_h, start_x:start_x + content_w]
+
+                paste_area[processed_roi_content == 255] = 0
+            else:
+                logger.warning(f"Content for asset '{asset.name}' is larger than ROI. Skipping centering.")
+                # 内容过大,直接使用原始二值图的反转作为结果
+                centered_roi = cv2.bitwise_not(binary_correct)
+        else:
+            logger.warning(f"No content found in ROI for asset: {asset.name}. ROI will be blank.")
+
+        processed_roi_bgr = cv2.cvtColor(centered_roi, cv2.COLOR_GRAY2BGR)
+        image_copy[y:y + h, x:x + w] = processed_roi_bgr
+
+        return image_copy
+
+
+
     """
     def battle_wait(self, random_click_swipt_enable: bool) -> bool:
         # 重写,出现达摩版本月度常用