| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489 |
- # This Python file uses the following encoding: utf-8
- # @author runhey
- # github https://github.com/runhey
- import cv2
- import numpy as np
- from numpy import float32, int32, uint8, fromfile
- from pathlib import Path
- from module.atom.RuleImageMallResourceMixin import RuleImageMallResourceMixin
- from module.base.decorator import cached_property
- from module.logger import logger
- from module.base.utils import is_approx_rectangle
- from module.base.utils.utils import random_normal_distribution_int
- class RuleImage(RuleImageMallResourceMixin):
- debug_mode: bool = False
- def __init__(self, roi_front: tuple, roi_back: tuple, method: str, threshold: float, file: str) -> None:
- """
- 初始化
- :param roi_front: 前置roi
- :param roi_back: 后置roi 用于匹配的区域
- :param method: 匹配方法 "Template matching"
- :param threshold: 阈值 0.8
- :param file: 相对路径, 带后缀
- """
- self._match_init = False # 这个是给后面的 等待图片稳定
- self._image = None # 这个是匹配的目标
- self._kp = None #
- self._des = None
- self.method = method
- self.roi_front: list = list(roi_front)
- self.roi_back = roi_back
- self.threshold = threshold
- self.file = file
- @cached_property
- def name(self) -> str:
- """
- :return:
- """
- return Path(self.file).stem.upper()
- def __str__(self):
- return self.name
- __repr__ = __str__
- def __eq__(self, other):
- return str(self) == str(other)
- def __hash__(self):
- return hash(self.name)
- def __bool__(self):
- return True
- def load_image(self) -> None:
- """
- 加载图片
- :return:
- """
- if self._image is not None:
- return
- img = cv2.imdecode(fromfile(self.file, dtype=uint8), -1)
- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
- self._image = img
- height, width, channels = self._image.shape
- if height != self.roi_front[3] or width != self.roi_front[2]:
- self.roi_front[2] = width
- self.roi_front[3] = height
- logger.debug(f"{self.name} roi_front size changed to {width}x{height}")
- def load_kp_des(self) -> None:
- if self._kp is not None and self._des is not None:
- return
- self._kp, self._des = self.sift.detectAndCompute(self.image, None)
- @property
- def image(self):
- """
- 获取图片
- :return:
- """
- if self._image is None:
- self.load_image()
- return self._image
- @cached_property
- def is_template_match(self) -> bool:
- """
- 是否是模板匹配
- :return:
- """
- return self.method == "Template matching"
- @cached_property
- def is_sift_flann(self) -> bool:
- return self.method == "Sift Flann"
- @cached_property
- def sift(self):
- return cv2.SIFT_create()
- @cached_property
- def kp(self):
- if self._kp is None:
- self.load_kp_des()
- return self._kp
- @cached_property
- def des(self):
- if self._des is None:
- self.load_kp_des()
- return self._des
- def corp(self, image: np.array, roi: list = None) -> np.array:
- """
- 截取图片
- :param image:
- :param roi
- :return:
- """
- if roi is None:
- x, y, w, h = self.roi_back
- else:
- x, y, w, h = roi
- x, y, w, h = int(x), int(y), int(w), int(h)
- return image[y:y + h, x:x + w]
- def match(self, image: np.array, threshold: float = None) -> bool:
- """
- :param threshold:
- :param image:
- :return:
- """
- if threshold is None:
- threshold = self.threshold
- if not self.is_template_match:
- return self.sift_match(image)
- # raise Exception(f"unknown method {self.method}")
- source = self.corp(image)
- mat = self.image
- if mat is None or mat.shape[0] == 0 or mat.shape[1] == 0:
- logger.error(f"Template image is invalid: {mat.shape}")
- return False # 模板无效, 匹配失败
- if mat.shape[0] > source.shape[0] or mat.shape[1] > source.shape[1]:
- # 模板大于源图, 视为无效匹配(避免 matchTemplate 的异常行为)
- return False
- res = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
- min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) # 最小匹配度,最大匹配度,最小匹配度的坐标,最大匹配度的坐标
- if self.debug_mode:
- logger.attr(self.name, f'matching score {max_val:.5f}')
- if max_val > threshold:
- self.roi_front[0] = max_loc[0] + self.roi_back[0]
- self.roi_front[1] = max_loc[1] + self.roi_back[1]
- return True
- else:
- return False
- def match_multi_scale(self, image: np.array, threshold: float = None,
- scales: list = None, scale_range: tuple = None) -> bool:
- """
- 多尺度模板匹配,自动尝试多个缩放比例以适应图片大小的变化
- :param image: 原始截图
- :param threshold: 匹配阈值
- :param scales: 缩放比例列表
- :param scale_range: 缩放范围 (start, end, step),例如 (0.8, 1.2, 0.1),step 默认为 0.1
- :return: 匹配是否成功
- """
- if threshold is None:
- threshold = self.threshold
- # 如果指定了 scale_range,自动生成 scales 列表
- if scale_range is not None:
- start, end = scale_range[:2]
- step = scale_range[2] if len(scale_range) > 2 else 0.1
- scales = sorted(set(round(x, 1) for x in np.arange(start, end + step, step)))
- if scales is None:
- scales = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2]
- else:
- scales = sorted(set(round(x, 1) for x in scales))
- source = self.corp(image)
- mat = self.image
- if mat is None or mat.shape[0] == 0 or mat.shape[1] == 0:
- logger.error(f"Template image is invalid: {mat.shape}")
- return False
- # 预计算模板尺寸
- mat_h, mat_w = mat.shape[:2]
- source_h, source_w = source.shape[:2]
- best_score = 0
- best_loc = None
- best_scale = 1.0
- for scale in scales:
- scaled_w = int(mat_w * scale)
- scaled_h = int(mat_h * scale)
- # 跳过无效缩放
- if scaled_w < 10 or scaled_h < 10:
- continue
- # 缩放后仍大于源图则跳过该尺度(避免 matchTemplate 契约问题)
- if scaled_w > source_w or scaled_h > source_h:
- continue
- try:
- scaled_mat = cv2.resize(mat, (scaled_w, scaled_h))
- res = cv2.matchTemplate(source, scaled_mat, cv2.TM_CCOEFF_NORMED)
- _, max_val, _, max_loc = cv2.minMaxLoc(res)
- if max_val > best_score:
- best_score = max_val
- best_loc = max_loc
- best_scale = scale
- except Exception as e:
- continue
- if self.debug_mode:
- logger.attr(self.name, f'best scale: {best_scale:.2f}, best score: {best_score:.5f}')
- if best_score > threshold and best_loc is not None:
- self.roi_front[0] = best_loc[0] + self.roi_back[0]
- self.roi_front[1] = best_loc[1] + self.roi_back[1]
- self.roi_front[2] = scaled_w
- self.roi_front[3] = scaled_h
- return True
- else:
- return False
- def match_all(self, image: np.array, threshold: float = None, roi: list = None) -> list[tuple]:
- """
- 区别于match,这个是返回所有的匹配结果
- :param roi:
- :param image:
- :param threshold:
- :return:
- """
- if roi is not None:
- self.roi_back = roi
- if threshold is None:
- threshold = self.threshold
- if not self.is_template_match:
- raise Exception(f"unknown method {self.method}")
- source = self.corp(image)
- mat = self.image
- results = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
- locations = np.where(results >= threshold)
- matches = []
- for pt in zip(*locations[::-1]): # (x, y) coordinates
- score = results[pt[1], pt[0]]
- # 得分, x, y, w, h
- x = self.roi_back[0] + pt[0]
- y = self.roi_back[1] + pt[1]
- matches.append((score, x, y, mat.shape[1], mat.shape[0]))
- return matches
- def match_all_any(self, image: np.array, threshold: float = None, roi: list = None, nms_threshold: float = 0.3) -> list[tuple]:
- """
- 区别于match,这个是返回所有的匹配结果,去除冗余匹配项(例如:多个框选区域重叠的情况)时使用。
- :param roi:
- :param image:
- :param threshold:
- :return:
- """
- if roi is not None:
- self.roi_back = roi
- if threshold is None:
- threshold = self.threshold
- if not self.is_template_match:
- raise Exception(f"unknown method {self.method}")
- source = self.corp(image)
- mat = self.image
- results = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
- locations = np.where(results >= threshold)
- matches = []
- for pt in zip(*locations[::-1]): # (x, y) coordinates
- score = results[pt[1], pt[0]]
- # 得分, x, y, w, h
- x = self.roi_back[0] + pt[0]
- y = self.roi_back[1] + pt[1]
- matches.append((score, x, y, mat.shape[1], mat.shape[0]))
- if len(matches) > 0:
- scores = np.array([m[0] for m in matches])
- boxes = np.array([[m[1], m[2], m[3], m[4]] for m in matches])
- # 使用OpenCV的NMSBoxes
- indices = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), score_threshold=threshold, nms_threshold=nms_threshold)
- filtered_matches = [matches[i] for i in indices]
- return filtered_matches
- return matches
- def coord(self) -> tuple:
- """
- 获取roi_front的随机的点击的坐标
- :return:
- """
- x, y, w, h = self.roi_front
- return x + random_normal_distribution_int(0, w), y + random_normal_distribution_int(0, h)
- def coord_more(self) -> tuple:
- """
- 获取roi_back的随机的点击的坐标
- :return:
- """
- x, y, w, h = self.roi_back
- return x + random_normal_distribution_int(0, w), y + random_normal_distribution_int(0, h)
- def front_center(self) -> tuple:
- """
- 获取roi_front的中心坐标
- :return:
- """
- x, y, w, h = self.roi_front
- return int(x + w//2), int(y + h//2)
- def test_match(self, image: np.array):
- self.debug_mode = True
- if self.is_template_match:
- return self.match(image)
- if self.is_sift_flann:
- return self.sift_match(image, show=True)
- def sift_match(self, image: np.array, show=False) -> bool:
- """
- 特征匹配,同样会修改 roi_front
- :param image: 是游戏的截图,就是转通道后的截图
- :param show: 测试用的
- :return:
- """
- source = self.corp(image)
- kp, des = self.sift.detectAndCompute(source, None)
- # 参数1:index_params
- # 对于SIFT和SURF,可以传入参数index_params=dict(algorithm=FLANN_INDEX_KDTREE, trees=5)。
- # 对于ORB,可以传入参数index_params=dict(algorithm=FLANN_INDEX_LSH, table_number=6, key_size=12)。
- index_params = dict(algorithm=1, trees=5)
- # 参数2:search_params 指定递归遍历的次数,值越高结果越准确,但是消耗的时间也越多。
- search_params = dict(checks=50)
- # 根据设置的参数创建特征匹配器 指定匹配的算法和kd树的层数,指定返回的个数
- flann = cv2.FlannBasedMatcher(index_params, search_params)
- # 利用创建好的特征匹配器利用k近邻算法来用模板的特征描述符去匹配图像的特征描述符,k指的是返回前k个最匹配的特征区域
- # 返回的是最匹配的两个特征点的信息,返回的类型是一个列表,列表元素的类型是Dmatch数据类型,具体是什么我也不知道
- # 第一个参数是小图的des, 第二个参数是大图的des
- matches = flann.knnMatch(self.des, des, k=2)
- good = []
- result = True
- for i, (m, n) in enumerate(matches):
- # 设定阈值, 距离小于对方的距离的0.7倍我们认为是好的匹配点.
- if m.distance < 0.6 * n.distance:
- good.append(m)
- if len(good) >= 10:
- src_pts = float32([self.kp[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
- dst_pts = float32([kp[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
- # 计算透视变换矩阵m, 要求点的数量>=4
- m, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
- # 创建一个包含模板图像四个角坐标的数组
- w, h = self.roi_front[2], self.roi_front[3]
- pts = float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
- if m is None:
- result = False
- else:
- dst = int32(cv2.perspectiveTransform(pts, m))
- self.roi_front[0] = dst[0, 0, 0] + self.roi_back[0]
- self.roi_front[1] = dst[0, 0, 1] + self.roi_back[1]
- if show:
- cv2.polylines(source, [dst], isClosed=True, color=(0, 0, 255), thickness=2)
- if not is_approx_rectangle(np.array([pos[0] for pos in dst])):
- result = False
- else:
- result = False
- # https://blog.csdn.net/cungudafa/article/details/105399278
- # https://blog.csdn.net/qq_45832961/article/details/122776322
- if show:
- # 准备一个空的掩膜来绘制好的匹配
- mask_matches = [[0, 0] for i in range(len(matches))]
- # 向掩膜中添加数据
- for i, (m, n) in enumerate(matches):
- if m.distance < 0.6 * n.distance: # 理论上0.7最好
- mask_matches[i] = [1, 0]
- img_matches = cv2.drawMatchesKnn(self.image, self.kp, source, kp, matches, None,
- matchColor=(0, 255, 0), singlePointColor=(255, 0, 0),
- matchesMask=mask_matches, flags=0)
- cv2.imshow(f'Sift Flann: {self.name}', img_matches)
- cv2.waitKey(0)
- cv2.destroyAllWindows()
- return result
- def match_mean_color(self, image, color: tuple, bias=10) -> bool:
- """
- :param image:
- :param color: rgb
- :param bias:
- :return:
- """
- image = self.corp(image)
- average_color = cv2.mean(image)
- if self.debug_mode:
- logger.info(f'{self.name} average_color: {average_color}')
- for i in range(3):
- if abs(average_color[i] - color[i]) > bias:
- return False
- return True
- def match_brightness(
- self, image: np.array,
- threshold: float = 0.9,
- roi: list = None,
- gray: bool = False) -> bool:
- source = self.corp(image, roi) if roi is not None else self.corp(image)
- template = self.image
- if len(source.shape) != 3:
- raise Exception(f'{self.name} source image must be 3-channel RGB, got shape={source.shape}')
- source_gray = cv2.cvtColor(source, cv2.COLOR_RGB2GRAY)
- source_hsv = cv2.cvtColor(source, cv2.COLOR_RGB2HSV)
- if len(template.shape) != 3:
- raise Exception(f'{self.name} template image must be 3-channel RGB, got shape={template.shape}')
- template_gray = cv2.cvtColor(template, cv2.COLOR_RGB2GRAY)
- template_hsv = cv2.cvtColor(template, cv2.COLOR_RGB2HSV)
- source_value = float(source_gray.mean()) if gray else float(source_hsv[:, :, 2].mean())
- template_value = float(template_gray.mean()) if gray else float(template_hsv[:, :, 2].mean())
- score = 1.0 - abs(source_value - template_value) / 255.0
- score = max(0.0, min(1.0, score))
- if self.debug_mode:
- logger.attr(self.name, f'brightness similarity {score:.5f}')
- logger.info(f'Template value: {template_value}, Source value: {source_value}')
- return score >= threshold
- def match_saturation(self, image: np.array, threshold: float = 0.9, roi: list = None) -> bool:
- source = self.corp(image, roi) if roi is not None else self.corp(image)
- template = self.image
- if len(source.shape) != 3:
- raise Exception(f'{self.name} source image must be 3-channel RGB, got shape={source.shape}')
- source_hsv = cv2.cvtColor(source, cv2.COLOR_RGB2HSV)
- if len(template.shape) != 3:
- raise Exception(f'{self.name} template image must be 3-channel RGB, got shape={template.shape}')
- template_hsv = cv2.cvtColor(template, cv2.COLOR_RGB2HSV)
- source_value = float(source_hsv[:, :, 1].mean())
- template_value = float(template_hsv[:, :, 1].mean())
- score = 1.0 - abs(source_value - template_value) / 255.0
- score = max(0.0, min(1.0, score))
- if self.debug_mode:
- logger.attr(self.name, f'saturation similarity {score:.5f}')
- logger.info(f'Template value: {template_value}, Source value: {source_value}')
- return score >= threshold
- if __name__ == "__main__":
- from dev_tools.assets_test import detect_image
- IMAGE_FILE = './log/test/QQ截图20240223151924.png'
- from tasks.Restart.assets import RestartAssets
- jade = RestartAssets.I_HARVEST_JADE
- jade.method = 'Sift Flann'
- sign = RestartAssets.I_HARVEST_SIGN
- sign.method = 'Sift Flann'
- print(jade.roi_front)
- detect_image(IMAGE_FILE, jade)
- detect_image(IMAGE_FILE, sign)
- print(jade.roi_front)
|