image.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. # This Python file uses the following encoding: utf-8
  2. # @author runhey
  3. # github https://github.com/runhey
  4. import cv2
  5. import numpy as np
  6. from numpy import float32, int32, uint8, fromfile
  7. from pathlib import Path
  8. from module.atom.RuleImageMallResourceMixin import RuleImageMallResourceMixin
  9. from module.base.decorator import cached_property
  10. from module.logger import logger
  11. from module.base.utils import is_approx_rectangle
  12. from module.base.utils.utils import random_normal_distribution_int
  13. class RuleImage(RuleImageMallResourceMixin):
  14. debug_mode: bool = False
  15. def __init__(self, roi_front: tuple, roi_back: tuple, method: str, threshold: float, file: str) -> None:
  16. """
  17. 初始化
  18. :param roi_front: 前置roi
  19. :param roi_back: 后置roi 用于匹配的区域
  20. :param method: 匹配方法 "Template matching"
  21. :param threshold: 阈值 0.8
  22. :param file: 相对路径, 带后缀
  23. """
  24. self._match_init = False # 这个是给后面的 等待图片稳定
  25. self._image = None # 这个是匹配的目标
  26. self._kp = None #
  27. self._des = None
  28. self.method = method
  29. self.roi_front: list = list(roi_front)
  30. self.roi_back = roi_back
  31. self.threshold = threshold
  32. self.file = file
  33. @cached_property
  34. def name(self) -> str:
  35. """
  36. :return:
  37. """
  38. return Path(self.file).stem.upper()
  39. def __str__(self):
  40. return self.name
  41. __repr__ = __str__
  42. def __eq__(self, other):
  43. return str(self) == str(other)
  44. def __hash__(self):
  45. return hash(self.name)
  46. def __bool__(self):
  47. return True
  48. def load_image(self) -> None:
  49. """
  50. 加载图片
  51. :return:
  52. """
  53. if self._image is not None:
  54. return
  55. img = cv2.imdecode(fromfile(self.file, dtype=uint8), -1)
  56. img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
  57. self._image = img
  58. height, width, channels = self._image.shape
  59. if height != self.roi_front[3] or width != self.roi_front[2]:
  60. self.roi_front[2] = width
  61. self.roi_front[3] = height
  62. logger.debug(f"{self.name} roi_front size changed to {width}x{height}")
  63. def load_kp_des(self) -> None:
  64. if self._kp is not None and self._des is not None:
  65. return
  66. self._kp, self._des = self.sift.detectAndCompute(self.image, None)
  67. @property
  68. def image(self):
  69. """
  70. 获取图片
  71. :return:
  72. """
  73. if self._image is None:
  74. self.load_image()
  75. return self._image
  76. @cached_property
  77. def is_template_match(self) -> bool:
  78. """
  79. 是否是模板匹配
  80. :return:
  81. """
  82. return self.method == "Template matching"
  83. @cached_property
  84. def is_sift_flann(self) -> bool:
  85. return self.method == "Sift Flann"
  86. @cached_property
  87. def sift(self):
  88. return cv2.SIFT_create()
  89. @cached_property
  90. def kp(self):
  91. if self._kp is None:
  92. self.load_kp_des()
  93. return self._kp
  94. @cached_property
  95. def des(self):
  96. if self._des is None:
  97. self.load_kp_des()
  98. return self._des
  99. def corp(self, image: np.array, roi: list = None) -> np.array:
  100. """
  101. 截取图片
  102. :param image:
  103. :param roi
  104. :return:
  105. """
  106. if roi is None:
  107. x, y, w, h = self.roi_back
  108. else:
  109. x, y, w, h = roi
  110. x, y, w, h = int(x), int(y), int(w), int(h)
  111. return image[y:y + h, x:x + w]
  112. def match(self, image: np.array, threshold: float = None) -> bool:
  113. """
  114. :param threshold:
  115. :param image:
  116. :return:
  117. """
  118. if threshold is None:
  119. threshold = self.threshold
  120. if not self.is_template_match:
  121. return self.sift_match(image)
  122. # raise Exception(f"unknown method {self.method}")
  123. source = self.corp(image)
  124. mat = self.image
  125. if mat is None or mat.shape[0] == 0 or mat.shape[1] == 0:
  126. logger.error(f"Template image is invalid: {mat.shape}")
  127. return False # 模板无效,匹配失败
  128. # 截图竖屏(宽<高, 登录阶段游戏尚未横屏)或roiBack越界时, 裁剪图可能小于模板,
  129. # 直接返回False防止matchTemplate断言崩溃, 横屏后自动恢复
  130. if source.shape[0] < mat.shape[0] or source.shape[1] < mat.shape[1]:
  131. return False
  132. res = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
  133. min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) # 最小匹配度,最大匹配度,最小匹配度的坐标,最大匹配度的坐标
  134. if self.debug_mode:
  135. logger.attr(self.name, f'matching score {max_val:.5f}')
  136. if max_val > threshold:
  137. self.roi_front[0] = max_loc[0] + self.roi_back[0]
  138. self.roi_front[1] = max_loc[1] + self.roi_back[1]
  139. return True
  140. else:
  141. return False
  142. def match_multi_scale(self, image: np.array, threshold: float = None,
  143. scales: list = None, scale_range: tuple = None) -> bool:
  144. """
  145. 多尺度模板匹配,自动尝试多个缩放比例以适应图片大小的变化
  146. :param image: 原始截图
  147. :param threshold: 匹配阈值
  148. :param scales: 缩放比例列表
  149. :param scale_range: 缩放范围 (start, end, step),例如 (0.8, 1.2, 0.1),step 默认为 0.1
  150. :return: 匹配是否成功
  151. """
  152. if threshold is None:
  153. threshold = self.threshold
  154. # 如果指定了 scale_range,自动生成 scales 列表
  155. if scale_range is not None:
  156. start, end = scale_range[:2]
  157. step = scale_range[2] if len(scale_range) > 2 else 0.1
  158. scales = sorted(set(round(x, 1) for x in np.arange(start, end + step, step)))
  159. if scales is None:
  160. scales = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2]
  161. else:
  162. scales = sorted(set(round(x, 1) for x in scales))
  163. source = self.corp(image)
  164. mat = self.image
  165. if mat is None or mat.shape[0] == 0 or mat.shape[1] == 0:
  166. logger.error(f"Template image is invalid: {mat.shape}")
  167. return False
  168. # 预计算模板尺寸
  169. mat_h, mat_w = mat.shape[:2]
  170. source_h, source_w = source.shape[:2]
  171. best_score = 0
  172. best_loc = None
  173. best_scale = 1.0
  174. for scale in scales:
  175. scaled_w = int(mat_w * scale)
  176. scaled_h = int(mat_h * scale)
  177. # 跳过无效缩放
  178. if scaled_w < 10 or scaled_h < 10:
  179. continue
  180. try:
  181. scaled_mat = cv2.resize(mat, (scaled_w, scaled_h))
  182. res = cv2.matchTemplate(source, scaled_mat, cv2.TM_CCOEFF_NORMED)
  183. _, max_val, _, max_loc = cv2.minMaxLoc(res)
  184. if max_val > best_score:
  185. best_score = max_val
  186. best_loc = max_loc
  187. best_scale = scale
  188. except Exception as e:
  189. continue
  190. if self.debug_mode:
  191. logger.attr(self.name, f'best scale: {best_scale:.2f}, best score: {best_score:.5f}')
  192. if best_score > threshold and best_loc is not None:
  193. self.roi_front[0] = best_loc[0] + self.roi_back[0]
  194. self.roi_front[1] = best_loc[1] + self.roi_back[1]
  195. self.roi_front[2] = scaled_w
  196. self.roi_front[3] = scaled_h
  197. return True
  198. else:
  199. return False
  200. def match_all(self, image: np.array, threshold: float = None, roi: list = None) -> list[tuple]:
  201. """
  202. 区别于match,这个是返回所有的匹配结果
  203. :param roi:
  204. :param image:
  205. :param threshold:
  206. :return:
  207. """
  208. if roi is not None:
  209. self.roi_back = roi
  210. if threshold is None:
  211. threshold = self.threshold
  212. if not self.is_template_match:
  213. raise Exception(f"unknown method {self.method}")
  214. source = self.corp(image)
  215. mat = self.image
  216. results = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
  217. locations = np.where(results >= threshold)
  218. matches = []
  219. for pt in zip(*locations[::-1]): # (x, y) coordinates
  220. score = results[pt[1], pt[0]]
  221. # 得分, x, y, w, h
  222. x = self.roi_back[0] + pt[0]
  223. y = self.roi_back[1] + pt[1]
  224. matches.append((score, x, y, mat.shape[1], mat.shape[0]))
  225. return matches
  226. def match_all_any(self, image: np.array, threshold: float = None, roi: list = None, nms_threshold: float = 0.3) -> list[tuple]:
  227. """
  228. 区别于match,这个是返回所有的匹配结果,去除冗余匹配项(例如:多个框选区域重叠的情况)时使用。
  229. :param roi:
  230. :param image:
  231. :param threshold:
  232. :return:
  233. """
  234. if roi is not None:
  235. self.roi_back = roi
  236. if threshold is None:
  237. threshold = self.threshold
  238. if not self.is_template_match:
  239. raise Exception(f"unknown method {self.method}")
  240. source = self.corp(image)
  241. mat = self.image
  242. results = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
  243. locations = np.where(results >= threshold)
  244. matches = []
  245. for pt in zip(*locations[::-1]): # (x, y) coordinates
  246. score = results[pt[1], pt[0]]
  247. # 得分, x, y, w, h
  248. x = self.roi_back[0] + pt[0]
  249. y = self.roi_back[1] + pt[1]
  250. matches.append((score, x, y, mat.shape[1], mat.shape[0]))
  251. if len(matches) > 0:
  252. scores = np.array([m[0] for m in matches])
  253. boxes = np.array([[m[1], m[2], m[3], m[4]] for m in matches])
  254. # 使用OpenCV的NMSBoxes
  255. indices = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), score_threshold=threshold, nms_threshold=nms_threshold)
  256. filtered_matches = [matches[i] for i in indices]
  257. return filtered_matches
  258. return matches
  259. def coord(self) -> tuple:
  260. """
  261. 获取roi_front的随机的点击的坐标
  262. :return:
  263. """
  264. x, y, w, h = self.roi_front
  265. return x + random_normal_distribution_int(0, w), y + random_normal_distribution_int(0, h)
  266. def coord_more(self) -> tuple:
  267. """
  268. 获取roi_back的随机的点击的坐标
  269. :return:
  270. """
  271. x, y, w, h = self.roi_back
  272. return x + random_normal_distribution_int(0, w), y + random_normal_distribution_int(0, h)
  273. def front_center(self) -> tuple:
  274. """
  275. 获取roi_front的中心坐标
  276. :return:
  277. """
  278. x, y, w, h = self.roi_front
  279. return int(x + w//2), int(y + h//2)
  280. def test_match(self, image: np.array):
  281. self.debug_mode = True
  282. if self.is_template_match:
  283. return self.match(image)
  284. if self.is_sift_flann:
  285. return self.sift_match(image, show=True)
  286. def sift_match(self, image: np.array, show=False) -> bool:
  287. """
  288. 特征匹配,同样会修改 roi_front
  289. :param image: 是游戏的截图,就是转通道后的截图
  290. :param show: 测试用的
  291. :return:
  292. """
  293. source = self.corp(image)
  294. kp, des = self.sift.detectAndCompute(source, None)
  295. # 参数1:index_params
  296. # 对于SIFT和SURF,可以传入参数index_params=dict(algorithm=FLANN_INDEX_KDTREE, trees=5)。
  297. # 对于ORB,可以传入参数index_params=dict(algorithm=FLANN_INDEX_LSH, table_number=6, key_size=12)。
  298. index_params = dict(algorithm=1, trees=5)
  299. # 参数2:search_params 指定递归遍历的次数,值越高结果越准确,但是消耗的时间也越多。
  300. search_params = dict(checks=50)
  301. # 根据设置的参数创建特征匹配器 指定匹配的算法和kd树的层数,指定返回的个数
  302. flann = cv2.FlannBasedMatcher(index_params, search_params)
  303. # 利用创建好的特征匹配器利用k近邻算法来用模板的特征描述符去匹配图像的特征描述符,k指的是返回前k个最匹配的特征区域
  304. # 返回的是最匹配的两个特征点的信息,返回的类型是一个列表,列表元素的类型是Dmatch数据类型,具体是什么我也不知道
  305. # 第一个参数是小图的des, 第二个参数是大图的des
  306. matches = flann.knnMatch(self.des, des, k=2)
  307. good = []
  308. result = True
  309. for i, (m, n) in enumerate(matches):
  310. # 设定阈值, 距离小于对方的距离的0.7倍我们认为是好的匹配点.
  311. if m.distance < 0.6 * n.distance:
  312. good.append(m)
  313. if len(good) >= 10:
  314. src_pts = float32([self.kp[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
  315. dst_pts = float32([kp[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
  316. # 计算透视变换矩阵m, 要求点的数量>=4
  317. m, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
  318. # 创建一个包含模板图像四个角坐标的数组
  319. w, h = self.roi_front[2], self.roi_front[3]
  320. pts = float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
  321. if m is None:
  322. result = False
  323. else:
  324. dst = int32(cv2.perspectiveTransform(pts, m))
  325. self.roi_front[0] = dst[0, 0, 0] + self.roi_back[0]
  326. self.roi_front[1] = dst[0, 0, 1] + self.roi_back[1]
  327. if show:
  328. cv2.polylines(source, [dst], isClosed=True, color=(0, 0, 255), thickness=2)
  329. if not is_approx_rectangle(np.array([pos[0] for pos in dst])):
  330. result = False
  331. else:
  332. result = False
  333. # https://blog.csdn.net/cungudafa/article/details/105399278
  334. # https://blog.csdn.net/qq_45832961/article/details/122776322
  335. if show:
  336. # 准备一个空的掩膜来绘制好的匹配
  337. mask_matches = [[0, 0] for i in range(len(matches))]
  338. # 向掩膜中添加数据
  339. for i, (m, n) in enumerate(matches):
  340. if m.distance < 0.6 * n.distance: # 理论上0.7最好
  341. mask_matches[i] = [1, 0]
  342. img_matches = cv2.drawMatchesKnn(self.image, self.kp, source, kp, matches, None,
  343. matchColor=(0, 255, 0), singlePointColor=(255, 0, 0),
  344. matchesMask=mask_matches, flags=0)
  345. cv2.imshow(f'Sift Flann: {self.name}', img_matches)
  346. cv2.waitKey(0)
  347. cv2.destroyAllWindows()
  348. return result
  349. def match_mean_color(self, image, color: tuple, bias=10) -> bool:
  350. """
  351. :param image:
  352. :param color: rgb
  353. :param bias:
  354. :return:
  355. """
  356. image = self.corp(image)
  357. average_color = cv2.mean(image)
  358. if self.debug_mode:
  359. logger.info(f'{self.name} average_color: {average_color}')
  360. for i in range(3):
  361. if abs(average_color[i] - color[i]) > bias:
  362. return False
  363. return True
  364. def match_brightness(
  365. self, image: np.array,
  366. threshold: float = 0.9,
  367. roi: list = None,
  368. gray: bool = False) -> bool:
  369. source = self.corp(image, roi) if roi is not None else self.corp(image)
  370. template = self.image
  371. if len(source.shape) != 3:
  372. raise Exception(f'{self.name} source image must be 3-channel RGB, got shape={source.shape}')
  373. source_gray = cv2.cvtColor(source, cv2.COLOR_RGB2GRAY)
  374. source_hsv = cv2.cvtColor(source, cv2.COLOR_RGB2HSV)
  375. if len(template.shape) != 3:
  376. raise Exception(f'{self.name} template image must be 3-channel RGB, got shape={template.shape}')
  377. template_gray = cv2.cvtColor(template, cv2.COLOR_RGB2GRAY)
  378. template_hsv = cv2.cvtColor(template, cv2.COLOR_RGB2HSV)
  379. source_value = float(source_gray.mean()) if gray else float(source_hsv[:, :, 2].mean())
  380. template_value = float(template_gray.mean()) if gray else float(template_hsv[:, :, 2].mean())
  381. score = 1.0 - abs(source_value - template_value) / 255.0
  382. score = max(0.0, min(1.0, score))
  383. if self.debug_mode:
  384. logger.attr(self.name, f'brightness similarity {score:.5f}')
  385. logger.info(f'Template value: {template_value}, Source value: {source_value}')
  386. return score >= threshold
  387. def match_saturation(self, image: np.array, threshold: float = 0.9, roi: list = None) -> bool:
  388. source = self.corp(image, roi) if roi is not None else self.corp(image)
  389. template = self.image
  390. if len(source.shape) != 3:
  391. raise Exception(f'{self.name} source image must be 3-channel RGB, got shape={source.shape}')
  392. source_hsv = cv2.cvtColor(source, cv2.COLOR_RGB2HSV)
  393. if len(template.shape) != 3:
  394. raise Exception(f'{self.name} template image must be 3-channel RGB, got shape={template.shape}')
  395. template_hsv = cv2.cvtColor(template, cv2.COLOR_RGB2HSV)
  396. source_value = float(source_hsv[:, :, 1].mean())
  397. template_value = float(template_hsv[:, :, 1].mean())
  398. score = 1.0 - abs(source_value - template_value) / 255.0
  399. score = max(0.0, min(1.0, score))
  400. if self.debug_mode:
  401. logger.attr(self.name, f'saturation similarity {score:.5f}')
  402. logger.info(f'Template value: {template_value}, Source value: {source_value}')
  403. return score >= threshold
  404. if __name__ == "__main__":
  405. from dev_tools.assets_test import detect_image
  406. IMAGE_FILE = './log/test/QQ截图20240223151924.png'
  407. from tasks.Restart.assets import RestartAssets
  408. jade = RestartAssets.I_HARVEST_JADE
  409. jade.method = 'Sift Flann'
  410. sign = RestartAssets.I_HARVEST_SIGN
  411. sign.method = 'Sift Flann'
  412. print(jade.roi_front)
  413. detect_image(IMAGE_FILE, jade)
  414. detect_image(IMAGE_FILE, sign)
  415. print(jade.roi_front)