image.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. # This Python file uses the following encoding: utf-8
  2. # @author runhey
  3. # github https://github.com/runhey
  4. import cv2
  5. import numpy as np
  6. from numpy import float32, int32, uint8, fromfile
  7. from pathlib import Path
  8. from module.base.decorator import cached_property
  9. from module.logger import logger
  10. from module.base.utils import is_approx_rectangle
  11. class RuleImage:
  12. def __init__(self, roi_front: tuple, roi_back: tuple, method: str, threshold: float, file: str) -> None:
  13. """
  14. 初始化
  15. :param roi_front: 前置roi
  16. :param roi_back: 后置roi 用于匹配的区域
  17. :param method: 匹配方法 "Template matching"
  18. :param threshold: 阈值 0.8
  19. :param file: 相对路径, 带后缀
  20. """
  21. self._match_init = False # 这个是给后面的 等待图片稳定
  22. self._image = None # 这个是匹配的目标
  23. self._kp = None #
  24. self._des = None
  25. self.method = method
  26. self.roi_front: list = list(roi_front)
  27. self.roi_back = roi_back
  28. self.threshold = threshold
  29. self.file = file
  30. @cached_property
  31. def name(self) -> str:
  32. """
  33. :return:
  34. """
  35. return Path(self.file).stem.upper()
  36. def __str__(self):
  37. return self.name
  38. __repr__ = __str__
  39. def __eq__(self, other):
  40. return str(self) == str(other)
  41. def __hash__(self):
  42. return hash(self.name)
  43. def __bool__(self):
  44. return True
  45. def load_image(self) -> None:
  46. """
  47. 加载图片
  48. :return:
  49. """
  50. if self._image is not None:
  51. return
  52. img = cv2.imdecode(fromfile(self.file, dtype=uint8), -1)
  53. img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
  54. self._image = img
  55. height, width, channels = self._image.shape
  56. if height != self.roi_front[3] or width != self.roi_front[2]:
  57. self.roi_front[2] = width
  58. self.roi_front[3] = height
  59. logger.debug(f"{self.name} roi_front size changed to {width}x{height}")
  60. def load_kp_des(self) -> None:
  61. if self._kp is not None and self._des is not None:
  62. return
  63. self._kp, self._des = self.sift.detectAndCompute(self.image, None)
  64. @property
  65. def image(self):
  66. """
  67. 获取图片
  68. :return:
  69. """
  70. if self._image is None:
  71. self.load_image()
  72. return self._image
  73. @cached_property
  74. def is_template_match(self) -> bool:
  75. """
  76. 是否是模板匹配
  77. :return:
  78. """
  79. return self.method == "Template matching"
  80. @cached_property
  81. def is_sift_flann(self) -> bool:
  82. return self.method == "Sift Flann"
  83. @cached_property
  84. def sift(self):
  85. return cv2.SIFT_create()
  86. @cached_property
  87. def kp(self):
  88. if self._kp is None:
  89. self.load_kp_des()
  90. return self._kp
  91. @cached_property
  92. def des(self):
  93. if self._des is None:
  94. self.load_kp_des()
  95. return self._des
  96. def corp(self, image: np.array, roi: list = None) -> np.array:
  97. """
  98. 截取图片
  99. :param image:
  100. :param roi
  101. :return:
  102. """
  103. if roi is None:
  104. x, y, w, h = self.roi_back
  105. else:
  106. x, y, w, h = roi
  107. x, y, w, h = int(x), int(y), int(w), int(h)
  108. return image[y:y + h, x:x + w]
  109. def match(self, image: np.array, threshold: float = None) -> bool:
  110. """
  111. :param threshold:
  112. :param image:
  113. :return:
  114. """
  115. if threshold is None:
  116. threshold = self.threshold
  117. if not self.is_template_match:
  118. return self.sift_match(image)
  119. # raise Exception(f"unknown method {self.method}")
  120. # if self.name == 'SCENE_SHENSHE':
  121. # source = image
  122. # else:
  123. # source = self.corp(image)
  124. source = self.corp(image)
  125. mat = self.image
  126. if mat is None or mat.shape[0] == 0 or mat.shape[1] == 0:
  127. logger.error(f"Template image is invalid: {mat.shape}") #检测模板尺寸,不合法则不进行匹配,避免两次截图画面完全相同造成模板不合法
  128. return True # 如果模板图像无效,直接返回 True
  129. res = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
  130. min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) # 最小匹配度,最大匹配度,最小匹配度的坐标,最大匹配度的坐标
  131. # logger.attr(self.name, max_val)
  132. # if self.name == 'SCENE_SHENSHE' and max_val > threshold:
  133. # self.roi_front[0] = max_loc[0]
  134. # self.roi_front[1] = max_loc[1]
  135. # self.roi_front[2] = mat.shape[1]
  136. # self.roi_front[3] = mat.shape[0]
  137. # return True
  138. # elif max_val > threshold:
  139. # self.roi_front[0] = max_loc[0] + self.roi_back[0]
  140. # self.roi_front[1] = max_loc[1] + self.roi_back[1]
  141. # return True
  142. # else:
  143. # return False
  144. if max_val > threshold:
  145. self.roi_front[0] = max_loc[0] + self.roi_back[0]
  146. self.roi_front[1] = max_loc[1] + self.roi_back[1]
  147. return True
  148. else:
  149. return False
  150. def match_all(self, image: np.array, threshold: float = None, roi: list = None) -> list[tuple]:
  151. """
  152. 区别于match,这个是返回所有的匹配结果
  153. :param roi:
  154. :param image:
  155. :param threshold:
  156. :return:
  157. """
  158. if roi is not None:
  159. self.roi_back = roi
  160. if threshold is None:
  161. threshold = self.threshold
  162. if not self.is_template_match:
  163. raise Exception(f"unknown method {self.method}")
  164. source = self.corp(image)
  165. mat = self.image
  166. results = cv2.matchTemplate(source, mat, cv2.TM_CCOEFF_NORMED)
  167. locations = np.where(results >= threshold)
  168. matches = []
  169. for pt in zip(*locations[::-1]): # (x, y) coordinates
  170. score = results[pt[1], pt[0]]
  171. # 得分, x, y, w, h
  172. x = self.roi_back[0] + pt[0]
  173. y = self.roi_back[1] + pt[1]
  174. matches.append((score, x, y, mat.shape[1], mat.shape[0]))
  175. return matches
  176. def coord(self) -> tuple:
  177. """
  178. 获取roi_front的随机的点击的坐标
  179. :return:
  180. """
  181. x, y, w, h = self.roi_front
  182. return x + np.random.randint(0, w), y + np.random.randint(0, h)
  183. def coord_more(self) -> tuple:
  184. """
  185. 获取roi_back的随机的点击的坐标
  186. :return:
  187. """
  188. x, y, w, h = self.roi_back
  189. return x + np.random.randint(0, w), y + np.random.randint(0, h)
  190. def front_center(self) -> tuple:
  191. """
  192. 获取roi_front的中心坐标
  193. :return:
  194. """
  195. x, y, w, h = self.roi_front
  196. return int(x + w//2), int(y + h//2)
  197. def test_match(self, image: np.array):
  198. if self.is_template_match:
  199. return self.match(image)
  200. if self.is_sift_flann:
  201. return self.sift_match(image, show=True)
  202. def sift_match(self, image: np.array, show=False) -> bool:
  203. """
  204. 特征匹配,同样会修改 roi_front
  205. :param image: 是游戏的截图,就是转通道后的截图
  206. :param show: 测试用的
  207. :return:
  208. """
  209. source = self.corp(image)
  210. kp, des = self.sift.detectAndCompute(source, None)
  211. # 参数1:index_params
  212. # 对于SIFT和SURF,可以传入参数index_params=dict(algorithm=FLANN_INDEX_KDTREE, trees=5)。
  213. # 对于ORB,可以传入参数index_params=dict(algorithm=FLANN_INDEX_LSH, table_number=6, key_size=12)。
  214. index_params = dict(algorithm=1, trees=5)
  215. # 参数2:search_params 指定递归遍历的次数,值越高结果越准确,但是消耗的时间也越多。
  216. search_params = dict(checks=50)
  217. # 根据设置的参数创建特征匹配器 指定匹配的算法和kd树的层数,指定返回的个数
  218. flann = cv2.FlannBasedMatcher(index_params, search_params)
  219. # 利用创建好的特征匹配器利用k近邻算法来用模板的特征描述符去匹配图像的特征描述符,k指的是返回前k个最匹配的特征区域
  220. # 返回的是最匹配的两个特征点的信息,返回的类型是一个列表,列表元素的类型是Dmatch数据类型,具体是什么我也不知道
  221. # 第一个参数是小图的des, 第二个参数是大图的des
  222. matches = flann.knnMatch(self.des, des, k=2)
  223. good = []
  224. result = True
  225. for i, (m, n) in enumerate(matches):
  226. # 设定阈值, 距离小于对方的距离的0.7倍我们认为是好的匹配点.
  227. if m.distance < 0.6 * n.distance:
  228. good.append(m)
  229. if len(good) >= 10:
  230. src_pts = float32([self.kp[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
  231. dst_pts = float32([kp[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
  232. # 计算透视变换矩阵m, 要求点的数量>=4
  233. m, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
  234. # 创建一个包含模板图像四个角坐标的数组
  235. w, h = self.roi_front[2], self.roi_front[3]
  236. pts = float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
  237. if m is None:
  238. result = False
  239. else:
  240. dst = int32(cv2.perspectiveTransform(pts, m))
  241. self.roi_front[0] = dst[0, 0, 0] + self.roi_back[0]
  242. self.roi_front[1] = dst[0, 0, 1] + self.roi_back[1]
  243. if show:
  244. cv2.polylines(source, [dst], isClosed=True, color=(0, 0, 255), thickness=2)
  245. if not is_approx_rectangle(np.array([pos[0] for pos in dst])):
  246. result = False
  247. else:
  248. result = False
  249. # https://blog.csdn.net/cungudafa/article/details/105399278
  250. # https://blog.csdn.net/qq_45832961/article/details/122776322
  251. if show:
  252. # 准备一个空的掩膜来绘制好的匹配
  253. mask_matches = [[0, 0] for i in range(len(matches))]
  254. # 向掩膜中添加数据
  255. for i, (m, n) in enumerate(matches):
  256. if m.distance < 0.6 * n.distance: # 理论上0.7最好
  257. mask_matches[i] = [1, 0]
  258. img_matches = cv2.drawMatchesKnn(self.image, self.kp, source, kp, matches, None,
  259. matchColor=(0, 255, 0), singlePointColor=(255, 0, 0),
  260. matchesMask=mask_matches, flags=0)
  261. cv2.imshow(f'Sift Flann: {self.name}', img_matches)
  262. cv2.waitKey(0)
  263. cv2.destroyAllWindows()
  264. return result
  265. def match_mean_color(self, image, color: tuple, bias=10) -> bool:
  266. """
  267. :param image:
  268. :param color: rgb
  269. :param bias:
  270. :return:
  271. """
  272. image = self.corp(image)
  273. average_color = cv2.mean(image)
  274. # logger.info(f'{self.name} average_color: {average_color}')
  275. for i in range(3):
  276. if abs(average_color[i] - color[i]) > bias:
  277. return False
  278. return True
  279. if __name__ == "__main__":
  280. from dev_tools.assets_test import detect_image
  281. IMAGE_FILE = './log/test/QQ截图20240223151924.png'
  282. from tasks.Restart.assets import RestartAssets
  283. jade = RestartAssets.I_HARVEST_JADE
  284. jade.method = 'Sift Flann'
  285. sign = RestartAssets.I_HARVEST_SIGN
  286. sign.method = 'Sift Flann'
  287. print(jade.roi_front)
  288. detect_image(IMAGE_FILE, jade)
  289. detect_image(IMAGE_FILE, sign)
  290. print(jade.roi_front)