windows_impl.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374
  1. # This Python file uses the following encoding: utf-8
  2. # @author runhey
  3. # github https://github.com/runhey
  4. import cv2
  5. from numpy import frombuffer, uint8, array, random
  6. import numpy as np
  7. import time
  8. from math import dist
  9. from cached_property import cached_property
  10. from win32gui import (GetWindowText, EnumWindows, FindWindow, FindWindowEx,
  11. IsWindow, GetWindowRect, GetWindowDC, DeleteObject,
  12. SetForegroundWindow, IsWindowVisible, GetDC, GetParent,
  13. EnumChildWindows, SetForegroundWindow)
  14. from win32con import (SRCCOPY, DESKTOPHORZRES, DESKTOPVERTRES, WM_LBUTTONUP,
  15. WM_LBUTTONDOWN, WM_ACTIVATE, WA_ACTIVE, MK_LBUTTON,
  16. WM_NCHITTEST, WM_SETCURSOR, HTCLIENT, WM_MOUSEMOVE,
  17. WM_PARENTNOTIFY, WM_MOUSEACTIVATE, WM_MOUSEWHEEL,
  18. WM_SETFOCUS)
  19. from win32ui import CreateDCFromHandle, CreateBitmap
  20. from win32api import GetSystemMetrics, SendMessage, MAKELONG, PostMessage
  21. from win32con import SRCCOPY
  22. from module.base.cBezier import BezierTrajectory
  23. from module.exception import RequestHumanTakeover, ScriptError
  24. from module.base.decorator import Config
  25. from module.base.timer import timer
  26. from module.logger import logger
  27. from module.device.handle import Handle, window_scale_rate, EmulatorFamily
  28. class Window(Handle):
  29. def __init__(self, *args, **kwargs):
  30. logger.info("Window init")
  31. super().__init__(*args, **kwargs)
  32. def screenshot_window_background(self):
  33. """
  34. 后台截屏
  35. :return:
  36. """
  37. widthScreen, heightScreen = self.screenshot_size
  38. # 返回句柄窗口的设备环境,覆盖整个窗口,包括非客户区,标题栏,菜单,边框
  39. hwndDc = GetWindowDC(self.screenshot_handle_num)
  40. # 创建设备描述表
  41. mfcDc = CreateDCFromHandle(hwndDc)
  42. # 创建内存设备描述表
  43. saveDc = mfcDc.CreateCompatibleDC()
  44. # 创建位图对象准备保存图片
  45. saveBitMap = CreateBitmap()
  46. # 为bitmap开辟存储空间
  47. saveBitMap.CreateCompatibleBitmap(mfcDc, widthScreen, heightScreen)
  48. # 将截图保存到saveBitMap中
  49. saveDc.SelectObject(saveBitMap)
  50. # 保存bitmap到内存设备描述表
  51. saveDc.BitBlt((0, 0), (widthScreen, heightScreen), mfcDc, (0, 0), SRCCOPY)
  52. # 保存图像
  53. signedIntsArray = saveBitMap.GetBitmapBits(True)
  54. imgSrceen = frombuffer(signedIntsArray, dtype='uint8')
  55. imgSrceen.shape = (heightScreen, widthScreen, 4)
  56. # 这点很重要 在alas中图片以np.ndarray(RGB)的顺序存储。但是opencv是以BGR
  57. imgSrceen = cv2.cvtColor(imgSrceen, cv2.COLOR_BGR2RGB)
  58. # imgSrceen = cv2.resize(imgSrceen, (win_size[0], win_size[1]))
  59. # 很奇怪的
  60. # # 测试显示截图图片
  61. # cv2.namedWindow('imgSrceen') # 命名窗口
  62. # cv2.imshow("imgSrceen", imgSrceen) # 显示
  63. # cv2.waitKey(0)
  64. # cv2.destroyAllWindows()
  65. # 内存释放
  66. DeleteObject(saveBitMap.GetHandle())
  67. saveDc.DeleteDC()
  68. mfcDc.DeleteDC()
  69. return imgSrceen
  70. @cached_property
  71. def control_handle_list(self) -> list:
  72. """
  73. 不同的模拟器需要的子句柄不同
  74. mumu模拟器的控制需要 根窗口和第一个子窗口
  75. 雷电模拟器只需要TheRender窗口
  76. 夜神模拟器
  77. :return:
  78. """
  79. result = []
  80. if self.emulator_family == EmulatorFamily.FAMILY_MUMU:
  81. result.append(self.root_node.num)
  82. result.append(self.root_node.children[0].num)
  83. return result
  84. elif self.emulator_family == EmulatorFamily.FAMILY_NOX:
  85. result.append(self.root_node.num)
  86. try:
  87. result.append(self.root_node.children[1].num)
  88. result.append(self.root_node.children[1].children[1].num)
  89. result.append(self.root_node.children[1].children[1].children[0].num)
  90. except:
  91. result.append(self.root_node.children[2].num)
  92. result.append(self.root_node.children[2].children[1].num)
  93. result.append(self.root_node.children[2].children[1].children[0].num)
  94. return result
  95. elif self.emulator_family == EmulatorFamily.FAMILY_LD:
  96. result.append(self.root_node.children[0].num)
  97. return result
  98. elif self.emulator_family == EmulatorFamily.FAMILY_MEMU:
  99. pass
  100. elif self.emulator_family == EmulatorFamily.FAMILY_BLUESTACKS:
  101. pass
  102. @cached_property
  103. def mumu_head_height(self):
  104. """
  105. 不同mumu模拟器的头部高度不同
  106. :return:
  107. """
  108. father_win_Rect = GetWindowRect(self.control_handle_list[0])
  109. father_height: int = father_win_Rect[3] - father_win_Rect[1] # 下y - 上y 计算高度
  110. children_win_Rect = GetWindowRect(self.control_handle_list[1])
  111. children_height: int = children_win_Rect[3] - children_win_Rect[1] # 下y - 上y 计算高度
  112. height = father_height - children_height
  113. if int(height * self.window_scale_rate) == 37:
  114. # 说明是mumu模拟器 不做处理
  115. pass
  116. if int(height * self.window_scale_rate) == 45:
  117. # 说明是mumu12模拟器 不做处理
  118. pass
  119. logger.info(f"Mumu emulator head height: {height}")
  120. return height
  121. def click_window_message(self, x: int, y: int, fast: bool = False):
  122. """
  123. :param x:
  124. :param y:
  125. :param fast:
  126. :return:
  127. """
  128. # 我不知道为什么的使用的pywin32==306的版本会导致获取的图片的是(1024, 576)
  129. # 所有我在点击的时候会除以这个缩放比例
  130. # 但是后面发现又不是影响的很奇怪
  131. x = int(x / self.window_scale_rate)
  132. y = int(y / self.window_scale_rate)
  133. if fast:
  134. press_time: float = (random.randint(10, 40)) / 1000.0
  135. else:
  136. press_time: float = (random.randint(100, 200)) / 1000.0
  137. emulator_type = len(self.control_handle_list)
  138. if emulator_type == 2: # mumu模拟器
  139. SendMessage(self.control_handle_list[0], WM_ACTIVATE, WA_ACTIVE, 0) # 激活窗口
  140. # SendMessage(self.control_handle_list[1], WM_ACTIVATE, WA_ACTIVE, 0) # 激活窗口
  141. # SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y+self.mumu_head_height)) # 模拟鼠标按下 先是父窗口 上面的框高度是57
  142. # mumu12模拟器 V3.5.16 之后后可以用下面的方式
  143. SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  144. time.sleep(press_time)
  145. SendMessage(self.control_handle_list[1], WM_LBUTTONUP, 0, MAKELONG(x, y)) # 模拟鼠标弹起 后是子窗口
  146. elif emulator_type > 2: # 夜神模拟器
  147. SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y)) # 模拟鼠标按下 先是父窗口 上面的框高度是57
  148. SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  149. SendMessage(self.control_handle_list[2], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  150. SendMessage(self.control_handle_list[3], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  151. time.sleep(press_time)
  152. SendMessage(self.control_handle_list[3], WM_LBUTTONUP, 0, MAKELONG(x, y)) # 模拟鼠标弹起 后是子窗口
  153. elif emulator_type == 1: # 雷电模拟器
  154. clickPos = MAKELONG(x, y)
  155. SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, clickPos)
  156. time.sleep(press_time)
  157. SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos)
  158. def long_click_window_message(self, x: int, y: int, duration: float):
  159. """
  160. :param x:
  161. :param y:
  162. :param duration: 持续时间 单位秒
  163. :return:
  164. """
  165. # 我不知道为什么的使用的pywin32==306的版本会导致获取的图片的是(1024, 576)
  166. # 所有我在点击的时候会除以这个缩放比例
  167. x = int(x / self.window_scale_rate)
  168. y = int(y / self.window_scale_rate)
  169. emulator_type = len(self.control_handle_list)
  170. if self.emulator_family == EmulatorFamily.FAMILY_MUMU: # mumu模拟器
  171. SendMessage(self.control_handle_list[1], WM_ACTIVATE, WA_ACTIVE, 0) # 激活窗口
  172. # SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y+self.mumu_head_height)) # 模拟鼠标按下 先是父窗口 上面的框高度是57
  173. SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  174. time.sleep(duration) # 长按时间1000ms-1500ms
  175. SendMessage(self.control_handle_list[1], WM_LBUTTONUP, 0, MAKELONG(x, y)) # 模拟鼠标弹起 后是子窗口
  176. elif emulator_type > 2: # 夜神模拟器
  177. SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y)) # 模拟鼠标按下 先是父窗口 上面的框高度是57
  178. SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  179. SendMessage(self.control_handle_list[2], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  180. SendMessage(self.control_handle_list[3], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
  181. time.sleep(duration) # 长按时间1000ms-1500ms
  182. SendMessage(self.control_handle_list[3], WM_LBUTTONUP, 0, MAKELONG(x, y)) # 模拟鼠标弹起 后是子窗口
  183. elif emulator_type == 1: # 雷电模拟器
  184. clickPos = MAKELONG(x, y)
  185. SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, clickPos) # 模拟鼠标按下
  186. time.sleep(duration) # 长按时间1000ms-1500ms
  187. SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos) # 模拟鼠标弹起
  188. # @timer
  189. def swipe_window_message(self, startPos: list, endPos: list) -> None:
  190. """
  191. 后台滑动
  192. :param startPos:
  193. :param endPos:
  194. :return:
  195. """
  196. # 生成的坐标点列表
  197. interval: int = 10 # 每次移动的间隔时间
  198. numberList: int = int(dist(startPos, endPos) / (1 * interval)) # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
  199. le = random.randint(2, 4) #
  200. deviation = random.randint(20, 40) # 幅度
  201. _type: int = 3
  202. obbsType = random.random() # 0.8的概率是先快中间慢后面快, 0.1概率是先快后慢, 0.1概率先慢后快
  203. if 0 < obbsType <= 0.8:
  204. _type = 3
  205. elif obbsType < 0.9:
  206. _type = 2
  207. else:
  208. _type = 1
  209. trace: list = BezierTrajectory.trackArray(start=startPos, end=endPos, numberList=numberList, le=le,
  210. deviation=deviation, bias=0.5, type=_type, cbb=0, yhh=20)
  211. # 使用生成的点列表进行拖拽
  212. handleNum = None
  213. if self.emulator_family == EmulatorFamily.FAMILY_MUMU: # mumu模拟器
  214. handleNum = self.control_handle_list[1]
  215. elif self.emulator_family == EmulatorFamily.FAMILY_NOX: # 夜神模拟器
  216. handleNum = self.control_handle_list[3]
  217. elif self.emulator_family == EmulatorFamily.FAMILY_LD: # 雷电模拟器
  218. handleNum = self.control_handle_list[0]
  219. # 激活窗口
  220. # SendMessage(self.control_handle_list[0], WM_PARENTNOTIFY, WM_LBUTTONDOWN, tmpPos)
  221. # SendMessage(self.control_handle_list[0], WM_MOUSEACTIVATE, WM_LBUTTONDOWN, tmpPos)
  222. # PostMessage(handleNum, WM_ACTIVATE, WA_ACTIVE, 0)
  223. # 先移动到第一个点
  224. tmpPos = MAKELONG(trace[0][0], trace[0][1])
  225. SendMessage(handleNum, WM_NCHITTEST, 0, tmpPos)
  226. SendMessage(handleNum, WM_SETCURSOR, handleNum, MAKELONG(HTCLIENT, WM_LBUTTONDOWN))
  227. PostMessage(handleNum, WM_LBUTTONDOWN, 0, tmpPos)
  228. # 一点一点移动鼠标
  229. manual_control: int = 3 # 手动控制最后几个点的数量
  230. total_len: int = len(trace)
  231. for index, pos in enumerate(trace):
  232. lparam = MAKELONG(pos[0], pos[1])
  233. PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, lparam)
  234. if manual_control >= total_len - index:
  235. time.sleep(0.08)
  236. else:
  237. time.sleep((interval + random.randint(-2, 2)) / 1000.0)
  238. # 最后释放鼠标
  239. time.sleep(0.05)
  240. end_lparam = MAKELONG(trace[-1][0], trace[-1][1])
  241. PostMessage(handleNum, WM_LBUTTONUP, 0, end_lparam)
  242. def swipe_vector_window_message2(self, startPos: list, endPos: list) -> None:
  243. """
  244. 后台滑动, 直线滑动
  245. :param startPos:
  246. :param endPos:
  247. :return:
  248. """
  249. # 生成的坐标点列表
  250. interval: int = 8 # 每次移动的间隔时间
  251. numberList: int = int(dist(startPos, endPos) / (1.5 * interval)) # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
  252. def generate_points(start_pos: list, end_pos: list, number: int) -> list:
  253. # 确定两点之间的步长
  254. step_x = (end_pos[0] - start_pos[0]) / (number + 1)
  255. step_y = (end_pos[1] - start_pos[1]) / (number + 1)
  256. # 生成中间点坐标列表
  257. points = []
  258. for i in range(number):
  259. x = start_pos[0] + step_x * (i + 1)
  260. y = start_pos[1] + step_y * (i + 1)
  261. points.append((x, y))
  262. # 添加起始点和最终点到列表中
  263. points.insert(0, tuple(start_pos))
  264. points.append(tuple(end_pos))
  265. return points
  266. trace: list = generate_points(start_pos=startPos, end_pos=endPos, number=numberList)
  267. # 使用生成的点列表进行拖拽
  268. emulator_type = len(self.control_handle_list)
  269. if emulator_type == 1: # 雷电模拟器
  270. handleNum = self.control_handle_list[0]
  271. elif emulator_type == 2: # mumu模拟器
  272. handleNum = self.control_handle_list[1]
  273. elif emulator_type > 2: # 夜神模拟器
  274. handleNum = self.control_handle_list[3]
  275. PostMessage(handleNum, WM_ACTIVATE, WA_ACTIVE, 0) # 激活窗口
  276. # 先移动到第一个点
  277. tmpPos = MAKELONG(trace[0][0], trace[0][1])
  278. SendMessage(handleNum, WM_NCHITTEST, 0, tmpPos)
  279. SendMessage(handleNum, WM_SETCURSOR, handleNum, MAKELONG(HTCLIENT, WM_LBUTTONDOWN))
  280. PostMessage(handleNum, WM_LBUTTONDOWN, MK_LBUTTON, tmpPos)
  281. # 一点一点移动鼠标
  282. for pos in trace:
  283. tmpPos = MAKELONG(pos[0], pos[1])
  284. PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, tmpPos)
  285. time.sleep((interval + random.randint(-2, 2)) / 1000.0)
  286. # 最后释放鼠标
  287. tmpPos = MAKELONG(trace[-1][0], trace[-1][1])
  288. PostMessage(handleNum, WM_LBUTTONUP, 0, tmpPos)
  289. def scroll_window_message(self, x: int, y: int, delta: int=-120) -> None:
  290. """
  291. 弃置
  292. https://github.com/runhey/OnmyojiAutoScript/issues/43
  293. :param x:
  294. :param y:
  295. :param delta:
  296. :return:
  297. """
  298. wparam = MAKELONG(0, delta)
  299. lparam = MAKELONG(x, y)
  300. handle_num = None
  301. if self.emulator_family == EmulatorFamily.FAMILY_MUMU: # mumu模拟器
  302. handle_num = self.control_handle_list[1]
  303. elif self.emulator_family == EmulatorFamily.FAMILY_NOX: # 夜神模拟器
  304. handle_num = self.control_handle_list[3]
  305. elif self.emulator_family == EmulatorFamily.FAMILY_LD: # 雷电模拟器
  306. handle_num = self.control_handle_list[0]
  307. SetForegroundWindow(handle_num)
  308. time.sleep(2)
  309. # SendMessage(handle_num, WM_SETFOCUS, 0, 0)
  310. # SendMessage(handle_num, WM_ACTIVATE, WA_ACTIVE, 0)
  311. # SendMessage(handle_num, WM_MOUSEACTIVATE, WM_LBUTTONDOWN, lparam)
  312. SendMessage(handle_num, WM_NCHITTEST, 0, lparam)
  313. SendMessage(handle_num, WM_SETCURSOR, handle_num, lparam)
  314. PostMessage(handle_num, WM_MOUSEMOVE, 0, lparam)
  315. self.click_window_message(x, y)
  316. for i in range(5):
  317. PostMessage(handle_num, WM_SETCURSOR, handle_num, lparam)
  318. PostMessage(handle_num, WM_MOUSEMOVE, 0, lparam)
  319. PostMessage(handle_num, WM_MOUSEWHEEL, wparam, lparam)
  320. time.sleep(0.5)
  321. # 抄网上的
  322. # SendMessage(handle_num, WM_NCHITTEST, 0, lparam)
  323. if __name__ == "__main__":
  324. w = Window(config='oas1')
  325. # img = w.screenshot_window_background()
  326. # handle = 459852
  327. # wparam = MAKELONG(0, -120)
  328. # lparam = MAKELONG(300, 300)
  329. # PostMessage(handle, WM_MOUSEWHEEL, wparam, lparam)
  330. w.long_click_window_message(142, 310, 1.5)