nemu_ipc.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540
  1. import asyncio
  2. import ctypes
  3. import os
  4. import sys
  5. from functools import partial, wraps
  6. import cv2
  7. import numpy as np
  8. from module.base.decorator import cached_property, del_cached_property, has_cached_property
  9. from module.base.utils import ensure_time
  10. from module.device.method.minitouch import insert_swipe, random_rectangle_point
  11. from module.device.method.utils import RETRY_TRIES, retry_sleep
  12. from module.exception import RequestHumanTakeover
  13. from module.logger import logger
  14. class NemuIpcIncompatible(Exception):
  15. pass
  16. class NemuIpcError(Exception):
  17. pass
  18. class CaptureStd:
  19. """
  20. Capture stdout and stderr from both python and C library
  21. https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769
  22. ```
  23. with CaptureStd() as capture:
  24. # String wasn't printed
  25. print('whatever')
  26. # But captured in ``capture.stdout``
  27. print(f'Got stdout: "{capture.stdout}"')
  28. print(f'Got stderr: "{capture.stderr}"')
  29. ```
  30. """
  31. def __init__(self):
  32. self.stdout = b''
  33. self.stderr = b''
  34. def _redirect_stdout(self, to):
  35. sys.stdout.close()
  36. os.dup2(to, self.fdout)
  37. sys.stdout = os.fdopen(self.fdout, 'w')
  38. def _redirect_stderr(self, to):
  39. sys.stderr.close()
  40. os.dup2(to, self.fderr)
  41. sys.stderr = os.fdopen(self.fderr, 'w')
  42. def __enter__(self):
  43. self.fdout = sys.stdout.fileno()
  44. self.fderr = sys.stderr.fileno()
  45. self.reader_out, self.writer_out = os.pipe()
  46. self.reader_err, self.writer_err = os.pipe()
  47. self.old_stdout = os.dup(self.fdout)
  48. self.old_stderr = os.dup(self.fderr)
  49. file_out = os.fdopen(self.writer_out, 'w')
  50. file_err = os.fdopen(self.writer_err, 'w')
  51. self._redirect_stdout(to=file_out.fileno())
  52. self._redirect_stderr(to=file_err.fileno())
  53. return self
  54. def __exit__(self, exc_type, exc_val, exc_tb):
  55. self._redirect_stdout(to=self.old_stdout)
  56. self._redirect_stderr(to=self.old_stderr)
  57. os.close(self.old_stdout)
  58. os.close(self.old_stderr)
  59. self.stdout = self.recvall(self.reader_out)
  60. self.stderr = self.recvall(self.reader_err)
  61. os.close(self.reader_out)
  62. os.close(self.reader_err)
  63. @staticmethod
  64. def recvall(reader, length=1024) -> bytes:
  65. fragments = []
  66. while 1:
  67. chunk = os.read(reader, length)
  68. if chunk:
  69. fragments.append(chunk)
  70. else:
  71. break
  72. output = b''.join(fragments)
  73. return output
  74. class CaptureNemuIpc(CaptureStd):
  75. instance = None
  76. def is_capturing(self):
  77. """
  78. Only capture at the topmost wrapper to avoid nested capturing
  79. If a capture is ongoing, this instance does nothing
  80. """
  81. cls = self.__class__
  82. return isinstance(cls.instance, cls) and cls.instance != self
  83. def __enter__(self):
  84. if self.is_capturing():
  85. return self
  86. super().__enter__()
  87. CaptureNemuIpc.instance = self
  88. return self
  89. def __exit__(self, exc_type, exc_val, exc_tb):
  90. if self.is_capturing():
  91. return
  92. CaptureNemuIpc.instance = None
  93. super().__exit__(exc_type, exc_val, exc_tb)
  94. self.check_stdout()
  95. self.check_stderr()
  96. def check_stdout(self):
  97. if not self.stdout:
  98. return
  99. logger.info(f'NemuIpc stdout: {self.stdout}')
  100. def check_stderr(self):
  101. if not self.stderr:
  102. return
  103. logger.error(f'NemuIpc stderr: {self.stderr}')
  104. # Calling an old MuMu12 player
  105. # Tested on 3.4.0
  106. # b'nemu_capture_display rpc error: 1783\r\n'
  107. # Tested on 3.7.3
  108. # b'nemu_capture_display rpc error: 1745\r\n'
  109. if b'error: 1783' in self.stderr or b'error: 1745' in self.stderr:
  110. raise NemuIpcIncompatible(
  111. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  112. # contact_id incorrect
  113. # b'nemu_capture_display cannot find rpc connection\r\n'
  114. if b'cannot find rpc connection' in self.stderr:
  115. raise NemuIpcError(self.stderr)
  116. # Emulator died
  117. # b'nemu_capture_display rpc error: 1722\r\n'
  118. # MuMuVMMSVC.exe died
  119. # b'nemu_capture_display rpc error: 1726\r\n'
  120. # No idea how to handle yet
  121. if b'error: 1722' in self.stderr or b'error: 1726' in self.stderr:
  122. raise NemuIpcError('Emulator instance is probably dead')
  123. def retry(func):
  124. @wraps(func)
  125. def retry_wrapper(self, *args, **kwargs):
  126. """
  127. Args:
  128. self (NemuIpcImpl):
  129. """
  130. init = None
  131. for _ in range(RETRY_TRIES):
  132. try:
  133. if callable(init):
  134. retry_sleep(_)
  135. init()
  136. return func(self, *args, **kwargs)
  137. # Can't handle
  138. except RequestHumanTakeover:
  139. break
  140. # Can't handle
  141. except NemuIpcIncompatible as e:
  142. logger.error(e)
  143. break
  144. # Function call timeout
  145. except asyncio.TimeoutError:
  146. logger.warning(f'Func {func.__name__}() call timeout, retrying: {_}')
  147. def init():
  148. self.reconnect()
  149. # NemuIpcError
  150. except NemuIpcError as e:
  151. logger.error(e)
  152. def init():
  153. self.reconnect()
  154. # Unknown, probably a trucked image
  155. except Exception as e:
  156. logger.exception(e)
  157. def init():
  158. pass
  159. logger.critical(f'Retry {func.__name__}() failed')
  160. raise RequestHumanTakeover
  161. return retry_wrapper
  162. class NemuIpcImpl:
  163. def __init__(self, nemu_folder: str, instance_id: int, display_id: int = 0):
  164. """
  165. Args:
  166. nemu_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/MuMuPlayer-12.0
  167. instance_id: Emulator instance ID, starting from 0
  168. display_id: Always 0 if keep app alive was disabled
  169. """
  170. self.nemu_folder: str = nemu_folder
  171. self.instance_id: int = instance_id
  172. self.display_id: int = display_id
  173. ipc_dll = os.path.abspath(os.path.join(nemu_folder, './shell/sdk/external_renderer_ipc.dll'))
  174. logger.info(
  175. f'NemuIpcImpl init, '
  176. f'nemu_folder={nemu_folder}, '
  177. f'ipc_dll={ipc_dll}, '
  178. f'instance_id={instance_id}, '
  179. f'display_id={display_id}'
  180. )
  181. try:
  182. self.lib = ctypes.CDLL(ipc_dll)
  183. except OSError as e:
  184. logger.error(e)
  185. # OSError: [WinError 126] 找不到指定的模块。
  186. if not os.path.exists(ipc_dll):
  187. raise NemuIpcIncompatible(
  188. f'ipc_dll={ipc_dll} does not exist, '
  189. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  190. else:
  191. raise NemuIpcIncompatible(
  192. f'ipc_dll={ipc_dll} exists, but cannot be loaded')
  193. self.connect_id: int = 0
  194. self.width = 0
  195. self.height = 0
  196. def connect(self):
  197. if self.connect_id > 0:
  198. return
  199. connect_id = self.ev_run_sync(
  200. self.lib.nemu_connect,
  201. self.nemu_folder, self.instance_id
  202. )
  203. if connect_id == 0:
  204. raise NemuIpcError(
  205. 'Connection failed, please check if nemu_folder is correct and emulator is running'
  206. )
  207. self.connect_id = connect_id
  208. # logger.info(f'NemuIpc connected: {self.connect_id}')
  209. def disconnect(self):
  210. if self.connect_id == 0:
  211. return
  212. self.ev_run_sync(
  213. self.lib.nemu_disconnect,
  214. self.connect_id
  215. )
  216. # logger.info(f'NemuIpc disconnected: {self.connect_id}')
  217. self.connect_id = 0
  218. def reconnect(self):
  219. self.disconnect()
  220. self.connect()
  221. def __enter__(self):
  222. self.connect()
  223. return self
  224. def __exit__(self, exc_type, exc_val, exc_tb):
  225. self.disconnect()
  226. @cached_property
  227. def _ev(self):
  228. return asyncio.new_event_loop()
  229. async def ev_run_async(self, func, *args, **kwargs):
  230. """
  231. Args:
  232. func: Sync function to call
  233. *args:
  234. **kwargs:
  235. Raises:
  236. asyncio.TimeoutError: If function call timeout
  237. """
  238. func_wrapped = partial(func, *args, **kwargs)
  239. # Increased timeout for slow PCs
  240. # Default screenshot interval is 0.2s, so a 0.15s timeout would have a fast retry without extra time costs
  241. result = await asyncio.wait_for(self._ev.run_in_executor(None, func_wrapped), timeout=0.15)
  242. return result
  243. def ev_run_sync(self, func, *args, **kwargs):
  244. """
  245. Args:
  246. func: Sync function to call
  247. *args:
  248. **kwargs:
  249. Raises:
  250. asyncio.TimeoutError: If function call timeout
  251. NemuIpcIncompatible:
  252. NemuIpcError
  253. """
  254. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  255. err = False
  256. if func.__name__ == 'nemu_connect':
  257. if result == 0:
  258. err = True
  259. else:
  260. if result > 0:
  261. err = True
  262. # Get to actual error message printed in std
  263. if err:
  264. logger.warning(f'Failed to call {func.__name__}, result={result}')
  265. with CaptureNemuIpc():
  266. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  267. return result
  268. def get_resolution(self):
  269. """
  270. Get emulator resolution, `self.width` and `self.height` will be set
  271. """
  272. if self.connect_id == 0:
  273. self.connect()
  274. width_ptr = ctypes.pointer(ctypes.c_int(0))
  275. height_ptr = ctypes.pointer(ctypes.c_int(0))
  276. nullptr = ctypes.POINTER(ctypes.c_int)()
  277. ret = self.ev_run_sync(
  278. self.lib.nemu_capture_display,
  279. self.connect_id, self.display_id, 0, width_ptr, height_ptr, nullptr
  280. )
  281. if ret > 0:
  282. raise NemuIpcError('nemu_capture_display failed during get_resolution()')
  283. self.width = width_ptr.contents.value
  284. self.height = height_ptr.contents.value
  285. @retry
  286. def screenshot(self):
  287. """
  288. Returns:
  289. np.ndarray: Image array in RGBA color space
  290. Note that image is upside down
  291. """
  292. if self.connect_id == 0:
  293. self.connect()
  294. self.get_resolution()
  295. width_ptr = ctypes.pointer(ctypes.c_int(self.width))
  296. height_ptr = ctypes.pointer(ctypes.c_int(self.height))
  297. length = self.width * self.height * 4
  298. pixels_pointer = ctypes.pointer((ctypes.c_ubyte * length)())
  299. ret = self.ev_run_sync(
  300. self.lib.nemu_capture_display,
  301. self.connect_id, self.display_id, length, width_ptr, height_ptr, pixels_pointer
  302. )
  303. if ret > 0:
  304. raise NemuIpcError('nemu_capture_display failed during screenshot()')
  305. # image = np.ctypeslib.as_array(pixels_pointer, shape=(self.height, self.width, 4))
  306. image = np.ctypeslib.as_array(pixels_pointer.contents).reshape((self.height, self.width, 4))
  307. return image
  308. def convert_xy(self, x, y):
  309. """
  310. Convert classic ADB coordinates to Nemu's
  311. `self.height` must be updated before calling this method
  312. Returns:
  313. int, int
  314. """
  315. x, y = int(x), int(y)
  316. x, y = self.height - y, x
  317. return x, y
  318. @retry
  319. def down(self, x, y):
  320. """
  321. Contact down, continuous contact down will be considered as swipe
  322. """
  323. if self.connect_id == 0:
  324. self.connect()
  325. if self.height == 0:
  326. self.get_resolution()
  327. x, y = self.convert_xy(x, y)
  328. ret = self.ev_run_sync(
  329. self.lib.nemu_input_event_touch_down,
  330. self.connect_id, self.display_id, x, y
  331. )
  332. if ret > 0:
  333. raise NemuIpcError('nemu_input_event_touch_down failed')
  334. @retry
  335. def up(self):
  336. """
  337. Contact up
  338. """
  339. if self.connect_id == 0:
  340. self.connect()
  341. ret = self.ev_run_sync(
  342. self.lib.nemu_input_event_touch_up,
  343. self.connect_id, self.display_id
  344. )
  345. if ret > 0:
  346. raise NemuIpcError('nemu_input_event_touch_up failed')
  347. def serial_to_id(serial: str):
  348. """
  349. Predict instance ID from serial
  350. E.g.
  351. "127.0.0.1:16384" -> 0
  352. "127.0.0.1:16416" -> 1
  353. Returns:
  354. int: instance_id, or None if failed to predict
  355. """
  356. try:
  357. port = int(serial.split(':')[1])
  358. except (IndexError, ValueError):
  359. return None
  360. index, offset = divmod(port - 16384, 32)
  361. if 0 <= index < 32 and offset in [0, 1, 2]:
  362. return index
  363. else:
  364. return None
  365. class NemuIpc():
  366. @cached_property
  367. def nemu_ipc(self) -> NemuIpcImpl:
  368. """
  369. Initialize a nemu ipc implementation
  370. """
  371. # Try existing settings first
  372. if self.config.script.device.emulatorinfo_path:
  373. folder = os.path.abspath(os.path.join(self.config.script.device.emulatorinfo_path, '../../'))
  374. index = serial_to_id(self.serial)
  375. if index is not None:
  376. try:
  377. return NemuIpcImpl(
  378. nemu_folder=folder,
  379. instance_id=index,
  380. display_id=0
  381. ).__enter__()
  382. except (NemuIpcIncompatible, NemuIpcError) as e:
  383. logger.error(e)
  384. logger.error('Emulator info incorrect')
  385. # Search emulator instance
  386. # with E:\ProgramFiles\MuMuPlayer-12.0\shell\MuMuPlayer.exe
  387. # installation path is E:\ProgramFiles\MuMuPlayer-12.0
  388. if self.emulator_instance is None:
  389. logger.error('Unable to use NemuIpc because emulator instance not found')
  390. raise RequestHumanTakeover
  391. try:
  392. return NemuIpcImpl(
  393. nemu_folder=self.emulator_instance.emulator.abspath('../'),
  394. instance_id=self.emulator_instance.MuMuPlayer12_id,
  395. display_id=0
  396. ).__enter__()
  397. except (NemuIpcIncompatible, NemuIpcError) as e:
  398. logger.error(e)
  399. logger.error('Unable to initialize NemuIpc')
  400. raise RequestHumanTakeover
  401. def nemu_ipc_available(self) -> bool:
  402. if not self.is_mumu_family:
  403. return False
  404. if self.nemud_app_keep_alive == '':
  405. return False
  406. try:
  407. _ = self.nemu_ipc
  408. except RequestHumanTakeover:
  409. return False
  410. return True
  411. def nemu_ipc_release(self):
  412. if has_cached_property(self, 'nemu_ipc'):
  413. self.nemu_ipc.disconnect()
  414. del_cached_property(self, 'nemu_ipc')
  415. logger.info('nemu_ipc released')
  416. def screenshot_nemu_ipc(self):
  417. image = self.nemu_ipc.screenshot()
  418. image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
  419. cv2.flip(image, 0, dst=image)
  420. return image
  421. def click_nemu_ipc(self, x, y):
  422. down = ensure_time((0.010, 0.020))
  423. self.nemu_ipc.down(x, y)
  424. self.sleep(down)
  425. self.nemu_ipc.up()
  426. self.sleep(0.050 - down)
  427. def long_click_nemu_ipc(self, x, y, duration=1.0):
  428. self.nemu_ipc.down(x, y)
  429. self.sleep(duration)
  430. self.nemu_ipc.up()
  431. self.sleep(0.050)
  432. def swipe_nemu_ipc(self, p1, p2):
  433. points = insert_swipe(p0=p1, p3=p2)
  434. for point in points:
  435. self.nemu_ipc.down(*point)
  436. self.sleep(0.010)
  437. self.nemu_ipc.up()
  438. self.sleep(0.050)
  439. def drag_nemu_ipc(self, p1, p2, point_random=(-10, -10, 10, 10)):
  440. p1 = np.array(p1) - random_rectangle_point(point_random)
  441. p2 = np.array(p2) - random_rectangle_point(point_random)
  442. points = insert_swipe(p0=p1, p3=p2, speed=20)
  443. for point in points:
  444. self.nemu_ipc.down(*point)
  445. self.sleep(0.010)
  446. self.nemu_ipc.down(*p2)
  447. self.sleep(0.140)
  448. self.nemu_ipc.down(*p2)
  449. self.sleep(0.140)
  450. self.nemu_ipc.up()
  451. self.sleep(0.050)