nemu_ipc.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. import asyncio
  2. import ctypes
  3. import os
  4. import sys
  5. from functools import partial, wraps
  6. from pathlib import Path
  7. import cv2
  8. import numpy as np
  9. from module.base.decorator import cached_property, del_cached_property, has_cached_property
  10. from module.base.utils import ensure_time
  11. from module.device.method.minitouch import insert_swipe, random_rectangle_point
  12. from module.device.method.utils import RETRY_TRIES, retry_sleep
  13. from module.exception import RequestHumanTakeover
  14. from module.logger import logger
  15. class NemuIpcIncompatible(Exception):
  16. pass
  17. class NemuIpcError(Exception):
  18. pass
  19. class CaptureStd:
  20. """
  21. Capture stdout and stderr from both python and C library
  22. https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769
  23. ```
  24. with CaptureStd() as capture:
  25. # String wasn't printed
  26. print('whatever')
  27. # But captured in ``capture.stdout``
  28. print(f'Got stdout: "{capture.stdout}"')
  29. print(f'Got stderr: "{capture.stderr}"')
  30. ```
  31. """
  32. def __init__(self):
  33. self.stdout = b''
  34. self.stderr = b''
  35. def _redirect_stdout(self, to):
  36. sys.stdout.close()
  37. os.dup2(to, self.fdout)
  38. sys.stdout = os.fdopen(self.fdout, 'w')
  39. def _redirect_stderr(self, to):
  40. sys.stderr.close()
  41. os.dup2(to, self.fderr)
  42. sys.stderr = os.fdopen(self.fderr, 'w')
  43. def __enter__(self):
  44. self.fdout = sys.stdout.fileno()
  45. self.fderr = sys.stderr.fileno()
  46. self.reader_out, self.writer_out = os.pipe()
  47. self.reader_err, self.writer_err = os.pipe()
  48. self.old_stdout = os.dup(self.fdout)
  49. self.old_stderr = os.dup(self.fderr)
  50. file_out = os.fdopen(self.writer_out, 'w')
  51. file_err = os.fdopen(self.writer_err, 'w')
  52. self._redirect_stdout(to=file_out.fileno())
  53. self._redirect_stderr(to=file_err.fileno())
  54. return self
  55. def __exit__(self, exc_type, exc_val, exc_tb):
  56. self._redirect_stdout(to=self.old_stdout)
  57. self._redirect_stderr(to=self.old_stderr)
  58. os.close(self.old_stdout)
  59. os.close(self.old_stderr)
  60. self.stdout = self.recvall(self.reader_out)
  61. self.stderr = self.recvall(self.reader_err)
  62. os.close(self.reader_out)
  63. os.close(self.reader_err)
  64. @staticmethod
  65. def recvall(reader, length=1024) -> bytes:
  66. fragments = []
  67. while 1:
  68. chunk = os.read(reader, length)
  69. if chunk:
  70. fragments.append(chunk)
  71. else:
  72. break
  73. output = b''.join(fragments)
  74. return output
  75. class CaptureNemuIpc(CaptureStd):
  76. instance = None
  77. def is_capturing(self):
  78. """
  79. Only capture at the topmost wrapper to avoid nested capturing
  80. If a capture is ongoing, this instance does nothing
  81. """
  82. cls = self.__class__
  83. return isinstance(cls.instance, cls) and cls.instance != self
  84. def __enter__(self):
  85. if self.is_capturing():
  86. return self
  87. super().__enter__()
  88. CaptureNemuIpc.instance = self
  89. return self
  90. def __exit__(self, exc_type, exc_val, exc_tb):
  91. if self.is_capturing():
  92. return
  93. CaptureNemuIpc.instance = None
  94. super().__exit__(exc_type, exc_val, exc_tb)
  95. self.check_stdout()
  96. self.check_stderr()
  97. def check_stdout(self):
  98. if not self.stdout:
  99. return
  100. logger.info(f'NemuIpc stdout: {self.stdout}')
  101. def check_stderr(self):
  102. if not self.stderr:
  103. return
  104. logger.error(f'NemuIpc stderr: {self.stderr}')
  105. # Calling an old MuMu12 player
  106. # Tested on 3.4.0
  107. # b'nemu_capture_display rpc error: 1783\r\n'
  108. # Tested on 3.7.3
  109. # b'nemu_capture_display rpc error: 1745\r\n'
  110. if b'error: 1783' in self.stderr or b'error: 1745' in self.stderr:
  111. raise NemuIpcIncompatible(
  112. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  113. # contact_id incorrect
  114. # b'nemu_capture_display cannot find rpc connection\r\n'
  115. if b'cannot find rpc connection' in self.stderr:
  116. raise NemuIpcError(self.stderr)
  117. # Emulator died
  118. # b'nemu_capture_display rpc error: 1722\r\n'
  119. # MuMuVMMSVC.exe died
  120. # b'nemu_capture_display rpc error: 1726\r\n'
  121. # No idea how to handle yet
  122. if b'error: 1722' in self.stderr or b'error: 1726' in self.stderr:
  123. raise NemuIpcError('Emulator instance is probably dead')
  124. def retry(func):
  125. @wraps(func)
  126. def retry_wrapper(self, *args, **kwargs):
  127. """
  128. Args:
  129. self (NemuIpcImpl):
  130. """
  131. init = None
  132. for _ in range(RETRY_TRIES):
  133. try:
  134. if callable(init):
  135. retry_sleep(_)
  136. init()
  137. return func(self, *args, **kwargs)
  138. # Can't handle
  139. except RequestHumanTakeover:
  140. break
  141. # Can't handle
  142. except NemuIpcIncompatible as e:
  143. logger.error(e)
  144. break
  145. # Function call timeout
  146. except asyncio.TimeoutError:
  147. logger.warning(f'Func {func.__name__}() call timeout, retrying: {_}')
  148. def init():
  149. self.reconnect()
  150. # NemuIpcError
  151. except NemuIpcError as e:
  152. logger.error(e)
  153. def init():
  154. self.reconnect()
  155. # Unknown, probably a trucked image
  156. except Exception as e:
  157. logger.exception(e)
  158. def init():
  159. pass
  160. logger.critical(f'Retry {func.__name__}() failed')
  161. raise RequestHumanTakeover
  162. return retry_wrapper
  163. class NemuIpcImpl:
  164. def __init__(self, nemu_folder: str, instance_id: int, display_id: int = 0):
  165. """
  166. Args:
  167. nemu_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/MuMuPlayer-12.0
  168. instance_id: Emulator instance ID, starting from 0
  169. display_id: Always 0 if keep app alive was disabled
  170. """
  171. self.nemu_folder: str = nemu_folder
  172. self.instance_id: int = instance_id
  173. self.display_id: int = display_id
  174. list_dll = [
  175. # MuMuPlayer12
  176. os.path.abspath(os.path.join(nemu_folder, './shell/sdk/external_renderer_ipc.dll')),
  177. # MuMuPlayer12 5.0
  178. os.path.abspath(os.path.join(nemu_folder, './nx_device/12.0/shell/sdk/external_renderer_ipc.dll')),
  179. # MuMuPlayer15
  180. os.path.abspath(os.path.join(nemu_folder, './nx_device/15.0/shell/sdk/external_renderer_ipc.dll')),
  181. ]
  182. self.lib = None
  183. for ipc_dll in list_dll:
  184. if not os.path.exists(ipc_dll):
  185. continue
  186. try:
  187. self.lib = ctypes.CDLL(ipc_dll)
  188. break
  189. except OSError as e:
  190. logger.error(e)
  191. logger.error(f'ipc_dll={ipc_dll} exists, but cannot be loaded')
  192. continue
  193. if self.lib is None:
  194. # not found
  195. raise NemuIpcIncompatible(
  196. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version. '
  197. f'None of the following path exists: {list_dll}')
  198. # success
  199. logger.info(
  200. f'NemuIpcImpl init, '
  201. f'nemu_folder={nemu_folder}, '
  202. f'ipc_dll={ipc_dll}, '
  203. f'instance_id={instance_id}, '
  204. f'display_id={display_id}'
  205. )
  206. self.connect_id: int = 0
  207. self.width = 0
  208. self.height = 0
  209. def connect(self):
  210. if self.connect_id > 0:
  211. return
  212. connect_id = self.ev_run_sync(
  213. self.lib.nemu_connect,
  214. self.nemu_folder, self.instance_id
  215. )
  216. if connect_id == 0:
  217. raise NemuIpcError(
  218. 'Connection failed, please check if nemu_folder is correct and emulator is running'
  219. )
  220. self.connect_id = connect_id
  221. # logger.info(f'NemuIpc connected: {self.connect_id}')
  222. def disconnect(self):
  223. if self.connect_id == 0:
  224. return
  225. self.ev_run_sync(
  226. self.lib.nemu_disconnect,
  227. self.connect_id
  228. )
  229. # logger.info(f'NemuIpc disconnected: {self.connect_id}')
  230. self.connect_id = 0
  231. def reconnect(self):
  232. self.disconnect()
  233. self.connect()
  234. def __enter__(self):
  235. self.connect()
  236. return self
  237. def __exit__(self, exc_type, exc_val, exc_tb):
  238. self.disconnect()
  239. @cached_property
  240. def _ev(self):
  241. return asyncio.new_event_loop()
  242. async def ev_run_async(self, func, *args, **kwargs):
  243. """
  244. Args:
  245. func: Sync function to call
  246. *args:
  247. **kwargs:
  248. Raises:
  249. asyncio.TimeoutError: If function call timeout
  250. """
  251. func_wrapped = partial(func, *args, **kwargs)
  252. # Increased timeout for slow PCs
  253. # Default screenshot interval is 0.2s, so a 0.15s timeout would have a fast retry without extra time costs
  254. result = await asyncio.wait_for(self._ev.run_in_executor(None, func_wrapped), timeout=0.15)
  255. return result
  256. def ev_run_sync(self, func, *args, **kwargs):
  257. """
  258. Args:
  259. func: Sync function to call
  260. *args:
  261. **kwargs:
  262. Raises:
  263. asyncio.TimeoutError: If function call timeout
  264. NemuIpcIncompatible:
  265. NemuIpcError
  266. """
  267. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  268. err = False
  269. if func.__name__ == 'nemu_connect':
  270. if result == 0:
  271. err = True
  272. else:
  273. if result > 0:
  274. err = True
  275. # Get to actual error message printed in std
  276. if err:
  277. logger.warning(f'Failed to call {func.__name__}, result={result}')
  278. with CaptureNemuIpc():
  279. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  280. return result
  281. def get_resolution(self):
  282. """
  283. Get emulator resolution, `self.width` and `self.height` will be set
  284. """
  285. if self.connect_id == 0:
  286. self.connect()
  287. width_ptr = ctypes.pointer(ctypes.c_int(0))
  288. height_ptr = ctypes.pointer(ctypes.c_int(0))
  289. nullptr = ctypes.POINTER(ctypes.c_int)()
  290. ret = self.ev_run_sync(
  291. self.lib.nemu_capture_display,
  292. self.connect_id, self.display_id, 0, width_ptr, height_ptr, nullptr
  293. )
  294. if ret > 0:
  295. raise NemuIpcError('nemu_capture_display failed during get_resolution()')
  296. self.width = width_ptr.contents.value
  297. self.height = height_ptr.contents.value
  298. @retry
  299. def screenshot(self):
  300. """
  301. Returns:
  302. np.ndarray: Image array in RGBA color space
  303. Note that image is upside down
  304. """
  305. if self.connect_id == 0:
  306. self.connect()
  307. self.get_resolution()
  308. width_ptr = ctypes.pointer(ctypes.c_int(self.width))
  309. height_ptr = ctypes.pointer(ctypes.c_int(self.height))
  310. length = self.width * self.height * 4
  311. pixels_pointer = ctypes.pointer((ctypes.c_ubyte * length)())
  312. ret = self.ev_run_sync(
  313. self.lib.nemu_capture_display,
  314. self.connect_id, self.display_id, length, width_ptr, height_ptr, pixels_pointer
  315. )
  316. if ret > 0:
  317. raise NemuIpcError('nemu_capture_display failed during screenshot()')
  318. # image = np.ctypeslib.as_array(pixels_pointer, shape=(self.height, self.width, 4))
  319. image = np.ctypeslib.as_array(pixels_pointer.contents).reshape((self.height, self.width, 4))
  320. return image
  321. def convert_xy(self, x, y):
  322. """
  323. Convert classic ADB coordinates to Nemu's
  324. `self.height` must be updated before calling this method
  325. Returns:
  326. int, int
  327. """
  328. x, y = int(x), int(y)
  329. x, y = self.height - y, x
  330. return x, y
  331. @retry
  332. def down(self, x, y):
  333. """
  334. Contact down, continuous contact down will be considered as swipe
  335. """
  336. if self.connect_id == 0:
  337. self.connect()
  338. if self.height == 0:
  339. self.get_resolution()
  340. x, y = self.convert_xy(x, y)
  341. ret = self.ev_run_sync(
  342. self.lib.nemu_input_event_touch_down,
  343. self.connect_id, self.display_id, x, y
  344. )
  345. if ret > 0:
  346. raise NemuIpcError('nemu_input_event_touch_down failed')
  347. @retry
  348. def up(self):
  349. """
  350. Contact up
  351. """
  352. if self.connect_id == 0:
  353. self.connect()
  354. ret = self.ev_run_sync(
  355. self.lib.nemu_input_event_touch_up,
  356. self.connect_id, self.display_id
  357. )
  358. if ret > 0:
  359. raise NemuIpcError('nemu_input_event_touch_up failed')
  360. def serial_to_id(serial: str):
  361. """
  362. Predict instance ID from serial
  363. E.g.
  364. "127.0.0.1:16384" -> 0
  365. "127.0.0.1:16416" -> 1
  366. Returns:
  367. int: instance_id, or None if failed to predict
  368. """
  369. try:
  370. port = int(serial.split(':')[1])
  371. except (IndexError, ValueError):
  372. return None
  373. index, offset = divmod(port - 16384, 32)
  374. if 0 <= index < 32 and offset in [0, 1, 2]:
  375. return index
  376. else:
  377. return None
  378. class NemuIpc():
  379. @cached_property
  380. def nemu_ipc(self) -> NemuIpcImpl:
  381. """
  382. Initialize a nemu ipc implementation
  383. """
  384. # Try existing settings first
  385. if self.config.script.device.emulatorinfo_path:
  386. folder = str(Path(self.config.script.device.emulatorinfo_path).parent.parent)
  387. index = serial_to_id(self.serial)
  388. if index is not None:
  389. try:
  390. return NemuIpcImpl(
  391. nemu_folder=folder,
  392. instance_id=index,
  393. display_id=0
  394. ).__enter__()
  395. except (NemuIpcIncompatible, NemuIpcError) as e:
  396. logger.error(e)
  397. logger.error('Emulator info incorrect')
  398. # Search emulator instance
  399. # with E:\ProgramFiles\MuMuPlayer-12.0\shell\MuMuPlayer.exe
  400. # installation path is E:\ProgramFiles\MuMuPlayer-12.0
  401. if self.emulator_instance is None:
  402. logger.error('Unable to use NemuIpc because emulator instance not found')
  403. raise RequestHumanTakeover
  404. try:
  405. return NemuIpcImpl(
  406. nemu_folder=self.emulator_instance.emulator.abspath('../'),
  407. instance_id=self.emulator_instance.MuMuPlayer12_id,
  408. display_id=0
  409. ).__enter__()
  410. except (NemuIpcIncompatible, NemuIpcError) as e:
  411. logger.error(e)
  412. logger.error('Unable to initialize NemuIpc')
  413. raise RequestHumanTakeover
  414. def nemu_ipc_available(self) -> bool:
  415. if not self.is_mumu_family:
  416. return False
  417. if self.nemud_app_keep_alive == '':
  418. return False
  419. try:
  420. _ = self.nemu_ipc
  421. except RequestHumanTakeover:
  422. return False
  423. return True
  424. def nemu_ipc_release(self):
  425. if has_cached_property(self, 'nemu_ipc'):
  426. self.nemu_ipc.disconnect()
  427. del_cached_property(self, 'nemu_ipc')
  428. logger.info('nemu_ipc released')
  429. def screenshot_nemu_ipc(self):
  430. image = self.nemu_ipc.screenshot()
  431. image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
  432. cv2.flip(image, 0, dst=image)
  433. return image
  434. def click_nemu_ipc(self, x, y):
  435. down = ensure_time((0.010, 0.020))
  436. self.nemu_ipc.down(x, y)
  437. self.sleep(down)
  438. self.nemu_ipc.up()
  439. self.sleep(0.050 - down)
  440. def long_click_nemu_ipc(self, x, y, duration=1.0):
  441. self.nemu_ipc.down(x, y)
  442. self.sleep(duration)
  443. self.nemu_ipc.up()
  444. self.sleep(0.050)
  445. def swipe_nemu_ipc(self, p1, p2):
  446. points = insert_swipe(p0=p1, p3=p2)
  447. for point in points:
  448. self.nemu_ipc.down(*point)
  449. self.sleep(0.010)
  450. self.nemu_ipc.up()
  451. self.sleep(0.050)
  452. def drag_nemu_ipc(self, p1, p2, point_random=(-10, -10, 10, 10)):
  453. p1 = np.array(p1) - random_rectangle_point(point_random)
  454. p2 = np.array(p2) - random_rectangle_point(point_random)
  455. points = insert_swipe(p0=p1, p3=p2, speed=20)
  456. for point in points:
  457. self.nemu_ipc.down(*point)
  458. self.sleep(0.010)
  459. self.nemu_ipc.down(*p2)
  460. self.sleep(0.140)
  461. self.nemu_ipc.down(*p2)
  462. self.sleep(0.140)
  463. self.nemu_ipc.up()
  464. self.sleep(0.050)