nemu_ipc.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551
  1. import asyncio
  2. import ctypes
  3. import os
  4. import sys
  5. from functools import partial, wraps
  6. from pathlib import Path
  7. import cv2
  8. import numpy as np
  9. from module.base.decorator import cached_property, del_cached_property, has_cached_property
  10. from module.base.utils import ensure_time
  11. from module.device.method.minitouch import insert_swipe, random_rectangle_point
  12. from module.device.method.utils import RETRY_TRIES, retry_sleep
  13. from module.exception import RequestHumanTakeover
  14. from module.logger import logger
  15. class NemuIpcIncompatible(Exception):
  16. pass
  17. class NemuIpcError(Exception):
  18. pass
  19. class CaptureStd:
  20. """
  21. Capture stdout and stderr from both python and C library
  22. https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769
  23. ```
  24. with CaptureStd() as capture:
  25. # String wasn't printed
  26. print('whatever')
  27. # But captured in ``capture.stdout``
  28. print(f'Got stdout: "{capture.stdout}"')
  29. print(f'Got stderr: "{capture.stderr}"')
  30. ```
  31. """
  32. def __init__(self):
  33. self.stdout = b''
  34. self.stderr = b''
  35. def _redirect_stdout(self, to):
  36. sys.stdout.close()
  37. os.dup2(to, self.fdout)
  38. sys.stdout = os.fdopen(self.fdout, 'w')
  39. def _redirect_stderr(self, to):
  40. sys.stderr.close()
  41. os.dup2(to, self.fderr)
  42. sys.stderr = os.fdopen(self.fderr, 'w')
  43. def __enter__(self):
  44. self.fdout = sys.stdout.fileno()
  45. self.fderr = sys.stderr.fileno()
  46. self.reader_out, self.writer_out = os.pipe()
  47. self.reader_err, self.writer_err = os.pipe()
  48. self.old_stdout = os.dup(self.fdout)
  49. self.old_stderr = os.dup(self.fderr)
  50. file_out = os.fdopen(self.writer_out, 'w')
  51. file_err = os.fdopen(self.writer_err, 'w')
  52. self._redirect_stdout(to=file_out.fileno())
  53. self._redirect_stderr(to=file_err.fileno())
  54. return self
  55. def __exit__(self, exc_type, exc_val, exc_tb):
  56. self._redirect_stdout(to=self.old_stdout)
  57. self._redirect_stderr(to=self.old_stderr)
  58. os.close(self.old_stdout)
  59. os.close(self.old_stderr)
  60. self.stdout = self.recvall(self.reader_out)
  61. self.stderr = self.recvall(self.reader_err)
  62. os.close(self.reader_out)
  63. os.close(self.reader_err)
  64. @staticmethod
  65. def recvall(reader, length=1024) -> bytes:
  66. fragments = []
  67. while 1:
  68. chunk = os.read(reader, length)
  69. if chunk:
  70. fragments.append(chunk)
  71. else:
  72. break
  73. output = b''.join(fragments)
  74. return output
  75. class CaptureNemuIpc(CaptureStd):
  76. instance = None
  77. def is_capturing(self):
  78. """
  79. Only capture at the topmost wrapper to avoid nested capturing
  80. If a capture is ongoing, this instance does nothing
  81. """
  82. cls = self.__class__
  83. return isinstance(cls.instance, cls) and cls.instance != self
  84. def __enter__(self):
  85. if self.is_capturing():
  86. return self
  87. super().__enter__()
  88. CaptureNemuIpc.instance = self
  89. return self
  90. def __exit__(self, exc_type, exc_val, exc_tb):
  91. if self.is_capturing():
  92. return
  93. CaptureNemuIpc.instance = None
  94. super().__exit__(exc_type, exc_val, exc_tb)
  95. self.check_stdout()
  96. self.check_stderr()
  97. def check_stdout(self):
  98. if not self.stdout:
  99. return
  100. logger.info(f'NemuIpc stdout: {self.stdout}')
  101. def check_stderr(self):
  102. if not self.stderr:
  103. return
  104. logger.error(f'NemuIpc stderr: {self.stderr}')
  105. # Calling an old MuMu12 player
  106. # Tested on 3.4.0
  107. # b'nemu_capture_display rpc error: 1783\r\n'
  108. # Tested on 3.7.3
  109. # b'nemu_capture_display rpc error: 1745\r\n'
  110. if b'error: 1783' in self.stderr or b'error: 1745' in self.stderr:
  111. raise NemuIpcIncompatible(
  112. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  113. # contact_id incorrect
  114. # b'nemu_capture_display cannot find rpc connection\r\n'
  115. if b'cannot find rpc connection' in self.stderr:
  116. raise NemuIpcError(self.stderr)
  117. # Emulator died
  118. # b'nemu_capture_display rpc error: 1722\r\n'
  119. # MuMuVMMSVC.exe died
  120. # b'nemu_capture_display rpc error: 1726\r\n'
  121. # No idea how to handle yet
  122. if b'error: 1722' in self.stderr or b'error: 1726' in self.stderr:
  123. raise NemuIpcError('Emulator instance is probably dead')
  124. def retry(func):
  125. @wraps(func)
  126. def retry_wrapper(self, *args, **kwargs):
  127. """
  128. Args:
  129. self (NemuIpcImpl):
  130. """
  131. init = None
  132. for _ in range(RETRY_TRIES):
  133. try:
  134. if callable(init):
  135. retry_sleep(_)
  136. init()
  137. return func(self, *args, **kwargs)
  138. # Can't handle
  139. except RequestHumanTakeover:
  140. break
  141. # Can't handle
  142. except NemuIpcIncompatible as e:
  143. logger.error(e)
  144. break
  145. # Function call timeout
  146. except asyncio.TimeoutError:
  147. logger.warning(f'Func {func.__name__}() call timeout, retrying: {_}')
  148. def init():
  149. self.reconnect()
  150. # NemuIpcError
  151. except NemuIpcError as e:
  152. logger.error(e)
  153. def init():
  154. self.reconnect()
  155. # Unknown, probably a trucked image
  156. except Exception as e:
  157. logger.exception(e)
  158. def init():
  159. pass
  160. logger.critical(f'Retry {func.__name__}() failed')
  161. raise RequestHumanTakeover
  162. return retry_wrapper
  163. class NemuIpcImpl:
  164. def __init__(self, nemu_folder: str, instance_id: int, display_id: int = 0):
  165. """
  166. Args:
  167. nemu_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/MuMuPlayer-12.0
  168. instance_id: Emulator instance ID, starting from 0
  169. display_id: Always 0 if keep app alive was disabled
  170. """
  171. self.nemu_folder: str = nemu_folder
  172. self.instance_id: int = instance_id
  173. self.display_id: int = display_id
  174. list_dll = [
  175. # MuMuPlayer12
  176. os.path.abspath(os.path.join(nemu_folder, './shell/sdk/external_renderer_ipc.dll')),
  177. # MuMuPlayer12 5.0
  178. os.path.abspath(os.path.join(nemu_folder, './nx_device/12.0/shell/sdk/external_renderer_ipc.dll')),
  179. ]
  180. self.lib = None
  181. for ipc_dll in list_dll:
  182. if not os.path.exists(ipc_dll):
  183. continue
  184. try:
  185. self.lib = ctypes.CDLL(ipc_dll)
  186. break
  187. except OSError as e:
  188. logger.error(e)
  189. logger.error(f'ipc_dll={ipc_dll} exists, but cannot be loaded')
  190. continue
  191. if self.lib is None:
  192. # not found
  193. raise NemuIpcIncompatible(
  194. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version. '
  195. f'None of the following path exists: {list_dll}')
  196. # success
  197. logger.info(
  198. f'NemuIpcImpl init, '
  199. f'nemu_folder={nemu_folder}, '
  200. f'ipc_dll={ipc_dll}, '
  201. f'instance_id={instance_id}, '
  202. f'display_id={display_id}'
  203. )
  204. self.connect_id: int = 0
  205. self.width = 0
  206. self.height = 0
  207. def connect(self):
  208. if self.connect_id > 0:
  209. return
  210. connect_id = self.ev_run_sync(
  211. self.lib.nemu_connect,
  212. self.nemu_folder, self.instance_id
  213. )
  214. if connect_id == 0:
  215. raise NemuIpcError(
  216. 'Connection failed, please check if nemu_folder is correct and emulator is running'
  217. )
  218. self.connect_id = connect_id
  219. # logger.info(f'NemuIpc connected: {self.connect_id}')
  220. def disconnect(self):
  221. if self.connect_id == 0:
  222. return
  223. self.ev_run_sync(
  224. self.lib.nemu_disconnect,
  225. self.connect_id
  226. )
  227. # logger.info(f'NemuIpc disconnected: {self.connect_id}')
  228. self.connect_id = 0
  229. def reconnect(self):
  230. self.disconnect()
  231. self.connect()
  232. def __enter__(self):
  233. self.connect()
  234. return self
  235. def __exit__(self, exc_type, exc_val, exc_tb):
  236. self.disconnect()
  237. @cached_property
  238. def _ev(self):
  239. return asyncio.new_event_loop()
  240. async def ev_run_async(self, func, *args, **kwargs):
  241. """
  242. Args:
  243. func: Sync function to call
  244. *args:
  245. **kwargs:
  246. Raises:
  247. asyncio.TimeoutError: If function call timeout
  248. """
  249. func_wrapped = partial(func, *args, **kwargs)
  250. # Increased timeout for slow PCs
  251. # Default screenshot interval is 0.2s, so a 0.15s timeout would have a fast retry without extra time costs
  252. result = await asyncio.wait_for(self._ev.run_in_executor(None, func_wrapped), timeout=0.15)
  253. return result
  254. def ev_run_sync(self, func, *args, **kwargs):
  255. """
  256. Args:
  257. func: Sync function to call
  258. *args:
  259. **kwargs:
  260. Raises:
  261. asyncio.TimeoutError: If function call timeout
  262. NemuIpcIncompatible:
  263. NemuIpcError
  264. """
  265. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  266. err = False
  267. if func.__name__ == 'nemu_connect':
  268. if result == 0:
  269. err = True
  270. else:
  271. if result > 0:
  272. err = True
  273. # Get to actual error message printed in std
  274. if err:
  275. logger.warning(f'Failed to call {func.__name__}, result={result}')
  276. with CaptureNemuIpc():
  277. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  278. return result
  279. def get_resolution(self):
  280. """
  281. Get emulator resolution, `self.width` and `self.height` will be set
  282. """
  283. if self.connect_id == 0:
  284. self.connect()
  285. width_ptr = ctypes.pointer(ctypes.c_int(0))
  286. height_ptr = ctypes.pointer(ctypes.c_int(0))
  287. nullptr = ctypes.POINTER(ctypes.c_int)()
  288. ret = self.ev_run_sync(
  289. self.lib.nemu_capture_display,
  290. self.connect_id, self.display_id, 0, width_ptr, height_ptr, nullptr
  291. )
  292. if ret > 0:
  293. raise NemuIpcError('nemu_capture_display failed during get_resolution()')
  294. self.width = width_ptr.contents.value
  295. self.height = height_ptr.contents.value
  296. @retry
  297. def screenshot(self):
  298. """
  299. Returns:
  300. np.ndarray: Image array in RGBA color space
  301. Note that image is upside down
  302. """
  303. if self.connect_id == 0:
  304. self.connect()
  305. self.get_resolution()
  306. width_ptr = ctypes.pointer(ctypes.c_int(self.width))
  307. height_ptr = ctypes.pointer(ctypes.c_int(self.height))
  308. length = self.width * self.height * 4
  309. pixels_pointer = ctypes.pointer((ctypes.c_ubyte * length)())
  310. ret = self.ev_run_sync(
  311. self.lib.nemu_capture_display,
  312. self.connect_id, self.display_id, length, width_ptr, height_ptr, pixels_pointer
  313. )
  314. if ret > 0:
  315. raise NemuIpcError('nemu_capture_display failed during screenshot()')
  316. # image = np.ctypeslib.as_array(pixels_pointer, shape=(self.height, self.width, 4))
  317. image = np.ctypeslib.as_array(pixels_pointer.contents).reshape((self.height, self.width, 4))
  318. return image
  319. def convert_xy(self, x, y):
  320. """
  321. Convert classic ADB coordinates to Nemu's
  322. `self.height` must be updated before calling this method
  323. Returns:
  324. int, int
  325. """
  326. x, y = int(x), int(y)
  327. x, y = self.height - y, x
  328. return x, y
  329. @retry
  330. def down(self, x, y):
  331. """
  332. Contact down, continuous contact down will be considered as swipe
  333. """
  334. if self.connect_id == 0:
  335. self.connect()
  336. if self.height == 0:
  337. self.get_resolution()
  338. x, y = self.convert_xy(x, y)
  339. ret = self.ev_run_sync(
  340. self.lib.nemu_input_event_touch_down,
  341. self.connect_id, self.display_id, x, y
  342. )
  343. if ret > 0:
  344. raise NemuIpcError('nemu_input_event_touch_down failed')
  345. @retry
  346. def up(self):
  347. """
  348. Contact up
  349. """
  350. if self.connect_id == 0:
  351. self.connect()
  352. ret = self.ev_run_sync(
  353. self.lib.nemu_input_event_touch_up,
  354. self.connect_id, self.display_id
  355. )
  356. if ret > 0:
  357. raise NemuIpcError('nemu_input_event_touch_up failed')
  358. def serial_to_id(serial: str):
  359. """
  360. Predict instance ID from serial
  361. E.g.
  362. "127.0.0.1:16384" -> 0
  363. "127.0.0.1:16416" -> 1
  364. Returns:
  365. int: instance_id, or None if failed to predict
  366. """
  367. try:
  368. port = int(serial.split(':')[1])
  369. except (IndexError, ValueError):
  370. return None
  371. index, offset = divmod(port - 16384, 32)
  372. if 0 <= index < 32 and offset in [0, 1, 2]:
  373. return index
  374. else:
  375. return None
  376. class NemuIpc():
  377. @cached_property
  378. def nemu_ipc(self) -> NemuIpcImpl:
  379. """
  380. Initialize a nemu ipc implementation
  381. """
  382. # Try existing settings first
  383. if self.config.script.device.emulatorinfo_path:
  384. folder = str(Path(self.config.script.device.emulatorinfo_path).parent.parent)
  385. index = serial_to_id(self.serial)
  386. if index is not None:
  387. try:
  388. return NemuIpcImpl(
  389. nemu_folder=folder,
  390. instance_id=index,
  391. display_id=0
  392. ).__enter__()
  393. except (NemuIpcIncompatible, NemuIpcError) as e:
  394. logger.error(e)
  395. logger.error('Emulator info incorrect')
  396. # Search emulator instance
  397. # with E:\ProgramFiles\MuMuPlayer-12.0\shell\MuMuPlayer.exe
  398. # installation path is E:\ProgramFiles\MuMuPlayer-12.0
  399. if self.emulator_instance is None:
  400. logger.error('Unable to use NemuIpc because emulator instance not found')
  401. raise RequestHumanTakeover
  402. try:
  403. return NemuIpcImpl(
  404. nemu_folder=self.emulator_instance.emulator.abspath('../'),
  405. instance_id=self.emulator_instance.MuMuPlayer12_id,
  406. display_id=0
  407. ).__enter__()
  408. except (NemuIpcIncompatible, NemuIpcError) as e:
  409. logger.error(e)
  410. logger.error('Unable to initialize NemuIpc')
  411. raise RequestHumanTakeover
  412. def nemu_ipc_available(self) -> bool:
  413. if not self.is_mumu_family:
  414. return False
  415. if self.nemud_app_keep_alive == '':
  416. return False
  417. try:
  418. _ = self.nemu_ipc
  419. except RequestHumanTakeover:
  420. return False
  421. return True
  422. def nemu_ipc_release(self):
  423. if has_cached_property(self, 'nemu_ipc'):
  424. self.nemu_ipc.disconnect()
  425. del_cached_property(self, 'nemu_ipc')
  426. logger.info('nemu_ipc released')
  427. def screenshot_nemu_ipc(self):
  428. image = self.nemu_ipc.screenshot()
  429. image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
  430. cv2.flip(image, 0, dst=image)
  431. return image
  432. def click_nemu_ipc(self, x, y):
  433. down = ensure_time((0.010, 0.020))
  434. self.nemu_ipc.down(x, y)
  435. self.sleep(down)
  436. self.nemu_ipc.up()
  437. self.sleep(0.050 - down)
  438. def long_click_nemu_ipc(self, x, y, duration=1.0):
  439. self.nemu_ipc.down(x, y)
  440. self.sleep(duration)
  441. self.nemu_ipc.up()
  442. self.sleep(0.050)
  443. def swipe_nemu_ipc(self, p1, p2):
  444. points = insert_swipe(p0=p1, p3=p2)
  445. for point in points:
  446. self.nemu_ipc.down(*point)
  447. self.sleep(0.010)
  448. self.nemu_ipc.up()
  449. self.sleep(0.050)
  450. def drag_nemu_ipc(self, p1, p2, point_random=(-10, -10, 10, 10)):
  451. p1 = np.array(p1) - random_rectangle_point(point_random)
  452. p2 = np.array(p2) - random_rectangle_point(point_random)
  453. points = insert_swipe(p0=p1, p3=p2, speed=20)
  454. for point in points:
  455. self.nemu_ipc.down(*point)
  456. self.sleep(0.010)
  457. self.nemu_ipc.down(*p2)
  458. self.sleep(0.140)
  459. self.nemu_ipc.down(*p2)
  460. self.sleep(0.140)
  461. self.nemu_ipc.up()
  462. self.sleep(0.050)