nemu_ipc.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555
  1. import asyncio
  2. import ctypes
  3. import os
  4. import random
  5. import sys
  6. from functools import partial, wraps
  7. from pathlib import Path
  8. import cv2
  9. import numpy as np
  10. from module.base.decorator import cached_property, del_cached_property, has_cached_property
  11. from module.base.utils import ensure_time
  12. from module.device.method.minitouch import insert_swipe, random_rectangle_point
  13. from module.device.method.utils import RETRY_TRIES, retry_sleep
  14. from module.exception import RequestHumanTakeover
  15. from module.logger import logger
  16. class NemuIpcIncompatible(Exception):
  17. pass
  18. class NemuIpcError(Exception):
  19. pass
  20. class CaptureStd:
  21. """
  22. Capture stdout and stderr from both python and C library
  23. https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769
  24. ```
  25. with CaptureStd() as capture:
  26. # String wasn't printed
  27. print('whatever')
  28. # But captured in ``capture.stdout``
  29. print(f'Got stdout: "{capture.stdout}"')
  30. print(f'Got stderr: "{capture.stderr}"')
  31. ```
  32. """
  33. def __init__(self):
  34. self.stdout = b''
  35. self.stderr = b''
  36. def _redirect_stdout(self, to):
  37. sys.stdout.close()
  38. os.dup2(to, self.fdout)
  39. sys.stdout = os.fdopen(self.fdout, 'w')
  40. def _redirect_stderr(self, to):
  41. sys.stderr.close()
  42. os.dup2(to, self.fderr)
  43. sys.stderr = os.fdopen(self.fderr, 'w')
  44. def __enter__(self):
  45. self.fdout = sys.stdout.fileno()
  46. self.fderr = sys.stderr.fileno()
  47. self.reader_out, self.writer_out = os.pipe()
  48. self.reader_err, self.writer_err = os.pipe()
  49. self.old_stdout = os.dup(self.fdout)
  50. self.old_stderr = os.dup(self.fderr)
  51. file_out = os.fdopen(self.writer_out, 'w')
  52. file_err = os.fdopen(self.writer_err, 'w')
  53. self._redirect_stdout(to=file_out.fileno())
  54. self._redirect_stderr(to=file_err.fileno())
  55. return self
  56. def __exit__(self, exc_type, exc_val, exc_tb):
  57. self._redirect_stdout(to=self.old_stdout)
  58. self._redirect_stderr(to=self.old_stderr)
  59. os.close(self.old_stdout)
  60. os.close(self.old_stderr)
  61. self.stdout = self.recvall(self.reader_out)
  62. self.stderr = self.recvall(self.reader_err)
  63. os.close(self.reader_out)
  64. os.close(self.reader_err)
  65. @staticmethod
  66. def recvall(reader, length=1024) -> bytes:
  67. fragments = []
  68. while 1:
  69. chunk = os.read(reader, length)
  70. if chunk:
  71. fragments.append(chunk)
  72. else:
  73. break
  74. output = b''.join(fragments)
  75. return output
  76. class CaptureNemuIpc(CaptureStd):
  77. instance = None
  78. def is_capturing(self):
  79. """
  80. Only capture at the topmost wrapper to avoid nested capturing
  81. If a capture is ongoing, this instance does nothing
  82. """
  83. cls = self.__class__
  84. return isinstance(cls.instance, cls) and cls.instance != self
  85. def __enter__(self):
  86. if self.is_capturing():
  87. return self
  88. super().__enter__()
  89. CaptureNemuIpc.instance = self
  90. return self
  91. def __exit__(self, exc_type, exc_val, exc_tb):
  92. if self.is_capturing():
  93. return
  94. CaptureNemuIpc.instance = None
  95. super().__exit__(exc_type, exc_val, exc_tb)
  96. self.check_stdout()
  97. self.check_stderr()
  98. def check_stdout(self):
  99. if not self.stdout:
  100. return
  101. logger.info(f'NemuIpc stdout: {self.stdout}')
  102. def check_stderr(self):
  103. if not self.stderr:
  104. return
  105. logger.error(f'NemuIpc stderr: {self.stderr}')
  106. # Calling an old MuMu12 player
  107. # Tested on 3.4.0
  108. # b'nemu_capture_display rpc error: 1783\r\n'
  109. # Tested on 3.7.3
  110. # b'nemu_capture_display rpc error: 1745\r\n'
  111. if b'error: 1783' in self.stderr or b'error: 1745' in self.stderr:
  112. raise NemuIpcIncompatible(
  113. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  114. # contact_id incorrect
  115. # b'nemu_capture_display cannot find rpc connection\r\n'
  116. if b'cannot find rpc connection' in self.stderr:
  117. raise NemuIpcError(self.stderr)
  118. # Emulator died
  119. # b'nemu_capture_display rpc error: 1722\r\n'
  120. # MuMuVMMSVC.exe died
  121. # b'nemu_capture_display rpc error: 1726\r\n'
  122. # No idea how to handle yet
  123. if b'error: 1722' in self.stderr or b'error: 1726' in self.stderr:
  124. raise NemuIpcError('Emulator instance is probably dead')
  125. def retry(func):
  126. @wraps(func)
  127. def retry_wrapper(self, *args, **kwargs):
  128. """
  129. Args:
  130. self (NemuIpcImpl):
  131. """
  132. init = None
  133. for _ in range(RETRY_TRIES):
  134. try:
  135. if callable(init):
  136. retry_sleep(_)
  137. init()
  138. return func(self, *args, **kwargs)
  139. # Can't handle
  140. except RequestHumanTakeover:
  141. break
  142. # Can't handle
  143. except NemuIpcIncompatible as e:
  144. logger.error(e)
  145. break
  146. # Function call timeout
  147. except asyncio.TimeoutError:
  148. logger.warning(f'Func {func.__name__}() call timeout, retrying: {_}')
  149. def init():
  150. self.reconnect()
  151. # NemuIpcError
  152. except NemuIpcError as e:
  153. logger.error(e)
  154. def init():
  155. self.reconnect()
  156. # Unknown, probably a trucked image
  157. except Exception as e:
  158. logger.exception(e)
  159. def init():
  160. pass
  161. logger.critical(f'Retry {func.__name__}() failed')
  162. raise RequestHumanTakeover
  163. return retry_wrapper
  164. class NemuIpcImpl:
  165. def __init__(self, nemu_folder: str, instance_id: int, display_id: int = 0):
  166. """
  167. Args:
  168. nemu_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/MuMuPlayer-12.0
  169. instance_id: Emulator instance ID, starting from 0
  170. display_id: Always 0 if keep app alive was disabled
  171. """
  172. self.nemu_folder: str = nemu_folder
  173. self.instance_id: int = instance_id
  174. self.display_id: int = display_id
  175. list_dll = [
  176. # MuMuPlayer12
  177. os.path.abspath(os.path.join(nemu_folder, './shell/sdk/external_renderer_ipc.dll')),
  178. # MuMuPlayer12 5.0
  179. os.path.abspath(os.path.join(nemu_folder, './nx_device/12.0/shell/sdk/external_renderer_ipc.dll')),
  180. # MuMuPlayer15
  181. os.path.abspath(os.path.join(nemu_folder, './nx_device/15.0/shell/sdk/external_renderer_ipc.dll')),
  182. ]
  183. self.lib = None
  184. for ipc_dll in list_dll:
  185. if not os.path.exists(ipc_dll):
  186. continue
  187. try:
  188. self.lib = ctypes.CDLL(ipc_dll)
  189. break
  190. except OSError as e:
  191. logger.error(e)
  192. logger.error(f'ipc_dll={ipc_dll} exists, but cannot be loaded')
  193. continue
  194. if self.lib is None:
  195. # not found
  196. raise NemuIpcIncompatible(
  197. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version. '
  198. f'None of the following path exists: {list_dll}')
  199. # success
  200. logger.info(
  201. f'NemuIpcImpl init, '
  202. f'nemu_folder={nemu_folder}, '
  203. f'ipc_dll={ipc_dll}, '
  204. f'instance_id={instance_id}, '
  205. f'display_id={display_id}'
  206. )
  207. self.connect_id: int = 0
  208. self.width = 0
  209. self.height = 0
  210. def connect(self):
  211. if self.connect_id > 0:
  212. return
  213. connect_id = self.ev_run_sync(
  214. self.lib.nemu_connect,
  215. self.nemu_folder, self.instance_id
  216. )
  217. if connect_id == 0:
  218. raise NemuIpcError(
  219. 'Connection failed, please check if nemu_folder is correct and emulator is running'
  220. )
  221. self.connect_id = connect_id
  222. # logger.info(f'NemuIpc connected: {self.connect_id}')
  223. def disconnect(self):
  224. if self.connect_id == 0:
  225. return
  226. self.ev_run_sync(
  227. self.lib.nemu_disconnect,
  228. self.connect_id
  229. )
  230. # logger.info(f'NemuIpc disconnected: {self.connect_id}')
  231. self.connect_id = 0
  232. def reconnect(self):
  233. self.disconnect()
  234. self.connect()
  235. def __enter__(self):
  236. self.connect()
  237. return self
  238. def __exit__(self, exc_type, exc_val, exc_tb):
  239. self.disconnect()
  240. @cached_property
  241. def _ev(self):
  242. return asyncio.new_event_loop()
  243. async def ev_run_async(self, func, *args, **kwargs):
  244. """
  245. Args:
  246. func: Sync function to call
  247. *args:
  248. **kwargs:
  249. Raises:
  250. asyncio.TimeoutError: If function call timeout
  251. """
  252. func_wrapped = partial(func, *args, **kwargs)
  253. # Increased timeout for slow PCs
  254. # Default screenshot interval is 0.2s, so a 0.15s timeout would have a fast retry without extra time costs
  255. result = await asyncio.wait_for(self._ev.run_in_executor(None, func_wrapped), timeout=0.15)
  256. return result
  257. def ev_run_sync(self, func, *args, **kwargs):
  258. """
  259. Args:
  260. func: Sync function to call
  261. *args:
  262. **kwargs:
  263. Raises:
  264. asyncio.TimeoutError: If function call timeout
  265. NemuIpcIncompatible:
  266. NemuIpcError
  267. """
  268. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  269. err = False
  270. if func.__name__ == 'nemu_connect':
  271. if result == 0:
  272. err = True
  273. else:
  274. if result > 0:
  275. err = True
  276. # Get to actual error message printed in std
  277. if err:
  278. logger.warning(f'Failed to call {func.__name__}, result={result}')
  279. with CaptureNemuIpc():
  280. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  281. return result
  282. def get_resolution(self):
  283. """
  284. Get emulator resolution, `self.width` and `self.height` will be set
  285. """
  286. if self.connect_id == 0:
  287. self.connect()
  288. width_ptr = ctypes.pointer(ctypes.c_int(0))
  289. height_ptr = ctypes.pointer(ctypes.c_int(0))
  290. nullptr = ctypes.POINTER(ctypes.c_int)()
  291. ret = self.ev_run_sync(
  292. self.lib.nemu_capture_display,
  293. self.connect_id, self.display_id, 0, width_ptr, height_ptr, nullptr
  294. )
  295. if ret > 0:
  296. raise NemuIpcError('nemu_capture_display failed during get_resolution()')
  297. self.width = width_ptr.contents.value
  298. self.height = height_ptr.contents.value
  299. @retry
  300. def screenshot(self):
  301. """
  302. Returns:
  303. np.ndarray: Image array in RGBA color space
  304. Note that image is upside down
  305. """
  306. if self.connect_id == 0:
  307. self.connect()
  308. self.get_resolution()
  309. width_ptr = ctypes.pointer(ctypes.c_int(self.width))
  310. height_ptr = ctypes.pointer(ctypes.c_int(self.height))
  311. length = self.width * self.height * 4
  312. pixels_pointer = ctypes.pointer((ctypes.c_ubyte * length)())
  313. ret = self.ev_run_sync(
  314. self.lib.nemu_capture_display,
  315. self.connect_id, self.display_id, length, width_ptr, height_ptr, pixels_pointer
  316. )
  317. if ret > 0:
  318. raise NemuIpcError('nemu_capture_display failed during screenshot()')
  319. # image = np.ctypeslib.as_array(pixels_pointer, shape=(self.height, self.width, 4))
  320. image = np.ctypeslib.as_array(pixels_pointer.contents).reshape((self.height, self.width, 4))
  321. return image
  322. def convert_xy(self, x, y):
  323. """
  324. Convert classic ADB coordinates to Nemu's
  325. `self.height` must be updated before calling this method
  326. Returns:
  327. int, int
  328. """
  329. x, y = int(x), int(y)
  330. x, y = self.height - y, x
  331. return x, y
  332. @retry
  333. def down(self, x, y):
  334. """
  335. Contact down, continuous contact down will be considered as swipe
  336. """
  337. if self.connect_id == 0:
  338. self.connect()
  339. if self.height == 0:
  340. self.get_resolution()
  341. x, y = self.convert_xy(x, y)
  342. ret = self.ev_run_sync(
  343. self.lib.nemu_input_event_touch_down,
  344. self.connect_id, self.display_id, x, y
  345. )
  346. if ret > 0:
  347. raise NemuIpcError('nemu_input_event_touch_down failed')
  348. @retry
  349. def up(self):
  350. """
  351. Contact up
  352. """
  353. if self.connect_id == 0:
  354. self.connect()
  355. ret = self.ev_run_sync(
  356. self.lib.nemu_input_event_touch_up,
  357. self.connect_id, self.display_id
  358. )
  359. if ret > 0:
  360. raise NemuIpcError('nemu_input_event_touch_up failed')
  361. def serial_to_id(serial: str):
  362. """
  363. Predict instance ID from serial
  364. E.g.
  365. "127.0.0.1:16384" -> 0
  366. "127.0.0.1:16416" -> 1
  367. Returns:
  368. int: instance_id, or None if failed to predict
  369. """
  370. try:
  371. port = int(serial.split(':')[1])
  372. except (IndexError, ValueError):
  373. return None
  374. index, offset = divmod(port - 16384, 32)
  375. if 0 <= index < 32 and offset in [0, 1, 2]:
  376. return index
  377. else:
  378. return None
  379. class NemuIpc():
  380. @cached_property
  381. def nemu_ipc(self) -> NemuIpcImpl:
  382. """
  383. Initialize a nemu ipc implementation
  384. """
  385. # Try existing settings first
  386. if self.config.script.device.emulatorinfo_path:
  387. folder = str(Path(self.config.script.device.emulatorinfo_path).parent.parent)
  388. index = serial_to_id(self.serial)
  389. if index is not None:
  390. try:
  391. return NemuIpcImpl(
  392. nemu_folder=folder,
  393. instance_id=index,
  394. display_id=0
  395. ).__enter__()
  396. except (NemuIpcIncompatible, NemuIpcError) as e:
  397. logger.error(e)
  398. logger.error('Emulator info incorrect')
  399. # Search emulator instance
  400. # with E:\ProgramFiles\MuMuPlayer-12.0\shell\MuMuPlayer.exe
  401. # installation path is E:\ProgramFiles\MuMuPlayer-12.0
  402. if self.emulator_instance is None:
  403. logger.error('Unable to use NemuIpc because emulator instance not found')
  404. raise RequestHumanTakeover
  405. try:
  406. return NemuIpcImpl(
  407. nemu_folder=self.emulator_instance.emulator.abspath('../'),
  408. instance_id=self.emulator_instance.MuMuPlayer12_id,
  409. display_id=0
  410. ).__enter__()
  411. except (NemuIpcIncompatible, NemuIpcError) as e:
  412. logger.error(e)
  413. logger.error('Unable to initialize NemuIpc')
  414. raise RequestHumanTakeover
  415. def nemu_ipc_available(self) -> bool:
  416. if not self.is_mumu_family:
  417. return False
  418. if self.nemud_app_keep_alive == '':
  419. return False
  420. try:
  421. _ = self.nemu_ipc
  422. except RequestHumanTakeover:
  423. return False
  424. return True
  425. def nemu_ipc_release(self):
  426. if has_cached_property(self, 'nemu_ipc'):
  427. self.nemu_ipc.disconnect()
  428. del_cached_property(self, 'nemu_ipc')
  429. logger.info('nemu_ipc released')
  430. def screenshot_nemu_ipc(self):
  431. image = self.nemu_ipc.screenshot()
  432. image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
  433. cv2.flip(image, 0, dst=image)
  434. return image
  435. def click_nemu_ipc(self, x, y):
  436. self.nemu_ipc.down(x, y)
  437. self.sleep(ensure_time((0.050, 0.110)))
  438. self.nemu_ipc.down(x + random.randint(-2, 2), y + random.randint(-2, 2))
  439. self.sleep(ensure_time((0.008, 0.020)))
  440. self.nemu_ipc.up()
  441. self.sleep(ensure_time((0.020, 0.045)))
  442. def long_click_nemu_ipc(self, x, y, duration=1.0):
  443. self.nemu_ipc.down(x, y)
  444. self.sleep(duration)
  445. self.nemu_ipc.up()
  446. self.sleep(0.050)
  447. def swipe_nemu_ipc(self, p1, p2):
  448. points = insert_swipe(p0=p1, p3=p2)
  449. for point in points:
  450. self.nemu_ipc.down(*point)
  451. self.sleep(ensure_time((0.006, 0.015)))
  452. self.nemu_ipc.up()
  453. self.sleep(0.050)
  454. def drag_nemu_ipc(self, p1, p2, point_random=(-10, -10, 10, 10)):
  455. p1 = np.array(p1) - random_rectangle_point(point_random)
  456. p2 = np.array(p2) - random_rectangle_point(point_random)
  457. points = insert_swipe(p0=p1, p3=p2, speed=20)
  458. for point in points:
  459. self.nemu_ipc.down(*point)
  460. self.sleep(0.010)
  461. self.nemu_ipc.down(*p2)
  462. self.sleep(0.140)
  463. self.nemu_ipc.down(*p2)
  464. self.sleep(0.140)
  465. self.nemu_ipc.up()
  466. self.sleep(0.050)