nemu_ipc.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541
  1. import asyncio
  2. import ctypes
  3. import os
  4. import sys
  5. from functools import partial, wraps
  6. from pathlib import Path
  7. import cv2
  8. import numpy as np
  9. from module.base.decorator import cached_property, del_cached_property, has_cached_property
  10. from module.base.utils import ensure_time
  11. from module.device.method.minitouch import insert_swipe, random_rectangle_point
  12. from module.device.method.utils import RETRY_TRIES, retry_sleep
  13. from module.exception import RequestHumanTakeover
  14. from module.logger import logger
  15. class NemuIpcIncompatible(Exception):
  16. pass
  17. class NemuIpcError(Exception):
  18. pass
  19. class CaptureStd:
  20. """
  21. Capture stdout and stderr from both python and C library
  22. https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769
  23. ```
  24. with CaptureStd() as capture:
  25. # String wasn't printed
  26. print('whatever')
  27. # But captured in ``capture.stdout``
  28. print(f'Got stdout: "{capture.stdout}"')
  29. print(f'Got stderr: "{capture.stderr}"')
  30. ```
  31. """
  32. def __init__(self):
  33. self.stdout = b''
  34. self.stderr = b''
  35. def _redirect_stdout(self, to):
  36. sys.stdout.close()
  37. os.dup2(to, self.fdout)
  38. sys.stdout = os.fdopen(self.fdout, 'w')
  39. def _redirect_stderr(self, to):
  40. sys.stderr.close()
  41. os.dup2(to, self.fderr)
  42. sys.stderr = os.fdopen(self.fderr, 'w')
  43. def __enter__(self):
  44. self.fdout = sys.stdout.fileno()
  45. self.fderr = sys.stderr.fileno()
  46. self.reader_out, self.writer_out = os.pipe()
  47. self.reader_err, self.writer_err = os.pipe()
  48. self.old_stdout = os.dup(self.fdout)
  49. self.old_stderr = os.dup(self.fderr)
  50. file_out = os.fdopen(self.writer_out, 'w')
  51. file_err = os.fdopen(self.writer_err, 'w')
  52. self._redirect_stdout(to=file_out.fileno())
  53. self._redirect_stderr(to=file_err.fileno())
  54. return self
  55. def __exit__(self, exc_type, exc_val, exc_tb):
  56. self._redirect_stdout(to=self.old_stdout)
  57. self._redirect_stderr(to=self.old_stderr)
  58. os.close(self.old_stdout)
  59. os.close(self.old_stderr)
  60. self.stdout = self.recvall(self.reader_out)
  61. self.stderr = self.recvall(self.reader_err)
  62. os.close(self.reader_out)
  63. os.close(self.reader_err)
  64. @staticmethod
  65. def recvall(reader, length=1024) -> bytes:
  66. fragments = []
  67. while 1:
  68. chunk = os.read(reader, length)
  69. if chunk:
  70. fragments.append(chunk)
  71. else:
  72. break
  73. output = b''.join(fragments)
  74. return output
  75. class CaptureNemuIpc(CaptureStd):
  76. instance = None
  77. def is_capturing(self):
  78. """
  79. Only capture at the topmost wrapper to avoid nested capturing
  80. If a capture is ongoing, this instance does nothing
  81. """
  82. cls = self.__class__
  83. return isinstance(cls.instance, cls) and cls.instance != self
  84. def __enter__(self):
  85. if self.is_capturing():
  86. return self
  87. super().__enter__()
  88. CaptureNemuIpc.instance = self
  89. return self
  90. def __exit__(self, exc_type, exc_val, exc_tb):
  91. if self.is_capturing():
  92. return
  93. CaptureNemuIpc.instance = None
  94. super().__exit__(exc_type, exc_val, exc_tb)
  95. self.check_stdout()
  96. self.check_stderr()
  97. def check_stdout(self):
  98. if not self.stdout:
  99. return
  100. logger.info(f'NemuIpc stdout: {self.stdout}')
  101. def check_stderr(self):
  102. if not self.stderr:
  103. return
  104. logger.error(f'NemuIpc stderr: {self.stderr}')
  105. # Calling an old MuMu12 player
  106. # Tested on 3.4.0
  107. # b'nemu_capture_display rpc error: 1783\r\n'
  108. # Tested on 3.7.3
  109. # b'nemu_capture_display rpc error: 1745\r\n'
  110. if b'error: 1783' in self.stderr or b'error: 1745' in self.stderr:
  111. raise NemuIpcIncompatible(
  112. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  113. # contact_id incorrect
  114. # b'nemu_capture_display cannot find rpc connection\r\n'
  115. if b'cannot find rpc connection' in self.stderr:
  116. raise NemuIpcError(self.stderr)
  117. # Emulator died
  118. # b'nemu_capture_display rpc error: 1722\r\n'
  119. # MuMuVMMSVC.exe died
  120. # b'nemu_capture_display rpc error: 1726\r\n'
  121. # No idea how to handle yet
  122. if b'error: 1722' in self.stderr or b'error: 1726' in self.stderr:
  123. raise NemuIpcError('Emulator instance is probably dead')
  124. def retry(func):
  125. @wraps(func)
  126. def retry_wrapper(self, *args, **kwargs):
  127. """
  128. Args:
  129. self (NemuIpcImpl):
  130. """
  131. init = None
  132. for _ in range(RETRY_TRIES):
  133. try:
  134. if callable(init):
  135. retry_sleep(_)
  136. init()
  137. return func(self, *args, **kwargs)
  138. # Can't handle
  139. except RequestHumanTakeover:
  140. break
  141. # Can't handle
  142. except NemuIpcIncompatible as e:
  143. logger.error(e)
  144. break
  145. # Function call timeout
  146. except asyncio.TimeoutError:
  147. logger.warning(f'Func {func.__name__}() call timeout, retrying: {_}')
  148. def init():
  149. self.reconnect()
  150. # NemuIpcError
  151. except NemuIpcError as e:
  152. logger.error(e)
  153. def init():
  154. self.reconnect()
  155. # Unknown, probably a trucked image
  156. except Exception as e:
  157. logger.exception(e)
  158. def init():
  159. pass
  160. logger.critical(f'Retry {func.__name__}() failed')
  161. raise RequestHumanTakeover
  162. return retry_wrapper
  163. class NemuIpcImpl:
  164. def __init__(self, nemu_folder: str, instance_id: int, display_id: int = 0):
  165. """
  166. Args:
  167. nemu_folder: Installation path of MuMu12, e.g. E:/ProgramFiles/MuMuPlayer-12.0
  168. instance_id: Emulator instance ID, starting from 0
  169. display_id: Always 0 if keep app alive was disabled
  170. """
  171. self.nemu_folder: str = nemu_folder
  172. self.instance_id: int = instance_id
  173. self.display_id: int = display_id
  174. ipc_dll = os.path.abspath(os.path.join(nemu_folder, './shell/sdk/external_renderer_ipc.dll'))
  175. logger.info(
  176. f'NemuIpcImpl init, '
  177. f'nemu_folder={nemu_folder}, '
  178. f'ipc_dll={ipc_dll}, '
  179. f'instance_id={instance_id}, '
  180. f'display_id={display_id}'
  181. )
  182. try:
  183. self.lib = ctypes.CDLL(ipc_dll)
  184. except OSError as e:
  185. logger.error(e)
  186. # OSError: [WinError 126] 找不到指定的模块。
  187. if not os.path.exists(ipc_dll):
  188. raise NemuIpcIncompatible(
  189. f'ipc_dll={ipc_dll} does not exist, '
  190. f'NemuIpc requires MuMu12 version >= 3.8.13, please check your version')
  191. else:
  192. raise NemuIpcIncompatible(
  193. f'ipc_dll={ipc_dll} exists, but cannot be loaded')
  194. self.connect_id: int = 0
  195. self.width = 0
  196. self.height = 0
  197. def connect(self):
  198. if self.connect_id > 0:
  199. return
  200. connect_id = self.ev_run_sync(
  201. self.lib.nemu_connect,
  202. self.nemu_folder, self.instance_id
  203. )
  204. if connect_id == 0:
  205. raise NemuIpcError(
  206. 'Connection failed, please check if nemu_folder is correct and emulator is running'
  207. )
  208. self.connect_id = connect_id
  209. # logger.info(f'NemuIpc connected: {self.connect_id}')
  210. def disconnect(self):
  211. if self.connect_id == 0:
  212. return
  213. self.ev_run_sync(
  214. self.lib.nemu_disconnect,
  215. self.connect_id
  216. )
  217. # logger.info(f'NemuIpc disconnected: {self.connect_id}')
  218. self.connect_id = 0
  219. def reconnect(self):
  220. self.disconnect()
  221. self.connect()
  222. def __enter__(self):
  223. self.connect()
  224. return self
  225. def __exit__(self, exc_type, exc_val, exc_tb):
  226. self.disconnect()
  227. @cached_property
  228. def _ev(self):
  229. return asyncio.new_event_loop()
  230. async def ev_run_async(self, func, *args, **kwargs):
  231. """
  232. Args:
  233. func: Sync function to call
  234. *args:
  235. **kwargs:
  236. Raises:
  237. asyncio.TimeoutError: If function call timeout
  238. """
  239. func_wrapped = partial(func, *args, **kwargs)
  240. # Increased timeout for slow PCs
  241. # Default screenshot interval is 0.2s, so a 0.15s timeout would have a fast retry without extra time costs
  242. result = await asyncio.wait_for(self._ev.run_in_executor(None, func_wrapped), timeout=0.15)
  243. return result
  244. def ev_run_sync(self, func, *args, **kwargs):
  245. """
  246. Args:
  247. func: Sync function to call
  248. *args:
  249. **kwargs:
  250. Raises:
  251. asyncio.TimeoutError: If function call timeout
  252. NemuIpcIncompatible:
  253. NemuIpcError
  254. """
  255. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  256. err = False
  257. if func.__name__ == 'nemu_connect':
  258. if result == 0:
  259. err = True
  260. else:
  261. if result > 0:
  262. err = True
  263. # Get to actual error message printed in std
  264. if err:
  265. logger.warning(f'Failed to call {func.__name__}, result={result}')
  266. with CaptureNemuIpc():
  267. result = self._ev.run_until_complete(self.ev_run_async(func, *args, **kwargs))
  268. return result
  269. def get_resolution(self):
  270. """
  271. Get emulator resolution, `self.width` and `self.height` will be set
  272. """
  273. if self.connect_id == 0:
  274. self.connect()
  275. width_ptr = ctypes.pointer(ctypes.c_int(0))
  276. height_ptr = ctypes.pointer(ctypes.c_int(0))
  277. nullptr = ctypes.POINTER(ctypes.c_int)()
  278. ret = self.ev_run_sync(
  279. self.lib.nemu_capture_display,
  280. self.connect_id, self.display_id, 0, width_ptr, height_ptr, nullptr
  281. )
  282. if ret > 0:
  283. raise NemuIpcError('nemu_capture_display failed during get_resolution()')
  284. self.width = width_ptr.contents.value
  285. self.height = height_ptr.contents.value
  286. @retry
  287. def screenshot(self):
  288. """
  289. Returns:
  290. np.ndarray: Image array in RGBA color space
  291. Note that image is upside down
  292. """
  293. if self.connect_id == 0:
  294. self.connect()
  295. self.get_resolution()
  296. width_ptr = ctypes.pointer(ctypes.c_int(self.width))
  297. height_ptr = ctypes.pointer(ctypes.c_int(self.height))
  298. length = self.width * self.height * 4
  299. pixels_pointer = ctypes.pointer((ctypes.c_ubyte * length)())
  300. ret = self.ev_run_sync(
  301. self.lib.nemu_capture_display,
  302. self.connect_id, self.display_id, length, width_ptr, height_ptr, pixels_pointer
  303. )
  304. if ret > 0:
  305. raise NemuIpcError('nemu_capture_display failed during screenshot()')
  306. # image = np.ctypeslib.as_array(pixels_pointer, shape=(self.height, self.width, 4))
  307. image = np.ctypeslib.as_array(pixels_pointer.contents).reshape((self.height, self.width, 4))
  308. return image
  309. def convert_xy(self, x, y):
  310. """
  311. Convert classic ADB coordinates to Nemu's
  312. `self.height` must be updated before calling this method
  313. Returns:
  314. int, int
  315. """
  316. x, y = int(x), int(y)
  317. x, y = self.height - y, x
  318. return x, y
  319. @retry
  320. def down(self, x, y):
  321. """
  322. Contact down, continuous contact down will be considered as swipe
  323. """
  324. if self.connect_id == 0:
  325. self.connect()
  326. if self.height == 0:
  327. self.get_resolution()
  328. x, y = self.convert_xy(x, y)
  329. ret = self.ev_run_sync(
  330. self.lib.nemu_input_event_touch_down,
  331. self.connect_id, self.display_id, x, y
  332. )
  333. if ret > 0:
  334. raise NemuIpcError('nemu_input_event_touch_down failed')
  335. @retry
  336. def up(self):
  337. """
  338. Contact up
  339. """
  340. if self.connect_id == 0:
  341. self.connect()
  342. ret = self.ev_run_sync(
  343. self.lib.nemu_input_event_touch_up,
  344. self.connect_id, self.display_id
  345. )
  346. if ret > 0:
  347. raise NemuIpcError('nemu_input_event_touch_up failed')
  348. def serial_to_id(serial: str):
  349. """
  350. Predict instance ID from serial
  351. E.g.
  352. "127.0.0.1:16384" -> 0
  353. "127.0.0.1:16416" -> 1
  354. Returns:
  355. int: instance_id, or None if failed to predict
  356. """
  357. try:
  358. port = int(serial.split(':')[1])
  359. except (IndexError, ValueError):
  360. return None
  361. index, offset = divmod(port - 16384, 32)
  362. if 0 <= index < 32 and offset in [0, 1, 2]:
  363. return index
  364. else:
  365. return None
  366. class NemuIpc():
  367. @cached_property
  368. def nemu_ipc(self) -> NemuIpcImpl:
  369. """
  370. Initialize a nemu ipc implementation
  371. """
  372. # Try existing settings first
  373. if self.config.script.device.emulatorinfo_path:
  374. folder = str(Path(self.config.script.device.emulatorinfo_path).parent.parent)
  375. index = serial_to_id(self.serial)
  376. if index is not None:
  377. try:
  378. return NemuIpcImpl(
  379. nemu_folder=folder,
  380. instance_id=index,
  381. display_id=0
  382. ).__enter__()
  383. except (NemuIpcIncompatible, NemuIpcError) as e:
  384. logger.error(e)
  385. logger.error('Emulator info incorrect')
  386. # Search emulator instance
  387. # with E:\ProgramFiles\MuMuPlayer-12.0\shell\MuMuPlayer.exe
  388. # installation path is E:\ProgramFiles\MuMuPlayer-12.0
  389. if self.emulator_instance is None:
  390. logger.error('Unable to use NemuIpc because emulator instance not found')
  391. raise RequestHumanTakeover
  392. try:
  393. return NemuIpcImpl(
  394. nemu_folder=self.emulator_instance.emulator.abspath('../'),
  395. instance_id=self.emulator_instance.MuMuPlayer12_id,
  396. display_id=0
  397. ).__enter__()
  398. except (NemuIpcIncompatible, NemuIpcError) as e:
  399. logger.error(e)
  400. logger.error('Unable to initialize NemuIpc')
  401. raise RequestHumanTakeover
  402. def nemu_ipc_available(self) -> bool:
  403. if not self.is_mumu_family:
  404. return False
  405. if self.nemud_app_keep_alive == '':
  406. return False
  407. try:
  408. _ = self.nemu_ipc
  409. except RequestHumanTakeover:
  410. return False
  411. return True
  412. def nemu_ipc_release(self):
  413. if has_cached_property(self, 'nemu_ipc'):
  414. self.nemu_ipc.disconnect()
  415. del_cached_property(self, 'nemu_ipc')
  416. logger.info('nemu_ipc released')
  417. def screenshot_nemu_ipc(self):
  418. image = self.nemu_ipc.screenshot()
  419. image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
  420. cv2.flip(image, 0, dst=image)
  421. return image
  422. def click_nemu_ipc(self, x, y):
  423. down = ensure_time((0.010, 0.020))
  424. self.nemu_ipc.down(x, y)
  425. self.sleep(down)
  426. self.nemu_ipc.up()
  427. self.sleep(0.050 - down)
  428. def long_click_nemu_ipc(self, x, y, duration=1.0):
  429. self.nemu_ipc.down(x, y)
  430. self.sleep(duration)
  431. self.nemu_ipc.up()
  432. self.sleep(0.050)
  433. def swipe_nemu_ipc(self, p1, p2):
  434. points = insert_swipe(p0=p1, p3=p2)
  435. for point in points:
  436. self.nemu_ipc.down(*point)
  437. self.sleep(0.010)
  438. self.nemu_ipc.up()
  439. self.sleep(0.050)
  440. def drag_nemu_ipc(self, p1, p2, point_random=(-10, -10, 10, 10)):
  441. p1 = np.array(p1) - random_rectangle_point(point_random)
  442. p2 = np.array(p2) - random_rectangle_point(point_random)
  443. points = insert_swipe(p0=p1, p3=p2, speed=20)
  444. for point in points:
  445. self.nemu_ipc.down(*point)
  446. self.sleep(0.010)
  447. self.nemu_ipc.down(*p2)
  448. self.sleep(0.140)
  449. self.nemu_ipc.down(*p2)
  450. self.sleep(0.140)
  451. self.nemu_ipc.up()
  452. self.sleep(0.050)