uiautomator_2.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333
  1. import typing as t
  2. import numpy as np
  3. import cv2
  4. from dataclasses import dataclass
  5. from functools import wraps
  6. from json.decoder import JSONDecodeError
  7. from subprocess import list2cmdline
  8. import uiautomator2 as u2
  9. from adbutils.errors import AdbError
  10. from lxml import etree
  11. from module.base.utils import point2str, random_rectangle_point, random_line_segments
  12. from module.device.connection import Connection
  13. from module.device.method.utils import (RETRY_TRIES, retry_sleep, handle_adb_error,
  14. ImageTruncated, PackageNotInstalled, possible_reasons)
  15. from module.exception import RequestHumanTakeover
  16. from module.logger import logger
  17. def retry(func):
  18. @wraps(func)
  19. def retry_wrapper(self, *args, **kwargs):
  20. """
  21. Args:
  22. self (Uiautomator2):
  23. """
  24. init = None
  25. for _ in range(RETRY_TRIES):
  26. try:
  27. if callable(init):
  28. retry_sleep(_)
  29. init()
  30. return func(self, *args, **kwargs)
  31. # Can't handle
  32. except RequestHumanTakeover:
  33. break
  34. # When adb server was killed
  35. except ConnectionResetError as e:
  36. logger.error(e)
  37. def init():
  38. self.adb_reconnect()
  39. # In `device.set_new_command_timeout(604800)`
  40. # json.decoder.JSONDecodeError: Expecting value: line 1 column 2 (char 1)
  41. except JSONDecodeError as e:
  42. logger.error(e)
  43. def init():
  44. self.install_uiautomator2()
  45. # AdbError
  46. except AdbError as e:
  47. if handle_adb_error(e):
  48. def init():
  49. self.adb_reconnect()
  50. else:
  51. break
  52. # RuntimeError: USB device 127.0.0.1:5555 is offline
  53. except RuntimeError as e:
  54. if handle_adb_error(e):
  55. def init():
  56. self.adb_reconnect()
  57. else:
  58. break
  59. # In `assert c.read string(4) == _OKAY`
  60. # ADB on emulator not enabled
  61. except AssertionError as e:
  62. logger.exception(e)
  63. possible_reasons(
  64. 'If you are using BlueStacks or LD player or WSA, '
  65. 'please enable ADB in the settings of your emulator'
  66. )
  67. break
  68. # Package not installed
  69. except PackageNotInstalled as e:
  70. logger.error(e)
  71. def init():
  72. self.detect_package()
  73. # ImageTruncated
  74. except ImageTruncated as e:
  75. logger.error(e)
  76. def init():
  77. pass
  78. # Unknown
  79. except Exception as e:
  80. logger.exception(e)
  81. def init():
  82. pass
  83. logger.critical(f'Retry {func.__name__}() failed')
  84. raise RequestHumanTakeover
  85. return retry_wrapper
  86. @dataclass
  87. class ProcessInfo:
  88. pid: int
  89. ppid: int
  90. thread_count: int
  91. cmdline: str
  92. name: str
  93. @dataclass
  94. class ShellBackgroundResponse:
  95. success: bool
  96. pid: int
  97. description: str
  98. class Uiautomator2(Connection):
  99. @retry
  100. def screenshot_uiautomator2(self):
  101. image = self.u2.screenshot(format='raw')
  102. image = np.frombuffer(image, np.uint8)
  103. if image is None:
  104. raise ImageTruncated('Empty image after reading from buffer')
  105. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  106. if image is None:
  107. raise ImageTruncated('Empty image after cv2.imdecode')
  108. image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  109. if image is None:
  110. raise ImageTruncated('Empty image after cv2.cvtColor')
  111. return image
  112. @retry
  113. def click_uiautomator2(self, x, y):
  114. self.u2.click(x, y)
  115. @retry
  116. def long_click_uiautomator2(self, x, y, duration=(1, 1.2)):
  117. self.u2.long_click(x, y, duration=duration)
  118. @retry
  119. def swipe_uiautomator2(self, p1, p2, duration=0.1):
  120. self.u2.swipe(*p1, *p2, duration=duration)
  121. @retry
  122. def _drag_along(self, path):
  123. """Swipe following path.
  124. Args:
  125. path (list): (x, y, sleep)
  126. Examples:
  127. al.drag_along([
  128. (403, 421, 0.2),
  129. (821, 326, 0.1),
  130. (821, 326-10, 0.1),
  131. (821, 326+10, 0.1),
  132. (821, 326, 0),
  133. ])
  134. Equals to:
  135. al.device.touch.down(403, 421)
  136. time.sleep(0.2)
  137. al.device.touch.move(821, 326)
  138. time.sleep(0.1)
  139. al.device.touch.move(821, 326-10)
  140. time.sleep(0.1)
  141. al.device.touch.move(821, 326+10)
  142. time.sleep(0.1)
  143. al.device.touch.up(821, 326)
  144. """
  145. length = len(path)
  146. for index, data in enumerate(path):
  147. x, y, second = data
  148. if index == 0:
  149. self.u2.touch.down(x, y)
  150. logger.info(point2str(x, y) + ' down')
  151. elif index - length == -1:
  152. self.u2.touch.up(x, y)
  153. logger.info(point2str(x, y) + ' up')
  154. else:
  155. self.u2.touch.move(x, y)
  156. logger.info(point2str(x, y) + ' move')
  157. self.sleep(second)
  158. def drag_uiautomator2(self, p1, p2, segments=1, shake=(0, 15), point_random=(-10, -10, 10, 10),
  159. shake_random=(-5, -5, 5, 5), swipe_duration=0.25, shake_duration=0.1):
  160. """Drag and shake, like:
  161. /\
  162. +-----------+ + +
  163. \/
  164. A simple swipe or drag don't work well, because it only has two points.
  165. Add some way point to make it more like swipe.
  166. Args:
  167. p1 (tuple): Start point, (x, y).
  168. p2 (tuple): End point, (x, y).
  169. segments (int):
  170. shake (tuple): Shake after arrive end point.
  171. point_random: Add random to start point and end point.
  172. shake_random: Add random to shake array.
  173. swipe_duration: Duration between way points.
  174. shake_duration: Duration between shake points.
  175. """
  176. p1 = np.array(p1) - random_rectangle_point(point_random)
  177. p2 = np.array(p2) - random_rectangle_point(point_random)
  178. path = [(x, y, swipe_duration) for x, y in random_line_segments(p1, p2, n=segments, random_range=point_random)]
  179. path += [
  180. (*p2 + shake + random_rectangle_point(shake_random), shake_duration),
  181. (*p2 - shake - random_rectangle_point(shake_random), shake_duration),
  182. (*p2, shake_duration)
  183. ]
  184. path = [(int(x), int(y), d) for x, y, d in path]
  185. self._drag_along(path)
  186. @retry
  187. def app_current_uiautomator2(self):
  188. """
  189. Returns:
  190. str: Package name.
  191. """
  192. result = self.u2.app_current()
  193. return result['package']
  194. @retry
  195. def app_start_uiautomator2(self, package_name=None):
  196. if not package_name:
  197. package_name = self.package
  198. try:
  199. self.u2.app_start(package_name)
  200. except u2.exceptions.BaseError as e:
  201. # BaseError: package "com.bilibili.azurlane" not found
  202. logger.error(e)
  203. raise PackageNotInstalled(package_name)
  204. @retry
  205. def app_stop_uiautomator2(self, package_name=None):
  206. if not package_name:
  207. package_name = self.package
  208. self.u2.app_stop(package_name)
  209. @retry
  210. def dump_hierarchy_uiautomator2(self) -> etree._Element:
  211. content = self.u2.dump_hierarchy(compressed=True)
  212. hierarchy = etree.fromstring(content.encode('utf-8'))
  213. return hierarchy
  214. @retry
  215. def resolution_uiautomator2(self, cal_rotation=True) -> t.Tuple[int, int]:
  216. """
  217. Faster u2.window_size(), cause that calls `dumpsys display` twice.
  218. Returns:
  219. (width, height)
  220. """
  221. info = self.u2.http.get('/info').json()
  222. w, h = info['display']['width'], info['display']['height']
  223. if cal_rotation:
  224. rotation = self.get_orientation()
  225. if (w > h) != (rotation % 2 == 1):
  226. w, h = h, w
  227. return w, h
  228. def resolution_check_uiautomator2(self):
  229. """
  230. Alas does not actively check resolution but the width and height of screenshots.
  231. However, some screenshot methods do not provide device resolution, so check it here.
  232. Returns:
  233. (width, height)
  234. Raises:
  235. RequestHumanTakeover: If resolution is not 1280x720
  236. """
  237. width, height = self.resolution_uiautomator2()
  238. logger.attr('Screen_size', f'{width}x{height}')
  239. if width == 1280 and height == 720:
  240. return (width, height)
  241. if width == 720 and height == 1280:
  242. return (width, height)
  243. logger.critical(f'Resolution not supported: {width}x{height}')
  244. logger.critical('Please set emulator resolution to 1280x720')
  245. raise RequestHumanTakeover
  246. @retry
  247. def proc_list_uiautomator2(self) -> t.List[ProcessInfo]:
  248. """
  249. Get info about current processes.
  250. """
  251. resp = self.u2.http.get("/proc/list", timeout=10)
  252. resp.raise_for_status()
  253. result = [
  254. ProcessInfo(
  255. pid=proc['pid'],
  256. ppid=proc['ppid'],
  257. thread_count=proc['threadCount'],
  258. cmdline=' '.join(proc['cmdline']) if proc['cmdline'] is not None else '',
  259. name=proc['name'],
  260. ) for proc in resp.json()
  261. ]
  262. return result
  263. @retry
  264. def u2_shell_background(self, cmdline, timeout=10) -> ShellBackgroundResponse:
  265. """
  266. Run at background.
  267. Note that this function will always return a success response,
  268. as this is a untested and hidden method in ATX.
  269. """
  270. if isinstance(cmdline, (list, tuple)):
  271. cmdline = list2cmdline(cmdline)
  272. elif isinstance(cmdline, str):
  273. cmdline = cmdline
  274. else:
  275. raise TypeError("cmdargs type invalid", type(cmdline))
  276. data = dict(command=cmdline, timeout=str(timeout))
  277. ret = self.u2.http.post("/shell/background", data=data, timeout=timeout + 10)
  278. ret.raise_for_status()
  279. resp = ret.json()
  280. resp = ShellBackgroundResponse(
  281. success=bool(resp.get('success', False)),
  282. pid=resp.get('pid', 0),
  283. description=resp.get('description', '')
  284. )
  285. return resp
  286. if __name__ == '__main__':
  287. ui2 = Uiautomator2(config='oas1')
  288. cv2.imshow("iiii", ui2.screenshot_uiautomator2())
  289. cv2.waitKey(0)