droidcast.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. import typing as t
  2. from functools import wraps
  3. import cv2
  4. import numpy as np
  5. import requests
  6. from adbutils.errors import AdbError
  7. from module.base.decorator import Config, cached_property, del_cached_property
  8. from module.base.timer import Timer
  9. from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
  10. from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
  11. ImageTruncated, PackageNotInstalled)
  12. from module.exception import RequestHumanTakeover
  13. from module.logger import logger
  14. class DroidCastVersionIncompatible(Exception):
  15. pass
  16. def retry(func):
  17. @wraps(func)
  18. def retry_wrapper(self, *args, **kwargs):
  19. """
  20. Args:
  21. self (Adb):
  22. """
  23. init = None
  24. for _ in range(RETRY_TRIES):
  25. try:
  26. if callable(init):
  27. retry_sleep(_)
  28. init()
  29. return func(self, *args, **kwargs)
  30. # Can't handle
  31. except RequestHumanTakeover:
  32. break
  33. # When adb server was killed
  34. except ConnectionResetError as e:
  35. logger.error(e)
  36. def init():
  37. self.adb_reconnect()
  38. # AdbError
  39. except AdbError as e:
  40. if handle_adb_error(e):
  41. def init():
  42. self.adb_reconnect()
  43. else:
  44. break
  45. # Package not installed
  46. except PackageNotInstalled as e:
  47. logger.error(e)
  48. def init():
  49. self.detect_package()
  50. # DroidCast not running
  51. # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
  52. # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
  53. except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
  54. logger.error(e)
  55. def init():
  56. self.droidcast_init()
  57. # DroidCastVersionIncompatible
  58. except DroidCastVersionIncompatible as e:
  59. logger.warning(e)
  60. def init():
  61. self.droidcast_init()
  62. # ImageTruncated
  63. except ImageTruncated as e:
  64. logger.error(e)
  65. def init():
  66. pass
  67. # Unknown
  68. except Exception as e:
  69. logger.exception(e)
  70. def init():
  71. pass
  72. logger.critical(f'Retry {func.__name__}() failed')
  73. raise RequestHumanTakeover
  74. return retry_wrapper
  75. class DroidCast(Uiautomator2):
  76. """
  77. DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
  78. DroidCast_raw, a modified version of DroidCast sending raw bitmap and png, https://github.com/Torther/DroidCastS
  79. """
  80. _droidcast_port: int = 0
  81. droidcast_width: int = 0
  82. droidcast_height: int = 0
  83. @cached_property
  84. def droidcast_session(self):
  85. session = requests.Session()
  86. session.trust_env = False # Ignore proxy
  87. self._droidcast_port = self.adb_forward('tcp:53516')
  88. return session
  89. """
  90. Check APIs from source code:
  91. https://github.com/Torther/DroidCast_raw/blob/DroidCast_raw/app/src/main/java/ink/mol/droidcast_raw/KtMain.kt
  92. Available APIs:
  93. - /screenshot
  94. To get a RGB565 bitmap
  95. - /preview
  96. To get PNG screenshots.
  97. """
  98. def droidcast_url(self, url='/preview'):
  99. if self.is_mumu_over_version_356:
  100. w, h = self.droidcast_width, self.droidcast_height
  101. if self.orientation == 0:
  102. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
  103. elif self.orientation == 1:
  104. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
  105. else:
  106. # logger.warning('DroidCast receives invalid device orientation')
  107. pass
  108. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  109. def droidcast_raw_url(self, url='/screenshot'):
  110. if self.is_mumu_over_version_356:
  111. w, h = self.droidcast_width, self.droidcast_height
  112. if self.orientation == 0:
  113. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
  114. elif self.orientation == 1:
  115. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
  116. else:
  117. # logger.warning('DroidCast receives invalid device orientation')
  118. pass
  119. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  120. def droidcast_init(self):
  121. logger.hr('DroidCast init')
  122. self.droidcast_stop()
  123. self._droidcast_update_resolution()
  124. logger.info('Pushing DroidCast apk')
  125. self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
  126. logger.info('Starting DroidCast apk')
  127. # DroidCast_raw-release-1.0.apk
  128. # CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main > /dev/null
  129. # adb shell CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main
  130. resp = self.u2_shell_background([
  131. 'CLASSPATH=/data/local/tmp/DroidCast_raw.apk',
  132. 'app_process',
  133. '/',
  134. 'ink.mol.droidcast_raw.Main',
  135. '>',
  136. '/dev/null'
  137. ])
  138. logger.info(resp)
  139. del_cached_property(self, 'droidcast_session')
  140. _ = self.droidcast_session
  141. if self.config.DROIDCAST_VERSION == 'DroidCast':
  142. logger.attr('DroidCast', self.droidcast_url())
  143. self.droidcast_wait_startup()
  144. elif self.config.DROIDCAST_VERSION == 'DroidCast_raw':
  145. logger.attr('DroidCast_raw', self.droidcast_raw_url())
  146. self.droidcast_wait_startup()
  147. else:
  148. logger.error(f'Unknown DROIDCAST_VERSION: {self.config.DROIDCAST_VERSION}')
  149. def _droidcast_update_resolution(self):
  150. if self.is_mumu_over_version_356:
  151. logger.info('Update droidcast resolution')
  152. w, h = self.resolution_uiautomator2(cal_rotation=False)
  153. self.get_orientation()
  154. # 720, 1280
  155. # mumu12 > 3.5.6 is always a vertical device
  156. self.droidcast_width, self.droidcast_height = w, h
  157. logger.info(f'Droicast resolution: {(w, h)}')
  158. @retry
  159. def screenshot_droidcast(self):
  160. self.config.DROIDCAST_VERSION = 'DroidCast'
  161. if self.is_mumu_over_version_356:
  162. if not self.droidcast_width or not self.droidcast_height:
  163. self._droidcast_update_resolution()
  164. resp = self.droidcast_session.get(self.droidcast_url(), timeout=3)
  165. if resp.status_code == 404:
  166. raise DroidCastVersionIncompatible('DroidCast server does not have /preview')
  167. image = resp.content
  168. image = np.frombuffer(image, np.uint8)
  169. if image is None:
  170. raise ImageTruncated('Empty image after reading from buffer')
  171. if image.shape == (1843200,):
  172. raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
  173. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  174. if image is None:
  175. raise ImageTruncated('Empty image after cv2.imdecode')
  176. image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  177. if image is None:
  178. raise ImageTruncated('Empty image after cv2.cvtColor')
  179. if self.is_mumu_over_version_356:
  180. if self.orientation == 1:
  181. image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
  182. return image
  183. @retry
  184. def screenshot_droidcast_raw(self):
  185. self.config.DROIDCAST_VERSION = 'DroidCast_raw'
  186. shape = (720, 1280)
  187. if self.is_mumu_over_version_356:
  188. if not self.droidcast_width or not self.droidcast_height:
  189. self._droidcast_update_resolution()
  190. if self.droidcast_height and self.droidcast_width:
  191. shape = (self.droidcast_height, self.droidcast_width)
  192. rotate = self.is_mumu_over_version_356 and self.orientation == 1
  193. image = self.droidcast_session.get(self.droidcast_raw_url(), timeout=3).content
  194. # DroidCast_raw returns a RGB565 bitmap
  195. try:
  196. arr = np.frombuffer(image, dtype=np.uint16)
  197. if rotate:
  198. arr = arr.reshape(shape)
  199. # arr = cv2.rotate(arr, cv2.ROTATE_90_CLOCKWISE)
  200. # A little bit faster?
  201. arr = cv2.transpose(arr)
  202. cv2.flip(arr, 1, dst=arr)
  203. else:
  204. arr = arr.reshape(shape)
  205. except ValueError as e:
  206. if len(image) < 500:
  207. logger.warning(f'Unexpected screenshot: {image}')
  208. # Try to load as `DroidCast`
  209. image = np.frombuffer(image, np.uint8)
  210. if image is not None:
  211. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  212. if image is not None:
  213. raise DroidCastVersionIncompatible(
  214. 'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
  215. # ValueError: cannot reshape array of size 0 into shape (720,1280)
  216. raise ImageTruncated(str(e))
  217. # Convert RGB565 to RGB888
  218. # https://blog.csdn.net/happy08god/article/details/10516871
  219. # r = (arr & 0b1111100000000000) >> (11 - 3)
  220. # g = (arr & 0b0000011111100000) >> (5 - 2)
  221. # b = (arr & 0b0000000000011111) << 3
  222. # r |= (r & 0b11100000) >> 5
  223. # g |= (g & 0b11000000) >> 6
  224. # b |= (b & 0b11100000) >> 5
  225. # r = r.astype(np.uint8)
  226. # g = g.astype(np.uint8)
  227. # b = b.astype(np.uint8)
  228. # image = cv2.merge([r, g, b])
  229. # The same as the code above but costs about 3~4ms instead of 10ms.
  230. # Note that cv2.convertScaleAbs is 5x fast as cv2.multiply, cv2.add is 8x fast as cv2.convertScaleAbs
  231. # Note that cv2.convertScaleAbs includes rounding
  232. r = cv2.bitwise_and(arr, 0b1111100000000000)
  233. r = cv2.convertScaleAbs(r, alpha=0.00390625)
  234. m = cv2.convertScaleAbs(r, alpha=0.03125)
  235. cv2.add(r, m, dst=r)
  236. g = cv2.bitwise_and(arr, 0b0000011111100000)
  237. g = cv2.convertScaleAbs(g, alpha=0.125)
  238. m = cv2.convertScaleAbs(g, alpha=0.015625, dst=m)
  239. cv2.add(g, m, dst=g)
  240. b = cv2.bitwise_and(arr, 0b0000000000011111)
  241. b = cv2.convertScaleAbs(b, alpha=8)
  242. m = cv2.convertScaleAbs(b, alpha=0.03125, dst=m)
  243. cv2.add(b, m, dst=b)
  244. image = cv2.merge([r, g, b])
  245. return image
  246. def droidcast_wait_startup(self):
  247. """
  248. Wait until DroidCast startup completed.
  249. """
  250. timeout = Timer(10).start()
  251. while 1:
  252. self.sleep(0.25)
  253. if timeout.reached():
  254. break
  255. try:
  256. resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
  257. # Route `/` is unavailable, but 404 means startup completed
  258. if resp.status_code == 404:
  259. logger.attr('DroidCast', 'online')
  260. return True
  261. except requests.exceptions.ConnectionError:
  262. logger.attr('DroidCast', 'offline')
  263. logger.warning('Wait DroidCast startup timeout, assume started')
  264. return False
  265. def droidcast_uninstall(self):
  266. """
  267. Stop DroidCast processes and remove DroidCast APK.
  268. DroidCast hasn't been installed but a JAVA class call, uninstall is a file delete.
  269. """
  270. self.droidcast_stop()
  271. logger.info('Removing DroidCast')
  272. self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
  273. def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
  274. """
  275. List all DroidCast processes.
  276. """
  277. processes = self.proc_list_uiautomator2()
  278. for proc in processes:
  279. if 'com.rayworks.droidcast.Main' in proc.cmdline:
  280. yield proc
  281. if 'com.torther.droidcasts.Main' in proc.cmdline:
  282. yield proc
  283. if 'ink.mol.droidcast_raw.Main' in proc.cmdline:
  284. yield proc
  285. def droidcast_stop(self):
  286. """
  287. Stop DroidCast processes.
  288. """
  289. logger.info('Stopping DroidCast')
  290. for proc in self._iter_droidcast_proc():
  291. logger.info(f'Kill pid={proc.pid}')
  292. self.adb_shell(['kill', '-s', 9, proc.pid])
  293. if __name__ == '__main__':
  294. dc = DroidCast(config='oas1')
  295. cv2.imshow("iiii", dc.screenshot_droidcast_raw())
  296. cv2.waitKey(0)