droidcast.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. import typing as t
  2. from functools import wraps
  3. import cv2
  4. import numpy as np
  5. import requests
  6. from adbutils.errors import AdbError
  7. from module.base.decorator import Config, cached_property, del_cached_property
  8. from module.base.timer import Timer
  9. from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
  10. from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
  11. ImageTruncated, PackageNotInstalled)
  12. from module.exception import RequestHumanTakeover
  13. from module.logger import logger
  14. class DroidCastVersionIncompatible(Exception):
  15. pass
  16. def retry(func):
  17. @wraps(func)
  18. def retry_wrapper(self, *args, **kwargs):
  19. """
  20. Args:
  21. self (Adb):
  22. """
  23. init = None
  24. for _ in range(RETRY_TRIES):
  25. try:
  26. if callable(init):
  27. retry_sleep(_)
  28. init()
  29. return func(self, *args, **kwargs)
  30. # Can't handle
  31. except RequestHumanTakeover:
  32. break
  33. # When adb server was killed
  34. except ConnectionResetError as e:
  35. logger.error(e)
  36. def init():
  37. self.adb_reconnect()
  38. # AdbError
  39. except AdbError as e:
  40. if handle_adb_error(e):
  41. def init():
  42. self.adb_reconnect()
  43. else:
  44. break
  45. # Package not installed
  46. except PackageNotInstalled as e:
  47. logger.error(e)
  48. def init():
  49. self.detect_package()
  50. # DroidCast not running
  51. # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
  52. # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
  53. except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
  54. logger.error(e)
  55. def init():
  56. self.droidcast_init()
  57. # DroidCastVersionIncompatible
  58. except DroidCastVersionIncompatible as e:
  59. logger.warning(e)
  60. def init():
  61. self.droidcast_init()
  62. # ImageTruncated
  63. except ImageTruncated as e:
  64. logger.error(e)
  65. def init():
  66. pass
  67. # Unknown
  68. except Exception as e:
  69. logger.exception(e)
  70. def init():
  71. pass
  72. logger.critical(f'Retry {func.__name__}() failed')
  73. raise RequestHumanTakeover
  74. return retry_wrapper
  75. class DroidCast(Uiautomator2):
  76. """
  77. DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
  78. DroidCast_raw, a modified version of DroidCast sending raw bitmap https://github.com/Torther/DroidCastS
  79. """
  80. _droidcast_port: int = 0
  81. droidcast_width: int = 0
  82. droidcast_height: int = 0
  83. @cached_property
  84. def droidcast_session(self):
  85. session = requests.Session()
  86. session.trust_env = False # Ignore proxy
  87. self._droidcast_port = self.adb_forward('tcp:53516')
  88. return session
  89. def droidcast_url(self, url='/screenshot?format=png'):
  90. """
  91. Check APIs from source code:
  92. https://github.com/rayworks/DroidCast/blob/master/app/src/main/java/com/rayworks/droidcast/Main.java
  93. Available APIs:
  94. - /screenshot
  95. To get JPG screenshots.
  96. - /screenshot?format=png
  97. To get PNG screenshots.
  98. - /screenshot?format=webp
  99. To get WEBP screenshots.
  100. - /src
  101. Websocket to get JPG screenshots.
  102. Note that /screenshot?format=jpg is unavailable.
  103. """
  104. if self.is_mumu_over_version_356:
  105. w, h = self.droidcast_width, self.droidcast_height
  106. if self.orientation == 0:
  107. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
  108. elif self.orientation == 1:
  109. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
  110. else:
  111. # logger.warning('DroidCast receives invalid device orientation')
  112. pass
  113. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  114. def droidcast_raw_url(self, url='/screenshot'):
  115. if self.is_mumu_over_version_356:
  116. w, h = self.droidcast_width, self.droidcast_height
  117. if self.orientation == 0:
  118. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
  119. elif self.orientation == 1:
  120. return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
  121. else:
  122. # logger.warning('DroidCast receives invalid device orientation')
  123. pass
  124. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  125. @Config.when(DROIDCAST_VERSION='DroidCast')
  126. def droidcast_init(self):
  127. logger.hr('Droidcast init')
  128. self.droidcast_stop()
  129. self._droidcast_update_resolution()
  130. logger.info('Pushing DroidCast apk')
  131. self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
  132. logger.info('Starting DroidCast apk')
  133. # CLASSPATH=/data/local/tmp/DroidCast.apk app_process / com.rayworks.droidcast.Main > /dev/null
  134. resp = self.u2_shell_background([
  135. 'CLASSPATH=/data/local/tmp/DroidCast.apk',
  136. 'app_process',
  137. '/',
  138. 'com.rayworks.droidcast.Main',
  139. '>',
  140. '/dev/null'
  141. ])
  142. logger.info(resp)
  143. del_cached_property(self, 'droidcast_session')
  144. _ = self.droidcast_session
  145. logger.attr('DroidCast', self.droidcast_url())
  146. self.droidcast_wait_startup()
  147. def _droidcast_update_resolution(self):
  148. if self.is_mumu_over_version_356:
  149. logger.info('Update droidcast resolution')
  150. w, h = self.resolution_uiautomator2(cal_rotation=False)
  151. self.get_orientation()
  152. # 720, 1280
  153. # mumu12 > 3.5.6 is always a vertical device
  154. self.droidcast_width, self.droidcast_height = w, h
  155. logger.info(f'Droicast resolution: {(w, h)}')
  156. @Config.when(DROIDCAST_VERSION='DroidCast_raw')
  157. def droidcast_init(self):
  158. logger.hr('Droidcast init')
  159. self.resolution_check_uiautomator2()
  160. self.droidcast_stop()
  161. logger.info('Pushing DroidCast apk')
  162. self.adb_push(self.config.DROIDCAST_RAW_FILEPATH_LOCAL, self.config.DROIDCAST_RAW_FILEPATH_REMOTE)
  163. logger.info('Starting DroidCast apk')
  164. # DroidCastS-release-1.1.5.apk
  165. # CLASSPATH=/data/local/tmp/DroidCastS-release-1.1.5.apk app_process / com.torther.droidcasts.Main > /dev/null
  166. resp = self.u2_shell_background([
  167. 'CLASSPATH=/data/local/tmp/DroidCastS.apk',
  168. 'app_process',
  169. '/',
  170. 'com.torther.droidcasts.Main',
  171. '>',
  172. '/dev/null'
  173. ])
  174. logger.info(resp)
  175. del_cached_property(self, 'droidcast_session')
  176. _ = self.droidcast_session
  177. logger.attr('DroidCast', self.droidcast_url())
  178. self.droidcast_wait_startup()
  179. @retry
  180. def screenshot_droidcast(self):
  181. self.config.DROIDCAST_VERSION = 'DroidCast'
  182. if self.is_mumu_over_version_356:
  183. if not self.droidcast_width or not self.droidcast_height:
  184. self._droidcast_update_resolution()
  185. image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
  186. image = np.frombuffer(image, np.uint8)
  187. if image is None:
  188. raise ImageTruncated('Empty image after reading from buffer')
  189. if image.shape == (1843200,):
  190. raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
  191. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  192. if image is None:
  193. raise ImageTruncated('Empty image after cv2.imdecode')
  194. image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  195. if image is None:
  196. raise ImageTruncated('Empty image after cv2.cvtColor')
  197. if self.is_mumu_over_version_356:
  198. if self.orientation == 1:
  199. image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
  200. return image
  201. @retry
  202. def screenshot_droidcast_raw(self):
  203. self.config.DROIDCAST_VERSION = 'DroidCast_raw'
  204. shape = (720, 1280)
  205. if self.is_mumu_over_version_356:
  206. if not self.droidcast_width or not self.droidcast_height:
  207. self._droidcast_update_resolution()
  208. if self.droidcast_height and self.droidcast_width:
  209. shape = (self.droidcast_height, self.droidcast_width)
  210. image = self.droidcast_session.get(self.droidcast_raw_url(), timeout=3).content
  211. # DroidCast_raw returns a RGB565 bitmap
  212. try:
  213. arr = np.frombuffer(image, dtype=np.uint16).reshape(shape)
  214. except ValueError as e:
  215. # Try to load as `DroidCast`
  216. image = np.frombuffer(image, np.uint8)
  217. if image is not None:
  218. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  219. if image is not None:
  220. raise DroidCastVersionIncompatible(
  221. 'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
  222. # ValueError: cannot reshape array of size 0 into shape (720,1280)
  223. raise ImageTruncated(str(e))
  224. # Convert RGB565 to RGB888
  225. # https://blog.csdn.net/happy08god/article/details/10516871
  226. # r = (arr & 0b1111100000000000) >> (11 - 3)
  227. # g = (arr & 0b0000011111100000) >> (5 - 2)
  228. # b = (arr & 0b0000000000011111) << 3
  229. # r |= (r & 0b11100000) >> 5
  230. # g |= (g & 0b11000000) >> 6
  231. # b |= (b & 0b11100000) >> 5
  232. # r = r.astype(np.uint8)
  233. # g = g.astype(np.uint8)
  234. # b = b.astype(np.uint8)
  235. # image = cv2.merge([r, g, b])
  236. # The same as the code above but costs about 5ms instead of 10ms.
  237. r = cv2.multiply(arr & 0b1111100000000000, 0.00390625).astype(np.uint8)
  238. g = cv2.multiply(arr & 0b0000011111100000, 0.125).astype(np.uint8)
  239. b = cv2.multiply(arr & 0b0000000000011111, 8).astype(np.uint8)
  240. r = cv2.add(r, cv2.multiply(r, 0.03125))
  241. g = cv2.add(g, cv2.multiply(g, 0.015625))
  242. b = cv2.add(b, cv2.multiply(b, 0.03125))
  243. image = cv2.merge([r, g, b])
  244. if self.is_mumu_over_version_356:
  245. if self.orientation == 1:
  246. image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
  247. return image
  248. def droidcast_wait_startup(self):
  249. """
  250. Wait until DroidCast startup completed.
  251. """
  252. timeout = Timer(10).start()
  253. while 1:
  254. self.sleep(0.25)
  255. if timeout.reached():
  256. break
  257. try:
  258. resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
  259. # Route `/` is unavailable, but 404 means startup completed
  260. if resp.status_code == 404:
  261. logger.attr('DroidCast', 'online')
  262. return True
  263. except requests.exceptions.ConnectionError:
  264. logger.attr('DroidCast', 'offline')
  265. logger.warning('Wait DroidCast startup timeout, assume started')
  266. return False
  267. def droidcast_uninstall(self):
  268. """
  269. Stop all DroidCast processes and remove DroidCast APK.
  270. DroidCast has't been installed but a JAVA class call, uninstall is a file delete.
  271. """
  272. self.droidcast_stop()
  273. logger.info('Removing DroidCast')
  274. self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
  275. self.adb_shell(["rm", self.config.DROIDCAST_RAW_FILEPATH_REMOTE])
  276. def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
  277. """
  278. List all DroidCast processes.
  279. """
  280. processes = self.proc_list_uiautomator2()
  281. for proc in processes:
  282. if 'com.rayworks.droidcast.Main' in proc.cmdline:
  283. yield proc
  284. if 'com.torther.droidcasts.Main' in proc.cmdline:
  285. yield proc
  286. def droidcast_stop(self):
  287. """
  288. Stop all DroidCast processes.
  289. """
  290. logger.info('Stopping DroidCast')
  291. for proc in self._iter_droidcast_proc():
  292. logger.info(f'Kill pid={proc.pid}')
  293. self.adb_shell(['kill', '-s', 9, proc.pid])
  294. if __name__ == '__main__':
  295. dc = DroidCast(config='oas1')
  296. cv2.imshow("iiii", dc.screenshot_droidcast())
  297. cv2.waitKey(0)