droidcast.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. import typing as t
  2. from functools import wraps
  3. import cv2
  4. import numpy as np
  5. import requests
  6. from adbutils.errors import AdbError
  7. from module.base.decorator import Config, cached_property, del_cached_property
  8. from module.base.timer import Timer
  9. from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
  10. from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
  11. ImageTruncated, PackageNotInstalled)
  12. from module.exception import RequestHumanTakeover
  13. from module.logger import logger
  14. class DroidCastVersionIncompatible(Exception):
  15. pass
  16. def retry(func):
  17. @wraps(func)
  18. def retry_wrapper(self, *args, **kwargs):
  19. """
  20. Args:
  21. self (Adb):
  22. """
  23. init = None
  24. for _ in range(RETRY_TRIES):
  25. try:
  26. if callable(init):
  27. retry_sleep(_)
  28. init()
  29. return func(self, *args, **kwargs)
  30. # Can't handle
  31. except RequestHumanTakeover:
  32. break
  33. # When adb server was killed
  34. except ConnectionResetError as e:
  35. logger.error(e)
  36. def init():
  37. self.adb_reconnect()
  38. # AdbError
  39. except AdbError as e:
  40. if handle_adb_error(e):
  41. def init():
  42. self.adb_reconnect()
  43. else:
  44. break
  45. # Package not installed
  46. except PackageNotInstalled as e:
  47. logger.error(e)
  48. def init():
  49. self.detect_package()
  50. # DroidCast not running
  51. # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
  52. # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
  53. except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
  54. logger.error(e)
  55. def init():
  56. self.droidcast_init()
  57. # DroidCastVersionIncompatible
  58. except DroidCastVersionIncompatible as e:
  59. logger.warning(e)
  60. def init():
  61. self.droidcast_init()
  62. # ImageTruncated
  63. except ImageTruncated as e:
  64. logger.error(e)
  65. def init():
  66. pass
  67. # Unknown
  68. except Exception as e:
  69. logger.exception(e)
  70. def init():
  71. pass
  72. logger.critical(f'Retry {func.__name__}() failed')
  73. raise RequestHumanTakeover
  74. return retry_wrapper
  75. class DroidCast(Uiautomator2):
  76. """
  77. DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
  78. DroidCast_raw, a modified version of DroidCast sending raw bitmap https://github.com/Torther/DroidCastS
  79. """
  80. _droidcast_port: int = 0
  81. @cached_property
  82. def droidcast_session(self):
  83. session = requests.Session()
  84. session.trust_env = False # Ignore proxy
  85. self._droidcast_port = self.adb_forward('tcp:53516')
  86. return session
  87. def droidcast_url(self, url='/screenshot?format=png'):
  88. """
  89. Check APIs from source code:
  90. https://github.com/rayworks/DroidCast/blob/master/app/src/main/java/com/rayworks/droidcast/Main.java
  91. Available APIs:
  92. - /screenshot
  93. To get JPG screenshots.
  94. - /screenshot?format=png
  95. To get PNG screenshots.
  96. - /screenshot?format=webp
  97. To get WEBP screenshots.
  98. - /src
  99. Websocket to get JPG screenshots.
  100. Note that /screenshot?format=jpg is unavailable.
  101. """
  102. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  103. @Config.when(DROIDCAST_VERSION='DroidCast')
  104. def droidcast_init(self):
  105. logger.hr('Droidcast init')
  106. self.droidcast_stop()
  107. logger.info('Pushing DroidCast apk')
  108. self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
  109. logger.info('Starting DroidCast apk')
  110. # CLASSPATH=/data/local/tmp/DroidCast.apk app_process / com.rayworks.droidcast.Main > /dev/null
  111. resp = self.u2_shell_background([
  112. 'CLASSPATH=/data/local/tmp/DroidCast.apk',
  113. 'app_process',
  114. '/',
  115. 'com.rayworks.droidcast.Main',
  116. '>',
  117. '/dev/null'
  118. ])
  119. logger.info(resp)
  120. del_cached_property(self, 'droidcast_session')
  121. _ = self.droidcast_session
  122. logger.attr('DroidCast', self.droidcast_url())
  123. self.droidcast_wait_startup()
  124. @Config.when(DROIDCAST_VERSION='DroidCast_raw')
  125. def droidcast_init(self):
  126. logger.hr('Droidcast init')
  127. self.resolution_check_uiautomator2()
  128. self.droidcast_stop()
  129. logger.info('Pushing DroidCast apk')
  130. self.adb_push(self.config.DROIDCAST_RAW_FILEPATH_LOCAL, self.config.DROIDCAST_RAW_FILEPATH_REMOTE)
  131. logger.info('Starting DroidCast apk')
  132. # DroidCastS-release-1.1.5.apk
  133. # CLASSPATH=/data/local/tmp/DroidCastS-release-1.1.5.apk app_process / com.torther.droidcasts.Main > /dev/null
  134. resp = self.u2_shell_background([
  135. 'CLASSPATH=/data/local/tmp/DroidCastS.apk',
  136. 'app_process',
  137. '/',
  138. 'com.torther.droidcasts.Main',
  139. '>',
  140. '/dev/null'
  141. ])
  142. logger.info(resp)
  143. del_cached_property(self, 'droidcast_session')
  144. _ = self.droidcast_session
  145. logger.attr('DroidCast', self.droidcast_url())
  146. self.droidcast_wait_startup()
  147. @retry
  148. def screenshot_droidcast(self):
  149. self.config.DROIDCAST_VERSION = 'DroidCast'
  150. image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
  151. image = np.frombuffer(image, np.uint8)
  152. if image is None:
  153. raise ImageTruncated('Empty image after reading from buffer')
  154. if image.shape == (1843200,):
  155. raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
  156. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  157. if image is None:
  158. raise ImageTruncated('Empty image after cv2.imdecode')
  159. image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  160. if image is None:
  161. raise ImageTruncated('Empty image after cv2.cvtColor')
  162. return image
  163. @retry
  164. def screenshot_droidcast_raw(self):
  165. self.config.DROIDCAST_VERSION = 'DroidCast_raw'
  166. image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
  167. # DroidCast_raw returns a RGB565 bitmap
  168. try:
  169. arr = np.frombuffer(image, dtype=np.uint16).reshape((720, 1280))
  170. except ValueError as e:
  171. # Try to load as `DroidCast`
  172. image = np.frombuffer(image, np.uint8)
  173. if image is not None:
  174. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  175. if image is not None:
  176. raise DroidCastVersionIncompatible(
  177. 'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
  178. # ValueError: cannot reshape array of size 0 into shape (720,1280)
  179. raise ImageTruncated(str(e))
  180. # Convert RGB565 to RGB888
  181. # https://blog.csdn.net/happy08god/article/details/10516871
  182. # r = (arr & 0b1111100000000000) >> (11 - 3)
  183. # g = (arr & 0b0000011111100000) >> (5 - 2)
  184. # b = (arr & 0b0000000000011111) << 3
  185. # r |= (r & 0b11100000) >> 5
  186. # g |= (g & 0b11000000) >> 6
  187. # b |= (b & 0b11100000) >> 5
  188. # r = r.astype(np.uint8)
  189. # g = g.astype(np.uint8)
  190. # b = b.astype(np.uint8)
  191. # image = cv2.merge([r, g, b])
  192. # The same as the code above but costs about 5ms instead of 10ms.
  193. r = cv2.multiply(arr & 0b1111100000000000, 0.00390625).astype(np.uint8)
  194. g = cv2.multiply(arr & 0b0000011111100000, 0.125).astype(np.uint8)
  195. b = cv2.multiply(arr & 0b0000000000011111, 8).astype(np.uint8)
  196. r = cv2.add(r, cv2.multiply(r, 0.03125))
  197. g = cv2.add(g, cv2.multiply(g, 0.015625))
  198. b = cv2.add(b, cv2.multiply(b, 0.03125))
  199. image = cv2.merge([r, g, b])
  200. return image
  201. def droidcast_wait_startup(self):
  202. """
  203. Wait until DroidCast startup completed.
  204. """
  205. timeout = Timer(10).start()
  206. while 1:
  207. self.sleep(0.25)
  208. if timeout.reached():
  209. break
  210. try:
  211. resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
  212. # Route `/` is unavailable, but 404 means startup completed
  213. if resp.status_code == 404:
  214. logger.attr('DroidCast', 'online')
  215. return True
  216. except requests.exceptions.ConnectionError:
  217. logger.attr('DroidCast', 'offline')
  218. logger.warning('Wait DroidCast startup timeout, assume started')
  219. return False
  220. def droidcast_uninstall(self):
  221. """
  222. Stop all DroidCast processes and remove DroidCast APK.
  223. DroidCast has't been installed but a JAVA class call, uninstall is a file delete.
  224. """
  225. self.droidcast_stop()
  226. logger.info('Removing DroidCast')
  227. self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
  228. self.adb_shell(["rm", self.config.DROIDCAST_RAW_FILEPATH_REMOTE])
  229. def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
  230. """
  231. List all DroidCast processes.
  232. """
  233. processes = self.proc_list_uiautomator2()
  234. for proc in processes:
  235. if 'com.rayworks.droidcast.Main' in proc.cmdline:
  236. yield proc
  237. if 'com.torther.droidcasts.Main' in proc.cmdline:
  238. yield proc
  239. def droidcast_stop(self):
  240. """
  241. Stop all DroidCast processes.
  242. """
  243. logger.info('Stopping DroidCast')
  244. for proc in self._iter_droidcast_proc():
  245. logger.info(f'Kill pid={proc.pid}')
  246. self.adb_shell(['kill', '-s', 9, proc.pid])
  247. if __name__ == '__main__':
  248. dc = DroidCast(config='oas1')
  249. cv2.imshow("iiii", dc.screenshot_uiautomator2())
  250. cv2.waitKey(0)