droidcast.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. import typing as t
  2. from functools import wraps
  3. import cv2
  4. import numpy as np
  5. import requests
  6. from adbutils.errors import AdbError
  7. from module.base.decorator import Config, cached_property, del_cached_property
  8. from module.base.timer import Timer
  9. from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
  10. from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
  11. ImageTruncated, PackageNotInstalled)
  12. from module.exception import RequestHumanTakeover
  13. from module.logger import logger
  14. class DroidCastVersionIncompatible(Exception):
  15. pass
  16. def retry(func):
  17. @wraps(func)
  18. def retry_wrapper(self, *args, **kwargs):
  19. """
  20. Args:
  21. self (Adb):
  22. """
  23. init = None
  24. for _ in range(RETRY_TRIES):
  25. try:
  26. if callable(init):
  27. retry_sleep(_)
  28. init()
  29. return func(self, *args, **kwargs)
  30. # Can't handle
  31. except RequestHumanTakeover:
  32. break
  33. # When adb server was killed
  34. except ConnectionResetError as e:
  35. logger.error(e)
  36. def init():
  37. self.adb_reconnect()
  38. # AdbError
  39. except AdbError as e:
  40. if handle_adb_error(e):
  41. def init():
  42. self.adb_reconnect()
  43. else:
  44. break
  45. # Package not installed
  46. except PackageNotInstalled as e:
  47. logger.error(e)
  48. def init():
  49. self.detect_package()
  50. # DroidCast not running
  51. # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
  52. # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
  53. except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
  54. logger.error(e)
  55. def init():
  56. self.droidcast_init()
  57. # DroidCastVersionIncompatible
  58. except DroidCastVersionIncompatible as e:
  59. logger.warning(e)
  60. def init():
  61. self.droidcast_init()
  62. # ImageTruncated
  63. except ImageTruncated as e:
  64. logger.error(e)
  65. def init():
  66. pass
  67. # Unknown
  68. except Exception as e:
  69. logger.exception(e)
  70. def init():
  71. pass
  72. logger.critical(f'Retry {func.__name__}() failed')
  73. raise RequestHumanTakeover
  74. return retry_wrapper
  75. class DroidCast(Uiautomator2):
  76. """
  77. DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
  78. DroidCast_raw, a modified version of DroidCast sending raw bitmap and png, https://github.com/Torther/DroidCastS
  79. """
  80. _droidcast_port: int = 0
  81. @cached_property
  82. def droidcast_session(self):
  83. session = requests.Session()
  84. session.trust_env = False # Ignore proxy
  85. self._droidcast_port = self.adb_forward('tcp:53516')
  86. return session
  87. """
  88. Check APIs from source code:
  89. https://github.com/Torther/DroidCast_raw/blob/DroidCast_raw/app/src/main/java/ink/mol/droidcast_raw/KtMain.kt
  90. Available APIs:
  91. - /screenshot
  92. To get a RGB565 bitmap
  93. - /preview
  94. To get PNG screenshots.
  95. """
  96. def droidcast_url(self, url='/preview'):
  97. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  98. def droidcast_raw_url(self, url='/screenshot'):
  99. return f'http://127.0.0.1:{self._droidcast_port}{url}'
  100. def droidcast_init(self):
  101. logger.hr('DroidCast init')
  102. self.droidcast_stop()
  103. logger.info('Pushing DroidCast apk')
  104. self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
  105. logger.info('Starting DroidCast apk')
  106. # DroidCast_raw-release-1.0.apk
  107. # CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main > /dev/null
  108. # adb shell CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main
  109. resp = self.u2_shell_background([
  110. 'CLASSPATH=/data/local/tmp/DroidCast_raw.apk',
  111. 'app_process',
  112. '/',
  113. 'ink.mol.droidcast_raw.Main',
  114. '>',
  115. '/dev/null'
  116. ])
  117. logger.info(resp)
  118. del_cached_property(self, 'droidcast_session')
  119. _ = self.droidcast_session
  120. @Config.when(DROIDCAST_VERSION='DroidCast')
  121. def droidcast_start(self):
  122. self.droidcast_init()
  123. logger.attr('DroidCast', self.droidcast_url())
  124. self.droidcast_wait_startup()
  125. @Config.when(DROIDCAST_VERSION='DroidCast_raw')
  126. def droidcast_raw_start(self):
  127. self.droidcast_init()
  128. logger.attr('DroidCast_raw', self.droidcast_raw_url())
  129. self.droidcast_wait_startup()
  130. @retry
  131. def screenshot_droidcast(self):
  132. self.config.DROIDCAST_VERSION = 'DroidCast'
  133. image = self.droidcast_session.get(self.droidcast_raw_url(), timeout=3).content
  134. image = np.frombuffer(image, np.uint8)
  135. if image is None:
  136. raise ImageTruncated('Empty image after reading from buffer')
  137. if image.shape == (1843200,):
  138. raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
  139. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  140. if image is None:
  141. raise ImageTruncated('Empty image after cv2.imdecode')
  142. image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  143. if image is None:
  144. raise ImageTruncated('Empty image after cv2.cvtColor')
  145. return image
  146. @retry
  147. def screenshot_droidcast_raw(self):
  148. self.config.DROIDCAST_VERSION = 'DroidCast_raw'
  149. image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
  150. # DroidCast_raw returns a RGB565 bitmap
  151. try:
  152. arr = np.frombuffer(image, dtype=np.uint16).reshape((720, 1280))
  153. except ValueError as e:
  154. # Try to load as `DroidCast`
  155. image = np.frombuffer(image, np.uint8)
  156. if image is not None:
  157. image = cv2.imdecode(image, cv2.IMREAD_COLOR)
  158. if image is not None:
  159. raise DroidCastVersionIncompatible(
  160. 'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
  161. # ValueError: cannot reshape array of size 0 into shape (720,1280)
  162. raise ImageTruncated(str(e))
  163. # Convert RGB565 to RGB888
  164. # https://blog.csdn.net/happy08god/article/details/10516871
  165. # r = (arr & 0b1111100000000000) >> (11 - 3)
  166. # g = (arr & 0b0000011111100000) >> (5 - 2)
  167. # b = (arr & 0b0000000000011111) << 3
  168. # r |= (r & 0b11100000) >> 5
  169. # g |= (g & 0b11000000) >> 6
  170. # b |= (b & 0b11100000) >> 5
  171. # r = r.astype(np.uint8)
  172. # g = g.astype(np.uint8)
  173. # b = b.astype(np.uint8)
  174. # image = cv2.merge([r, g, b])
  175. # The same as the code above but costs about 5ms instead of 10ms.
  176. r = cv2.multiply(arr & 0b1111100000000000, 0.00390625).astype(np.uint8)
  177. g = cv2.multiply(arr & 0b0000011111100000, 0.125).astype(np.uint8)
  178. b = cv2.multiply(arr & 0b0000000000011111, 8).astype(np.uint8)
  179. r = cv2.add(r, cv2.multiply(r, 0.03125))
  180. g = cv2.add(g, cv2.multiply(g, 0.015625))
  181. b = cv2.add(b, cv2.multiply(b, 0.03125))
  182. image = cv2.merge([r, g, b])
  183. return image
  184. def droidcast_wait_startup(self):
  185. """
  186. Wait until DroidCast startup completed.
  187. """
  188. timeout = Timer(10).start()
  189. while 1:
  190. self.sleep(0.25)
  191. if timeout.reached():
  192. break
  193. try:
  194. resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
  195. # Route `/` is unavailable, but 404 means startup completed
  196. if resp.status_code == 404:
  197. logger.attr('DroidCast', 'online')
  198. return True
  199. except requests.exceptions.ConnectionError:
  200. logger.attr('DroidCast', 'offline')
  201. logger.warning('Wait DroidCast startup timeout, assume started')
  202. return False
  203. def droidcast_uninstall(self):
  204. """
  205. Stop DroidCast processes and remove DroidCast APK.
  206. DroidCast hasn't been installed but a JAVA class call, uninstall is a file delete.
  207. """
  208. self.droidcast_stop()
  209. logger.info('Removing DroidCast')
  210. self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
  211. def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
  212. """
  213. List all DroidCast processes.
  214. """
  215. processes = self.proc_list_uiautomator2()
  216. for proc in processes:
  217. if 'com.rayworks.droidcast.Main' in proc.cmdline:
  218. yield proc
  219. if 'com.torther.droidcasts.Main' in proc.cmdline:
  220. yield proc
  221. if 'ink.mol.droidcast_raw.Main' in proc.cmdline:
  222. yield proc
  223. def droidcast_stop(self):
  224. """
  225. Stop DroidCast processes.
  226. """
  227. logger.info('Stopping DroidCast')
  228. for proc in self._iter_droidcast_proc():
  229. logger.info(f'Kill pid={proc.pid}')
  230. self.adb_shell(['kill', '-s', 9, proc.pid])
  231. if __name__ == '__main__':
  232. dc = DroidCast(config='oas1')
  233. cv2.imshow("iiii", dc.screenshot_droidcast())
  234. cv2.waitKey(0)