core.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. import socket
  2. import struct
  3. import threading
  4. import time
  5. import typing as t
  6. from time import sleep
  7. import numpy as np
  8. from adbutils import _AdbStreamConnection, AdbError, Network
  9. from module.base.decorator import cached_property
  10. from module.base.timer import Timer
  11. from module.device.connection import Connection
  12. from module.device.method.scrcpy.control import ControlSender
  13. from module.device.method.scrcpy.options import ScrcpyOptions
  14. from module.device.method.utils import recv_all
  15. from module.exception import RequestHumanTakeover
  16. from module.logger import logger
  17. class ScrcpyError(Exception):
  18. pass
  19. class ScrcpyCore(Connection):
  20. """
  21. Scrcpy: https://github.com/Genymobile/scrcpy
  22. Module from https://github.com/leng-yue/py-scrcpy-client
  23. """
  24. _scrcpy_last_frame: t.Optional[np.ndarray] = None
  25. _scrcpy_last_frame_time: float = 0.
  26. _scrcpy_alive = False
  27. _scrcpy_server_stream: t.Optional[_AdbStreamConnection] = None
  28. _scrcpy_video_socket: t.Optional[socket.socket] = None
  29. _scrcpy_control_socket: t.Optional[socket.socket] = None
  30. _scrcpy_control_socket_lock = threading.Lock()
  31. _scrcpy_stream_loop_thread = None
  32. _scrcpy_resolution: t.Tuple[int, int] = (1280, 720)
  33. @cached_property
  34. def _scrcpy_control(self) -> ControlSender:
  35. return ControlSender(self)
  36. def scrcpy_init(self):
  37. self._scrcpy_server_stop()
  38. logger.hr('Scrcpy init')
  39. logger.info(f'pushing {self.config.SCRCPY_FILEPATH_LOCAL}')
  40. self.adb_push(self.config.SCRCPY_FILEPATH_LOCAL, self.config.SCRCPY_FILEPATH_REMOTE)
  41. self._scrcpy_alive = False
  42. self.scrcpy_ensure_running()
  43. def scrcpy_ensure_running(self):
  44. if not self._scrcpy_alive:
  45. with self._scrcpy_control_socket_lock:
  46. self._scrcpy_server_start()
  47. def _scrcpy_server_start(self):
  48. """
  49. Connect to scrcpy server, there will be two sockets, video and control socket.
  50. Raises:
  51. ScrcpyError:
  52. """
  53. logger.hr('Scrcpy server start')
  54. commands = ScrcpyOptions.command_v120(jar_path=self.config.SCRCPY_FILEPATH_REMOTE)
  55. self._scrcpy_server_stream: _AdbStreamConnection = self.adb.shell(
  56. commands,
  57. stream=True,
  58. )
  59. logger.info('Create server stream')
  60. ret = self._scrcpy_server_stream.read(10)
  61. # b'Aborted \r\n'
  62. # Probably because file not exists
  63. if b'Aborted' in ret:
  64. raise ScrcpyError('Aborted')
  65. if ret == b'[server] E':
  66. # [server] ERROR: ...
  67. ret += recv_all(self._scrcpy_server_stream)
  68. logger.error(ret)
  69. # java.lang.IllegalArgumentException: The server version (1.25) does not match the client (...)
  70. if b'does not match the client' in ret:
  71. raise ScrcpyError('Server version does not match the client')
  72. else:
  73. raise ScrcpyError('Unknown scrcpy error')
  74. else:
  75. # [server] INFO: Device: ...
  76. ret += self._scrcpy_receive_from_server_stream()
  77. logger.info(ret)
  78. pass
  79. logger.info('Create video socket')
  80. timeout = Timer(3).start()
  81. while 1:
  82. if timeout.reached():
  83. raise ScrcpyError('Connect scrcpy-server timeout')
  84. try:
  85. self._scrcpy_video_socket = self.adb.create_connection(
  86. Network.LOCAL_ABSTRACT, "scrcpy"
  87. )
  88. break
  89. except AdbError:
  90. sleep(0.1)
  91. dummy_byte = self._scrcpy_video_socket.recv(1)
  92. if not len(dummy_byte) or dummy_byte != b"\x00":
  93. raise ScrcpyError('Did not receive Dummy Byte from video stream')
  94. logger.info('Create control socket')
  95. self._scrcpy_control_socket = self.adb.create_connection(
  96. Network.LOCAL_ABSTRACT, "scrcpy"
  97. )
  98. logger.info('Fetch device info')
  99. device_name = self._scrcpy_video_socket.recv(64).decode("utf-8").rstrip("\x00")
  100. if len(device_name):
  101. logger.attr('Scrcpy Device', device_name)
  102. else:
  103. raise ScrcpyError('Did not receive Device Name')
  104. ret = self._scrcpy_video_socket.recv(4)
  105. self._scrcpy_resolution = struct.unpack(">HH", ret)
  106. logger.attr('Scrcpy Resolution', self._scrcpy_resolution)
  107. self._scrcpy_video_socket.setblocking(False)
  108. self._scrcpy_alive = True
  109. logger.info('Start video stream loop thread')
  110. self._scrcpy_stream_loop_thread = threading.Thread(
  111. target=self._scrcpy_stream_loop, daemon=True
  112. )
  113. self._scrcpy_stream_loop_thread.start()
  114. while 1:
  115. if self._scrcpy_stream_loop_thread is not None and self._scrcpy_stream_loop_thread.is_alive():
  116. break
  117. self.sleep(0.001)
  118. logger.info('Scrcpy server is up')
  119. def _scrcpy_server_stop(self):
  120. """
  121. Stop listening (both threaded and blocked)
  122. """
  123. logger.hr('Scrcpy server stop')
  124. # err = self._scrcpy_receive_from_server_stream()
  125. # if err:
  126. # logger.error(err)
  127. self._scrcpy_alive = False
  128. if self._scrcpy_server_stream is not None:
  129. try:
  130. self._scrcpy_server_stream.close()
  131. except Exception:
  132. pass
  133. if self._scrcpy_control_socket is not None:
  134. try:
  135. self._scrcpy_control_socket.close()
  136. except Exception:
  137. pass
  138. if self._scrcpy_video_socket is not None:
  139. try:
  140. self._scrcpy_video_socket.close()
  141. except Exception:
  142. pass
  143. logger.info('Scrcpy server stopped')
  144. def _scrcpy_receive_from_server_stream(self):
  145. if self._scrcpy_server_stream is not None:
  146. try:
  147. return self._scrcpy_server_stream.conn.recv(4096)
  148. except Exception:
  149. pass
  150. def _scrcpy_stream_loop(self) -> None:
  151. """
  152. Core loop for video parsing
  153. """
  154. try:
  155. from av.codec import CodecContext
  156. from av.error import InvalidDataError
  157. except ImportError as e:
  158. logger.error(e)
  159. logger.error('You must have `av` installed to use scrcpy screenshot, please update dependencies')
  160. raise RequestHumanTakeover
  161. codec = CodecContext.create("h264", "r")
  162. while self._scrcpy_alive:
  163. try:
  164. raw_h264 = self._scrcpy_video_socket.recv(0x10000)
  165. if raw_h264 == b"":
  166. raise ScrcpyError("Video stream is disconnected")
  167. packets = codec.parse(raw_h264)
  168. for packet in packets:
  169. frames = codec.decode(packet)
  170. for frame in frames:
  171. # logger.info('frame received')
  172. frame = frame.to_ndarray(format="rgb24")
  173. self._scrcpy_last_frame = frame
  174. self._scrcpy_last_frame_time = time.time()
  175. self._scrcpy_resolution = (frame.shape[1], frame.shape[0])
  176. except (BlockingIOError, InvalidDataError):
  177. # only return nonempty frames, may block cv2 render thread
  178. time.sleep(0.001)
  179. except (ConnectionError, OSError) as e: # Socket Closed
  180. if self._scrcpy_alive:
  181. logger.error(f'_scrcpy_stream_loop_thread: {repr(e)}')
  182. raise
  183. raise ScrcpyError('_scrcpy_stream_loop stopped')