screenshot.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. import os
  2. import time
  3. from collections import deque
  4. from datetime import datetime
  5. from pathlib import Path
  6. import cv2
  7. import numpy as np
  8. from PIL import Image
  9. from module.base.decorator import cached_property
  10. from module.base.timer import Timer
  11. from module.base.utils import get_color, image_size, limit_in, save_image
  12. from module.device.env import IS_WINDOWS
  13. from module.device.method.adb import Adb
  14. from module.device.method.windows import Window
  15. from module.device.method.droidcast import DroidCast
  16. from module.device.method.scrcpy import Scrcpy
  17. from module.device.method.nemu_ipc import NemuIpc
  18. from module.exception import RequestHumanTakeover, ScriptError
  19. from module.logger import logger
  20. class Screenshot(Adb, DroidCast, Scrcpy, Window, NemuIpc):
  21. _screen_size_checked = False
  22. _screen_black_checked = False
  23. _minicap_uninstalled = False
  24. _screenshot_interval = Timer(0.1)
  25. _last_save_time = {}
  26. image: np.ndarray
  27. def __init__(self, *args, **kwargs):
  28. super().__init__(*args, **kwargs)
  29. super(Window, self).__init__(*args, **kwargs)
  30. @cached_property
  31. def screenshot_methods(self):
  32. return {
  33. 'ADB': self.screenshot_adb,
  34. 'ADB_nc': self.screenshot_adb_nc,
  35. 'uiautomator2': self.screenshot_uiautomator2,
  36. # 'aScreenCap': self.screenshot_ascreencap,
  37. # 'aScreenCap_nc': self.screenshot_ascreencap_nc,
  38. 'DroidCast': self.screenshot_droidcast,
  39. 'DroidCast_raw': self.screenshot_droidcast_raw,
  40. 'scrcpy': self.screenshot_scrcpy,
  41. 'window_background': self.screenshot_window_background if IS_WINDOWS else None,
  42. 'nemu_ipc': self.screenshot_nemu_ipc
  43. }
  44. def screenshot(self):
  45. """
  46. Returns:
  47. np.ndarray:
  48. """
  49. self._screenshot_interval.wait()
  50. self._screenshot_interval.reset()
  51. for _ in range(2):
  52. method = self.screenshot_methods.get(
  53. self.config.script.device.screenshot_method,
  54. self.screenshot_adb # 第二个参数默认的是screenshot_adb
  55. )
  56. self.image = method()
  57. # if self.config.Emulator_ScreenshotDedithering:
  58. # # This will take 40-60ms
  59. # cv2.fastNlMeansDenoising(self.image, self.image, h=17, templateWindowSize=1, searchWindowSize=2)
  60. # self.image = self._handle_orientated_image(self.image)
  61. # if self.config.Error_SaveError:
  62. if self.config.script.error.save_error:
  63. self.screenshot_deque.append({'time': datetime.now(), 'image': self.image})
  64. if self.check_screen_size() and self.check_screen_black():
  65. break
  66. else:
  67. continue
  68. return self.image
  69. def _handle_orientated_image(self, image):
  70. """
  71. Args:
  72. image (np.ndarray):
  73. Returns:
  74. np.ndarray:
  75. """
  76. width, height = image_size(self.image)
  77. if width == 1280 and height == 720:
  78. return image
  79. # Rotate screenshots only when they're not 1280x720
  80. if self.orientation == 0:
  81. pass
  82. elif self.orientation == 1:
  83. image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
  84. elif self.orientation == 2:
  85. image = cv2.rotate(image, cv2.ROTATE_180)
  86. elif self.orientation == 3:
  87. image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
  88. else:
  89. raise ScriptError(f'Invalid device orientation: {self.orientation}')
  90. return image
  91. @cached_property
  92. def screenshot_deque(self):
  93. # return deque(maxlen=int(self.config.Error_ScreenshotLength))
  94. return deque(maxlen=int(self.config.script.error.screenshot_length))
  95. def save_screenshot(self, genre='items', interval=None, to_base_folder=False):
  96. """Save a screenshot. Use millisecond timestamp as file name.
  97. Args:
  98. genre (str, optional): Screenshot type.
  99. interval (int, float): Seconds between two save. Saves in the interval will be dropped.
  100. to_base_folder (bool): If save to base folder.
  101. Returns:
  102. bool: True if save succeed.
  103. """
  104. now = time.time()
  105. if interval is None:
  106. interval = 1 # 可改
  107. if now - self._last_save_time.get(genre, 0) > interval:
  108. fmt = 'png'
  109. file: str = '%s.%s' % (int(now * 1000), fmt)
  110. folder = Path('./log/screenshots')
  111. folder.mkdir(parents=True, exist_ok=True)
  112. file: Path = folder / file
  113. file = file.resolve()
  114. # folder = os.path.join(folder, genre)
  115. # if not os.path.exists(folder):
  116. # os.mkdir(folder)
  117. # file = os.path.join(folder, file)
  118. self.image_save(file)
  119. self._last_save_time[genre] = now
  120. return True
  121. else:
  122. self._last_save_time[genre] = now
  123. return False
  124. def screenshot_last_save_time_reset(self, genre):
  125. self._last_save_time[genre] = 0
  126. def screenshot_interval_set(self, interval=None):
  127. """
  128. Args:
  129. interval (int, float, str):
  130. Minimum interval between 2 screenshots in seconds.
  131. Or None for Optimization_ScreenshotInterval, 'combat' for Optimization_CombatScreenshotInterval
  132. """
  133. if interval is None:
  134. origin = self.config.script.optimization.screenshot_interval
  135. interval = limit_in(origin, 0.1, 0.3)
  136. if interval != origin:
  137. logger.warning(f'Optimization.ScreenshotInterval {origin} is revised to {interval}')
  138. self.config.script.optimization.screenshot_interval = interval
  139. # Allow nemu_ipc to have a lower default
  140. if self.config.Emulator_ScreenshotMethod == 'nemu_ipc':
  141. interval = limit_in(origin, 0.1, 0.2)
  142. elif interval == 'combat':
  143. origin = self.config.script.optimization.combat_screenshot_interval
  144. interval = limit_in(origin, 0.3, 1.0)
  145. if interval != origin:
  146. logger.warning(f'Optimization.CombatScreenshotInterval {origin} is revised to {interval}')
  147. self.config.script.optimization.combat_screenshot_interval = interval
  148. elif isinstance(interval, (int, float)):
  149. # No limitation for manual set in code
  150. pass
  151. else:
  152. logger.warning(f'Unknown screenshot interval: {interval}')
  153. raise ScriptError(f'Unknown screenshot interval: {interval}')
  154. # Screenshot interval in scrcpy is meaningless,
  155. # video stream is received continuously no matter you use it or not.
  156. # if self.config.script.device.screenshot_method == 'scrcpy':
  157. if self.config.script.device.screenshot_method == 'scrcpy':
  158. interval = 0.1
  159. if interval != self._screenshot_interval.limit:
  160. logger.info(f'Screenshot interval set to {interval}s')
  161. self._screenshot_interval.limit = interval
  162. def image_show(self, image=None):
  163. if image is None:
  164. image = self.image
  165. Image.fromarray(image).show()
  166. def image_save(self, file):
  167. save_image(self.image, file)
  168. def check_screen_size(self):
  169. """
  170. Screen size must be 1280x720.
  171. Take a screenshot before call.
  172. """
  173. if self._screen_size_checked:
  174. return True
  175. orientated = False
  176. for _ in range(2):
  177. # Check screen size
  178. width, height = image_size(self.image)
  179. logger.attr('Screen_size', f'{width}x{height}')
  180. if width == 1280 and height == 720:
  181. self._screen_size_checked = True
  182. return True
  183. elif not orientated and (width == 720 and height == 1280):
  184. logger.info('Received orientated screenshot, handling')
  185. self.get_orientation()
  186. self.image = self._handle_orientated_image(self.image)
  187. orientated = True
  188. width, height = image_size(self.image)
  189. if width == 720 and height == 1280:
  190. logger.info('Unable to handle orientated screenshot, continue for now')
  191. return True
  192. else:
  193. continue
  194. # elif self.config.Emulator_Serial == 'wsa-0':
  195. # self.display_resize_wsa(0)
  196. # return False
  197. elif hasattr(self, 'app_is_running') and not self.app_is_running():
  198. logger.warning('Received orientated screenshot, game not running')
  199. return True
  200. else:
  201. logger.critical(f'Resolution not supported: {width}x{height}')
  202. logger.critical('Please set emulator resolution to 1280x720')
  203. raise RequestHumanTakeover
  204. def check_screen_size_sample(self):
  205. return self.image.shape == (720, 1280, 3)
  206. def check_screen_black(self):
  207. if self._screen_black_checked:
  208. return True
  209. # Check screen color
  210. # May get a pure black screenshot on some emulators.
  211. color = get_color(self.image, area=(0, 0, 1280, 720))
  212. if sum(color) < 1:
  213. # if self.config.Emulator_Serial == 'wsa-0':
  214. # for _ in range(2):
  215. # display = self.get_display_id()
  216. # if display == 0:
  217. # return True
  218. # logger.info(f'Game running on display {display}')
  219. # logger.warning('Game not running on display 0, will be restarted')
  220. # self.app_stop_uiautomator2()
  221. # return False
  222. if self.config.script.device.screenshot_method == 'uiautomator2':
  223. logger.warning(f'Received pure black screenshots from emulator, color: {color}')
  224. logger.warning('Uninstall minicap and retry')
  225. self.uninstall_minicap()
  226. self._screen_black_checked = False
  227. return False
  228. else:
  229. logger.warning(f'Received pure black screenshots from emulator, color: {color}')
  230. logger.warning(f'Screenshot method {self.config.script.device.screenshot_method}'
  231. f'may not work on emulator `{self.serial}`, or the emulator is not fully started')
  232. if self.is_mumu_family:
  233. if self.config.script.device.screenshot_method == 'DroidCast':
  234. self.droidcast_stop()
  235. else:
  236. logger.warning('If you are using MuMu X, please upgrade to version >= 12.1.5.0')
  237. self._screen_black_checked = False
  238. return False
  239. else:
  240. self._screen_black_checked = True
  241. return True
  242. if __name__ == "__main__":
  243. s = Screenshot(config="oas1")
  244. s.screenshot()
  245. s.image_show()