screenshot.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. import os
  2. import time
  3. from collections import deque
  4. from datetime import datetime
  5. from pathlib import Path
  6. import cv2
  7. import numpy as np
  8. from PIL import Image
  9. from module.base.decorator import cached_property
  10. from module.base.timer import Timer
  11. from module.base.utils import get_color, image_size, limit_in, save_image
  12. from module.device.method.adb import Adb
  13. from module.device.method.window import Window
  14. from module.device.method.droidcast import DroidCast
  15. from module.device.method.scrcpy import Scrcpy
  16. from module.device.method.nemu_ipc import NemuIpc
  17. from module.exception import RequestHumanTakeover, ScriptError
  18. from module.logger import logger
  19. class Screenshot(Adb, DroidCast, Scrcpy, Window, NemuIpc):
  20. _screen_size_checked = False
  21. _screen_black_checked = False
  22. _minicap_uninstalled = False
  23. _screenshot_interval = Timer(0.1)
  24. _last_save_time = {}
  25. image: np.ndarray
  26. def __init__(self, *args, **kwargs):
  27. super().__init__(*args, **kwargs)
  28. super(Window, self).__init__(*args, **kwargs)
  29. @cached_property
  30. def screenshot_methods(self):
  31. return {
  32. 'ADB': self.screenshot_adb,
  33. 'ADB_nc': self.screenshot_adb_nc,
  34. 'uiautomator2': self.screenshot_uiautomator2,
  35. # 'aScreenCap': self.screenshot_ascreencap,
  36. # 'aScreenCap_nc': self.screenshot_ascreencap_nc,
  37. 'DroidCast': self.screenshot_droidcast,
  38. 'DroidCast_raw': self.screenshot_droidcast_raw,
  39. 'scrcpy': self.screenshot_scrcpy,
  40. 'window_background': self.screenshot_window_background,
  41. 'nemu_ipc': self.screenshot_nemu_ipc
  42. }
  43. def screenshot(self):
  44. """
  45. Returns:
  46. np.ndarray:
  47. """
  48. self._screenshot_interval.wait()
  49. self._screenshot_interval.reset()
  50. for _ in range(2):
  51. method = self.screenshot_methods.get(
  52. self.config.script.device.screenshot_method,
  53. self.screenshot_adb # 第二个参数默认的是screenshot_adb
  54. )
  55. self.image = method()
  56. # if self.config.Emulator_ScreenshotDedithering:
  57. # # This will take 40-60ms
  58. # cv2.fastNlMeansDenoising(self.image, self.image, h=17, templateWindowSize=1, searchWindowSize=2)
  59. # self.image = self._handle_orientated_image(self.image)
  60. # if self.config.Error_SaveError:
  61. if self.config.script.error.save_error:
  62. self.screenshot_deque.append({'time': datetime.now(), 'image': self.image})
  63. if self.check_screen_size() and self.check_screen_black():
  64. break
  65. else:
  66. continue
  67. return self.image
  68. def _handle_orientated_image(self, image):
  69. """
  70. Args:
  71. image (np.ndarray):
  72. Returns:
  73. np.ndarray:
  74. """
  75. width, height = image_size(self.image)
  76. if width == 1280 and height == 720:
  77. return image
  78. # Rotate screenshots only when they're not 1280x720
  79. if self.orientation == 0:
  80. pass
  81. elif self.orientation == 1:
  82. image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
  83. elif self.orientation == 2:
  84. image = cv2.rotate(image, cv2.ROTATE_180)
  85. elif self.orientation == 3:
  86. image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
  87. else:
  88. raise ScriptError(f'Invalid device orientation: {self.orientation}')
  89. return image
  90. @cached_property
  91. def screenshot_deque(self):
  92. # return deque(maxlen=int(self.config.Error_ScreenshotLength))
  93. return deque(maxlen=int(self.config.script.error.screenshot_length))
  94. def save_screenshot(self, genre='items', interval=None, to_base_folder=False):
  95. """Save a screenshot. Use millisecond timestamp as file name.
  96. Args:
  97. genre (str, optional): Screenshot type.
  98. interval (int, float): Seconds between two save. Saves in the interval will be dropped.
  99. to_base_folder (bool): If save to base folder.
  100. Returns:
  101. bool: True if save succeed.
  102. """
  103. now = time.time()
  104. if interval is None:
  105. interval = 1 # 可改
  106. if now - self._last_save_time.get(genre, 0) > interval:
  107. fmt = 'png'
  108. file: str = '%s.%s' % (int(now * 1000), fmt)
  109. folder = Path('./log/screenshots')
  110. folder.mkdir(parents=True, exist_ok=True)
  111. file: Path = folder / file
  112. file = file.resolve()
  113. # folder = os.path.join(folder, genre)
  114. # if not os.path.exists(folder):
  115. # os.mkdir(folder)
  116. # file = os.path.join(folder, file)
  117. self.image_save(file)
  118. self._last_save_time[genre] = now
  119. return True
  120. else:
  121. self._last_save_time[genre] = now
  122. return False
  123. def screenshot_last_save_time_reset(self, genre):
  124. self._last_save_time[genre] = 0
  125. def screenshot_interval_set(self, interval=None):
  126. """
  127. Args:
  128. interval (int, float, str):
  129. Minimum interval between 2 screenshots in seconds.
  130. Or None for Optimization_ScreenshotInterval, 'combat' for Optimization_CombatScreenshotInterval
  131. """
  132. if interval is None:
  133. origin = self.config.script.optimization.screenshot_interval
  134. interval = limit_in(origin, 0.1, 0.3)
  135. if interval != origin:
  136. logger.warning(f'Optimization.ScreenshotInterval {origin} is revised to {interval}')
  137. self.config.script.optimization.screenshot_interval = interval
  138. # Allow nemu_ipc to have a lower default
  139. if self.config.Emulator_ScreenshotMethod == 'nemu_ipc':
  140. interval = limit_in(origin, 0.1, 0.2)
  141. elif interval == 'combat':
  142. origin = self.config.script.optimization.combat_screenshot_interval
  143. interval = limit_in(origin, 0.3, 1.0)
  144. if interval != origin:
  145. logger.warning(f'Optimization.CombatScreenshotInterval {origin} is revised to {interval}')
  146. self.config.script.optimization.combat_screenshot_interval = interval
  147. elif isinstance(interval, (int, float)):
  148. # No limitation for manual set in code
  149. pass
  150. else:
  151. logger.warning(f'Unknown screenshot interval: {interval}')
  152. raise ScriptError(f'Unknown screenshot interval: {interval}')
  153. # Screenshot interval in scrcpy is meaningless,
  154. # video stream is received continuously no matter you use it or not.
  155. # if self.config.script.device.screenshot_method == 'scrcpy':
  156. if self.config.script.device.screenshot_method == 'scrcpy':
  157. interval = 0.1
  158. if interval != self._screenshot_interval.limit:
  159. logger.info(f'Screenshot interval set to {interval}s')
  160. self._screenshot_interval.limit = interval
  161. def image_show(self, image=None):
  162. if image is None:
  163. image = self.image
  164. Image.fromarray(image).show()
  165. def image_save(self, file):
  166. save_image(self.image, file)
  167. def check_screen_size(self):
  168. """
  169. Screen size must be 1280x720.
  170. Take a screenshot before call.
  171. """
  172. if self._screen_size_checked:
  173. return True
  174. orientated = False
  175. for _ in range(2):
  176. # Check screen size
  177. width, height = image_size(self.image)
  178. logger.attr('Screen_size', f'{width}x{height}')
  179. if width == 1280 and height == 720:
  180. self._screen_size_checked = True
  181. return True
  182. elif not orientated and (width == 720 and height == 1280):
  183. logger.info('Received orientated screenshot, handling')
  184. self.get_orientation()
  185. self.image = self._handle_orientated_image(self.image)
  186. orientated = True
  187. width, height = image_size(self.image)
  188. if width == 720 and height == 1280:
  189. logger.info('Unable to handle orientated screenshot, continue for now')
  190. return True
  191. else:
  192. continue
  193. # elif self.config.Emulator_Serial == 'wsa-0':
  194. # self.display_resize_wsa(0)
  195. # return False
  196. elif hasattr(self, 'app_is_running') and not self.app_is_running():
  197. logger.warning('Received orientated screenshot, game not running')
  198. return True
  199. else:
  200. logger.critical(f'Resolution not supported: {width}x{height}')
  201. logger.critical('Please set emulator resolution to 1280x720')
  202. raise RequestHumanTakeover
  203. def check_screen_black(self):
  204. if self._screen_black_checked:
  205. return True
  206. # Check screen color
  207. # May get a pure black screenshot on some emulators.
  208. color = get_color(self.image, area=(0, 0, 1280, 720))
  209. if sum(color) < 1:
  210. # if self.config.Emulator_Serial == 'wsa-0':
  211. # for _ in range(2):
  212. # display = self.get_display_id()
  213. # if display == 0:
  214. # return True
  215. # logger.info(f'Game running on display {display}')
  216. # logger.warning('Game not running on display 0, will be restarted')
  217. # self.app_stop_uiautomator2()
  218. # return False
  219. if self.config.script.device.screenshot_method == 'uiautomator2':
  220. logger.warning(f'Received pure black screenshots from emulator, color: {color}')
  221. logger.warning('Uninstall minicap and retry')
  222. self.uninstall_minicap()
  223. self._screen_black_checked = False
  224. return False
  225. else:
  226. logger.warning(f'Received pure black screenshots from emulator, color: {color}')
  227. logger.warning(f'Screenshot method {self.config.script.device.screenshot_method}'
  228. f'may not work on emulator `{self.serial}`, or the emulator is not fully started')
  229. if self.is_mumu_family:
  230. if self.config.script.device.screenshot_method == 'DroidCast':
  231. self.droidcast_stop()
  232. else:
  233. logger.warning('If you are using MuMu X, please upgrade to version >= 12.1.5.0')
  234. self._screen_black_checked = False
  235. return False
  236. else:
  237. self._screen_black_checked = True
  238. return True
  239. if __name__ == "__main__":
  240. s = Screenshot(config="oas1")
  241. s.screenshot()
  242. s.image_show()