benchmark.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. import time
  2. import typing as t
  3. import numpy as np
  4. from rich.table import Table
  5. from rich.text import Text
  6. from module.base.utils import float2str as float2str_
  7. from module.base.utils import random_rectangle_point
  8. from module.daemon.daemon_base import DaemonBase
  9. from module.config.config import Config
  10. from module.device.device import Device
  11. from module.exception import RequestHumanTakeover
  12. from module.logger import logger
  13. def float2str(n, decimal=3):
  14. if not isinstance(n, (float, int)):
  15. return str(n)
  16. else:
  17. return float2str_(n, decimal=decimal) + 's'
  18. class Benchmark(DaemonBase):
  19. TEST_TOTAL = 15
  20. TEST_BEST = int(TEST_TOTAL * 0.8)
  21. def benchmark_test(self, func, *args, **kwargs):
  22. """
  23. Args:
  24. func: Function to test.
  25. *args: Passes to func.
  26. **kwargs: Passes to func.
  27. Returns:
  28. float: Time cost on average.
  29. """
  30. logger.hr(f'Benchmark test', level=2)
  31. logger.info(f'Testing function: {func.__name__}')
  32. record = []
  33. for n in range(1, self.TEST_TOTAL + 1):
  34. start = time.time()
  35. try:
  36. func(*args, **kwargs)
  37. except RequestHumanTakeover:
  38. logger.critical('RequestHumanTakeover')
  39. logger.warning(f'Benchmark tests failed on func: {func.__name__}')
  40. return 'Failed'
  41. except Exception as e:
  42. logger.exception(e)
  43. logger.warning(f'Benchmark tests failed on func: {func.__name__}')
  44. return 'Failed'
  45. cost = time.time() - start
  46. logger.attr(
  47. f'{str(n).rjust(2, "0")}/{self.TEST_TOTAL}',
  48. f'{float2str(cost)}'
  49. )
  50. record.append(cost)
  51. logger.info('Benchmark tests done')
  52. average = float(np.mean(np.sort(record)[:self.TEST_BEST]))
  53. logger.info(f'Time cost {float2str(average)} ({self.TEST_BEST} best results out of {self.TEST_TOTAL} tests)')
  54. return average
  55. @staticmethod
  56. def evaluate_screenshot(cost):
  57. if not isinstance(cost, (float, int)):
  58. return Text(cost, style="bold bright_red")
  59. if cost < 0.10:
  60. return Text('Ultra Fast', style="bold bright_green")
  61. if cost < 0.20:
  62. return Text('Very Fast', style="bright_green")
  63. if cost < 0.30:
  64. return Text('Fast', style="green")
  65. if cost < 0.50:
  66. return Text('Medium', style="yellow")
  67. if cost < 0.75:
  68. return Text('Slow', style="red")
  69. if cost < 1.00:
  70. return Text('Very Slow', style="bright_red")
  71. return Text('Ultra Slow', style="bold bright_red")
  72. @staticmethod
  73. def evaluate_click(cost):
  74. if not isinstance(cost, (float, int)):
  75. return Text(cost, style="bold bright_red")
  76. if cost < 0.1:
  77. return Text('Fast', style="bright_green")
  78. if cost < 0.2:
  79. return Text('Medium', style="yellow")
  80. if cost < 0.4:
  81. return Text('Slow', style="red")
  82. return Text('Very Slow', style="bright_red")
  83. @staticmethod
  84. def show(test, data, evaluate_func):
  85. """
  86. +--------------+--------+--------+
  87. | Screenshot | time | Speed |
  88. +--------------+--------+--------+
  89. | ADB | 0.319s | Fast |
  90. | uiautomator2 | 0.476s | Medium |
  91. | aScreenCap | Failed | Failed |
  92. +--------------+--------+--------+
  93. """
  94. # table = PrettyTable()
  95. # table.field_names = [test, 'Time', 'Speed']
  96. # for row in data:
  97. # table.add_row([row[0], f'{float2str(row[1])}', evaluate_func(row[1])])
  98. # for row in table.get_string().split('\n'):
  99. # logger.info(row)
  100. table = Table(show_lines=True)
  101. table.add_column(
  102. test, header_style="bright_cyan", style="cyan", no_wrap=True
  103. )
  104. table.add_column("Time", style="magenta")
  105. table.add_column("Speed", style="green")
  106. for row in data:
  107. table.add_row(
  108. row[0],
  109. float2str(row[1]),
  110. evaluate_func(row[1]),
  111. )
  112. logger.print(table, justify='center')
  113. def benchmark(self, screenshot: t.Tuple[str] = (), click: t.Tuple[str] = ()):
  114. logger.hr('Benchmark', level=1)
  115. logger.info(f'Testing screenshot methods: {screenshot}')
  116. logger.info(f'Testing click methods: {click}')
  117. screenshot_result = []
  118. for method in screenshot:
  119. result = self.benchmark_test(self.device.screenshot_methods[method])
  120. screenshot_result.append([method, result])
  121. area = (120, 20, 200, 50) # Somewhere safe to click.
  122. click_result = []
  123. for method in click:
  124. x, y = random_rectangle_point(area)
  125. result = self.benchmark_test(self.device.click_methods[method], x, y)
  126. click_result.append([method, result])
  127. def compare(res):
  128. res = res[1]
  129. if not isinstance(res, (int, float)):
  130. return 100
  131. else:
  132. return res
  133. logger.hr('Benchmark Results', level=1)
  134. fastest_screenshot = 'ADB_nc'
  135. fastest_click = 'minitouch'
  136. if screenshot_result:
  137. self.show(test='Screenshot', data=screenshot_result, evaluate_func=self.evaluate_screenshot)
  138. fastest = sorted(screenshot_result, key=lambda item: compare(item))[0]
  139. logger.info(f'Recommend screenshot method: {fastest[0]} ({float2str(fastest[1])})')
  140. fastest_screenshot = fastest[0]
  141. if click_result:
  142. self.show(test='Control', data=click_result, evaluate_func=self.evaluate_click)
  143. fastest = sorted(click_result, key=lambda item: compare(item))[0]
  144. logger.info(f'Recommend control method: {fastest[0]} ({float2str(fastest[1])})')
  145. fastest_click = fastest[0]
  146. return fastest_screenshot, fastest_click
  147. def get_test_methods(self) -> t.Tuple[t.Tuple[str], t.Tuple[str]]:
  148. device = self.config.Benchmark_DeviceType
  149. # device == 'emulator'
  150. screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background']
  151. click = ['ADB', 'uiautomator2', 'minitouch', 'window_message']
  152. def remove(*args):
  153. return [l for l in screenshot if l not in args]
  154. # No ascreencap on Android > 9
  155. if device in ['emulator_android_12', 'android_phone_12']:
  156. screenshot = remove('aScreenCap', 'aScreenCap_nc')
  157. # No nc loopback
  158. if device in ['plone_cloud_with_adb']:
  159. screenshot = remove('ADB_nc', 'aScreenCap_nc')
  160. # VMOS
  161. if device == 'android_phone_vmos':
  162. screenshot = ['ADB', 'aScreenCap', 'DroidCast', 'DroidCast_raw']
  163. click = ['ADB', 'Hermit', 'MaaTouch']
  164. # scene = self.config.Benchmark_TestScene
  165. # if 'screenshot' not in scene:
  166. # screenshot = []
  167. # if 'click' not in scene:
  168. # click = []
  169. return tuple(screenshot), tuple(click)
  170. def run(self):
  171. try:
  172. self.config.override(script.device.screenshot_method='ADB')
  173. self.device.uninstall_minicap()
  174. except RequestHumanTakeover:
  175. logger.critical('Request human takeover')
  176. return
  177. logger.attr('DeviceType', self.config.Benchmark_DeviceType)
  178. logger.attr('TestScene', self.config.Benchmark_TestScene)
  179. screenshot, click = self.get_test_methods()
  180. self.benchmark(screenshot, click)
  181. def run_simple_screenshot_benchmark(self):
  182. """
  183. Returns:
  184. str: The fastest screenshot method on current device.
  185. """
  186. screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background']
  187. def remove(*args):
  188. return [l for l in screenshot if l not in args]
  189. sdk = self.device.sdk_ver
  190. logger.info(f'sdk_ver: {sdk}')
  191. if not (21 <= sdk <= 28):
  192. screenshot = remove('aScreenCap', 'aScreenCap_nc')
  193. if self.device.is_chinac_phone_cloud:
  194. screenshot = remove('ADB_nc', 'aScreenCap_nc')
  195. screenshot = tuple(screenshot)
  196. self.TEST_TOTAL = 3
  197. self.TEST_BEST = 1
  198. method, _ = self.benchmark(screenshot, tuple())
  199. return method
  200. if __name__ == '__main__':
  201. config = Config('oas1')
  202. device = Device(config)
  203. b = Benchmark(config=config, device=device)
  204. # print(b.run_simple_screenshot_benchmark())
  205. screenshot, click = b.get_test_methods()
  206. b.benchmark(set(), click)