|
|
@@ -0,0 +1,245 @@
|
|
|
+import time
|
|
|
+import typing as t
|
|
|
+
|
|
|
+import numpy as np
|
|
|
+from rich.table import Table
|
|
|
+from rich.text import Text
|
|
|
+
|
|
|
+from module.base.utils import float2str as float2str_
|
|
|
+from module.base.utils import random_rectangle_point
|
|
|
+from module.daemon.daemon_base import DaemonBase
|
|
|
+from module.config.config import Config
|
|
|
+from module.device.device import Device
|
|
|
+from module.exception import RequestHumanTakeover
|
|
|
+from module.logger import logger
|
|
|
+
|
|
|
+
|
|
|
+def float2str(n, decimal=3):
|
|
|
+ if not isinstance(n, (float, int)):
|
|
|
+ return str(n)
|
|
|
+ else:
|
|
|
+ return float2str_(n, decimal=decimal) + 's'
|
|
|
+
|
|
|
+
|
|
|
+class Benchmark(DaemonBase):
|
|
|
+ TEST_TOTAL = 15
|
|
|
+ TEST_BEST = int(TEST_TOTAL * 0.8)
|
|
|
+
|
|
|
+ def benchmark_test(self, func, *args, **kwargs):
|
|
|
+ """
|
|
|
+ Args:
|
|
|
+ func: Function to test.
|
|
|
+ *args: Passes to func.
|
|
|
+ **kwargs: Passes to func.
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ float: Time cost on average.
|
|
|
+ """
|
|
|
+ logger.hr(f'Benchmark test', level=2)
|
|
|
+ logger.info(f'Testing function: {func.__name__}')
|
|
|
+ record = []
|
|
|
+
|
|
|
+ for n in range(1, self.TEST_TOTAL + 1):
|
|
|
+ start = time.time()
|
|
|
+
|
|
|
+ try:
|
|
|
+ func(*args, **kwargs)
|
|
|
+ except RequestHumanTakeover:
|
|
|
+ logger.critical('RequestHumanTakeover')
|
|
|
+ logger.warning(f'Benchmark tests failed on func: {func.__name__}')
|
|
|
+ return 'Failed'
|
|
|
+ except Exception as e:
|
|
|
+ logger.exception(e)
|
|
|
+ logger.warning(f'Benchmark tests failed on func: {func.__name__}')
|
|
|
+ return 'Failed'
|
|
|
+
|
|
|
+ cost = time.time() - start
|
|
|
+ logger.attr(
|
|
|
+ f'{str(n).rjust(2, "0")}/{self.TEST_TOTAL}',
|
|
|
+ f'{float2str(cost)}'
|
|
|
+ )
|
|
|
+ record.append(cost)
|
|
|
+
|
|
|
+ logger.info('Benchmark tests done')
|
|
|
+ average = float(np.mean(np.sort(record)[:self.TEST_BEST]))
|
|
|
+ logger.info(f'Time cost {float2str(average)} ({self.TEST_BEST} best results out of {self.TEST_TOTAL} tests)')
|
|
|
+ return average
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def evaluate_screenshot(cost):
|
|
|
+ if not isinstance(cost, (float, int)):
|
|
|
+ return Text(cost, style="bold bright_red")
|
|
|
+
|
|
|
+ if cost < 0.10:
|
|
|
+ return Text('Ultra Fast', style="bold bright_green")
|
|
|
+ if cost < 0.20:
|
|
|
+ return Text('Very Fast', style="bright_green")
|
|
|
+ if cost < 0.30:
|
|
|
+ return Text('Fast', style="green")
|
|
|
+ if cost < 0.50:
|
|
|
+ return Text('Medium', style="yellow")
|
|
|
+ if cost < 0.75:
|
|
|
+ return Text('Slow', style="red")
|
|
|
+ if cost < 1.00:
|
|
|
+ return Text('Very Slow', style="bright_red")
|
|
|
+ return Text('Ultra Slow', style="bold bright_red")
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def evaluate_click(cost):
|
|
|
+ if not isinstance(cost, (float, int)):
|
|
|
+ return Text(cost, style="bold bright_red")
|
|
|
+
|
|
|
+ if cost < 0.1:
|
|
|
+ return Text('Fast', style="bright_green")
|
|
|
+ if cost < 0.2:
|
|
|
+ return Text('Medium', style="yellow")
|
|
|
+ if cost < 0.4:
|
|
|
+ return Text('Slow', style="red")
|
|
|
+ return Text('Very Slow', style="bright_red")
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def show(test, data, evaluate_func):
|
|
|
+ """
|
|
|
+ +--------------+--------+--------+
|
|
|
+ | Screenshot | time | Speed |
|
|
|
+ +--------------+--------+--------+
|
|
|
+ | ADB | 0.319s | Fast |
|
|
|
+ | uiautomator2 | 0.476s | Medium |
|
|
|
+ | aScreenCap | Failed | Failed |
|
|
|
+ +--------------+--------+--------+
|
|
|
+ """
|
|
|
+ # table = PrettyTable()
|
|
|
+ # table.field_names = [test, 'Time', 'Speed']
|
|
|
+ # for row in data:
|
|
|
+ # table.add_row([row[0], f'{float2str(row[1])}', evaluate_func(row[1])])
|
|
|
+
|
|
|
+ # for row in table.get_string().split('\n'):
|
|
|
+ # logger.info(row)
|
|
|
+ table = Table(show_lines=True)
|
|
|
+ table.add_column(
|
|
|
+ test, header_style="bright_cyan", style="cyan", no_wrap=True
|
|
|
+ )
|
|
|
+ table.add_column("Time", style="magenta")
|
|
|
+ table.add_column("Speed", style="green")
|
|
|
+ for row in data:
|
|
|
+ table.add_row(
|
|
|
+ row[0],
|
|
|
+ float2str(row[1]),
|
|
|
+ evaluate_func(row[1]),
|
|
|
+ )
|
|
|
+ logger.print(table, justify='center')
|
|
|
+
|
|
|
+ def benchmark(self, screenshot: t.Tuple[str] = (), click: t.Tuple[str] = ()):
|
|
|
+ logger.hr('Benchmark', level=1)
|
|
|
+ logger.info(f'Testing screenshot methods: {screenshot}')
|
|
|
+ logger.info(f'Testing click methods: {click}')
|
|
|
+
|
|
|
+ screenshot_result = []
|
|
|
+ for method in screenshot:
|
|
|
+ result = self.benchmark_test(self.device.screenshot_methods[method])
|
|
|
+ screenshot_result.append([method, result])
|
|
|
+
|
|
|
+ area = (120, 20, 200, 50) # Somewhere safe to click.
|
|
|
+ click_result = []
|
|
|
+ for method in click:
|
|
|
+ x, y = random_rectangle_point(area)
|
|
|
+ result = self.benchmark_test(self.device.click_methods[method], x, y)
|
|
|
+ click_result.append([method, result])
|
|
|
+
|
|
|
+ def compare(res):
|
|
|
+ res = res[1]
|
|
|
+ if not isinstance(res, (int, float)):
|
|
|
+ return 100
|
|
|
+ else:
|
|
|
+ return res
|
|
|
+
|
|
|
+ logger.hr('Benchmark Results', level=1)
|
|
|
+ fastest_screenshot = 'ADB_nc'
|
|
|
+ fastest_click = 'minitouch'
|
|
|
+ if screenshot_result:
|
|
|
+ self.show(test='Screenshot', data=screenshot_result, evaluate_func=self.evaluate_screenshot)
|
|
|
+ fastest = sorted(screenshot_result, key=lambda item: compare(item))[0]
|
|
|
+ logger.info(f'Recommend screenshot method: {fastest[0]} ({float2str(fastest[1])})')
|
|
|
+ fastest_screenshot = fastest[0]
|
|
|
+ if click_result:
|
|
|
+ self.show(test='Control', data=click_result, evaluate_func=self.evaluate_click)
|
|
|
+ fastest = sorted(click_result, key=lambda item: compare(item))[0]
|
|
|
+ logger.info(f'Recommend control method: {fastest[0]} ({float2str(fastest[1])})')
|
|
|
+ fastest_click = fastest[0]
|
|
|
+
|
|
|
+ return fastest_screenshot, fastest_click
|
|
|
+
|
|
|
+ def get_test_methods(self) -> t.Tuple[t.Tuple[str], t.Tuple[str]]:
|
|
|
+ device = self.config.Benchmark_DeviceType
|
|
|
+ # device == 'emulator'
|
|
|
+ screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background']
|
|
|
+ click = ['ADB', 'uiautomator2', 'minitouch', 'window_message']
|
|
|
+
|
|
|
+ def remove(*args):
|
|
|
+ return [l for l in screenshot if l not in args]
|
|
|
+
|
|
|
+ # No ascreencap on Android > 9
|
|
|
+ if device in ['emulator_android_12', 'android_phone_12']:
|
|
|
+ screenshot = remove('aScreenCap', 'aScreenCap_nc')
|
|
|
+ # No nc loopback
|
|
|
+ if device in ['plone_cloud_with_adb']:
|
|
|
+ screenshot = remove('ADB_nc', 'aScreenCap_nc')
|
|
|
+ # VMOS
|
|
|
+ if device == 'android_phone_vmos':
|
|
|
+ screenshot = ['ADB', 'aScreenCap', 'DroidCast', 'DroidCast_raw']
|
|
|
+ click = ['ADB', 'Hermit', 'MaaTouch']
|
|
|
+
|
|
|
+ # scene = self.config.Benchmark_TestScene
|
|
|
+ # if 'screenshot' not in scene:
|
|
|
+ # screenshot = []
|
|
|
+ # if 'click' not in scene:
|
|
|
+ # click = []
|
|
|
+
|
|
|
+ return tuple(screenshot), tuple(click)
|
|
|
+
|
|
|
+ def run(self):
|
|
|
+ try:
|
|
|
+ self.config.override(Emulator_ScreenshotMethod='ADB')
|
|
|
+ self.device.uninstall_minicap()
|
|
|
+
|
|
|
+ except RequestHumanTakeover:
|
|
|
+ logger.critical('Request human takeover')
|
|
|
+ return
|
|
|
+
|
|
|
+ logger.attr('DeviceType', self.config.Benchmark_DeviceType)
|
|
|
+ logger.attr('TestScene', self.config.Benchmark_TestScene)
|
|
|
+ screenshot, click = self.get_test_methods()
|
|
|
+ self.benchmark(screenshot, click)
|
|
|
+
|
|
|
+ def run_simple_screenshot_benchmark(self):
|
|
|
+ """
|
|
|
+ Returns:
|
|
|
+ str: The fastest screenshot method on current device.
|
|
|
+ """
|
|
|
+ screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background']
|
|
|
+
|
|
|
+ def remove(*args):
|
|
|
+ return [l for l in screenshot if l not in args]
|
|
|
+
|
|
|
+ sdk = self.device.sdk_ver
|
|
|
+ logger.info(f'sdk_ver: {sdk}')
|
|
|
+ if not (21 <= sdk <= 28):
|
|
|
+ screenshot = remove('aScreenCap', 'aScreenCap_nc')
|
|
|
+ if self.device.is_chinac_phone_cloud:
|
|
|
+ screenshot = remove('ADB_nc', 'aScreenCap_nc')
|
|
|
+ screenshot = tuple(screenshot)
|
|
|
+
|
|
|
+ self.TEST_TOTAL = 3
|
|
|
+ self.TEST_BEST = 1
|
|
|
+ method, _ = self.benchmark(screenshot, tuple())
|
|
|
+
|
|
|
+ return method
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ config = Config('oas1')
|
|
|
+ device = Device(config)
|
|
|
+ b = Benchmark(config=config, device=device)
|
|
|
+ # print(b.run_simple_screenshot_benchmark())
|
|
|
+ screenshot, click = b.get_test_methods()
|
|
|
+ b.benchmark(set(), click)
|