| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247 |
- import time
- import typing as t
- import numpy as np
- from rich.table import Table
- from rich.text import Text
- from module.base.utils import float2str as float2str_
- from module.base.utils import random_rectangle_point
- from module.daemon.daemon_base import DaemonBase
- from module.config.config import Config
- from module.device.device import Device
- from module.exception import RequestHumanTakeover
- from module.logger import logger
- def float2str(n, decimal=3):
- if not isinstance(n, (float, int)):
- return str(n)
- else:
- return float2str_(n, decimal=decimal) + 's'
- class Benchmark(DaemonBase):
- TEST_TOTAL = 15
- TEST_BEST = int(TEST_TOTAL * 0.8)
- def benchmark_test(self, func, *args, **kwargs):
- """
- Args:
- func: Function to test.
- *args: Passes to func.
- **kwargs: Passes to func.
- Returns:
- float: Time cost on average.
- """
- logger.hr(f'Benchmark test', level=2)
- logger.info(f'Testing function: {func.__name__}')
- record = []
- for n in range(1, self.TEST_TOTAL + 1):
- start = time.time()
- try:
- func(*args, **kwargs)
- except RequestHumanTakeover:
- logger.critical('RequestHumanTakeover')
- logger.warning(f'Benchmark tests failed on func: {func.__name__}')
- return 'Failed'
- except Exception as e:
- logger.exception(e)
- logger.warning(f'Benchmark tests failed on func: {func.__name__}')
- return 'Failed'
- cost = time.time() - start
- logger.attr(
- f'{str(n).rjust(2, "0")}/{self.TEST_TOTAL}',
- f'{float2str(cost)}'
- )
- record.append(cost)
- logger.info('Benchmark tests done')
- average = float(np.mean(np.sort(record)[:self.TEST_BEST]))
- logger.info(f'Time cost {float2str(average)} ({self.TEST_BEST} best results out of {self.TEST_TOTAL} tests)')
- return average
- @staticmethod
- def evaluate_screenshot(cost):
- if not isinstance(cost, (float, int)):
- return Text(cost, style="bold bright_red")
- if cost < 0.10:
- return Text('Ultra Fast', style="bold bright_green")
- if cost < 0.20:
- return Text('Very Fast', style="bright_green")
- if cost < 0.30:
- return Text('Fast', style="green")
- if cost < 0.50:
- return Text('Medium', style="yellow")
- if cost < 0.75:
- return Text('Slow', style="red")
- if cost < 1.00:
- return Text('Very Slow', style="bright_red")
- return Text('Ultra Slow', style="bold bright_red")
- @staticmethod
- def evaluate_click(cost):
- if not isinstance(cost, (float, int)):
- return Text(cost, style="bold bright_red")
- if cost < 0.1:
- return Text('Fast', style="bright_green")
- if cost < 0.2:
- return Text('Medium', style="yellow")
- if cost < 0.4:
- return Text('Slow', style="red")
- return Text('Very Slow', style="bright_red")
- @staticmethod
- def show(test, data, evaluate_func):
- """
- +--------------+--------+--------+
- | Screenshot | time | Speed |
- +--------------+--------+--------+
- | ADB | 0.319s | Fast |
- | uiautomator2 | 0.476s | Medium |
- | aScreenCap | Failed | Failed |
- +--------------+--------+--------+
- """
- # table = PrettyTable()
- # table.field_names = [test, 'Time', 'Speed']
- # for row in data:
- # table.add_row([row[0], f'{float2str(row[1])}', evaluate_func(row[1])])
- # for row in table.get_string().split('\n'):
- # logger.info(row)
- table = Table(show_lines=True)
- table.add_column(
- test, header_style="bright_cyan", style="cyan", no_wrap=True
- )
- table.add_column("Time", style="magenta")
- table.add_column("Speed", style="green")
- for row in data:
- table.add_row(
- row[0],
- float2str(row[1]),
- evaluate_func(row[1]),
- )
- logger.print(table, justify='center')
- def benchmark(self, screenshot: t.Tuple[str] = (), click: t.Tuple[str] = ()):
- logger.hr('Benchmark', level=1)
- logger.info(f'Testing screenshot methods: {screenshot}')
- logger.info(f'Testing click methods: {click}')
- screenshot_result = []
- for method in screenshot:
- result = self.benchmark_test(self.device.screenshot_methods[method])
- screenshot_result.append([method, result])
- area = (120, 20, 200, 50) # Somewhere safe to click.
- click_result = []
- for method in click:
- x, y = random_rectangle_point(area)
- result = self.benchmark_test(self.device.click_methods[method], x, y)
- click_result.append([method, result])
- def compare(res):
- res = res[1]
- if not isinstance(res, (int, float)):
- return 100
- else:
- return res
- logger.hr('Benchmark Results', level=1)
- fastest_screenshot = 'ADB_nc'
- fastest_click = 'minitouch'
- if screenshot_result:
- self.show(test='Screenshot', data=screenshot_result, evaluate_func=self.evaluate_screenshot)
- fastest = sorted(screenshot_result, key=lambda item: compare(item))[0]
- logger.info(f'Recommend screenshot method: {fastest[0]} ({float2str(fastest[1])})')
- fastest_screenshot = fastest[0]
- if click_result:
- self.show(test='Control', data=click_result, evaluate_func=self.evaluate_click)
- fastest = sorted(click_result, key=lambda item: compare(item))[0]
- logger.info(f'Recommend control method: {fastest[0]} ({float2str(fastest[1])})')
- fastest_click = fastest[0]
- return fastest_screenshot, fastest_click
- def get_test_methods(self) -> t.Tuple[t.Tuple[str], t.Tuple[str]]:
- device = self.config.Benchmark_DeviceType
- # device == 'emulator'
- screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background']
- click = ['ADB', 'uiautomator2', 'minitouch', 'window_message']
- def remove(*args):
- return [l for l in screenshot if l not in args]
- # No ascreencap on Android > 9
- if device in ['emulator_android_12', 'android_phone_12']:
- screenshot = remove('aScreenCap', 'aScreenCap_nc')
- # No nc loopback
- if device in ['plone_cloud_with_adb']:
- screenshot = remove('ADB_nc', 'aScreenCap_nc')
- # VMOS
- if device == 'android_phone_vmos':
- screenshot = ['ADB', 'aScreenCap', 'DroidCast', 'DroidCast_raw']
- click = ['ADB', 'Hermit', 'MaaTouch']
- # scene = self.config.Benchmark_TestScene
- # if 'screenshot' not in scene:
- # screenshot = []
- # if 'click' not in scene:
- # click = []
- return tuple(screenshot), tuple(click)
- def run(self):
- try:
- self.config.override(self.device.screenshot_method == 'ADB')
- self.device.uninstall_minicap()
- except RequestHumanTakeover:
- logger.critical('Request human takeover')
- return
- logger.attr('DeviceType', self.config.Benchmark_DeviceType)
- logger.attr('TestScene', self.config.Benchmark_TestScene)
- screenshot, click = self.get_test_methods()
- self.benchmark(screenshot, click)
- def run_simple_screenshot_benchmark(self):
- """
- Returns:
- str: The fastest screenshot method on current device.
- """
- screenshot = ['ADB', 'ADB_nc', 'uiautomator2', 'DroidCast_raw', 'DroidCast', 'window_background', 'nemu_ipc']
- def remove(*args):
- return [l for l in screenshot if l not in args]
- sdk = self.device.sdk_ver
- logger.info(f'sdk_ver: {sdk}')
- if not (21 <= sdk <= 28):
- screenshot = remove('aScreenCap', 'aScreenCap_nc')
- if self.device.is_chinac_phone_cloud:
- screenshot = remove('ADB_nc', 'aScreenCap_nc')
- if self.config.script.device.handle == '':
- screenshot = remove('window_background')
- screenshot = tuple(screenshot)
- self.TEST_TOTAL = 3
- self.TEST_BEST = 1
- method, _ = self.benchmark(screenshot, tuple())
- return method
- if __name__ == '__main__':
- config = Config('oas1')
- device = Device(config)
- b = Benchmark(config=config, device=device)
- print(b.run_simple_screenshot_benchmark())
- # screenshot, click = b.get_test_methods()
- # b.benchmark(set(), click)
|