| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353 |
- import typing as t
- from functools import wraps
- import cv2
- import numpy as np
- import requests
- from adbutils.errors import AdbError
- from module.base.decorator import Config, cached_property, del_cached_property
- from module.base.timer import Timer
- from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
- from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
- ImageTruncated, PackageNotInstalled)
- from module.exception import RequestHumanTakeover
- from module.logger import logger
- class DroidCastVersionIncompatible(Exception):
- pass
- def retry(func):
- @wraps(func)
- def retry_wrapper(self, *args, **kwargs):
- """
- Args:
- self (Adb):
- """
- init = None
- for _ in range(RETRY_TRIES):
- try:
- if callable(init):
- retry_sleep(_)
- init()
- return func(self, *args, **kwargs)
- # Can't handle
- except RequestHumanTakeover:
- break
- # When adb server was killed
- except ConnectionResetError as e:
- logger.error(e)
- def init():
- self.adb_reconnect()
- # AdbError
- except AdbError as e:
- if handle_adb_error(e):
- def init():
- self.adb_reconnect()
- else:
- break
- # Package not installed
- except PackageNotInstalled as e:
- logger.error(e)
- def init():
- self.detect_package()
- # DroidCast not running
- # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
- # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
- except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
- logger.error(e)
- def init():
- self.droidcast_init()
- # DroidCastVersionIncompatible
- except DroidCastVersionIncompatible as e:
- logger.warning(e)
- def init():
- self.droidcast_init()
- # ImageTruncated
- except ImageTruncated as e:
- logger.error(e)
- def init():
- pass
- # Unknown
- except Exception as e:
- logger.exception(e)
- def init():
- pass
- logger.critical(f'Retry {func.__name__}() failed')
- raise RequestHumanTakeover
- return retry_wrapper
- class DroidCast(Uiautomator2):
- """
- DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
- DroidCast_raw, a modified version of DroidCast sending raw bitmap and png, https://github.com/Torther/DroidCastS
- """
- _droidcast_port: int = 0
- droidcast_width: int = 0
- droidcast_height: int = 0
- @cached_property
- def droidcast_session(self):
- session = requests.Session()
- session.trust_env = False # Ignore proxy
- self._droidcast_port = self.adb_forward('tcp:53516')
- return session
- """
- Check APIs from source code:
- https://github.com/Torther/DroidCast_raw/blob/DroidCast_raw/app/src/main/java/ink/mol/droidcast_raw/KtMain.kt
- Available APIs:
- - /screenshot
- To get a RGB565 bitmap
- - /preview
- To get PNG screenshots.
- """
- def droidcast_url(self, url='/preview'):
- if self.is_mumu_over_version_356:
- w, h = self.droidcast_width, self.droidcast_height
- if self.orientation == 0:
- return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
- elif self.orientation == 1:
- return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
- else:
- # logger.warning('DroidCast receives invalid device orientation')
- pass
- return f'http://127.0.0.1:{self._droidcast_port}{url}'
- def droidcast_raw_url(self, url='/screenshot'):
- if self.is_mumu_over_version_356:
- w, h = self.droidcast_width, self.droidcast_height
- if self.orientation == 0:
- return f'http://127.0.0.1:{self._droidcast_port}{url}?width={w}&height={h}'
- elif self.orientation == 1:
- return f'http://127.0.0.1:{self._droidcast_port}{url}?width={h}&height={w}'
- else:
- # logger.warning('DroidCast receives invalid device orientation')
- pass
- return f'http://127.0.0.1:{self._droidcast_port}{url}'
- def droidcast_init(self):
- logger.hr('DroidCast init')
- self.droidcast_stop()
- self._droidcast_update_resolution()
- logger.info('Pushing DroidCast apk')
- self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
- logger.info('Starting DroidCast apk')
- # DroidCast_raw-release-1.0.apk
- # CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main > /dev/null
- # adb shell CLASSPATH=/data/local/tmp/DroidCast_raw.apk app_process / ink.mol.droidcast_raw.Main
- resp = self.u2_shell_background([
- 'CLASSPATH=/data/local/tmp/DroidCast_raw.apk',
- 'app_process',
- '/',
- 'ink.mol.droidcast_raw.Main',
- '>',
- '/dev/null'
- ])
- logger.info(resp)
- del_cached_property(self, 'droidcast_session')
- _ = self.droidcast_session
- if self.config.DROIDCAST_VERSION == 'DroidCast':
- logger.attr('DroidCast', self.droidcast_url())
- self.droidcast_wait_startup()
- elif self.config.DROIDCAST_VERSION == 'DroidCast_raw':
- logger.attr('DroidCast_raw', self.droidcast_raw_url())
- self.droidcast_wait_startup()
- else:
- logger.error(f'Unknown DROIDCAST_VERSION: {self.config.DROIDCAST_VERSION}')
- def _droidcast_update_resolution(self):
- if self.is_mumu_over_version_356:
- logger.info('Update droidcast resolution')
- w, h = self.resolution_uiautomator2(cal_rotation=False)
- self.get_orientation()
- # 720, 1280
- # mumu12 > 3.5.6 is always a vertical device
- self.droidcast_width, self.droidcast_height = w, h
- logger.info(f'Droicast resolution: {(w, h)}')
- @retry
- def screenshot_droidcast(self):
- self.config.DROIDCAST_VERSION = 'DroidCast'
- if self.is_mumu_over_version_356:
- if not self.droidcast_width or not self.droidcast_height:
- self._droidcast_update_resolution()
- resp = self.droidcast_session.get(self.droidcast_url(), timeout=3)
- if resp.status_code == 404:
- raise DroidCastVersionIncompatible('DroidCast server does not have /preview')
- image = resp.content
- image = np.frombuffer(image, np.uint8)
- if image is None:
- raise ImageTruncated('Empty image after reading from buffer')
- if image.shape == (1843200,):
- raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
- image = cv2.imdecode(image, cv2.IMREAD_COLOR)
- if image is None:
- raise ImageTruncated('Empty image after cv2.imdecode')
- image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
- if image is None:
- raise ImageTruncated('Empty image after cv2.cvtColor')
- if self.is_mumu_over_version_356:
- if self.orientation == 1:
- image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
- return image
- @retry
- def screenshot_droidcast_raw(self):
- self.config.DROIDCAST_VERSION = 'DroidCast_raw'
- shape = (720, 1280)
- if self.is_mumu_over_version_356:
- if not self.droidcast_width or not self.droidcast_height:
- self._droidcast_update_resolution()
- if self.droidcast_height and self.droidcast_width:
- shape = (self.droidcast_height, self.droidcast_width)
- rotate = self.is_mumu_over_version_356 and self.orientation == 1
- image = self.droidcast_session.get(self.droidcast_raw_url(), timeout=3).content
- # DroidCast_raw returns a RGB565 bitmap
- try:
- arr = np.frombuffer(image, dtype=np.uint16)
- if rotate:
- arr = arr.reshape(shape)
- # arr = cv2.rotate(arr, cv2.ROTATE_90_CLOCKWISE)
- # A little bit faster?
- arr = cv2.transpose(arr)
- cv2.flip(arr, 1, dst=arr)
- else:
- arr = arr.reshape(shape)
- except ValueError as e:
- if len(image) < 500:
- logger.warning(f'Unexpected screenshot: {image}')
- # Try to load as `DroidCast`
- image = np.frombuffer(image, np.uint8)
- if image is not None:
- image = cv2.imdecode(image, cv2.IMREAD_COLOR)
- if image is not None:
- raise DroidCastVersionIncompatible(
- 'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
- # ValueError: cannot reshape array of size 0 into shape (720,1280)
- raise ImageTruncated(str(e))
- # Convert RGB565 to RGB888
- # https://blog.csdn.net/happy08god/article/details/10516871
- # r = (arr & 0b1111100000000000) >> (11 - 3)
- # g = (arr & 0b0000011111100000) >> (5 - 2)
- # b = (arr & 0b0000000000011111) << 3
- # r |= (r & 0b11100000) >> 5
- # g |= (g & 0b11000000) >> 6
- # b |= (b & 0b11100000) >> 5
- # r = r.astype(np.uint8)
- # g = g.astype(np.uint8)
- # b = b.astype(np.uint8)
- # image = cv2.merge([r, g, b])
- # The same as the code above but costs about 3~4ms instead of 10ms.
- # Note that cv2.convertScaleAbs is 5x fast as cv2.multiply, cv2.add is 8x fast as cv2.convertScaleAbs
- # Note that cv2.convertScaleAbs includes rounding
- r = cv2.bitwise_and(arr, 0b1111100000000000)
- r = cv2.convertScaleAbs(r, alpha=0.00390625)
- m = cv2.convertScaleAbs(r, alpha=0.03125)
- cv2.add(r, m, dst=r)
- g = cv2.bitwise_and(arr, 0b0000011111100000)
- g = cv2.convertScaleAbs(g, alpha=0.125)
- m = cv2.convertScaleAbs(g, alpha=0.015625, dst=m)
- cv2.add(g, m, dst=g)
- b = cv2.bitwise_and(arr, 0b0000000000011111)
- b = cv2.convertScaleAbs(b, alpha=8)
- m = cv2.convertScaleAbs(b, alpha=0.03125, dst=m)
- cv2.add(b, m, dst=b)
- image = cv2.merge([r, g, b])
- return image
- def droidcast_wait_startup(self):
- """
- Wait until DroidCast startup completed.
- """
- timeout = Timer(10).start()
- while 1:
- self.sleep(0.25)
- if timeout.reached():
- break
- try:
- resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
- # Route `/` is unavailable, but 404 means startup completed
- if resp.status_code == 404:
- logger.attr('DroidCast', 'online')
- return True
- except requests.exceptions.ConnectionError:
- logger.attr('DroidCast', 'offline')
- logger.warning('Wait DroidCast startup timeout, assume started')
- return False
- def droidcast_uninstall(self):
- """
- Stop DroidCast processes and remove DroidCast APK.
- DroidCast hasn't been installed but a JAVA class call, uninstall is a file delete.
- """
- self.droidcast_stop()
- logger.info('Removing DroidCast')
- self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
- def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
- """
- List all DroidCast processes.
- """
- processes = self.proc_list_uiautomator2()
- for proc in processes:
- if 'com.rayworks.droidcast.Main' in proc.cmdline:
- yield proc
- if 'com.torther.droidcasts.Main' in proc.cmdline:
- yield proc
- if 'ink.mol.droidcast_raw.Main' in proc.cmdline:
- yield proc
- def droidcast_stop(self):
- """
- Stop DroidCast processes.
- """
- logger.info('Stopping DroidCast')
- for proc in self._iter_droidcast_proc():
- logger.info(f'Kill pid={proc.pid}')
- self.adb_shell(['kill', '-s', 9, proc.pid])
- if __name__ == '__main__':
- dc = DroidCast(config='oas1')
- cv2.imshow("iiii", dc.screenshot_droidcast_raw())
- cv2.waitKey(0)
|