| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637 |
- import asyncio
- import json
- import random
- import re
- import socket
- import time
- import numpy as np
- from functools import wraps
- from typing import List
- import websockets
- from adbutils.errors import AdbError
- from uiautomator2 import _Service
- from module.base.decorator import Config, cached_property, del_cached_property
- from module.base.timer import Timer
- from module.base.utils import random_rectangle_point
- from module.device.connection import Connection
- from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
- from module.exception import RequestHumanTakeover, ScriptError
- from module.logger import logger
- def random_normal_distribution(a, b, n=5):
- output = np.mean(np.random.uniform(a, b, size=n))
- return output
- def random_theta():
- theta = np.random.uniform(0, 2 * np.pi)
- return np.array([np.sin(theta), np.cos(theta)])
- def random_rho(dis):
- return random_normal_distribution(-dis, dis)
- def insert_swipe(p0, p3, speed=15, min_distance=10):
- """
- Insert way point from start to end.
- First generate a cubic bézier curve
- Args:
- p0: Start point.
- p3: End point.
- speed: Average move speed, pixels per 10ms.
- min_distance:
- Returns:
- list[list[int]]: List of points.
- Examples:
- > insert_swipe((400, 400), (600, 600), speed=20)
- [[400, 400], [406, 406], [416, 415], [429, 428], [444, 442], [462, 459], [481, 478], [504, 500], [527, 522],
- [545, 540], [560, 557], [573, 570], [584, 582], [592, 590], [597, 596], [600, 600]]
- """
- p0 = np.array(p0)
- p3 = np.array(p3)
- # Random control points in Bézier curve
- distance = np.linalg.norm(p3 - p0)
- p1 = 2 / 3 * p0 + 1 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
- p2 = 1 / 3 * p0 + 2 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
- # Random `t` on Bézier curve, sparse in the middle, dense at start and end
- segments = max(int(distance / speed) + 1, 5)
- lower = random_normal_distribution(-85, -60)
- upper = random_normal_distribution(80, 90)
- theta = np.arange(lower + 0., upper + 0.0001, (upper - lower) / segments)
- ts = np.sin(theta / 180 * np.pi)
- ts = np.sign(ts) * abs(ts) ** 0.9
- ts = (ts - min(ts)) / (max(ts) - min(ts))
- # Generate cubic Bézier curve
- points = []
- prev = (-100, -100)
- for t in ts:
- point = p0 * (1 - t) ** 3 + 3 * p1 * t * (1 - t) ** 2 + 3 * p2 * t ** 2 * (1 - t) + p3 * t ** 3
- point = point.astype(int).tolist()
- if np.linalg.norm(np.subtract(point, prev)) < min_distance:
- continue
- points.append(point)
- prev = point
- # Delete nearing points
- if len(points[1:]):
- distance = np.linalg.norm(np.subtract(points[1:], points[0]), axis=1)
- mask = np.append(True, distance > min_distance)
- points = np.array(points)[mask].tolist()
- else:
- points = [p0, p3]
- return points
- def smooth_path(points: list, min_distance: float = 30.0, offset_range: float = 3.0):
- """
- 路径平滑,基于路径方向的垂直偏移
- Args:
- points: 原始点列表 [(x1, y1), (x2, y2), ...]
- min_distance: 最小插点距离
- offset_range: 垂直偏移范围
- Returns:
- list: 平滑后的点列表
- """
- if len(points) < 2:
- return points
- smooth_points = [points[0]]
- for i in range(len(points) - 1):
- start = np.array(points[i])
- end = np.array(points[i + 1])
- direction = end - start
- distance = np.linalg.norm(direction)
- if distance <= min_distance:
- continue
- # 归一化方向向量
- unit_direction = direction / distance
- # 计算垂直向量
- perpendicular = np.array([-unit_direction[1], unit_direction[0]])
- num_segments = int(distance / min_distance)
- for j in range(1, num_segments + 1):
- ratio = j / num_segments
- interpolated = start + ratio * direction
- # 垂直方向的随机偏移
- perpendicular_offset = np.random.uniform(-offset_range, offset_range)
- final_point = interpolated + perpendicular_offset * perpendicular
- smooth_points.append((int(final_point[0]), int(final_point[1])))
- if smooth_points[-1] != points[-1]:
- smooth_points.append(points[-1])
- return smooth_points
- class Command:
- def __init__(
- self,
- operation: str,
- contact: int = 0,
- x: int = 0,
- y: int = 0,
- ms: int = 10,
- pressure: int = 100
- ):
- """
- See https://github.com/openstf/minitouch#writable-to-the-socket
- Args:
- operation: c, r, d, m, u, w
- contact:
- x:
- y:
- ms:
- pressure:
- """
- self.operation = operation
- self.contact = contact
- self.x = x
- self.y = y
- self.ms = ms
- self.pressure = pressure
- def to_minitouch(self) -> str:
- """
- String that write into minitouch socket
- """
- if self.operation == 'c':
- return f'{self.operation}\n'
- elif self.operation == 'r':
- return f'{self.operation}\n'
- elif self.operation == 'd':
- return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
- elif self.operation == 'm':
- return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
- elif self.operation == 'u':
- return f'{self.operation} {self.contact}\n'
- elif self.operation == 'w':
- return f'{self.operation} {self.ms}\n'
- else:
- return ''
- def to_atx_agent(self, max_x=1280, max_y=720) -> str:
- """
- Dict that send to atx-agent, $DEVICE_URL/minitouch
- See https://github.com/openatx/atx-agent#minitouch%E6%93%8D%E4%BD%9C%E6%96%B9%E6%B3%95
- """
- x, y = self.x / max_x, self.y / max_y
- if self.operation == 'c':
- out = dict(operation=self.operation)
- elif self.operation == 'r':
- out = dict(operation=self.operation)
- elif self.operation == 'd':
- out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
- elif self.operation == 'm':
- out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
- elif self.operation == 'u':
- out = dict(operation=self.operation, index=self.contact)
- elif self.operation == 'w':
- out = dict(operation=self.operation, milliseconds=self.ms)
- else:
- out = dict()
- return json.dumps(out)
- class CommandBuilder:
- """Build command str for minitouch.
- You can use this, to custom actions as you wish::
- with safe_connection(_DEVICE_ID) as connection:
- builder = CommandBuilder()
- builder.down(0, 400, 400, 50)
- builder.commit()
- builder.move(0, 500, 500, 50)
- builder.commit()
- builder.move(0, 800, 400, 50)
- builder.commit()
- builder.up(0)
- builder.commit()
- builder.publish(connection)
- """
- DEFAULT_DELAY = 0.05
- max_x = 1280
- max_y = 720
- def __init__(self, device):
- """
- Args:
- device:
- """
- self.device = device
- self.commands = []
- self.delay = 0
- def convert(self, x, y):
- max_x, max_y = self.device.max_x, self.device.max_y
- orientation = self.device.orientation
- if orientation == 0:
- pass
- elif orientation == 1:
- x, y = 720 - y, x
- max_x, max_y = max_y, max_x
- elif orientation == 2:
- x, y = 1280 - x, 720 - y
- elif orientation == 3:
- x, y = y, 1280 - x
- max_x, max_y = max_y, max_x
- else:
- raise ScriptError(f'Invalid device orientation: {orientation}')
- self.max_x, self.max_y = max_x, max_y
- if not self.device.config.DEVICE_OVER_HTTP:
- # Maximum X and Y coordinates may, but usually do not, match the display size.
- x, y = int(x / 1280 * max_x), int(y / 720 * max_y)
- else:
- # When over http, max_x and max_y are default to 1280 and 720, skip matching display size
- x, y = int(x), int(y)
- return x, y
- def commit(self):
- """ add minitouch command: 'c\n' """
- self.commands.append(Command('c'))
- return self
- def reset(self):
- """ add minitouch command: 'r\n' """
- self.commands.append(Command('r'))
- return self
- def wait(self, ms=10):
- """ add minitouch command: 'w <ms>\n' """
- self.commands.append(Command('w', ms=ms))
- self.delay += ms
- return self
- def up(self, contact=0):
- """ add minitouch command: 'u <contact>\n' """
- self.commands.append(Command('u', contact=contact))
- return self
- def down(self, x, y, contact=0, pressure=100):
- """ add minitouch command: 'd <contact> <x> <y> <pressure>\n' """
- x, y = self.convert(x, y)
- self.commands.append(Command('d', x=x, y=y, contact=contact, pressure=pressure))
- return self
- def move(self, x, y, contact=0, pressure=100):
- """ add minitouch command: 'm <contact> <x> <y> <pressure>\n' """
- x, y = self.convert(x, y)
- self.commands.append(Command('m', x=x, y=y, contact=contact, pressure=pressure))
- return self
- def clear(self):
- """ clear current commands """
- self.commands = []
- self.delay = 0
- def to_minitouch(self) -> str:
- return ''.join([command.to_minitouch() for command in self.commands])
- def to_atx_agent(self) -> List[str]:
- return [command.to_atx_agent(self.max_x, self.max_y) for command in self.commands]
- class MinitouchNotInstalledError(Exception):
- pass
- class MinitouchOccupiedError(Exception):
- pass
- class U2Service(_Service):
- def __init__(self, name, u2obj):
- self.name = name
- self.u2obj = u2obj
- self.service_url = self.u2obj.path2url("/services/" + name)
- def retry(func):
- @wraps(func)
- def retry_wrapper(self, *args, **kwargs):
- """
- Args:
- self (Minitouch):
- """
- init = None
- for _ in range(RETRY_TRIES):
- try:
- if callable(init):
- retry_sleep(_)
- init()
- return func(self, *args, **kwargs)
- # Can't handle
- except RequestHumanTakeover:
- break
- # When adb server was killed
- except ConnectionResetError as e:
- logger.error(e)
- def init():
- self.adb_reconnect()
- # Emulator closed
- except ConnectionAbortedError as e:
- logger.error(e)
- def init():
- self.adb_reconnect()
- # MinitouchNotInstalledError: Received empty data from minitouch
- except MinitouchNotInstalledError as e:
- logger.error(e)
- def init():
- self.install_uiautomator2()
- if self._minitouch_port:
- self.adb_forward_remove(f'tcp:{self._minitouch_port}')
- del_cached_property(self, 'minitouch_builder')
- # MinitouchOccupiedError: Timeout when connecting to minitouch
- except MinitouchOccupiedError as e:
- logger.error(e)
- def init():
- self.restart_atx()
- if self._minitouch_port:
- self.adb_forward_remove(f'tcp:{self._minitouch_port}')
- del_cached_property(self, 'minitouch_builder')
- # AdbError
- except AdbError as e:
- if handle_adb_error(e):
- def init():
- self.adb_reconnect()
- else:
- break
- except BrokenPipeError as e:
- logger.error(e)
- def init():
- del_cached_property(self, 'minitouch_builder')
- # Unknown, probably a trucked image
- except Exception as e:
- logger.exception(e)
- def init():
- pass
- logger.critical(f'Retry {func.__name__}() failed')
- raise RequestHumanTakeover
- return retry_wrapper
- class Minitouch(Connection):
- _minitouch_port: int = 0
- _minitouch_client: socket.socket
- _minitouch_pid: int
- _minitouch_ws: websockets.WebSocketClientProtocol
- max_x: int
- max_y: int
- max_pressure: int = 100
- def _humanized_pressure(self) -> int:
- top = getattr(self, 'max_pressure', 100) or 100
- return random.randint(max(1, top // 2), top)
- def _humanized_dwell(self) -> int:
- return int(random.triangular(45, 130, 65))
- @cached_property
- def minitouch_builder(self):
- self.minitouch_init()
- return CommandBuilder(self)
- @Config.when(DEVICE_OVER_HTTP=False)
- def minitouch_init(self):
- logger.hr('MiniTouch init')
- max_x, max_y = 1280, 720
- max_contacts = 2
- max_pressure = 50
- self.get_orientation()
- self._minitouch_port = self.adb_forward("localabstract:minitouch")
- # No need, minitouch already started by uiautomator2
- # self.adb_shell([self.config.MINITOUCH_FILEPATH_REMOTE])
- retry_timeout = Timer(2).start()
- while 1:
- client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- client.settimeout(1)
- client.connect(('127.0.0.1', self._minitouch_port))
- self._minitouch_client = client
- # get minitouch server info
- socket_out = client.makefile()
- # v <version>
- # protocol version, usually it is 1. needn't use this
- try:
- out = socket_out.readline().replace("\n", "").replace("\r", "")
- except socket.timeout:
- client.close()
- raise MinitouchOccupiedError(
- 'Timeout when connecting to minitouch, '
- 'probably because another connection has been established'
- )
- logger.info(out)
- # ^ <max-contacts> <max-x> <max-y> <max-pressure>
- out = socket_out.readline().replace("\n", "").replace("\r", "")
- logger.info(out)
- try:
- _, max_contacts, max_x, max_y, max_pressure, *_ = out.split(" ")
- break
- except ValueError:
- client.close()
- if retry_timeout.reached():
- raise MinitouchNotInstalledError(
- 'Received empty data from minitouch, '
- 'probably because minitouch is not installed'
- )
- else:
- # Minitouch may not start that fast
- self.sleep(1)
- continue
- # self.max_contacts = max_contacts
- self.max_x = int(max_x)
- self.max_y = int(max_y)
- try:
- self.max_pressure = int(max_pressure)
- except (ValueError, TypeError):
- self.max_pressure = 100
- # $ <pid>
- out = socket_out.readline().replace("\n", "").replace("\r", "")
- logger.info(out)
- _, pid = out.split(" ")
- self._minitouch_pid = pid
- logger.info(
- "minitouch running on port: {}, pid: {}".format(self._minitouch_port, self._minitouch_pid)
- )
- logger.info(
- "max_contact: {}; max_x: {}; max_y: {}; max_pressure: {}".format(
- max_contacts, max_x, max_y, max_pressure
- )
- )
- @Config.when(DEVICE_OVER_HTTP=False)
- def minitouch_send(self):
- content = self.minitouch_builder.to_minitouch()
- # logger.info("send operation: {}".format(content.replace("\n", "\\n")))
- byte_content = content.encode('utf-8')
- self._minitouch_client.sendall(byte_content)
- self._minitouch_client.recv(0)
- time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
- self.minitouch_builder.clear()
- @cached_property
- def _minitouch_loop(self):
- return asyncio.new_event_loop()
- def _minitouch_loop_run(self, event):
- """
- Args:
- event: Async function
- Raises:
- MinitouchOccupiedError
- """
- try:
- return self._minitouch_loop.run_until_complete(event)
- except websockets.ConnectionClosedError as e:
- # ConnectionClosedError: no close frame received or sent
- # ConnectionClosedError: sent 1011 (unexpected error) keepalive ping timeout; no close frame received
- logger.error(e)
- raise MinitouchOccupiedError(
- 'ConnectionClosedError, '
- 'probably because another connection has been established'
- )
- @Config.when(DEVICE_OVER_HTTP=True)
- def minitouch_init(self):
- logger.hr('MiniTouch init')
- self.max_x, self.max_y = 1280, 720
- self.get_orientation()
- logger.info('Stop minitouch service')
- s = U2Service('minitouch', self.u2)
- s.stop()
- while 1:
- if not s.running():
- break
- self.sleep(0.05)
- logger.info('Start minitouch service')
- s.start()
- while 1:
- if s.running():
- break
- self.sleep(0.05)
- # 'ws://127.0.0.1:7912/minitouch'
- url = re.sub(r"^https?://", 'ws://', self.serial) + '/minitouch'
- logger.attr('Minitouch', url)
- async def connect():
- ws = await websockets.connect(url)
- # start @minitouch service
- logger.info(await ws.recv())
- # dial unix:@minitouch
- logger.info(await ws.recv())
- return ws
- self._minitouch_ws = self._minitouch_loop_run(connect())
- @Config.when(DEVICE_OVER_HTTP=True)
- def minitouch_send(self):
- content = self.minitouch_builder.to_atx_agent()
- async def send():
- for row in content:
- # logger.info("send operation: {}".format(row.replace("\n", "\\n")))
- await self._minitouch_ws.send(row)
- self._minitouch_loop_run(send())
- time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
- self.minitouch_builder.clear()
- @retry
- def click_minitouch(self, x, y):
- builder = self.minitouch_builder
- pressure = self._humanized_pressure()
- builder.down(x, y, pressure=pressure).commit().wait(self._humanized_dwell())
- mx, my = x + random.randint(-2, 2), y + random.randint(-2, 2)
- builder.move(mx, my, pressure=pressure).commit().wait(random.randint(8, 20))
- builder.up().commit()
- self.minitouch_send()
- @retry
- def long_click_minitouch(self, x, y, duration=1.0):
- duration = int(duration * 1000)
- builder = self.minitouch_builder
- builder.down(x, y, pressure=self._humanized_pressure()).commit().wait(duration)
- builder.up().commit()
- self.minitouch_send()
- @retry
- def swipe_minitouch(self, p1, p2):
- points = insert_swipe(p0=p1, p3=p2)
- builder = self.minitouch_builder
- builder.down(*points[0]).commit()
- self.minitouch_send()
- for point in points[1:]:
- builder.move(*point).commit().wait(random.randint(6, 15))
- self.minitouch_send()
- builder.up().commit()
- self.minitouch_send()
- @retry
- def drag_minitouch(self, p1, p2, point_random=(-10, -10, 10, 10)):
- p1 = np.array(p1) - random_rectangle_point(point_random)
- p2 = np.array(p2) - random_rectangle_point(point_random)
- points = insert_swipe(p0=p1, p3=p2, speed=20)
- builder = self.minitouch_builder
- builder.down(*points[0]).commit()
- self.minitouch_send()
- for point in points[1:]:
- builder.move(*point).commit().wait(random.randint(6, 15))
- self.minitouch_send()
- builder.move(*p2).commit().wait(140)
- builder.move(*p2).commit().wait(140)
- self.minitouch_send()
- builder.up().commit()
- self.minitouch_send()
- if __name__ == '__main__':
- mm = Minitouch(config='oas1')
- mm.click_minitouch(200, 150)
|