minitouch.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637
  1. import asyncio
  2. import json
  3. import random
  4. import re
  5. import socket
  6. import time
  7. import numpy as np
  8. from functools import wraps
  9. from typing import List
  10. import websockets
  11. from adbutils.errors import AdbError
  12. from uiautomator2 import _Service
  13. from module.base.decorator import Config, cached_property, del_cached_property
  14. from module.base.timer import Timer
  15. from module.base.utils import random_rectangle_point
  16. from module.device.connection import Connection
  17. from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
  18. from module.exception import RequestHumanTakeover, ScriptError
  19. from module.logger import logger
  20. def random_normal_distribution(a, b, n=5):
  21. output = np.mean(np.random.uniform(a, b, size=n))
  22. return output
  23. def random_theta():
  24. theta = np.random.uniform(0, 2 * np.pi)
  25. return np.array([np.sin(theta), np.cos(theta)])
  26. def random_rho(dis):
  27. return random_normal_distribution(-dis, dis)
  28. def insert_swipe(p0, p3, speed=15, min_distance=10):
  29. """
  30. Insert way point from start to end.
  31. First generate a cubic bézier curve
  32. Args:
  33. p0: Start point.
  34. p3: End point.
  35. speed: Average move speed, pixels per 10ms.
  36. min_distance:
  37. Returns:
  38. list[list[int]]: List of points.
  39. Examples:
  40. > insert_swipe((400, 400), (600, 600), speed=20)
  41. [[400, 400], [406, 406], [416, 415], [429, 428], [444, 442], [462, 459], [481, 478], [504, 500], [527, 522],
  42. [545, 540], [560, 557], [573, 570], [584, 582], [592, 590], [597, 596], [600, 600]]
  43. """
  44. p0 = np.array(p0)
  45. p3 = np.array(p3)
  46. # Random control points in Bézier curve
  47. distance = np.linalg.norm(p3 - p0)
  48. p1 = 2 / 3 * p0 + 1 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  49. p2 = 1 / 3 * p0 + 2 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  50. # Random `t` on Bézier curve, sparse in the middle, dense at start and end
  51. segments = max(int(distance / speed) + 1, 5)
  52. lower = random_normal_distribution(-85, -60)
  53. upper = random_normal_distribution(80, 90)
  54. theta = np.arange(lower + 0., upper + 0.0001, (upper - lower) / segments)
  55. ts = np.sin(theta / 180 * np.pi)
  56. ts = np.sign(ts) * abs(ts) ** 0.9
  57. ts = (ts - min(ts)) / (max(ts) - min(ts))
  58. # Generate cubic Bézier curve
  59. points = []
  60. prev = (-100, -100)
  61. for t in ts:
  62. point = p0 * (1 - t) ** 3 + 3 * p1 * t * (1 - t) ** 2 + 3 * p2 * t ** 2 * (1 - t) + p3 * t ** 3
  63. point = point.astype(int).tolist()
  64. if np.linalg.norm(np.subtract(point, prev)) < min_distance:
  65. continue
  66. points.append(point)
  67. prev = point
  68. # Delete nearing points
  69. if len(points[1:]):
  70. distance = np.linalg.norm(np.subtract(points[1:], points[0]), axis=1)
  71. mask = np.append(True, distance > min_distance)
  72. points = np.array(points)[mask].tolist()
  73. else:
  74. points = [p0, p3]
  75. return points
  76. def smooth_path(points: list, min_distance: float = 30.0, offset_range: float = 3.0):
  77. """
  78. 路径平滑,基于路径方向的垂直偏移
  79. Args:
  80. points: 原始点列表 [(x1, y1), (x2, y2), ...]
  81. min_distance: 最小插点距离
  82. offset_range: 垂直偏移范围
  83. Returns:
  84. list: 平滑后的点列表
  85. """
  86. if len(points) < 2:
  87. return points
  88. smooth_points = [points[0]]
  89. for i in range(len(points) - 1):
  90. start = np.array(points[i])
  91. end = np.array(points[i + 1])
  92. direction = end - start
  93. distance = np.linalg.norm(direction)
  94. if distance <= min_distance:
  95. continue
  96. # 归一化方向向量
  97. unit_direction = direction / distance
  98. # 计算垂直向量
  99. perpendicular = np.array([-unit_direction[1], unit_direction[0]])
  100. num_segments = int(distance / min_distance)
  101. for j in range(1, num_segments + 1):
  102. ratio = j / num_segments
  103. interpolated = start + ratio * direction
  104. # 垂直方向的随机偏移
  105. perpendicular_offset = np.random.uniform(-offset_range, offset_range)
  106. final_point = interpolated + perpendicular_offset * perpendicular
  107. smooth_points.append((int(final_point[0]), int(final_point[1])))
  108. if smooth_points[-1] != points[-1]:
  109. smooth_points.append(points[-1])
  110. return smooth_points
  111. class Command:
  112. def __init__(
  113. self,
  114. operation: str,
  115. contact: int = 0,
  116. x: int = 0,
  117. y: int = 0,
  118. ms: int = 10,
  119. pressure: int = 100
  120. ):
  121. """
  122. See https://github.com/openstf/minitouch#writable-to-the-socket
  123. Args:
  124. operation: c, r, d, m, u, w
  125. contact:
  126. x:
  127. y:
  128. ms:
  129. pressure:
  130. """
  131. self.operation = operation
  132. self.contact = contact
  133. self.x = x
  134. self.y = y
  135. self.ms = ms
  136. self.pressure = pressure
  137. def to_minitouch(self) -> str:
  138. """
  139. String that write into minitouch socket
  140. """
  141. if self.operation == 'c':
  142. return f'{self.operation}\n'
  143. elif self.operation == 'r':
  144. return f'{self.operation}\n'
  145. elif self.operation == 'd':
  146. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  147. elif self.operation == 'm':
  148. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  149. elif self.operation == 'u':
  150. return f'{self.operation} {self.contact}\n'
  151. elif self.operation == 'w':
  152. return f'{self.operation} {self.ms}\n'
  153. else:
  154. return ''
  155. def to_atx_agent(self, max_x=1280, max_y=720) -> str:
  156. """
  157. Dict that send to atx-agent, $DEVICE_URL/minitouch
  158. See https://github.com/openatx/atx-agent#minitouch%E6%93%8D%E4%BD%9C%E6%96%B9%E6%B3%95
  159. """
  160. x, y = self.x / max_x, self.y / max_y
  161. if self.operation == 'c':
  162. out = dict(operation=self.operation)
  163. elif self.operation == 'r':
  164. out = dict(operation=self.operation)
  165. elif self.operation == 'd':
  166. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  167. elif self.operation == 'm':
  168. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  169. elif self.operation == 'u':
  170. out = dict(operation=self.operation, index=self.contact)
  171. elif self.operation == 'w':
  172. out = dict(operation=self.operation, milliseconds=self.ms)
  173. else:
  174. out = dict()
  175. return json.dumps(out)
  176. class CommandBuilder:
  177. """Build command str for minitouch.
  178. You can use this, to custom actions as you wish::
  179. with safe_connection(_DEVICE_ID) as connection:
  180. builder = CommandBuilder()
  181. builder.down(0, 400, 400, 50)
  182. builder.commit()
  183. builder.move(0, 500, 500, 50)
  184. builder.commit()
  185. builder.move(0, 800, 400, 50)
  186. builder.commit()
  187. builder.up(0)
  188. builder.commit()
  189. builder.publish(connection)
  190. """
  191. DEFAULT_DELAY = 0.05
  192. max_x = 1280
  193. max_y = 720
  194. def __init__(self, device):
  195. """
  196. Args:
  197. device:
  198. """
  199. self.device = device
  200. self.commands = []
  201. self.delay = 0
  202. def convert(self, x, y):
  203. max_x, max_y = self.device.max_x, self.device.max_y
  204. orientation = self.device.orientation
  205. if orientation == 0:
  206. pass
  207. elif orientation == 1:
  208. x, y = 720 - y, x
  209. max_x, max_y = max_y, max_x
  210. elif orientation == 2:
  211. x, y = 1280 - x, 720 - y
  212. elif orientation == 3:
  213. x, y = y, 1280 - x
  214. max_x, max_y = max_y, max_x
  215. else:
  216. raise ScriptError(f'Invalid device orientation: {orientation}')
  217. self.max_x, self.max_y = max_x, max_y
  218. if not self.device.config.DEVICE_OVER_HTTP:
  219. # Maximum X and Y coordinates may, but usually do not, match the display size.
  220. x, y = int(x / 1280 * max_x), int(y / 720 * max_y)
  221. else:
  222. # When over http, max_x and max_y are default to 1280 and 720, skip matching display size
  223. x, y = int(x), int(y)
  224. return x, y
  225. def commit(self):
  226. """ add minitouch command: 'c\n' """
  227. self.commands.append(Command('c'))
  228. return self
  229. def reset(self):
  230. """ add minitouch command: 'r\n' """
  231. self.commands.append(Command('r'))
  232. return self
  233. def wait(self, ms=10):
  234. """ add minitouch command: 'w <ms>\n' """
  235. self.commands.append(Command('w', ms=ms))
  236. self.delay += ms
  237. return self
  238. def up(self, contact=0):
  239. """ add minitouch command: 'u <contact>\n' """
  240. self.commands.append(Command('u', contact=contact))
  241. return self
  242. def down(self, x, y, contact=0, pressure=100):
  243. """ add minitouch command: 'd <contact> <x> <y> <pressure>\n' """
  244. x, y = self.convert(x, y)
  245. self.commands.append(Command('d', x=x, y=y, contact=contact, pressure=pressure))
  246. return self
  247. def move(self, x, y, contact=0, pressure=100):
  248. """ add minitouch command: 'm <contact> <x> <y> <pressure>\n' """
  249. x, y = self.convert(x, y)
  250. self.commands.append(Command('m', x=x, y=y, contact=contact, pressure=pressure))
  251. return self
  252. def clear(self):
  253. """ clear current commands """
  254. self.commands = []
  255. self.delay = 0
  256. def to_minitouch(self) -> str:
  257. return ''.join([command.to_minitouch() for command in self.commands])
  258. def to_atx_agent(self) -> List[str]:
  259. return [command.to_atx_agent(self.max_x, self.max_y) for command in self.commands]
  260. class MinitouchNotInstalledError(Exception):
  261. pass
  262. class MinitouchOccupiedError(Exception):
  263. pass
  264. class U2Service(_Service):
  265. def __init__(self, name, u2obj):
  266. self.name = name
  267. self.u2obj = u2obj
  268. self.service_url = self.u2obj.path2url("/services/" + name)
  269. def retry(func):
  270. @wraps(func)
  271. def retry_wrapper(self, *args, **kwargs):
  272. """
  273. Args:
  274. self (Minitouch):
  275. """
  276. init = None
  277. for _ in range(RETRY_TRIES):
  278. try:
  279. if callable(init):
  280. retry_sleep(_)
  281. init()
  282. return func(self, *args, **kwargs)
  283. # Can't handle
  284. except RequestHumanTakeover:
  285. break
  286. # When adb server was killed
  287. except ConnectionResetError as e:
  288. logger.error(e)
  289. def init():
  290. self.adb_reconnect()
  291. # Emulator closed
  292. except ConnectionAbortedError as e:
  293. logger.error(e)
  294. def init():
  295. self.adb_reconnect()
  296. # MinitouchNotInstalledError: Received empty data from minitouch
  297. except MinitouchNotInstalledError as e:
  298. logger.error(e)
  299. def init():
  300. self.install_uiautomator2()
  301. if self._minitouch_port:
  302. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  303. del_cached_property(self, 'minitouch_builder')
  304. # MinitouchOccupiedError: Timeout when connecting to minitouch
  305. except MinitouchOccupiedError as e:
  306. logger.error(e)
  307. def init():
  308. self.restart_atx()
  309. if self._minitouch_port:
  310. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  311. del_cached_property(self, 'minitouch_builder')
  312. # AdbError
  313. except AdbError as e:
  314. if handle_adb_error(e):
  315. def init():
  316. self.adb_reconnect()
  317. else:
  318. break
  319. except BrokenPipeError as e:
  320. logger.error(e)
  321. def init():
  322. del_cached_property(self, 'minitouch_builder')
  323. # Unknown, probably a trucked image
  324. except Exception as e:
  325. logger.exception(e)
  326. def init():
  327. pass
  328. logger.critical(f'Retry {func.__name__}() failed')
  329. raise RequestHumanTakeover
  330. return retry_wrapper
  331. class Minitouch(Connection):
  332. _minitouch_port: int = 0
  333. _minitouch_client: socket.socket
  334. _minitouch_pid: int
  335. _minitouch_ws: websockets.WebSocketClientProtocol
  336. max_x: int
  337. max_y: int
  338. max_pressure: int = 100
  339. def _humanized_pressure(self) -> int:
  340. top = getattr(self, 'max_pressure', 100) or 100
  341. return random.randint(max(20, top // 2), top)
  342. def _humanized_dwell(self) -> int:
  343. return int(random.triangular(45, 130, 65))
  344. @cached_property
  345. def minitouch_builder(self):
  346. self.minitouch_init()
  347. return CommandBuilder(self)
  348. @Config.when(DEVICE_OVER_HTTP=False)
  349. def minitouch_init(self):
  350. logger.hr('MiniTouch init')
  351. max_x, max_y = 1280, 720
  352. max_contacts = 2
  353. max_pressure = 50
  354. self.get_orientation()
  355. self._minitouch_port = self.adb_forward("localabstract:minitouch")
  356. # No need, minitouch already started by uiautomator2
  357. # self.adb_shell([self.config.MINITOUCH_FILEPATH_REMOTE])
  358. retry_timeout = Timer(2).start()
  359. while 1:
  360. client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  361. client.settimeout(1)
  362. client.connect(('127.0.0.1', self._minitouch_port))
  363. self._minitouch_client = client
  364. # get minitouch server info
  365. socket_out = client.makefile()
  366. # v <version>
  367. # protocol version, usually it is 1. needn't use this
  368. try:
  369. out = socket_out.readline().replace("\n", "").replace("\r", "")
  370. except socket.timeout:
  371. client.close()
  372. raise MinitouchOccupiedError(
  373. 'Timeout when connecting to minitouch, '
  374. 'probably because another connection has been established'
  375. )
  376. logger.info(out)
  377. # ^ <max-contacts> <max-x> <max-y> <max-pressure>
  378. out = socket_out.readline().replace("\n", "").replace("\r", "")
  379. logger.info(out)
  380. try:
  381. _, max_contacts, max_x, max_y, max_pressure, *_ = out.split(" ")
  382. break
  383. except ValueError:
  384. client.close()
  385. if retry_timeout.reached():
  386. raise MinitouchNotInstalledError(
  387. 'Received empty data from minitouch, '
  388. 'probably because minitouch is not installed'
  389. )
  390. else:
  391. # Minitouch may not start that fast
  392. self.sleep(1)
  393. continue
  394. # self.max_contacts = max_contacts
  395. self.max_x = int(max_x)
  396. self.max_y = int(max_y)
  397. try:
  398. self.max_pressure = int(max_pressure)
  399. except (ValueError, TypeError):
  400. self.max_pressure = 100
  401. # $ <pid>
  402. out = socket_out.readline().replace("\n", "").replace("\r", "")
  403. logger.info(out)
  404. _, pid = out.split(" ")
  405. self._minitouch_pid = pid
  406. logger.info(
  407. "minitouch running on port: {}, pid: {}".format(self._minitouch_port, self._minitouch_pid)
  408. )
  409. logger.info(
  410. "max_contact: {}; max_x: {}; max_y: {}; max_pressure: {}".format(
  411. max_contacts, max_x, max_y, max_pressure
  412. )
  413. )
  414. @Config.when(DEVICE_OVER_HTTP=False)
  415. def minitouch_send(self):
  416. content = self.minitouch_builder.to_minitouch()
  417. # logger.info("send operation: {}".format(content.replace("\n", "\\n")))
  418. byte_content = content.encode('utf-8')
  419. self._minitouch_client.sendall(byte_content)
  420. self._minitouch_client.recv(0)
  421. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  422. self.minitouch_builder.clear()
  423. @cached_property
  424. def _minitouch_loop(self):
  425. return asyncio.new_event_loop()
  426. def _minitouch_loop_run(self, event):
  427. """
  428. Args:
  429. event: Async function
  430. Raises:
  431. MinitouchOccupiedError
  432. """
  433. try:
  434. return self._minitouch_loop.run_until_complete(event)
  435. except websockets.ConnectionClosedError as e:
  436. # ConnectionClosedError: no close frame received or sent
  437. # ConnectionClosedError: sent 1011 (unexpected error) keepalive ping timeout; no close frame received
  438. logger.error(e)
  439. raise MinitouchOccupiedError(
  440. 'ConnectionClosedError, '
  441. 'probably because another connection has been established'
  442. )
  443. @Config.when(DEVICE_OVER_HTTP=True)
  444. def minitouch_init(self):
  445. logger.hr('MiniTouch init')
  446. self.max_x, self.max_y = 1280, 720
  447. self.get_orientation()
  448. logger.info('Stop minitouch service')
  449. s = U2Service('minitouch', self.u2)
  450. s.stop()
  451. while 1:
  452. if not s.running():
  453. break
  454. self.sleep(0.05)
  455. logger.info('Start minitouch service')
  456. s.start()
  457. while 1:
  458. if s.running():
  459. break
  460. self.sleep(0.05)
  461. # 'ws://127.0.0.1:7912/minitouch'
  462. url = re.sub(r"^https?://", 'ws://', self.serial) + '/minitouch'
  463. logger.attr('Minitouch', url)
  464. async def connect():
  465. ws = await websockets.connect(url)
  466. # start @minitouch service
  467. logger.info(await ws.recv())
  468. # dial unix:@minitouch
  469. logger.info(await ws.recv())
  470. return ws
  471. self._minitouch_ws = self._minitouch_loop_run(connect())
  472. @Config.when(DEVICE_OVER_HTTP=True)
  473. def minitouch_send(self):
  474. content = self.minitouch_builder.to_atx_agent()
  475. async def send():
  476. for row in content:
  477. # logger.info("send operation: {}".format(row.replace("\n", "\\n")))
  478. await self._minitouch_ws.send(row)
  479. self._minitouch_loop_run(send())
  480. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  481. self.minitouch_builder.clear()
  482. @retry
  483. def click_minitouch(self, x, y):
  484. builder = self.minitouch_builder
  485. pressure = self._humanized_pressure()
  486. builder.down(x, y, pressure=pressure).commit().wait(self._humanized_dwell())
  487. mx, my = x + random.randint(-2, 2), y + random.randint(-2, 2)
  488. builder.move(mx, my, pressure=pressure).commit().wait(random.randint(8, 20))
  489. builder.up().commit()
  490. self.minitouch_send()
  491. @retry
  492. def long_click_minitouch(self, x, y, duration=1.0):
  493. duration = int(duration * 1000)
  494. builder = self.minitouch_builder
  495. builder.down(x, y, pressure=self._humanized_pressure()).commit().wait(duration)
  496. builder.up().commit()
  497. self.minitouch_send()
  498. @retry
  499. def swipe_minitouch(self, p1, p2):
  500. points = insert_swipe(p0=p1, p3=p2)
  501. builder = self.minitouch_builder
  502. builder.down(*points[0]).commit()
  503. self.minitouch_send()
  504. for point in points[1:]:
  505. builder.move(*point).commit().wait(random.randint(6, 15))
  506. self.minitouch_send()
  507. builder.up().commit()
  508. self.minitouch_send()
  509. @retry
  510. def drag_minitouch(self, p1, p2, point_random=(-10, -10, 10, 10)):
  511. p1 = np.array(p1) - random_rectangle_point(point_random)
  512. p2 = np.array(p2) - random_rectangle_point(point_random)
  513. points = insert_swipe(p0=p1, p3=p2, speed=20)
  514. builder = self.minitouch_builder
  515. builder.down(*points[0]).commit()
  516. self.minitouch_send()
  517. for point in points[1:]:
  518. builder.move(*point).commit().wait(random.randint(6, 15))
  519. self.minitouch_send()
  520. builder.move(*p2).commit().wait(140)
  521. builder.move(*p2).commit().wait(140)
  522. self.minitouch_send()
  523. builder.up().commit()
  524. self.minitouch_send()
  525. if __name__ == '__main__':
  526. mm = Minitouch(config='oas1')
  527. mm.click_minitouch(200, 150)