minitouch.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. import asyncio
  2. import json
  3. import re
  4. import socket
  5. import time
  6. import numpy as np
  7. from functools import wraps
  8. from typing import List
  9. import websockets
  10. from adbutils.errors import AdbError
  11. from uiautomator2 import _Service
  12. from module.base.decorator import Config, cached_property, del_cached_property
  13. from module.base.timer import Timer
  14. from module.base.utils import random_rectangle_point
  15. from module.device.connection import Connection
  16. from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
  17. from module.exception import RequestHumanTakeover, ScriptError
  18. from module.logger import logger
  19. def random_normal_distribution(a, b, n=5):
  20. output = np.mean(np.random.uniform(a, b, size=n))
  21. return output
  22. def random_theta():
  23. theta = np.random.uniform(0, 2 * np.pi)
  24. return np.array([np.sin(theta), np.cos(theta)])
  25. def random_rho(dis):
  26. return random_normal_distribution(-dis, dis)
  27. def insert_swipe(p0, p3, speed=15, min_distance=10):
  28. """
  29. Insert way point from start to end.
  30. First generate a cubic bézier curve
  31. Args:
  32. p0: Start point.
  33. p3: End point.
  34. speed: Average move speed, pixels per 10ms.
  35. min_distance:
  36. Returns:
  37. list[list[int]]: List of points.
  38. Examples:
  39. > insert_swipe((400, 400), (600, 600), speed=20)
  40. [[400, 400], [406, 406], [416, 415], [429, 428], [444, 442], [462, 459], [481, 478], [504, 500], [527, 522],
  41. [545, 540], [560, 557], [573, 570], [584, 582], [592, 590], [597, 596], [600, 600]]
  42. """
  43. p0 = np.array(p0)
  44. p3 = np.array(p3)
  45. # Random control points in Bézier curve
  46. distance = np.linalg.norm(p3 - p0)
  47. p1 = 2 / 3 * p0 + 1 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  48. p2 = 1 / 3 * p0 + 2 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  49. # Random `t` on Bézier curve, sparse in the middle, dense at start and end
  50. segments = max(int(distance / speed) + 1, 5)
  51. lower = random_normal_distribution(-85, -60)
  52. upper = random_normal_distribution(80, 90)
  53. theta = np.arange(lower + 0., upper + 0.0001, (upper - lower) / segments)
  54. ts = np.sin(theta / 180 * np.pi)
  55. ts = np.sign(ts) * abs(ts) ** 0.9
  56. ts = (ts - min(ts)) / (max(ts) - min(ts))
  57. # Generate cubic Bézier curve
  58. points = []
  59. prev = (-100, -100)
  60. for t in ts:
  61. point = p0 * (1 - t) ** 3 + 3 * p1 * t * (1 - t) ** 2 + 3 * p2 * t ** 2 * (1 - t) + p3 * t ** 3
  62. point = point.astype(int).tolist()
  63. if np.linalg.norm(np.subtract(point, prev)) < min_distance:
  64. continue
  65. points.append(point)
  66. prev = point
  67. # Delete nearing points
  68. if len(points[1:]):
  69. distance = np.linalg.norm(np.subtract(points[1:], points[0]), axis=1)
  70. mask = np.append(True, distance > min_distance)
  71. points = np.array(points)[mask].tolist()
  72. else:
  73. points = [p0, p3]
  74. return points
  75. def smooth_path(points: list, min_distance: float = 30.0, offset_range: float = 3.0):
  76. """
  77. 路径平滑,基于路径方向的垂直偏移
  78. Args:
  79. points: 原始点列表 [(x1, y1), (x2, y2), ...]
  80. min_distance: 最小插点距离
  81. offset_range: 垂直偏移范围
  82. Returns:
  83. list: 平滑后的点列表
  84. """
  85. if len(points) < 2:
  86. return points
  87. smooth_points = [points[0]]
  88. for i in range(len(points) - 1):
  89. start = np.array(points[i])
  90. end = np.array(points[i + 1])
  91. direction = end - start
  92. distance = np.linalg.norm(direction)
  93. if distance <= min_distance:
  94. continue
  95. # 归一化方向向量
  96. unit_direction = direction / distance
  97. # 计算垂直向量
  98. perpendicular = np.array([-unit_direction[1], unit_direction[0]])
  99. num_segments = int(distance / min_distance)
  100. for j in range(1, num_segments + 1):
  101. ratio = j / num_segments
  102. interpolated = start + ratio * direction
  103. # 垂直方向的随机偏移
  104. perpendicular_offset = np.random.uniform(-offset_range, offset_range)
  105. final_point = interpolated + perpendicular_offset * perpendicular
  106. smooth_points.append((int(final_point[0]), int(final_point[1])))
  107. if smooth_points[-1] != points[-1]:
  108. smooth_points.append(points[-1])
  109. return smooth_points
  110. class Command:
  111. def __init__(
  112. self,
  113. operation: str,
  114. contact: int = 0,
  115. x: int = 0,
  116. y: int = 0,
  117. ms: int = 10,
  118. pressure: int = 100
  119. ):
  120. """
  121. See https://github.com/openstf/minitouch#writable-to-the-socket
  122. Args:
  123. operation: c, r, d, m, u, w
  124. contact:
  125. x:
  126. y:
  127. ms:
  128. pressure:
  129. """
  130. self.operation = operation
  131. self.contact = contact
  132. self.x = x
  133. self.y = y
  134. self.ms = ms
  135. self.pressure = pressure
  136. def to_minitouch(self) -> str:
  137. """
  138. String that write into minitouch socket
  139. """
  140. if self.operation == 'c':
  141. return f'{self.operation}\n'
  142. elif self.operation == 'r':
  143. return f'{self.operation}\n'
  144. elif self.operation == 'd':
  145. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  146. elif self.operation == 'm':
  147. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  148. elif self.operation == 'u':
  149. return f'{self.operation} {self.contact}\n'
  150. elif self.operation == 'w':
  151. return f'{self.operation} {self.ms}\n'
  152. else:
  153. return ''
  154. def to_atx_agent(self, max_x=1280, max_y=720) -> str:
  155. """
  156. Dict that send to atx-agent, $DEVICE_URL/minitouch
  157. See https://github.com/openatx/atx-agent#minitouch%E6%93%8D%E4%BD%9C%E6%96%B9%E6%B3%95
  158. """
  159. x, y = self.x / max_x, self.y / max_y
  160. if self.operation == 'c':
  161. out = dict(operation=self.operation)
  162. elif self.operation == 'r':
  163. out = dict(operation=self.operation)
  164. elif self.operation == 'd':
  165. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  166. elif self.operation == 'm':
  167. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  168. elif self.operation == 'u':
  169. out = dict(operation=self.operation, index=self.contact)
  170. elif self.operation == 'w':
  171. out = dict(operation=self.operation, milliseconds=self.ms)
  172. else:
  173. out = dict()
  174. return json.dumps(out)
  175. class CommandBuilder:
  176. """Build command str for minitouch.
  177. You can use this, to custom actions as you wish::
  178. with safe_connection(_DEVICE_ID) as connection:
  179. builder = CommandBuilder()
  180. builder.down(0, 400, 400, 50)
  181. builder.commit()
  182. builder.move(0, 500, 500, 50)
  183. builder.commit()
  184. builder.move(0, 800, 400, 50)
  185. builder.commit()
  186. builder.up(0)
  187. builder.commit()
  188. builder.publish(connection)
  189. """
  190. DEFAULT_DELAY = 0.05
  191. max_x = 1280
  192. max_y = 720
  193. def __init__(self, device):
  194. """
  195. Args:
  196. device:
  197. """
  198. self.device = device
  199. self.commands = []
  200. self.delay = 0
  201. def convert(self, x, y):
  202. max_x, max_y = self.device.max_x, self.device.max_y
  203. orientation = self.device.orientation
  204. if orientation == 0:
  205. pass
  206. elif orientation == 1:
  207. x, y = 720 - y, x
  208. max_x, max_y = max_y, max_x
  209. elif orientation == 2:
  210. x, y = 1280 - x, 720 - y
  211. elif orientation == 3:
  212. x, y = y, 1280 - x
  213. max_x, max_y = max_y, max_x
  214. else:
  215. raise ScriptError(f'Invalid device orientation: {orientation}')
  216. self.max_x, self.max_y = max_x, max_y
  217. if not self.device.config.DEVICE_OVER_HTTP:
  218. # Maximum X and Y coordinates may, but usually do not, match the display size.
  219. x, y = int(x / 1280 * max_x), int(y / 720 * max_y)
  220. else:
  221. # When over http, max_x and max_y are default to 1280 and 720, skip matching display size
  222. x, y = int(x), int(y)
  223. return x, y
  224. def commit(self):
  225. """ add minitouch command: 'c\n' """
  226. self.commands.append(Command('c'))
  227. return self
  228. def reset(self):
  229. """ add minitouch command: 'r\n' """
  230. self.commands.append(Command('r'))
  231. return self
  232. def wait(self, ms=10):
  233. """ add minitouch command: 'w <ms>\n' """
  234. self.commands.append(Command('w', ms=ms))
  235. self.delay += ms
  236. return self
  237. def up(self, contact=0):
  238. """ add minitouch command: 'u <contact>\n' """
  239. self.commands.append(Command('u', contact=contact))
  240. return self
  241. def down(self, x, y, contact=0, pressure=100):
  242. """ add minitouch command: 'd <contact> <x> <y> <pressure>\n' """
  243. x, y = self.convert(x, y)
  244. self.commands.append(Command('d', x=x, y=y, contact=contact, pressure=pressure))
  245. return self
  246. def move(self, x, y, contact=0, pressure=100):
  247. """ add minitouch command: 'm <contact> <x> <y> <pressure>\n' """
  248. x, y = self.convert(x, y)
  249. self.commands.append(Command('m', x=x, y=y, contact=contact, pressure=pressure))
  250. return self
  251. def clear(self):
  252. """ clear current commands """
  253. self.commands = []
  254. self.delay = 0
  255. def to_minitouch(self) -> str:
  256. return ''.join([command.to_minitouch() for command in self.commands])
  257. def to_atx_agent(self) -> List[str]:
  258. return [command.to_atx_agent(self.max_x, self.max_y) for command in self.commands]
  259. class MinitouchNotInstalledError(Exception):
  260. pass
  261. class MinitouchOccupiedError(Exception):
  262. pass
  263. class U2Service(_Service):
  264. def __init__(self, name, u2obj):
  265. self.name = name
  266. self.u2obj = u2obj
  267. self.service_url = self.u2obj.path2url("/services/" + name)
  268. def retry(func):
  269. @wraps(func)
  270. def retry_wrapper(self, *args, **kwargs):
  271. """
  272. Args:
  273. self (Minitouch):
  274. """
  275. init = None
  276. for _ in range(RETRY_TRIES):
  277. try:
  278. if callable(init):
  279. retry_sleep(_)
  280. init()
  281. return func(self, *args, **kwargs)
  282. # Can't handle
  283. except RequestHumanTakeover:
  284. break
  285. # When adb server was killed
  286. except ConnectionResetError as e:
  287. logger.error(e)
  288. def init():
  289. self.adb_reconnect()
  290. # Emulator closed
  291. except ConnectionAbortedError as e:
  292. logger.error(e)
  293. def init():
  294. self.adb_reconnect()
  295. # MinitouchNotInstalledError: Received empty data from minitouch
  296. except MinitouchNotInstalledError as e:
  297. logger.error(e)
  298. def init():
  299. self.install_uiautomator2()
  300. if self._minitouch_port:
  301. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  302. del_cached_property(self, 'minitouch_builder')
  303. # MinitouchOccupiedError: Timeout when connecting to minitouch
  304. except MinitouchOccupiedError as e:
  305. logger.error(e)
  306. def init():
  307. self.restart_atx()
  308. if self._minitouch_port:
  309. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  310. del_cached_property(self, 'minitouch_builder')
  311. # AdbError
  312. except AdbError as e:
  313. if handle_adb_error(e):
  314. def init():
  315. self.adb_reconnect()
  316. else:
  317. break
  318. except BrokenPipeError as e:
  319. logger.error(e)
  320. def init():
  321. del_cached_property(self, 'minitouch_builder')
  322. # Unknown, probably a trucked image
  323. except Exception as e:
  324. logger.exception(e)
  325. def init():
  326. pass
  327. logger.critical(f'Retry {func.__name__}() failed')
  328. raise RequestHumanTakeover
  329. return retry_wrapper
  330. class Minitouch(Connection):
  331. _minitouch_port: int = 0
  332. _minitouch_client: socket.socket
  333. _minitouch_pid: int
  334. _minitouch_ws: websockets.WebSocketClientProtocol
  335. max_x: int
  336. max_y: int
  337. @cached_property
  338. def minitouch_builder(self):
  339. self.minitouch_init()
  340. return CommandBuilder(self)
  341. @Config.when(DEVICE_OVER_HTTP=False)
  342. def minitouch_init(self):
  343. logger.hr('MiniTouch init')
  344. max_x, max_y = 1280, 720
  345. max_contacts = 2
  346. max_pressure = 50
  347. self.get_orientation()
  348. self._minitouch_port = self.adb_forward("localabstract:minitouch")
  349. # No need, minitouch already started by uiautomator2
  350. # self.adb_shell([self.config.MINITOUCH_FILEPATH_REMOTE])
  351. retry_timeout = Timer(2).start()
  352. while 1:
  353. client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  354. client.settimeout(1)
  355. client.connect(('127.0.0.1', self._minitouch_port))
  356. self._minitouch_client = client
  357. # get minitouch server info
  358. socket_out = client.makefile()
  359. # v <version>
  360. # protocol version, usually it is 1. needn't use this
  361. try:
  362. out = socket_out.readline().replace("\n", "").replace("\r", "")
  363. except socket.timeout:
  364. client.close()
  365. raise MinitouchOccupiedError(
  366. 'Timeout when connecting to minitouch, '
  367. 'probably because another connection has been established'
  368. )
  369. logger.info(out)
  370. # ^ <max-contacts> <max-x> <max-y> <max-pressure>
  371. out = socket_out.readline().replace("\n", "").replace("\r", "")
  372. logger.info(out)
  373. try:
  374. _, max_contacts, max_x, max_y, max_pressure, *_ = out.split(" ")
  375. break
  376. except ValueError:
  377. client.close()
  378. if retry_timeout.reached():
  379. raise MinitouchNotInstalledError(
  380. 'Received empty data from minitouch, '
  381. 'probably because minitouch is not installed'
  382. )
  383. else:
  384. # Minitouch may not start that fast
  385. self.sleep(1)
  386. continue
  387. # self.max_contacts = max_contacts
  388. self.max_x = int(max_x)
  389. self.max_y = int(max_y)
  390. # self.max_pressure = max_pressure
  391. # $ <pid>
  392. out = socket_out.readline().replace("\n", "").replace("\r", "")
  393. logger.info(out)
  394. _, pid = out.split(" ")
  395. self._minitouch_pid = pid
  396. logger.info(
  397. "minitouch running on port: {}, pid: {}".format(self._minitouch_port, self._minitouch_pid)
  398. )
  399. logger.info(
  400. "max_contact: {}; max_x: {}; max_y: {}; max_pressure: {}".format(
  401. max_contacts, max_x, max_y, max_pressure
  402. )
  403. )
  404. @Config.when(DEVICE_OVER_HTTP=False)
  405. def minitouch_send(self):
  406. content = self.minitouch_builder.to_minitouch()
  407. # logger.info("send operation: {}".format(content.replace("\n", "\\n")))
  408. byte_content = content.encode('utf-8')
  409. self._minitouch_client.sendall(byte_content)
  410. self._minitouch_client.recv(0)
  411. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  412. self.minitouch_builder.clear()
  413. @cached_property
  414. def _minitouch_loop(self):
  415. return asyncio.new_event_loop()
  416. def _minitouch_loop_run(self, event):
  417. """
  418. Args:
  419. event: Async function
  420. Raises:
  421. MinitouchOccupiedError
  422. """
  423. try:
  424. return self._minitouch_loop.run_until_complete(event)
  425. except websockets.ConnectionClosedError as e:
  426. # ConnectionClosedError: no close frame received or sent
  427. # ConnectionClosedError: sent 1011 (unexpected error) keepalive ping timeout; no close frame received
  428. logger.error(e)
  429. raise MinitouchOccupiedError(
  430. 'ConnectionClosedError, '
  431. 'probably because another connection has been established'
  432. )
  433. @Config.when(DEVICE_OVER_HTTP=True)
  434. def minitouch_init(self):
  435. logger.hr('MiniTouch init')
  436. self.max_x, self.max_y = 1280, 720
  437. self.get_orientation()
  438. logger.info('Stop minitouch service')
  439. s = U2Service('minitouch', self.u2)
  440. s.stop()
  441. while 1:
  442. if not s.running():
  443. break
  444. self.sleep(0.05)
  445. logger.info('Start minitouch service')
  446. s.start()
  447. while 1:
  448. if s.running():
  449. break
  450. self.sleep(0.05)
  451. # 'ws://127.0.0.1:7912/minitouch'
  452. url = re.sub(r"^https?://", 'ws://', self.serial) + '/minitouch'
  453. logger.attr('Minitouch', url)
  454. async def connect():
  455. ws = await websockets.connect(url)
  456. # start @minitouch service
  457. logger.info(await ws.recv())
  458. # dial unix:@minitouch
  459. logger.info(await ws.recv())
  460. return ws
  461. self._minitouch_ws = self._minitouch_loop_run(connect())
  462. @Config.when(DEVICE_OVER_HTTP=True)
  463. def minitouch_send(self):
  464. content = self.minitouch_builder.to_atx_agent()
  465. async def send():
  466. for row in content:
  467. # logger.info("send operation: {}".format(row.replace("\n", "\\n")))
  468. await self._minitouch_ws.send(row)
  469. self._minitouch_loop_run(send())
  470. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  471. self.minitouch_builder.clear()
  472. @retry
  473. def click_minitouch(self, x, y):
  474. builder = self.minitouch_builder
  475. builder.down(x, y).commit()
  476. builder.up().commit()
  477. self.minitouch_send()
  478. @retry
  479. def long_click_minitouch(self, x, y, duration=1.0):
  480. duration = int(duration * 1000)
  481. builder = self.minitouch_builder
  482. builder.down(x, y).commit().wait(duration)
  483. builder.up().commit()
  484. self.minitouch_send()
  485. @retry
  486. def swipe_minitouch(self, p1, p2):
  487. points = insert_swipe(p0=p1, p3=p2)
  488. builder = self.minitouch_builder
  489. builder.down(*points[0]).commit()
  490. self.minitouch_send()
  491. for point in points[1:]:
  492. builder.move(*point).commit().wait(10)
  493. self.minitouch_send()
  494. builder.up().commit()
  495. self.minitouch_send()
  496. @retry
  497. def drag_minitouch(self, p1, p2, point_random=(-10, -10, 10, 10)):
  498. p1 = np.array(p1) - random_rectangle_point(point_random)
  499. p2 = np.array(p2) - random_rectangle_point(point_random)
  500. points = insert_swipe(p0=p1, p3=p2, speed=20)
  501. builder = self.minitouch_builder
  502. builder.down(*points[0]).commit()
  503. self.minitouch_send()
  504. for point in points[1:]:
  505. builder.move(*point).commit().wait(10)
  506. self.minitouch_send()
  507. builder.move(*p2).commit().wait(140)
  508. builder.move(*p2).commit().wait(140)
  509. self.minitouch_send()
  510. builder.up().commit()
  511. self.minitouch_send()
  512. if __name__ == '__main__':
  513. mm = Minitouch(config='oas1')
  514. mm.click_minitouch(200, 150)