minitouch.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. import asyncio
  2. import json
  3. import re
  4. import socket
  5. import time
  6. import numpy as np
  7. from functools import wraps
  8. from typing import List
  9. import websockets
  10. from adbutils.errors import AdbError
  11. from uiautomator2 import _Service
  12. from module.base.decorator import Config, cached_property, del_cached_property
  13. from module.base.timer import Timer
  14. from module.base.utils import random_rectangle_point
  15. from module.device.connection import Connection
  16. from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
  17. from module.exception import RequestHumanTakeover, ScriptError
  18. from module.logger import logger
  19. def random_normal_distribution(a, b, n=5):
  20. output = np.mean(np.random.uniform(a, b, size=n))
  21. return output
  22. def random_theta():
  23. theta = np.random.uniform(0, 2 * np.pi)
  24. return np.array([np.sin(theta), np.cos(theta)])
  25. def random_rho(dis):
  26. return random_normal_distribution(-dis, dis)
  27. def insert_swipe(p0, p3, speed=15, min_distance=10):
  28. """
  29. Insert way point from start to end.
  30. First generate a cubic bézier curve
  31. Args:
  32. p0: Start point.
  33. p3: End point.
  34. speed: Average move speed, pixels per 10ms.
  35. min_distance:
  36. Returns:
  37. list[list[int]]: List of points.
  38. Examples:
  39. > insert_swipe((400, 400), (600, 600), speed=20)
  40. [[400, 400], [406, 406], [416, 415], [429, 428], [444, 442], [462, 459], [481, 478], [504, 500], [527, 522],
  41. [545, 540], [560, 557], [573, 570], [584, 582], [592, 590], [597, 596], [600, 600]]
  42. """
  43. p0 = np.array(p0)
  44. p3 = np.array(p3)
  45. # Random control points in Bézier curve
  46. distance = np.linalg.norm(p3 - p0)
  47. p1 = 2 / 3 * p0 + 1 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  48. p2 = 1 / 3 * p0 + 2 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  49. # Random `t` on Bézier curve, sparse in the middle, dense at start and end
  50. segments = max(int(distance / speed) + 1, 5)
  51. lower = random_normal_distribution(-85, -60)
  52. upper = random_normal_distribution(80, 90)
  53. theta = np.arange(lower + 0., upper + 0.0001, (upper - lower) / segments)
  54. ts = np.sin(theta / 180 * np.pi)
  55. ts = np.sign(ts) * abs(ts) ** 0.9
  56. ts = (ts - min(ts)) / (max(ts) - min(ts))
  57. # Generate cubic Bézier curve
  58. points = []
  59. prev = (-100, -100)
  60. for t in ts:
  61. point = p0 * (1 - t) ** 3 + 3 * p1 * t * (1 - t) ** 2 + 3 * p2 * t ** 2 * (1 - t) + p3 * t ** 3
  62. point = point.astype(np.int).tolist()
  63. if np.linalg.norm(np.subtract(point, prev)) < min_distance:
  64. continue
  65. points.append(point)
  66. prev = point
  67. # Delete nearing points
  68. if len(points[1:]):
  69. distance = np.linalg.norm(np.subtract(points[1:], points[0]), axis=1)
  70. mask = np.append(True, distance > min_distance)
  71. points = np.array(points)[mask].tolist()
  72. else:
  73. points = [p0, p3]
  74. return points
  75. class Command:
  76. def __init__(
  77. self,
  78. operation: str,
  79. contact: int = 0,
  80. x: int = 0,
  81. y: int = 0,
  82. ms: int = 10,
  83. pressure: int = 100
  84. ):
  85. """
  86. See https://github.com/openstf/minitouch#writable-to-the-socket
  87. Args:
  88. operation: c, r, d, m, u, w
  89. contact:
  90. x:
  91. y:
  92. ms:
  93. pressure:
  94. """
  95. self.operation = operation
  96. self.contact = contact
  97. self.x = x
  98. self.y = y
  99. self.ms = ms
  100. self.pressure = pressure
  101. def to_minitouch(self) -> str:
  102. """
  103. String that write into minitouch socket
  104. """
  105. if self.operation == 'c':
  106. return f'{self.operation}\n'
  107. elif self.operation == 'r':
  108. return f'{self.operation}\n'
  109. elif self.operation == 'd':
  110. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  111. elif self.operation == 'm':
  112. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  113. elif self.operation == 'u':
  114. return f'{self.operation} {self.contact}\n'
  115. elif self.operation == 'w':
  116. return f'{self.operation} {self.ms}\n'
  117. else:
  118. return ''
  119. def to_atx_agent(self, max_x=1280, max_y=720) -> str:
  120. """
  121. Dict that send to atx-agent, $DEVICE_URL/minitouch
  122. See https://github.com/openatx/atx-agent#minitouch%E6%93%8D%E4%BD%9C%E6%96%B9%E6%B3%95
  123. """
  124. x, y = self.x / max_x, self.y / max_y
  125. if self.operation == 'c':
  126. out = dict(operation=self.operation)
  127. elif self.operation == 'r':
  128. out = dict(operation=self.operation)
  129. elif self.operation == 'd':
  130. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  131. elif self.operation == 'm':
  132. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  133. elif self.operation == 'u':
  134. out = dict(operation=self.operation, index=self.contact)
  135. elif self.operation == 'w':
  136. out = dict(operation=self.operation, milliseconds=self.ms)
  137. else:
  138. out = dict()
  139. return json.dumps(out)
  140. class CommandBuilder:
  141. """Build command str for minitouch.
  142. You can use this, to custom actions as you wish::
  143. with safe_connection(_DEVICE_ID) as connection:
  144. builder = CommandBuilder()
  145. builder.down(0, 400, 400, 50)
  146. builder.commit()
  147. builder.move(0, 500, 500, 50)
  148. builder.commit()
  149. builder.move(0, 800, 400, 50)
  150. builder.commit()
  151. builder.up(0)
  152. builder.commit()
  153. builder.publish(connection)
  154. """
  155. DEFAULT_DELAY = 0.05
  156. max_x = 1280
  157. max_y = 720
  158. def __init__(self, device):
  159. """
  160. Args:
  161. device:
  162. """
  163. self.device = device
  164. self.commands = []
  165. self.delay = 0
  166. def convert(self, x, y):
  167. max_x, max_y = self.device.max_x, self.device.max_y
  168. orientation = self.device.orientation
  169. if orientation == 0:
  170. pass
  171. elif orientation == 1:
  172. x, y = 720 - y, x
  173. max_x, max_y = max_y, max_x
  174. elif orientation == 2:
  175. x, y = 1280 - x, 720 - y
  176. elif orientation == 3:
  177. x, y = y, 1280 - x
  178. max_x, max_y = max_y, max_x
  179. else:
  180. raise ScriptError(f'Invalid device orientation: {orientation}')
  181. self.max_x, self.max_y = max_x, max_y
  182. if not self.device.config.DEVICE_OVER_HTTP:
  183. # Maximum X and Y coordinates may, but usually do not, match the display size.
  184. x, y = int(x / 1280 * max_x), int(y / 720 * max_y)
  185. else:
  186. # When over http, max_x and max_y are default to 1280 and 720, skip matching display size
  187. x, y = int(x), int(y)
  188. return x, y
  189. def commit(self):
  190. """ add minitouch command: 'c\n' """
  191. self.commands.append(Command('c'))
  192. return self
  193. def reset(self):
  194. """ add minitouch command: 'r\n' """
  195. self.commands.append(Command('r'))
  196. return self
  197. def wait(self, ms=10):
  198. """ add minitouch command: 'w <ms>\n' """
  199. self.commands.append(Command('w', ms=ms))
  200. self.delay += ms
  201. return self
  202. def up(self, contact=0):
  203. """ add minitouch command: 'u <contact>\n' """
  204. self.commands.append(Command('u', contact=contact))
  205. return self
  206. def down(self, x, y, contact=0, pressure=100):
  207. """ add minitouch command: 'd <contact> <x> <y> <pressure>\n' """
  208. x, y = self.convert(x, y)
  209. self.commands.append(Command('d', x=x, y=y, contact=contact, pressure=pressure))
  210. return self
  211. def move(self, x, y, contact=0, pressure=100):
  212. """ add minitouch command: 'm <contact> <x> <y> <pressure>\n' """
  213. x, y = self.convert(x, y)
  214. self.commands.append(Command('m', x=x, y=y, contact=contact, pressure=pressure))
  215. return self
  216. def clear(self):
  217. """ clear current commands """
  218. self.commands = []
  219. self.delay = 0
  220. def to_minitouch(self) -> str:
  221. return ''.join([command.to_minitouch() for command in self.commands])
  222. def to_atx_agent(self) -> List[str]:
  223. return [command.to_atx_agent(self.max_x, self.max_y) for command in self.commands]
  224. class MinitouchNotInstalledError(Exception):
  225. pass
  226. class MinitouchOccupiedError(Exception):
  227. pass
  228. class U2Service(_Service):
  229. def __init__(self, name, u2obj):
  230. self.name = name
  231. self.u2obj = u2obj
  232. self.service_url = self.u2obj.path2url("/services/" + name)
  233. def retry(func):
  234. @wraps(func)
  235. def retry_wrapper(self, *args, **kwargs):
  236. """
  237. Args:
  238. self (Minitouch):
  239. """
  240. init = None
  241. for _ in range(RETRY_TRIES):
  242. try:
  243. if callable(init):
  244. retry_sleep(_)
  245. init()
  246. return func(self, *args, **kwargs)
  247. # Can't handle
  248. except RequestHumanTakeover:
  249. break
  250. # When adb server was killed
  251. except ConnectionResetError as e:
  252. logger.error(e)
  253. def init():
  254. self.adb_reconnect()
  255. # Emulator closed
  256. except ConnectionAbortedError as e:
  257. logger.error(e)
  258. def init():
  259. self.adb_reconnect()
  260. # MinitouchNotInstalledError: Received empty data from minitouch
  261. except MinitouchNotInstalledError as e:
  262. logger.error(e)
  263. def init():
  264. self.install_uiautomator2()
  265. if self._minitouch_port:
  266. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  267. del_cached_property(self, 'minitouch_builder')
  268. # MinitouchOccupiedError: Timeout when connecting to minitouch
  269. except MinitouchOccupiedError as e:
  270. logger.error(e)
  271. def init():
  272. self.restart_atx()
  273. if self._minitouch_port:
  274. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  275. del_cached_property(self, 'minitouch_builder')
  276. # AdbError
  277. except AdbError as e:
  278. if handle_adb_error(e):
  279. def init():
  280. self.adb_reconnect()
  281. else:
  282. break
  283. except BrokenPipeError as e:
  284. logger.error(e)
  285. def init():
  286. del_cached_property(self, 'minitouch_builder')
  287. # Unknown, probably a trucked image
  288. except Exception as e:
  289. logger.exception(e)
  290. def init():
  291. pass
  292. logger.critical(f'Retry {func.__name__}() failed')
  293. raise RequestHumanTakeover
  294. return retry_wrapper
  295. class Minitouch(Connection):
  296. _minitouch_port: int = 0
  297. _minitouch_client: socket.socket
  298. _minitouch_pid: int
  299. _minitouch_ws: websockets.WebSocketClientProtocol
  300. max_x: int
  301. max_y: int
  302. @cached_property
  303. def minitouch_builder(self):
  304. self.minitouch_init()
  305. return CommandBuilder(self)
  306. @Config.when(DEVICE_OVER_HTTP=False)
  307. def minitouch_init(self):
  308. logger.hr('MiniTouch init')
  309. max_x, max_y = 1280, 720
  310. max_contacts = 2
  311. max_pressure = 50
  312. self.get_orientation()
  313. self._minitouch_port = self.adb_forward("localabstract:minitouch")
  314. # No need, minitouch already started by uiautomator2
  315. # self.adb_shell([self.config.MINITOUCH_FILEPATH_REMOTE])
  316. retry_timeout = Timer(2).start()
  317. while 1:
  318. client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  319. client.settimeout(1)
  320. client.connect(('127.0.0.1', self._minitouch_port))
  321. self._minitouch_client = client
  322. # get minitouch server info
  323. socket_out = client.makefile()
  324. # v <version>
  325. # protocol version, usually it is 1. needn't use this
  326. try:
  327. out = socket_out.readline().replace("\n", "").replace("\r", "")
  328. except socket.timeout:
  329. client.close()
  330. raise MinitouchOccupiedError(
  331. 'Timeout when connecting to minitouch, '
  332. 'probably because another connection has been established'
  333. )
  334. logger.info(out)
  335. # ^ <max-contacts> <max-x> <max-y> <max-pressure>
  336. out = socket_out.readline().replace("\n", "").replace("\r", "")
  337. logger.info(out)
  338. try:
  339. _, max_contacts, max_x, max_y, max_pressure, *_ = out.split(" ")
  340. break
  341. except ValueError:
  342. client.close()
  343. if retry_timeout.reached():
  344. raise MinitouchNotInstalledError(
  345. 'Received empty data from minitouch, '
  346. 'probably because minitouch is not installed'
  347. )
  348. else:
  349. # Minitouch may not start that fast
  350. self.sleep(1)
  351. continue
  352. # self.max_contacts = max_contacts
  353. self.max_x = int(max_x)
  354. self.max_y = int(max_y)
  355. # self.max_pressure = max_pressure
  356. # $ <pid>
  357. out = socket_out.readline().replace("\n", "").replace("\r", "")
  358. logger.info(out)
  359. _, pid = out.split(" ")
  360. self._minitouch_pid = pid
  361. logger.info(
  362. "minitouch running on port: {}, pid: {}".format(self._minitouch_port, self._minitouch_pid)
  363. )
  364. logger.info(
  365. "max_contact: {}; max_x: {}; max_y: {}; max_pressure: {}".format(
  366. max_contacts, max_x, max_y, max_pressure
  367. )
  368. )
  369. @Config.when(DEVICE_OVER_HTTP=False)
  370. def minitouch_send(self):
  371. content = self.minitouch_builder.to_minitouch()
  372. # logger.info("send operation: {}".format(content.replace("\n", "\\n")))
  373. byte_content = content.encode('utf-8')
  374. self._minitouch_client.sendall(byte_content)
  375. self._minitouch_client.recv(0)
  376. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  377. self.minitouch_builder.clear()
  378. @cached_property
  379. def _minitouch_loop(self):
  380. return asyncio.new_event_loop()
  381. def _minitouch_loop_run(self, event):
  382. """
  383. Args:
  384. event: Async function
  385. Raises:
  386. MinitouchOccupiedError
  387. """
  388. try:
  389. return self._minitouch_loop.run_until_complete(event)
  390. except websockets.ConnectionClosedError as e:
  391. # ConnectionClosedError: no close frame received or sent
  392. # ConnectionClosedError: sent 1011 (unexpected error) keepalive ping timeout; no close frame received
  393. logger.error(e)
  394. raise MinitouchOccupiedError(
  395. 'ConnectionClosedError, '
  396. 'probably because another connection has been established'
  397. )
  398. @Config.when(DEVICE_OVER_HTTP=True)
  399. def minitouch_init(self):
  400. logger.hr('MiniTouch init')
  401. self.max_x, self.max_y = 1280, 720
  402. self.get_orientation()
  403. logger.info('Stop minitouch service')
  404. s = U2Service('minitouch', self.u2)
  405. s.stop()
  406. while 1:
  407. if not s.running():
  408. break
  409. self.sleep(0.05)
  410. logger.info('Start minitouch service')
  411. s.start()
  412. while 1:
  413. if s.running():
  414. break
  415. self.sleep(0.05)
  416. # 'ws://127.0.0.1:7912/minitouch'
  417. url = re.sub(r"^https?://", 'ws://', self.serial) + '/minitouch'
  418. logger.attr('Minitouch', url)
  419. async def connect():
  420. ws = await websockets.connect(url)
  421. # start @minitouch service
  422. logger.info(await ws.recv())
  423. # dial unix:@minitouch
  424. logger.info(await ws.recv())
  425. return ws
  426. self._minitouch_ws = self._minitouch_loop_run(connect())
  427. @Config.when(DEVICE_OVER_HTTP=True)
  428. def minitouch_send(self):
  429. content = self.minitouch_builder.to_atx_agent()
  430. async def send():
  431. for row in content:
  432. # logger.info("send operation: {}".format(row.replace("\n", "\\n")))
  433. await self._minitouch_ws.send(row)
  434. self._minitouch_loop_run(send())
  435. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  436. self.minitouch_builder.clear()
  437. @retry
  438. def click_minitouch(self, x, y):
  439. builder = self.minitouch_builder
  440. builder.down(x, y).commit()
  441. builder.up().commit()
  442. self.minitouch_send()
  443. @retry
  444. def long_click_minitouch(self, x, y, duration=1.0):
  445. duration = int(duration * 1000)
  446. builder = self.minitouch_builder
  447. builder.down(x, y).commit().wait(duration)
  448. builder.up().commit()
  449. self.minitouch_send()
  450. @retry
  451. def swipe_minitouch(self, p1, p2):
  452. points = insert_swipe(p0=p1, p3=p2)
  453. builder = self.minitouch_builder
  454. builder.down(*points[0]).commit()
  455. self.minitouch_send()
  456. for point in points[1:]:
  457. builder.move(*point).commit().wait(10)
  458. self.minitouch_send()
  459. builder.up().commit()
  460. self.minitouch_send()
  461. @retry
  462. def drag_minitouch(self, p1, p2, point_random=(-10, -10, 10, 10)):
  463. p1 = np.array(p1) - random_rectangle_point(point_random)
  464. p2 = np.array(p2) - random_rectangle_point(point_random)
  465. points = insert_swipe(p0=p1, p3=p2, speed=20)
  466. builder = self.minitouch_builder
  467. builder.down(*points[0]).commit()
  468. self.minitouch_send()
  469. for point in points[1:]:
  470. builder.move(*point).commit().wait(10)
  471. self.minitouch_send()
  472. builder.move(*p2).commit().wait(140)
  473. builder.move(*p2).commit().wait(140)
  474. self.minitouch_send()
  475. builder.up().commit()
  476. self.minitouch_send()
  477. if __name__ == '__main__':
  478. mm = Minitouch(config='oas1')
  479. mm.click_minitouch(200, 150)