minitouch.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. import asyncio
  2. import json
  3. import re
  4. import socket
  5. import time
  6. from functools import wraps
  7. from typing import List
  8. import websockets
  9. from adbutils.errors import AdbError
  10. from uiautomator2 import _Service
  11. from module.base.decorator import Config, cached_property, del_cached_property
  12. from module.base.timer import Timer
  13. from module.base.utils import *
  14. from module.device.connection import Connection
  15. from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
  16. from module.exception import RequestHumanTakeover, ScriptError
  17. from module.logger import logger
  18. def random_normal_distribution(a, b, n=5):
  19. output = np.mean(np.random.uniform(a, b, size=n))
  20. return output
  21. def random_theta():
  22. theta = np.random.uniform(0, 2 * np.pi)
  23. return np.array([np.sin(theta), np.cos(theta)])
  24. def random_rho(dis):
  25. return random_normal_distribution(-dis, dis)
  26. def insert_swipe(p0, p3, speed=15, min_distance=10):
  27. """
  28. Insert way point from start to end.
  29. First generate a cubic bézier curve
  30. Args:
  31. p0: Start point.
  32. p3: End point.
  33. speed: Average move speed, pixels per 10ms.
  34. min_distance:
  35. Returns:
  36. list[list[int]]: List of points.
  37. Examples:
  38. > insert_swipe((400, 400), (600, 600), speed=20)
  39. [[400, 400], [406, 406], [416, 415], [429, 428], [444, 442], [462, 459], [481, 478], [504, 500], [527, 522],
  40. [545, 540], [560, 557], [573, 570], [584, 582], [592, 590], [597, 596], [600, 600]]
  41. """
  42. p0 = np.array(p0)
  43. p3 = np.array(p3)
  44. # Random control points in Bézier curve
  45. distance = np.linalg.norm(p3 - p0)
  46. p1 = 2 / 3 * p0 + 1 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  47. p2 = 1 / 3 * p0 + 2 / 3 * p3 + random_theta() * random_rho(distance * 0.1)
  48. # Random `t` on Bézier curve, sparse in the middle, dense at start and end
  49. segments = max(int(distance / speed) + 1, 5)
  50. lower = random_normal_distribution(-85, -60)
  51. upper = random_normal_distribution(80, 90)
  52. theta = np.arange(lower + 0., upper + 0.0001, (upper - lower) / segments)
  53. ts = np.sin(theta / 180 * np.pi)
  54. ts = np.sign(ts) * abs(ts) ** 0.9
  55. ts = (ts - min(ts)) / (max(ts) - min(ts))
  56. # Generate cubic Bézier curve
  57. points = []
  58. prev = (-100, -100)
  59. for t in ts:
  60. point = p0 * (1 - t) ** 3 + 3 * p1 * t * (1 - t) ** 2 + 3 * p2 * t ** 2 * (1 - t) + p3 * t ** 3
  61. point = point.astype(np.int).tolist()
  62. if np.linalg.norm(np.subtract(point, prev)) < min_distance:
  63. continue
  64. points.append(point)
  65. prev = point
  66. # Delete nearing points
  67. if len(points[1:]):
  68. distance = np.linalg.norm(np.subtract(points[1:], points[0]), axis=1)
  69. mask = np.append(True, distance > min_distance)
  70. points = np.array(points)[mask].tolist()
  71. else:
  72. points = [p0, p3]
  73. return points
  74. class Command:
  75. def __init__(
  76. self,
  77. operation: str,
  78. contact: int = 0,
  79. x: int = 0,
  80. y: int = 0,
  81. ms: int = 10,
  82. pressure: int = 100
  83. ):
  84. """
  85. See https://github.com/openstf/minitouch#writable-to-the-socket
  86. Args:
  87. operation: c, r, d, m, u, w
  88. contact:
  89. x:
  90. y:
  91. ms:
  92. pressure:
  93. """
  94. self.operation = operation
  95. self.contact = contact
  96. self.x = x
  97. self.y = y
  98. self.ms = ms
  99. self.pressure = pressure
  100. def to_minitouch(self) -> str:
  101. """
  102. String that write into minitouch socket
  103. """
  104. if self.operation == 'c':
  105. return f'{self.operation}\n'
  106. elif self.operation == 'r':
  107. return f'{self.operation}\n'
  108. elif self.operation == 'd':
  109. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  110. elif self.operation == 'm':
  111. return f'{self.operation} {self.contact} {self.x} {self.y} {self.pressure}\n'
  112. elif self.operation == 'u':
  113. return f'{self.operation} {self.contact}\n'
  114. elif self.operation == 'w':
  115. return f'{self.operation} {self.ms}\n'
  116. else:
  117. return ''
  118. def to_atx_agent(self, max_x=1280, max_y=720) -> str:
  119. """
  120. Dict that send to atx-agent, $DEVICE_URL/minitouch
  121. See https://github.com/openatx/atx-agent#minitouch%E6%93%8D%E4%BD%9C%E6%96%B9%E6%B3%95
  122. """
  123. x, y = self.x / max_x, self.y / max_y
  124. if self.operation == 'c':
  125. out = dict(operation=self.operation)
  126. elif self.operation == 'r':
  127. out = dict(operation=self.operation)
  128. elif self.operation == 'd':
  129. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  130. elif self.operation == 'm':
  131. out = dict(operation=self.operation, index=self.contact, pressure=self.pressure, xP=x, yP=y)
  132. elif self.operation == 'u':
  133. out = dict(operation=self.operation, index=self.contact)
  134. elif self.operation == 'w':
  135. out = dict(operation=self.operation, milliseconds=self.ms)
  136. else:
  137. out = dict()
  138. return json.dumps(out)
  139. class CommandBuilder:
  140. """Build command str for minitouch.
  141. You can use this, to custom actions as you wish::
  142. with safe_connection(_DEVICE_ID) as connection:
  143. builder = CommandBuilder()
  144. builder.down(0, 400, 400, 50)
  145. builder.commit()
  146. builder.move(0, 500, 500, 50)
  147. builder.commit()
  148. builder.move(0, 800, 400, 50)
  149. builder.commit()
  150. builder.up(0)
  151. builder.commit()
  152. builder.publish(connection)
  153. """
  154. DEFAULT_DELAY = 0.05
  155. max_x = 1280
  156. max_y = 720
  157. def __init__(self, device):
  158. """
  159. Args:
  160. device:
  161. """
  162. self.device = device
  163. self.commands = []
  164. self.delay = 0
  165. def convert(self, x, y):
  166. max_x, max_y = self.device.max_x, self.device.max_y
  167. orientation = self.device.orientation
  168. if orientation == 0:
  169. pass
  170. elif orientation == 1:
  171. x, y = 720 - y, x
  172. max_x, max_y = max_y, max_x
  173. elif orientation == 2:
  174. x, y = 1280 - x, 720 - y
  175. elif orientation == 3:
  176. x, y = y, 1280 - x
  177. max_x, max_y = max_y, max_x
  178. else:
  179. raise ScriptError(f'Invalid device orientation: {orientation}')
  180. self.max_x, self.max_y = max_x, max_y
  181. if not self.device.config.DEVICE_OVER_HTTP:
  182. # Maximum X and Y coordinates may, but usually do not, match the display size.
  183. x, y = int(x / 1280 * max_x), int(y / 720 * max_y)
  184. else:
  185. # When over http, max_x and max_y are default to 1280 and 720, skip matching display size
  186. x, y = int(x), int(y)
  187. return x, y
  188. def commit(self):
  189. """ add minitouch command: 'c\n' """
  190. self.commands.append(Command('c'))
  191. return self
  192. def reset(self):
  193. """ add minitouch command: 'r\n' """
  194. self.commands.append(Command('r'))
  195. return self
  196. def wait(self, ms=10):
  197. """ add minitouch command: 'w <ms>\n' """
  198. self.commands.append(Command('w', ms=ms))
  199. self.delay += ms
  200. return self
  201. def up(self, contact=0):
  202. """ add minitouch command: 'u <contact>\n' """
  203. self.commands.append(Command('u', contact=contact))
  204. return self
  205. def down(self, x, y, contact=0, pressure=100):
  206. """ add minitouch command: 'd <contact> <x> <y> <pressure>\n' """
  207. x, y = self.convert(x, y)
  208. self.commands.append(Command('d', x=x, y=y, contact=contact, pressure=pressure))
  209. return self
  210. def move(self, x, y, contact=0, pressure=100):
  211. """ add minitouch command: 'm <contact> <x> <y> <pressure>\n' """
  212. x, y = self.convert(x, y)
  213. self.commands.append(Command('m', x=x, y=y, contact=contact, pressure=pressure))
  214. return self
  215. def clear(self):
  216. """ clear current commands """
  217. self.commands = []
  218. self.delay = 0
  219. def to_minitouch(self) -> str:
  220. return ''.join([command.to_minitouch() for command in self.commands])
  221. def to_atx_agent(self) -> List[str]:
  222. return [command.to_atx_agent(self.max_x, self.max_y) for command in self.commands]
  223. class MinitouchNotInstalledError(Exception):
  224. pass
  225. class MinitouchOccupiedError(Exception):
  226. pass
  227. class U2Service(_Service):
  228. def __init__(self, name, u2obj):
  229. self.name = name
  230. self.u2obj = u2obj
  231. self.service_url = self.u2obj.path2url("/services/" + name)
  232. def retry(func):
  233. @wraps(func)
  234. def retry_wrapper(self, *args, **kwargs):
  235. """
  236. Args:
  237. self (Minitouch):
  238. """
  239. init = None
  240. for _ in range(RETRY_TRIES):
  241. try:
  242. if callable(init):
  243. retry_sleep(_)
  244. init()
  245. return func(self, *args, **kwargs)
  246. # Can't handle
  247. except RequestHumanTakeover:
  248. break
  249. # When adb server was killed
  250. except ConnectionResetError as e:
  251. logger.error(e)
  252. def init():
  253. self.adb_reconnect()
  254. # Emulator closed
  255. except ConnectionAbortedError as e:
  256. logger.error(e)
  257. def init():
  258. self.adb_reconnect()
  259. # MinitouchNotInstalledError: Received empty data from minitouch
  260. except MinitouchNotInstalledError as e:
  261. logger.error(e)
  262. def init():
  263. self.install_uiautomator2()
  264. if self._minitouch_port:
  265. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  266. del_cached_property(self, 'minitouch_builder')
  267. # MinitouchOccupiedError: Timeout when connecting to minitouch
  268. except MinitouchOccupiedError as e:
  269. logger.error(e)
  270. def init():
  271. self.restart_atx()
  272. if self._minitouch_port:
  273. self.adb_forward_remove(f'tcp:{self._minitouch_port}')
  274. del_cached_property(self, 'minitouch_builder')
  275. # AdbError
  276. except AdbError as e:
  277. if handle_adb_error(e):
  278. def init():
  279. self.adb_reconnect()
  280. else:
  281. break
  282. except BrokenPipeError as e:
  283. logger.error(e)
  284. def init():
  285. del_cached_property(self, 'minitouch_builder')
  286. # Unknown, probably a trucked image
  287. except Exception as e:
  288. logger.exception(e)
  289. def init():
  290. pass
  291. logger.critical(f'Retry {func.__name__}() failed')
  292. raise RequestHumanTakeover
  293. return retry_wrapper
  294. class Minitouch(Connection):
  295. _minitouch_port: int = 0
  296. _minitouch_client: socket.socket
  297. _minitouch_pid: int
  298. _minitouch_ws: websockets.WebSocketClientProtocol
  299. max_x: int
  300. max_y: int
  301. @cached_property
  302. def minitouch_builder(self):
  303. self.minitouch_init()
  304. return CommandBuilder(self)
  305. @Config.when(DEVICE_OVER_HTTP=False)
  306. def minitouch_init(self):
  307. logger.hr('MiniTouch init')
  308. max_x, max_y = 1280, 720
  309. max_contacts = 2
  310. max_pressure = 50
  311. self.get_orientation()
  312. self._minitouch_port = self.adb_forward("localabstract:minitouch")
  313. # No need, minitouch already started by uiautomator2
  314. # self.adb_shell([self.config.MINITOUCH_FILEPATH_REMOTE])
  315. retry_timeout = Timer(2).start()
  316. while 1:
  317. client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  318. client.settimeout(1)
  319. client.connect(('127.0.0.1', self._minitouch_port))
  320. self._minitouch_client = client
  321. # get minitouch server info
  322. socket_out = client.makefile()
  323. # v <version>
  324. # protocol version, usually it is 1. needn't use this
  325. try:
  326. out = socket_out.readline().replace("\n", "").replace("\r", "")
  327. except socket.timeout:
  328. client.close()
  329. raise MinitouchOccupiedError(
  330. 'Timeout when connecting to minitouch, '
  331. 'probably because another connection has been established'
  332. )
  333. logger.info(out)
  334. # ^ <max-contacts> <max-x> <max-y> <max-pressure>
  335. out = socket_out.readline().replace("\n", "").replace("\r", "")
  336. logger.info(out)
  337. try:
  338. _, max_contacts, max_x, max_y, max_pressure, *_ = out.split(" ")
  339. break
  340. except ValueError:
  341. client.close()
  342. if retry_timeout.reached():
  343. raise MinitouchNotInstalledError(
  344. 'Received empty data from minitouch, '
  345. 'probably because minitouch is not installed'
  346. )
  347. else:
  348. # Minitouch may not start that fast
  349. self.sleep(1)
  350. continue
  351. # self.max_contacts = max_contacts
  352. self.max_x = int(max_x)
  353. self.max_y = int(max_y)
  354. # self.max_pressure = max_pressure
  355. # $ <pid>
  356. out = socket_out.readline().replace("\n", "").replace("\r", "")
  357. logger.info(out)
  358. _, pid = out.split(" ")
  359. self._minitouch_pid = pid
  360. logger.info(
  361. "minitouch running on port: {}, pid: {}".format(self._minitouch_port, self._minitouch_pid)
  362. )
  363. logger.info(
  364. "max_contact: {}; max_x: {}; max_y: {}; max_pressure: {}".format(
  365. max_contacts, max_x, max_y, max_pressure
  366. )
  367. )
  368. @Config.when(DEVICE_OVER_HTTP=False)
  369. def minitouch_send(self):
  370. content = self.minitouch_builder.to_minitouch()
  371. # logger.info("send operation: {}".format(content.replace("\n", "\\n")))
  372. byte_content = content.encode('utf-8')
  373. self._minitouch_client.sendall(byte_content)
  374. self._minitouch_client.recv(0)
  375. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  376. self.minitouch_builder.clear()
  377. @cached_property
  378. def _minitouch_loop(self):
  379. return asyncio.new_event_loop()
  380. def _minitouch_loop_run(self, event):
  381. """
  382. Args:
  383. event: Async function
  384. Raises:
  385. MinitouchOccupiedError
  386. """
  387. try:
  388. return self._minitouch_loop.run_until_complete(event)
  389. except websockets.ConnectionClosedError as e:
  390. # ConnectionClosedError: no close frame received or sent
  391. # ConnectionClosedError: sent 1011 (unexpected error) keepalive ping timeout; no close frame received
  392. logger.error(e)
  393. raise MinitouchOccupiedError(
  394. 'ConnectionClosedError, '
  395. 'probably because another connection has been established'
  396. )
  397. @Config.when(DEVICE_OVER_HTTP=True)
  398. def minitouch_init(self):
  399. logger.hr('MiniTouch init')
  400. self.max_x, self.max_y = 1280, 720
  401. self.get_orientation()
  402. logger.info('Stop minitouch service')
  403. s = U2Service('minitouch', self.u2)
  404. s.stop()
  405. while 1:
  406. if not s.running():
  407. break
  408. self.sleep(0.05)
  409. logger.info('Start minitouch service')
  410. s.start()
  411. while 1:
  412. if s.running():
  413. break
  414. self.sleep(0.05)
  415. # 'ws://127.0.0.1:7912/minitouch'
  416. url = re.sub(r"^https?://", 'ws://', self.serial) + '/minitouch'
  417. logger.attr('Minitouch', url)
  418. async def connect():
  419. ws = await websockets.connect(url)
  420. # start @minitouch service
  421. logger.info(await ws.recv())
  422. # dial unix:@minitouch
  423. logger.info(await ws.recv())
  424. return ws
  425. self._minitouch_ws = self._minitouch_loop_run(connect())
  426. @Config.when(DEVICE_OVER_HTTP=True)
  427. def minitouch_send(self):
  428. content = self.minitouch_builder.to_atx_agent()
  429. async def send():
  430. for row in content:
  431. # logger.info("send operation: {}".format(row.replace("\n", "\\n")))
  432. await self._minitouch_ws.send(row)
  433. self._minitouch_loop_run(send())
  434. time.sleep(self.minitouch_builder.delay / 1000 + self.minitouch_builder.DEFAULT_DELAY)
  435. self.minitouch_builder.clear()
  436. @retry
  437. def click_minitouch(self, x, y):
  438. builder = self.minitouch_builder
  439. builder.down(x, y).commit()
  440. builder.up().commit()
  441. self.minitouch_send()
  442. @retry
  443. def long_click_minitouch(self, x, y, duration=1.0):
  444. duration = int(duration * 1000)
  445. builder = self.minitouch_builder
  446. builder.down(x, y).commit().wait(duration)
  447. builder.up().commit()
  448. self.minitouch_send()
  449. @retry
  450. def swipe_minitouch(self, p1, p2):
  451. points = insert_swipe(p0=p1, p3=p2)
  452. builder = self.minitouch_builder
  453. builder.down(*points[0]).commit()
  454. self.minitouch_send()
  455. for point in points[1:]:
  456. builder.move(*point).commit().wait(10)
  457. self.minitouch_send()
  458. builder.up().commit()
  459. self.minitouch_send()
  460. @retry
  461. def drag_minitouch(self, p1, p2, point_random=(-10, -10, 10, 10)):
  462. p1 = np.array(p1) - random_rectangle_point(point_random)
  463. p2 = np.array(p2) - random_rectangle_point(point_random)
  464. points = insert_swipe(p0=p1, p3=p2, speed=20)
  465. builder = self.minitouch_builder
  466. builder.down(*points[0]).commit()
  467. self.minitouch_send()
  468. for point in points[1:]:
  469. builder.move(*point).commit().wait(10)
  470. self.minitouch_send()
  471. builder.move(*p2).commit().wait(140)
  472. builder.move(*p2).commit().wait(140)
  473. self.minitouch_send()
  474. builder.up().commit()
  475. self.minitouch_send()