Преглед на файлове

Merge pull request #4 from runhey/dev

Dev
huangrunheng преди 3 години
родител
ревизия
e10d816861

+ 160 - 0
module/base/cBezier.py

@@ -0,0 +1,160 @@
+'''
+author: cbb
+这个库基于贝塞尔曲线实现模拟人手动滑动的轨迹
+可以用于selenium轨迹的模拟,或者生成轨迹数组用于js加密通过网站服务器后端分控检测
+QQ群 134064772
+里面的人说话好听,个个都是人才
+copy from https://github.com/2833844911/gurs
+'''
+
+import numpy as np
+import math
+import random
+
+class BezierTrajectory:
+
+    @classmethod
+    def _bztsg(cls, dataTrajectory):
+        lengthOfdata = len(dataTrajectory)
+
+        def staer(x):
+            t = ((x - dataTrajectory[0][0]) / (dataTrajectory[-1][0] - dataTrajectory[0][0]))
+            y = np.array([0, 0], dtype=np.float64)
+            for s in range(len(dataTrajectory)):
+                y += dataTrajectory[s] * ((math.factorial(lengthOfdata - 1) / (
+                            math.factorial(s) * math.factorial(lengthOfdata - 1 - s))) * math.pow(t, s) * math.pow(
+                    (1 - t), lengthOfdata - 1 - s))
+            return y[1]
+
+        return staer
+
+    @classmethod
+    def _type(cls, type, x, numberList):
+        numberListre = []
+        pin = (x[1] - x[0]) / numberList
+        if type == 0:
+            for i in range(numberList):
+                numberListre.append(i * pin)
+            if pin >= 0:
+                numberListre = numberListre[::-1]
+        elif type == 1:
+            for i in range(numberList):
+                numberListre.append(1 * ((i * pin) ** 2))
+            numberListre = numberListre[::-1]
+        elif type == 2:
+            for i in range(numberList):
+                numberListre.append(1 * ((i * pin - x[1]) ** 2))
+
+        elif type == 3:
+            dataTrajectory = [np.array([0,0]), np.array([(x[1]-x[0])*0.8, (x[1]-x[0])*0.6]), np.array([x[1]-x[0], 0])]
+            fun = cls._bztsg(dataTrajectory)
+            numberListre = [0]
+            for i in range(1,numberList):
+                numberListre.append(fun(i * pin) + numberListre[-1])
+            if pin >= 0:
+                numberListre = numberListre[::-1]
+        numberListre = np.abs(np.array(numberListre) - max(numberListre))
+        biaoNumberList = ((numberListre - numberListre[numberListre.argmin()]) / (
+                    numberListre[numberListre.argmax()] - numberListre[numberListre.argmin()])) * (x[1] - x[0]) + x[0]
+        biaoNumberList[0] = x[0]
+        biaoNumberList[-1] = x[1]
+        return biaoNumberList
+
+    @classmethod
+    def getFun(cls, s):
+        '''
+
+        :param s: 传入P点
+        :return: 返回公式
+        '''
+        dataTrajectory = []
+        for i in s:
+            dataTrajectory.append(np.array(i))
+        return cls._bztsg(dataTrajectory)
+
+    @classmethod
+    def simulation(cls, start, end, le=1, deviation=0, bias=0.5):
+        '''
+
+        :param start:开始点的坐标 如 start = [0, 0]
+        :param end:结束点的坐标 如 end = [100, 100]
+        :param le:几阶贝塞尔曲线,越大越复杂 如 le = 4
+        :param deviation:轨迹上下波动的范围 如 deviation = 10
+        :param bias:波动范围的分布位置 如 bias = 0.5
+        :return:返回一个字典equation对应该曲线的方程,P对应贝塞尔曲线的影响点
+        '''
+        start = np.array(start)
+        end = np.array(end)
+        cbb = []
+        if le != 1:
+            e = (1 - bias) / (le - 1)
+            cbb = [[bias + e * i, bias + e * (i + 1)] for i in range(le - 1)]
+
+        dataTrajectoryList = [start]
+
+        t = random.choice([-1, 1])
+        w = 0
+        for i in cbb:
+            px1 = start[0] + (end[0] - start[0]) * (random.random() * (i[1] - i[0]) + (i[0]))
+            p = np.array([px1, cls._bztsg([start, end])(px1) + t * deviation])
+            dataTrajectoryList.append(p)
+            w += 1
+            if w >= 2:
+                w = 0
+                t = -1 * t
+
+        dataTrajectoryList.append(end)
+        return {"equation": cls._bztsg(dataTrajectoryList), "P": np.array(dataTrajectoryList)}
+
+    @classmethod
+    def trackArray(cls, start, end, numberList, le=1, deviation=0, bias=0.5, type=0, cbb=0, yhh=10):
+        '''
+
+        :param start:开始点的坐标 如 start = [0, 0]
+        :param end:结束点的坐标 如 end = [100, 100]
+        :param numberList:返回的数组的轨迹点的数量 numberList = 150
+        :param le:几阶贝塞尔曲线,越大越复杂 如 le = 4
+        :param deviation:轨迹上下波动的范围 如 deviation = 10
+        :param bias:波动范围的分布位置 如 bias = 0.5
+        :param type:0表示均速滑动,1表示先慢后快,2表示先快后慢,3表示先慢中间快后慢 如 type = 1
+        :param cbb:在终点来回摆动的次数
+        :param yhh:在终点来回摆动的范围
+        :return:返回一个字典trackArray对应轨迹数组,P对应贝塞尔曲线的影响点
+        '''
+        s = []
+        fun = cls.simulation(start, end, le, deviation, bias)
+        w = fun['P']
+        fun = fun["equation"]
+        if cbb != 0:
+            numberListOfcbb = round(numberList*0.2/(cbb+1))
+            numberList -= (numberListOfcbb*(cbb+1))
+
+            xTrackArray = cls._type(type, [start[0], end[0]], numberList)
+            for i in xTrackArray:
+                s.append([i, fun(i)])
+            dq = yhh/cbb
+            kg = 0
+            ends = np.copy(end)
+            for i in range(cbb):
+                if kg == 0:
+                    d = np.array([end[0] + (yhh-dq*i), ((end[1]-start[1])/(end[0]-start[0]))*(end[0]+(yhh-dq*i)) +(end[1]-((end[1]-start[1])/(end[0]-start[0]))*end[0])   ])
+                    kg = 1
+                else:
+                    d = np.array([end[0] - (yhh - dq * i), ((end[1]-start[1])/(end[0]-start[0]))*(end[0]-(yhh-dq*i)) +(end[1]-((end[1]-start[1])/(end[0]-start[0]))*end[0])  ])
+                    kg = 0
+                print(d)
+                y = cls.trackArray(ends, d, numberListOfcbb, le=2, deviation=0, bias=0.5, type=0, cbb=0, yhh=10)
+                s += list(y['trackArray'])
+                ends = d
+            y = cls.trackArray(ends, end, numberListOfcbb, le=2, deviation=0, bias=0.5, type=0, cbb=0, yhh=10)
+            s += list(y['trackArray'])
+
+        else:
+            xTrackArray = cls._type(type, [start[0], end[0]], numberList)
+            for i in xTrackArray:
+                s.append([i, fun(i)])
+        # return {"trackArray": np.array(s), "P": w}
+        return [[int(s[0]), int(s[1])] for s in s]
+
+# print(BezierTrajectory.trackArray(start=[0, 0], end=[100, 100], numberList=50, le=2,
+#                     deviation=0, bias=0.5, type=0, cbb=0, yhh=10))

+ 808 - 18
module/base/utils.py

@@ -1,10 +1,12 @@
-# This Python file uses the following encoding: utf-8
 import re
 import re
 
 
 import cv2
 import cv2
 import numpy as np
 import numpy as np
 from PIL import Image
 from PIL import Image
 
 
+REGEX_NODE = re.compile(r'(-?[A-Za-z]+)(-?\d+)')
+
+
 def random_normal_distribution_int(a, b, n=3):
 def random_normal_distribution_int(a, b, n=3):
     """Generate a normal distribution int within the interval. Use the average value of several random numbers to
     """Generate a normal distribution int within the interval. Use the average value of several random numbers to
     simulate normal distribution.
     simulate normal distribution.
@@ -23,6 +25,119 @@ def random_normal_distribution_int(a, b, n=3):
     else:
     else:
         return b
         return b
 
 
+
+def random_rectangle_point(area, n=3):
+    """Choose a random point in an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        n (int): The amount of numbers in simulation. Default to 3.
+
+    Returns:
+        tuple(int): (x, y)
+    """
+    x = random_normal_distribution_int(area[0], area[2], n=n)
+    y = random_normal_distribution_int(area[1], area[3], n=n)
+    return x, y
+
+
+def random_rectangle_vector(vector, box, random_range=(0, 0, 0, 0), padding=15):
+    """Place a vector in a box randomly.
+
+    Args:
+        vector: (x, y)
+        box: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        random_range (tuple): Add a random_range to vector. (x_min, y_min, x_max, y_max).
+        padding (int):
+
+    Returns:
+        tuple(int), tuple(int): start_point, end_point.
+    """
+    vector = np.array(vector) + random_rectangle_point(random_range)
+    vector = np.round(vector).astype(np.int)
+    half_vector = np.round(vector / 2).astype(np.int)
+    box = np.array(box) + np.append(np.abs(half_vector) + padding, -np.abs(half_vector) - padding)
+    center = random_rectangle_point(box)
+    start_point = center - half_vector
+    end_point = start_point + vector
+    return tuple(start_point), tuple(end_point)
+
+
+def random_rectangle_vector_opted(
+        vector, box, random_range=(0, 0, 0, 0), padding=15, whitelist_area=None, blacklist_area=None):
+    """
+    Place a vector in a box randomly.
+
+    When emulator/game stuck, it treats a swipe as a click, clicking at the end of swipe path.
+    To prevent this, random results need to be filtered.
+
+    Args:
+        vector: (x, y)
+        box: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        random_range (tuple): Add a random_range to vector. (x_min, y_min, x_max, y_max).
+        padding (int):
+        whitelist_area: (list[tuple[int]]):
+            A list of area that safe to click. Swipe path will end there.
+        blacklist_area: (list[tuple[int]]):
+            If none of the whitelist_area satisfies current vector, blacklist_area will be used.
+            Delete random path that ends in any blacklist_area.
+
+    Returns:
+        tuple(int), tuple(int): start_point, end_point.
+    """
+    vector = np.array(vector) + random_rectangle_point(random_range)
+    vector = np.round(vector).astype(np.int)
+    half_vector = np.round(vector / 2).astype(np.int)
+    box_pad = np.array(box) + np.append(np.abs(half_vector) + padding, -np.abs(half_vector) - padding)
+    box_pad = area_offset(box_pad, half_vector)
+    segment = int(np.linalg.norm(vector) // 70) + 1
+
+    def in_blacklist(end):
+        if not blacklist_area:
+            return False
+        for x in range(segment + 1):
+            point = - vector * x / segment + end
+            for area in blacklist_area:
+                if point_in_area(point, area, threshold=0):
+                    return True
+        return False
+
+    if whitelist_area:
+        for area in whitelist_area:
+            area = area_limit(area, box_pad)
+            if all([x > 0 for x in area_size(area)]):
+                end_point = random_rectangle_point(area)
+                for _ in range(10):
+                    if in_blacklist(end_point):
+                        continue
+                    return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+    for _ in range(100):
+        end_point = random_rectangle_point(box_pad)
+        if in_blacklist(end_point):
+            continue
+        return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+    end_point = random_rectangle_point(box_pad)
+    return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+
+def random_line_segments(p1, p2, n, random_range=(0, 0, 0, 0)):
+    """Cut a line into multiple segments.
+
+    Args:
+        p1: (x, y).
+        p2: (x, y).
+        n: Number of slice.
+        random_range: Add a random_range to points.
+
+    Returns:
+        list[tuple]: [(x0, y0), (x1, y1), (x2, y2)]
+    """
+    return [tuple((((n - index) * p1 + index * p2) / n).astype(int) + random_rectangle_point(random_range))
+            for index in range(0, n + 1)]
+
+
 def ensure_time(second, n=3, precision=3):
 def ensure_time(second, n=3, precision=3):
     """Ensure to be time.
     """Ensure to be time.
 
 
@@ -53,6 +168,190 @@ def ensure_time(second, n=3, precision=3):
         return second
         return second
 
 
 
 
+def ensure_int(*args):
+    """
+    Convert all elements to int.
+    Return the same structure as nested objects.
+
+    Args:
+        *args:
+
+    Returns:
+        list:
+    """
+
+    def to_int(item):
+        try:
+            return int(item)
+        except TypeError:
+            result = [to_int(i) for i in item]
+            if len(result) == 1:
+                result = result[0]
+            return result
+
+    return to_int(args)
+
+
+def area_offset(area, offset):
+    """
+    Move an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        offset: (x, y).
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    upper_left_x, upper_left_y, bottom_right_x, bottom_right_y = area
+    x, y = offset
+    return upper_left_x + x, upper_left_y + y, bottom_right_x + x, bottom_right_y + y
+
+
+def area_pad(area, pad=10):
+    """
+    Inner offset an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        pad (int):
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    upper_left_x, upper_left_y, bottom_right_x, bottom_right_y = area
+    return upper_left_x + pad, upper_left_y + pad, bottom_right_x - pad, bottom_right_y - pad
+
+
+def limit_in(x, lower, upper):
+    """
+    Limit x within range (lower, upper)
+
+    Args:
+        x:
+        lower:
+        upper:
+
+    Returns:
+        int, float:
+    """
+    return max(min(x, upper), lower)
+
+
+def area_limit(area1, area2):
+    """
+    Limit an area in another area.
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    x_lower, y_lower, x_upper, y_upper = area2
+    return (
+        limit_in(area1[0], x_lower, x_upper),
+        limit_in(area1[1], y_lower, y_upper),
+        limit_in(area1[2], x_lower, x_upper),
+        limit_in(area1[3], y_lower, y_upper),
+    )
+
+
+def area_size(area):
+    """
+    Area size or shape.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (x, y).
+    """
+    return (
+        max(area[2] - area[0], 0),
+        max(area[3] - area[1], 0)
+    )
+
+
+def point_limit(point, area):
+    """
+    Limit point in an area.
+
+    Args:
+        point: (x, y).
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (x, y).
+    """
+    return (
+        limit_in(point[0], area[0], area[2]),
+        limit_in(point[1], area[1], area[3])
+    )
+
+
+def point_in_area(point, area, threshold=5):
+    """
+
+    Args:
+        point: (x, y).
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    return area[0] - threshold < point[0] < area[2] + threshold and area[1] - threshold < point[1] < area[3] + threshold
+
+
+def area_in_area(area1, area2, threshold=5):
+    """
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    return area2[0] - threshold <= area1[0] \
+           and area2[1] - threshold <= area1[1] \
+           and area1[2] <= area2[2] + threshold \
+           and area1[3] <= area2[3] + threshold
+
+
+def area_cross_area(area1, area2, threshold=5):
+    """
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    # https://www.yiiven.cn/rect-is-intersection.html
+    xa1, ya1, xa2, ya2 = area1
+    xb1, yb1, xb2, yb2 = area2
+    return abs(xb2 + xb1 - xa2 - xa1) <= xa2 - xa1 + xb2 - xb1 + threshold * 2 \
+           and abs(yb2 + yb1 - ya2 - ya1) <= ya2 - ya1 + yb2 - yb1 + threshold * 2
+
+
+def float2str(n, decimal=3):
+    """
+    Args:
+        n (float):
+        decimal (int):
+
+    Returns:
+        str:
+    """
+    return str(round(n, decimal)).ljust(decimal + 2, "0")
+
+
 def point2str(x, y, length=4):
 def point2str(x, y, length=4):
     """
     """
     Args:
     Args:
@@ -66,31 +365,522 @@ def point2str(x, y, length=4):
     return '(%s, %s)' % (str(int(x)).rjust(length), str(int(y)).rjust(length))
     return '(%s, %s)' % (str(int(x)).rjust(length), str(int(y)).rjust(length))
 
 
 
 
-def random_rectangle_point(area, n=3):
-    """Choose a random point in an area.
+def col2name(col):
+    """
+    Convert a zero indexed column cell reference to a string.
 
 
     Args:
     Args:
-        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
-        n (int): The amount of numbers in simulation. Default to 3.
+       col: The cell column. Int.
 
 
     Returns:
     Returns:
-        tuple(int): (x, y)
+        Column style string.
+
+    Examples:
+        0 -> A, 3 -> D, 35 -> AJ, -1 -> -A
     """
     """
-    x = random_normal_distribution_int(area[0], area[2], n=n)
-    y = random_normal_distribution_int(area[1], area[3], n=n)
-    return x, y
 
 
-def random_line_segments(p1, p2, n, random_range=(0, 0, 0, 0)):
-    """Cut a line into multiple segments.
+    col_neg = col < 0
+    if col_neg:
+        col_num = -col
+    else:
+        col_num = col + 1  # Change to 1-index.
+    col_str = ''
+
+    while col_num:
+        # Set remainder from 1 .. 26
+        remainder = col_num % 26
+
+        if remainder == 0:
+            remainder = 26
+
+        # Convert the remainder to a character.
+        col_letter = chr(remainder + 64)
+
+        # Accumulate the column letters, right to left.
+        col_str = col_letter + col_str
+
+        # Get the next order of magnitude.
+        col_num = int((col_num - 1) / 26)
+
+    if col_neg:
+        return '-' + col_str
+    else:
+        return col_str
+
+
+def name2col(col_str):
+    """
+    Convert a cell reference in A1 notation to a zero indexed row and column.
 
 
     Args:
     Args:
-        p1: (x, y).
-        p2: (x, y).
-        n: Number of slice.
-        random_range: Add a random_range to points.
+       col_str:  A1 style string.
 
 
     Returns:
     Returns:
-        list[tuple]: [(x0, y0), (x1, y1), (x2, y2)]
+        row, col: Zero indexed cell row and column indices.
     """
     """
-    return [tuple((((n - index) * p1 + index * p2) / n).astype(int) + random_rectangle_point(random_range))
-            for index in range(0, n + 1)]
+    # Convert base26 column string to number.
+    expn = 0
+    col = 0
+    col_neg = col_str.startswith('-')
+    col_str = col_str.strip('-').upper()
+
+    for char in reversed(col_str):
+        col += (ord(char) - 64) * (26 ** expn)
+        expn += 1
+
+    if col_neg:
+        return -col
+    else:
+        return col - 1  # Convert 1-index to zero-index
+
+
+def node2location(node):
+    """
+    See location2node()
+
+    Args:
+        node (str): Example: 'E3'
+
+    Returns:
+        tuple[int]: Example: (4, 2)
+    """
+    res = REGEX_NODE.search(node)
+    if res:
+        x, y = res.group(1), res.group(2)
+        y = int(y)
+        if y > 0:
+            y -= 1
+        return name2col(x), y
+    else:
+        # Whatever
+        return ord(node[0]) % 32 - 1, int(node[1:]) - 1
+
+
+def location2node(location):
+    """
+    Convert location tuple to an Excel-like cell.
+    Accept negative values also.
+
+         -2   -1    0    1    2    3
+    -2 -B-2 -A-2  A-2  B-2  C-2  D-2
+    -1 -B-1 -A-1  A-1  B-1  C-1  D-1
+     0  -B1  -A1   A1   B1   C1   D1
+     1  -B2  -A2   A2   B2   C2   D2
+     2  -B3  -A3   A3   B3   C3   D3
+     3  -B4  -A4   A4   B4   C4   D4
+
+    # To generate the table above
+    index = range(-2, 4)
+    row = '   ' + ' '.join([str(i).rjust(4) for i in index])
+    print(row)
+    for y in index:
+        row = str(y).rjust(2) + ' ' + ' '.join([location2node((x, y)).rjust(4) for x in index])
+        print(row)
+
+    def check(node):
+        return point2str(*node2location(location2node(node)), length=2)
+    row = '   ' + ' '.join([str(i).rjust(8) for i in index])
+    print(row)
+    for y in index:
+        row = str(y).rjust(2) + ' ' + ' '.join([check((x, y)).rjust(4) for x in index])
+        print(row)
+
+    Args:
+        location (tuple[int]):
+
+    Returns:
+        str:
+    """
+    x, y = location
+    if y >= 0:
+        y += 1
+    return col2name(x) + str(y)
+
+
+def load_image(file, area=None):
+    """
+    Load an image like pillow and drop alpha channel.
+
+    Args:
+        file (str):
+        area (tuple):
+
+    Returns:
+        np.ndarray:
+    """
+    image = Image.open(file)
+    if area is not None:
+        image = image.crop(area)
+    image = np.array(image)
+    channel = image.shape[2] if len(image.shape) > 2 else 1
+    if channel > 3:
+        image = image[:, :, :3].copy()
+    return image
+
+
+def save_image(image, file):
+    """
+    Save an image like pillow.
+
+    Args:
+        image (np.ndarray):
+        file (str):
+    """
+    # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    # cv2.imwrite(file, image)
+    Image.fromarray(image).save(file)
+
+
+def crop(image, area, copy=True):
+    """
+    Crop image like pillow, when using opencv / numpy.
+    Provides a black background if cropping outside of image.
+
+    Args:
+        image (np.ndarray):
+        area:
+        copy (bool):
+
+    Returns:
+        np.ndarray:
+    """
+    x1, y1, x2, y2 = map(int, map(round, area))
+    h, w = image.shape[:2]
+    border = np.maximum((0 - y1, y2 - h, 0 - x1, x2 - w), 0)
+    x1, y1, x2, y2 = np.maximum((x1, y1, x2, y2), 0)
+    image = image[y1:y2, x1:x2]
+    if sum(border) > 0:
+        image = cv2.copyMakeBorder(image, *border, borderType=cv2.BORDER_CONSTANT, value=(0, 0, 0))
+    if copy:
+        image = image.copy()
+    return image
+
+
+def resize(image, size):
+    """
+    Resize image like pillow image.resize(), but implement in opencv.
+    Pillow uses PIL.Image.NEAREST by default.
+
+    Args:
+        image (np.ndarray):
+        size: (x, y)
+
+    Returns:
+        np.ndarray:
+    """
+    return cv2.resize(image, size, interpolation=cv2.INTER_NEAREST)
+
+
+def image_channel(image):
+    """
+    Args:
+        image (np.ndarray):
+
+    Returns:
+        int: 0 for grayscale, 3 for RGB.
+    """
+    return image.shape[2] if len(image.shape) == 3 else 0
+
+
+def image_size(image):
+    """
+    Args:
+        image (np.ndarray):
+
+    Returns:
+        int, int: width, height
+    """
+    shape = image.shape
+    return shape[1], shape[0]
+
+
+def rgb2gray(image):
+    """
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(image)
+    return cv2.add(
+        cv2.multiply(cv2.max(cv2.max(r, g), b), 0.5),
+        cv2.multiply(cv2.min(cv2.min(r, g), b), 0.5)
+    )
+
+
+def rgb2hsv(image):
+    """
+    Convert RGB color space to HSV color space.
+    HSV is Hue Saturation Value.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Hue (0~360), Saturation (0~100), Value (0~100).
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV).astype(np.float)
+    image *= (360 / 180, 100 / 255, 100 / 255)
+    return image
+
+
+def rgb2yuv(image):
+    """
+    Convert RGB to YUV color space.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
+    return image
+
+
+def rgb2luma(image):
+    """
+    Convert RGB to the Y channel (Luminance) in YUV color space.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
+    luma, _, _ = cv2.split(image)
+    return luma
+
+
+def get_color(image, area):
+    """Calculate the average color of a particular area of the image.
+
+    Args:
+        image (np.ndarray): Screenshot.
+        area (tuple): (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y)
+
+    Returns:
+        tuple: (r, g, b)
+    """
+    temp = crop(image, area, copy=False)
+    color = cv2.mean(temp)
+    return color[:3]
+
+
+def get_bbox(image, threshold=0):
+    """
+    A numpy implementation of the getbbox() in pillow.
+
+    Args:
+        image (np.ndarray): Screenshot.
+        threshold (int): Color <= threshold will be considered black
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y)
+    """
+    if image_channel(image) == 3:
+        image = np.max(image, axis=2)
+    x = np.where(np.max(image, axis=0) > threshold)[0]
+    y = np.where(np.max(image, axis=1) > threshold)[0]
+    return x[0], y[0], x[-1] + 1, y[-1] + 1
+
+
+def color_similarity(color1, color2):
+    """
+    Args:
+        color1 (tuple): (r, g, b)
+        color2 (tuple): (r, g, b)
+
+    Returns:
+        int:
+    """
+    diff = np.array(color1).astype(int) - np.array(color2).astype(int)
+    diff = np.max(np.maximum(diff, 0)) - np.min(np.minimum(diff, 0))
+    return diff
+
+
+def color_similar(color1, color2, threshold=10):
+    """Consider two colors are similar, if tolerance lesser or equal threshold.
+    Tolerance = Max(Positive(difference_rgb)) + Max(- Negative(difference_rgb))
+    The same as the tolerance in Photoshop.
+
+    Args:
+        color1 (tuple): (r, g, b)
+        color2 (tuple): (r, g, b)
+        threshold (int): Default to 10.
+
+    Returns:
+        bool: True if two colors are similar.
+    """
+    # print(color1, color2)
+    diff = np.array(color1).astype(int) - np.array(color2).astype(int)
+    diff = np.max(np.maximum(diff, 0)) - np.min(np.minimum(diff, 0))
+    return diff <= threshold
+
+
+def color_similar_1d(image, color, threshold=10):
+    """
+    Args:
+        image (np.ndarray): 1D array.
+        color: (r, g, b)
+        threshold(int): Default to 10.
+
+    Returns:
+        np.ndarray: bool
+    """
+    diff = image.astype(int) - color
+    diff = np.max(np.maximum(diff, 0), axis=1) - np.min(np.minimum(diff, 0), axis=1)
+    return diff <= threshold
+
+
+def color_similarity_2d(image, color):
+    """
+    Args:
+        image: 2D array.
+        color: (r, g, b)
+
+    Returns:
+        np.ndarray: uint8
+    """
+    r, g, b = cv2.split(cv2.subtract(image, (*color, 0)))
+    positive = cv2.max(cv2.max(r, g), b)
+    r, g, b = cv2.split(cv2.subtract((*color, 0), image))
+    negative = cv2.max(cv2.max(r, g), b)
+    return cv2.subtract(255, cv2.add(positive, negative))
+
+
+def extract_letters(image, letter=(255, 255, 255), threshold=128):
+    """Set letter color to black, set background color to white.
+
+    Args:
+        image: Shape (height, width, channel)
+        letter (tuple): Letter RGB.
+        threshold (int):
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(cv2.subtract(image, (*letter, 0)))
+    positive = cv2.max(cv2.max(r, g), b)
+    r, g, b = cv2.split(cv2.subtract((*letter, 0), image))
+    negative = cv2.max(cv2.max(r, g), b)
+    return cv2.multiply(cv2.add(positive, negative), 255.0 / threshold)
+
+
+def extract_white_letters(image, threshold=128):
+    """Set letter color to black, set background color to white.
+    This function will discourage color pixels (Non-gray pixels)
+
+    Args:
+        image: Shape (height, width, channel)
+        threshold (int):
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(cv2.subtract((255, 255, 255, 0), image))
+    minimum = cv2.min(cv2.min(r, g), b)
+    maximum = cv2.max(cv2.max(r, g), b)
+    return cv2.multiply(cv2.add(maximum, cv2.subtract(maximum, minimum)), 255.0 / threshold)
+
+
+def color_mapping(image, max_multiply=2):
+    """
+    Mapping color to 0-255.
+    Minimum color to 0, maximum color to 255, multiply colors by 2 at max.
+
+    Args:
+        image (np.ndarray):
+        max_multiply (int, float):
+
+    Returns:
+        np.ndarray:
+    """
+    image = image.astype(float)
+    low, high = np.min(image), np.max(image)
+    multiply = min(255 / (high - low), max_multiply)
+    add = (255 - multiply * (low + high)) / 2
+    image = cv2.add(cv2.multiply(image, multiply), add)
+    image[image > 255] = 255
+    image[image < 0] = 0
+    return image.astype(np.uint8)
+
+
+def image_left_strip(image, threshold, length):
+    """
+    In `DAILY:200/200` strip `DAILY:` and leave `200/200`
+
+    Args:
+        image (np.ndarray): (height, width)
+        threshold (int):
+            0-255
+            The first column with brightness lower than this
+            will be considered as left edge.
+        length (int):
+            Strip this length of image after the left edge
+
+    Returns:
+        np.ndarray:
+    """
+    brightness = np.mean(image, axis=0)
+    match = np.where(brightness < threshold)[0]
+
+    if len(match):
+        left = match[0] + length
+        total = image.shape[1]
+        if left < total:
+            image = image[:, left:]
+    return image
+
+
+def red_overlay_transparency(color1, color2, red=247):
+    """Calculate the transparency of red overlay.
+
+    Args:
+        color1: origin color.
+        color2: changed color.
+        red(int): red color 0-255. Default to 247.
+
+    Returns:
+        float: 0-1
+    """
+    return (color2[0] - color1[0]) / (red - color1[0])
+
+
+def color_bar_percentage(image, area, prev_color, reverse=False, starter=0, threshold=30):
+    """
+    Args:
+        image:
+        area:
+        prev_color:
+        reverse: True if bar goes from right to left.
+        starter:
+        threshold:
+
+    Returns:
+        float: 0 to 1.
+    """
+    image = crop(image, area)
+    image = image[:, ::-1, :] if reverse else image
+    length = image.shape[1]
+    prev_index = starter
+
+    for _ in range(1280):
+        bar = color_similarity_2d(image, color=prev_color)
+        index = np.where(np.any(bar > 255 - threshold, axis=0))[0]
+        if not index.size:
+            return prev_index / length
+        else:
+            index = index[-1]
+        if index <= prev_index:
+            return index / length
+        prev_index = index
+
+        prev_row = bar[:, prev_index] > 255 - threshold
+        if not prev_row.size:
+            return prev_index / length
+        prev_color = np.mean(image[:, prev_index], axis=0)
+
+    return 0.

+ 35 - 0
module/config/utils.py

@@ -120,3 +120,38 @@ def deep_iter(data, depth=0, current_depth=1):
                 yield [key] + child_path, child_value
                 yield [key] + child_path, child_value
     else:
     else:
         yield [], data
         yield [], data
+
+def server_timezone() -> timedelta:
+    return timedelta(hours=8)
+
+def server_time_offset() -> timedelta:
+    """
+    To convert local time to server time:
+        server_time = local_time + server_time_offset()
+    To convert server time to local time:
+        local_time = server_time - server_time_offset()
+    """
+    return datetime.now(timezone.utc).astimezone().utcoffset() - server_timezone()
+
+def get_server_next_update(daily_trigger):
+    """
+    Args:
+        daily_trigger (list[str], str): [ "00:00", "12:00", "18:00",]
+
+    Returns:
+        datetime.datetime
+    """
+    if isinstance(daily_trigger, str):
+        daily_trigger = daily_trigger.replace(' ', '').split(',')
+
+    diff = server_time_offset()
+    local_now = datetime.now()
+    trigger = []
+    for t in daily_trigger:
+        h, m = [int(x) for x in t.split(':')]
+        future = local_now.replace(hour=h, minute=m, second=0, microsecond=0) + diff
+        s = (future - local_now).total_seconds() % 86400
+        future = local_now + timedelta(seconds=s)
+        trigger.append(future)
+    update = sorted(trigger)[0]
+    return update

+ 67 - 0
module/device/app_control.py

@@ -0,0 +1,67 @@
+from lxml import etree
+
+from module.device.method.adb import Adb
+from module.device.method.uiautomator_2 import Uiautomator2
+from module.device.method.utils import HierarchyButton
+# from module.device.method.wsa import WSA
+from module.logger import logger
+
+
+class AppControl(Adb, Uiautomator2):
+    hierarchy: etree._Element
+    _app_u2_family = ['uiautomator2', 'minitouch', 'scrcpy']
+
+    def app_is_running(self) -> bool:
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        # if self.is_wsa:
+        #     package = self.app_current_wsa()
+        if method in AppControl._app_u2_family:
+            package = self.app_current_uiautomator2()
+        else:
+            package = self.app_current_adb()
+
+        package = package.strip(' \t\r\n')
+        logger.attr('Package_name', package)
+        return package == self.package
+
+    def app_start(self):
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        logger.info(f'App start: {self.package}')
+        # if self.config.Emulator_Serial == 'wsa-0':
+        #     self.app_start_wsa(display=0)
+        if method in AppControl._app_u2_family:
+            self.app_start_uiautomator2()
+        else:
+            self.app_start_adb()
+
+    def app_stop(self):
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        logger.info(f'App stop: {self.package}')
+        if method in AppControl._app_u2_family:
+            self.app_stop_uiautomator2()
+        else:
+            self.app_stop_adb()
+
+    def dump_hierarchy(self) -> etree._Element:
+        """
+        Returns:
+            etree._Element: Select elements with `self.hierarchy.xpath('//*[@text="Hermit"]')` for example.
+        """
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        if method in AppControl._app_u2_family:
+            self.hierarchy = self.dump_hierarchy_uiautomator2()
+        else:
+            self.hierarchy = self.dump_hierarchy_adb()
+        return self.hierarchy
+
+    def xpath_to_button(self, xpath: str) -> HierarchyButton:
+        """
+        Args:
+            xpath (str):
+
+        Returns:
+            HierarchyButton:
+                An object with methods and properties similar to Button.
+                If element not found or multiple elements were found, return None.
+        """
+        return HierarchyButton(self.hierarchy, xpath)

+ 186 - 0
module/device/control.py

@@ -0,0 +1,186 @@
+# from module.base.button import Button
+from module.base.decorator import cached_property
+from module.base.timer import Timer
+from module.base.utils import *
+# from module.device.method.hermit import Hermit
+# from module.device.method.maatouch import MaaTouch
+from module.device.method.minitouch import Minitouch
+from module.device.method.adb import Adb
+from module.device.method.scrcpy import Scrcpy
+from module.device.method.window import Window
+from module.logger import logger
+
+
+class Control(Minitouch, Scrcpy, Window, Adb):
+    def handle_control_check(self, button):
+        # Will be overridden in Device
+        pass
+
+    @cached_property
+    def click_methods(self):
+        return {
+            'ADB': self.click_adb,
+            'uiautomator2': self.click_uiautomator2,
+            'minitouch': self.click_minitouch,
+            'window_message': self.click_window_message
+            # 'Hermit': self.click_hermit,
+            # 'MaaTouch': self.click_maatouch,
+        }
+
+    def click(self, button, control_check=True):
+        """
+        后面改一改  不用用button的逻辑
+        :param button:
+        :param control_check:
+        :return:
+        """
+        if control_check:
+            self.handle_control_check(button)
+        x, y = random_rectangle_point(button.button)
+        x, y = ensure_int(x, y)
+        logger.info(
+            'Click %s @ %s' % (point2str(x, y), button)
+        )
+        method = self.click_methods.get(
+            self.config.get_arg("Script", 'Emulator', 'ControlMethod'),
+            self.click_adb
+        )
+        method(x, y)
+
+    def multi_click(self, button, n, interval=(0.1, 0.2)):
+        """
+        也是不能用button的逻辑
+        :param button:
+        :param n:
+        :param interval:
+        :return:
+        """
+        self.handle_control_check(button)
+        click_timer = Timer(0.1)
+        for _ in range(n):
+            remain = ensure_time(interval) - click_timer.current()
+            if remain > 0:
+                self.sleep(remain)
+            click_timer.reset()
+
+            self.click(button, control_check=False)
+
+    def long_click(self, button, duration=(1, 1.2)):
+        """
+
+        :param button:
+        :param duration:
+        :return:
+        """
+        self.handle_control_check(button)
+        x, y = random_rectangle_point(button.button)
+        x, y = ensure_int(x, y)
+        duration = ensure_time(duration)
+        logger.info(
+            'Click %s @ %s, %s' % (point2str(x, y), button, duration)
+        )
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            self.long_click_minitouch(x, y, duration)
+        elif method == 'window_message':
+            self.long_click_window_message(x, y, duration)
+        elif method == 'uiautomator2':
+            self.long_click_uiautomator2(x, y, duration)
+        elif method == 'scrcpy':
+            self.long_click_scrcpy(x, y, duration)
+        # elif method == 'MaaTouch':
+        #     self.long_click_maatouch(x, y, duration)
+        else:
+            self.swipe_adb((x, y), (x, y), duration)
+
+    def swipe(self, p1, p2, duration=(0.1, 0.2), name='SWIPE', distance_check=True):
+        self.handle_control_check(name)
+        p1, p2 = ensure_int(p1, p2)
+        duration = ensure_time(duration)
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        elif method == 'window_message':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        elif method == 'uiautomator2':
+            logger.info('Swipe %s -> %s, %s' % (point2str(*p1), point2str(*p2), duration))
+        elif method == 'scrcpy':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        # elif method == 'MaaTouch':
+        #     logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        else:
+            # ADB needs to be slow, or swipe doesn't work
+            duration *= 2.5
+            logger.info('Swipe %s -> %s, %s' % (point2str(*p1), point2str(*p2), duration))
+
+        if distance_check:
+            if np.linalg.norm(np.subtract(p1, p2)) < 10:
+                # Should swipe a certain distance, otherwise AL will treat it as click.
+                # uiautomator2 should >= 6px, minitouch should >= 5px
+                logger.info('Swipe distance < 10px, dropped')
+                return
+
+        if method == 'minitouch':
+            self.swipe_minitouch(p1, p2)
+        elif method == 'window_message':
+            self.swipe_window_message(p1, p2)
+        elif method == 'uiautomator2':
+            self.swipe_uiautomator2(p1, p2, duration=duration)
+        elif method == 'scrcpy':
+            self.swipe_scrcpy(p1, p2)
+        # elif method == 'MaaTouch':
+        #     self.swipe_maatouch(p1, p2)
+        else:
+            self.swipe_adb(p1, p2, duration=duration)
+
+    def swipe_vector(self, vector, box=(123, 159, 1175, 628), random_range=(0, 0, 0, 0), padding=15,
+                     duration=(0.1, 0.2), whitelist_area=None, blacklist_area=None, name='SWIPE', distance_check=True):
+        """Method to swipe.
+
+        Args:
+            box (tuple): Swipe in box (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+            vector (tuple): (x, y).
+            random_range (tuple): (x_min, y_min, x_max, y_max).
+            padding (int):
+            duration (int, float, tuple):
+            whitelist_area: (list[tuple[int]]):
+                A list of area that safe to click. Swipe path will end there.
+            blacklist_area: (list[tuple[int]]):
+                If none of the whitelist_area satisfies current vector, blacklist_area will be used.
+                Delete random path that ends in any blacklist_area.
+            name (str): Swipe name
+            distance_check: (bool):
+        """
+        p1, p2 = random_rectangle_vector_opted(
+            vector,
+            box=box,
+            random_range=random_range,
+            padding=padding,
+            whitelist_area=whitelist_area,
+            blacklist_area=blacklist_area
+        )
+        self.swipe(p1, p2, duration=duration, name=name, distance_check=distance_check)
+
+    def drag(self, p1, p2, segments=1, shake=(0, 15), point_random=(-10, -10, 10, 10), shake_random=(-5, -5, 5, 5),
+             swipe_duration=0.25, shake_duration=0.1, name='DRAG'):
+        self.handle_control_check(name)
+        p1, p2 = ensure_int(p1, p2)
+        logger.info(
+            'Drag %s -> %s' % (point2str(*p1), point2str(*p2))
+        )
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            self.drag_minitouch(p1, p2, point_random=point_random)
+        elif method == 'uiautomator2':
+            self.drag_uiautomator2(
+                p1, p2, segments=segments, shake=shake, point_random=point_random, shake_random=shake_random,
+                swipe_duration=swipe_duration, shake_duration=shake_duration)
+        elif method == 'scrcpy':
+            self.drag_scrcpy(p1, p2, point_random=point_random)
+        # elif method == 'MaaTouch':
+        #     self.drag_maatouch(p1, p2, point_random=point_random)
+        else:
+            logger.warning(f'Control method {method} does not support drag well, '
+                           f'falling back to ADB swipe may cause unexpected behaviour')
+            self.swipe_adb(p1, p2, duration=ensure_time(swipe_duration * 2))
+            # self.click(Button(area=(), color=(), button=area_offset(point_random, p2), name=name))

+ 214 - 5
module/device/device.py

@@ -1,6 +1,215 @@
-# This Python file uses the following encoding: utf-8
-# @author runhey
-# github https://github.com/runhey
+import sys
+from collections import deque
+from datetime import datetime
 
 
-class Device:
-    pass
+from module.base.timer import Timer
+from module.config.utils import get_server_next_update
+from module.device.app_control import AppControl
+from module.device.control import Control
+from module.device.screenshot import Screenshot
+from module.exception import (GameNotRunningError, GameStuckError,
+                              GameTooManyClickError, RequestHumanTakeover)
+from module.handler.assets import GET_MISSION
+from module.logger import logger
+
+if sys.platform == 'win32':
+    from module.device.emulator import EmulatorManager
+else:
+    class EmulatorManager:
+        pass
+
+
+class Device(Screenshot, Control, AppControl, EmulatorManager):
+    _screen_size_checked = False
+    detect_record = set()
+    click_record = deque(maxlen=15)
+    stuck_timer = Timer(60, count=60).start()
+    stuck_timer_long = Timer(180, count=180).start()
+    stuck_long_wait_list = ['BATTLE_STATUS_S', 'PAUSE', 'LOGIN_CHECK']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.screenshot_interval_set()
+
+        # Auto-select the fastest screenshot method
+        if not self.config.is_template_config and self.config.get_arg("Script", 'Device', 'ScreenshotMethod') == 'auto':
+            self.run_simple_screenshot_benchmark()
+
+    def run_simple_screenshot_benchmark(self):
+        """
+        Perform a screenshot method benchmark, test 3 times on each method.
+        The fastest one will be set into config.
+        """
+        logger.info('run_simple_screenshot_benchmark')
+        # Check resolution first
+        self.resolution_check_uiautomator2()
+        # Perform benchmark
+        # from module.daemon.benchmark import Benchmark
+        # bench = Benchmark(config=self.config, device=self)
+        # method = bench.run_simple_screenshot_benchmark()
+        # # Set
+        # self.config.Emulator_ScreenshotMethod = method
+
+    def handle_night_commission(self, daily_trigger='21:00', threshold=30):
+        """
+        Args:
+            daily_trigger (int): Time for commission refresh.
+            threshold (int): Seconds around refresh time.
+
+        Returns:
+            bool: If handled.
+        """
+        update = get_server_next_update(daily_trigger=daily_trigger)
+        now = datetime.now()
+        diff = (update.timestamp() - now.timestamp()) % 86400
+        if threshold < diff < 86400 - threshold:
+            return False
+
+        if GET_MISSION.match(self.image, offset=True):
+            logger.info('Night commission appear.')
+            self.click(GET_MISSION)
+            return True
+
+        return False
+
+    def screenshot(self):
+        """
+        Returns:
+            np.ndarray:
+        """
+        self.stuck_record_check()
+
+        try:
+            super().screenshot()
+        except RequestHumanTakeover:
+            if not self.ascreencap_available:
+                logger.error('aScreenCap unavailable on current device, fallback to auto')
+                self.run_simple_screenshot_benchmark()
+                super().screenshot()
+            else:
+                raise
+
+        if self.handle_night_commission():
+            super().screenshot()
+
+        return self.image
+
+    def release_during_wait(self):
+        # Scrcpy server is still sending video stream,
+        # stop it during wait
+        if self.config.Emulator_ScreenshotMethod == 'scrcpy':
+            self._scrcpy_server_stop()
+
+    def stuck_record_add(self, button):
+        self.detect_record.add(str(button))
+
+    def stuck_record_clear(self):
+        self.detect_record = set()
+        self.stuck_timer.reset()
+        self.stuck_timer_long.reset()
+
+    def stuck_record_check(self):
+        """
+        Raises:
+            GameStuckError:
+        """
+        reached = self.stuck_timer.reached()
+        reached_long = self.stuck_timer_long.reached()
+
+        if not reached:
+            return False
+        if not reached_long:
+            for button in self.stuck_long_wait_list:
+                if button in self.detect_record:
+                    return False
+
+        logger.warning('Wait too long')
+        logger.warning(f'Waiting for {self.detect_record}')
+        self.stuck_record_clear()
+
+        if self.app_is_running():
+            raise GameStuckError(f'Wait too long')
+        else:
+            raise GameNotRunningError('Game died')
+
+    def handle_control_check(self, button):
+        self.stuck_record_clear()
+        self.click_record_add(button)
+        self.click_record_check()
+
+    def click_record_add(self, button):
+        self.click_record.append(str(button))
+
+    def click_record_clear(self):
+        self.click_record.clear()
+
+    def click_record_remove(self, button):
+        """
+        Remove a button from `click_record`
+
+        Args:
+            button (Button):
+
+        Returns:
+            int: Number of button removed
+        """
+        removed = 0
+        for _ in range(self.click_record.maxlen):
+            try:
+                self.click_record.remove(str(button))
+                removed += 1
+            except ValueError:
+                # Value not in queue
+                break
+
+        return removed
+
+    def click_record_check(self):
+        """
+        Raises:
+            GameTooManyClickError:
+        """
+        count = {}
+        for key in self.click_record:
+            count[key] = count.get(key, 0) + 1
+        count = sorted(count.items(), key=lambda item: item[1])
+        if count[0][1] >= 12:
+            logger.warning(f'Too many click for a button: {count[0][0]}')
+            logger.warning(f'History click: {[str(prev) for prev in self.click_record]}')
+            self.click_record_clear()
+            raise GameTooManyClickError(f'Too many click for a button: {count[0][0]}')
+        if len(count) >= 2 and count[0][1] >= 6 and count[1][1] >= 6:
+            logger.warning(f'Too many click between 2 buttons: {count[0][0]}, {count[1][0]}')
+            logger.warning(f'History click: {[str(prev) for prev in self.click_record]}')
+            self.click_record_clear()
+            raise GameTooManyClickError(f'Too many click between 2 buttons: {count[0][0]}, {count[1][0]}')
+
+    def disable_stuck_detection(self):
+        """
+        Disable stuck detection and its handler. Usually uses in semi auto and debugging.
+        """
+        logger.info('Disable stuck detection')
+
+        def empty_function(*arg, **kwargs):
+            return False
+
+        self.click_record_check = empty_function
+        self.stuck_record_check = empty_function
+
+    def app_start(self):
+        if not self.config.Error_HandleError:
+            logger.critical('No app stop/start, because HandleError disabled')
+            logger.critical('Please enable Alas.Error.HandleError or manually login to AzurLane')
+            raise RequestHumanTakeover
+        super().app_start()
+        self.stuck_record_clear()
+        self.click_record_clear()
+
+    def app_stop(self):
+        if not self.config.Error_HandleError:
+            logger.critical('No app stop/start, because HandleError disabled')
+            logger.critical('Please enable Alas.Error.HandleError or manually login to AzurLane')
+            raise RequestHumanTakeover
+        super().app_stop()
+        self.stuck_record_clear()
+        self.click_record_clear()

+ 328 - 0
module/device/emulator.py

@@ -0,0 +1,328 @@
+import os
+import re
+import winreg
+import subprocess
+
+from adbutils.errors import AdbError
+
+from deploy.emulator import VirtualBoxEmulator
+from module.base.decorator import cached_property
+from module.device.connection import Connection
+from module.device.method.utils import get_serial_pair
+from module.exception import RequestHumanTakeover, EmulatorNotRunningError
+from module.logger import logger
+
+
+class EmulatorInstance(VirtualBoxEmulator):
+
+    def __init__(self, name, root_path, emu_path,
+                 vbox_path=None, vbox_name=None, kill_para=None, multi_para=None):
+        """
+        Args:
+            name (str): Emulator name in windows uninstall list.
+            root_path (str): Relative path from uninstall.exe to emulator installation folder.
+            emu_path (str): Relative path to executable simulator file.
+            vbox_path (str): Relative path to virtual box folder.
+            vbox_name (str): Regular Expression to match the name of .vbox file.
+            kill_para (str): Parameters required by kill emulator.
+            multi_para (str): Parameters required by start multi open emulator,
+            #id will be replaced with the real ID.
+        """
+        super().__init__(
+            name=name,
+            root_path=root_path,
+            adb_path=None,
+            vbox_path=vbox_path,
+            vbox_name=vbox_name,
+        )
+        self.emu_path = emu_path
+        self.kill_para = kill_para
+        self.multi_para = multi_para
+
+    @cached_property
+    def id_and_serial(self):
+        """
+        Returns:
+            list[str, str]: List of multi_id and serial.
+        """
+        vbox = []
+        for path, folders, files in os.walk(os.path.join(self.root, self.vbox_path)):
+            for file in files:
+                if re.match(self.vbox_name, file):
+                    file = os.path.join(path, file)
+                    vbox.append(file)
+
+        serial = []
+        for file in vbox:
+            with open(file, 'r', encoding='utf-8', errors='ignore') as f:
+                for line in f.readlines():
+                    # <Forwarding name="port2" proto="1" hostip="127.0.0.1" hostport="62026" guestport="5555"/>
+                    res = re.search('<*?hostport="(.*?)".*?guestport="5555"/>', line)
+                    if res:
+                        serial.append([os.path.basename(file).split(".")[0], f'127.0.0.1:{res.group(1)}'])
+
+        return serial
+
+
+class Bluestacks5Instance(EmulatorInstance):
+    @cached_property
+    def root(self):
+        try:
+            return super().root
+        except FileNotFoundError:
+            self.name = 'BlueStacks_nxt_cn'
+            return super().root
+
+    @cached_property
+    def id_and_serial(self):
+        try:
+            reg = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\BlueStacks_nxt")
+        except FileNotFoundError:
+            reg = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\BlueStacks_nxt_cn")
+        directory = winreg.QueryValueEx(reg, 'UserDefinedDir')[0]
+
+        with open(os.path.join(directory, 'bluestacks.conf'), encoding='utf-8') as f:
+            content = f.read()
+        emulators = re.findall(r'bst.instance.(\w+).status.adb_port="(\d+)"', content)
+        serial = []
+        for emulator in emulators:
+            serial.append([emulator[0], f'127.0.0.1:{emulator[1]}'])
+        return serial
+
+
+class EmulatorManager(Connection):
+    pid = None
+    SUPPORTED_EMULATORS = {
+        'nox_player': EmulatorInstance(
+            name="Nox",
+            root_path=".",
+            emu_path="./Nox.exe",
+            vbox_path="./BignoxVMS",
+            vbox_name='.*.vbox$',
+            kill_para='-quit',
+            multi_para='-clone:#id',
+        ),
+        'mumu_player': EmulatorInstance(
+            name="Nemu",
+            root_path=".",
+            emu_path="./EmulatorShell/NemuPlayer.exe",
+            vbox_path="./vms",
+            vbox_name='.*.nemu$',
+        ),
+        'bluestacks_5': Bluestacks5Instance(
+            name='BlueStacks_nxt',
+            root_path='.',
+            emu_path='./HD-Player.exe',
+            multi_para='--instance #id',
+        ),
+    }
+
+    def detect_emulator(self, serial, emulator=None):
+        """
+        Args:
+            serial (str):
+            emulator (EmulatorInstance):
+
+        Returns:
+            list[EmulatorInstance, str]:Emulator and multi_id
+        """
+        if emulator is None:
+            logger.info('Detect emulator from all emulators installed')
+            emulators = []
+            for emulator in self.SUPPORTED_EMULATORS.values():
+                try:
+                    serials = emulator.id_and_serial
+                    for cur_serial in serials:
+                        if cur_serial[1] == serial:
+                            emulators.append([emulator, cur_serial[0]])
+                except FileNotFoundError:
+                    pass
+
+            logger.info('Detected emulators:')
+            for emulator in emulators:
+                logger.info(f'Name: {emulator[0].name}, Multi_id: {emulator[1]}')
+
+            if len(emulators) == 1 or \
+                    (len(emulators) > 0 and emulators[0][0] == self.SUPPORTED_EMULATORS['mumu_player']):
+                logger.info('Find the only emulator, using it')
+                return emulators[0][0], emulators[0][1]
+            elif len(emulators) == 0:
+                logger.warning('The emulator corresponding to serial is not found, '
+                               'please check the setting or use custom command')
+            else:
+                logger.warning('Multiple emulators with the same serial have been found, '
+                               'please select one manually or use custom command')
+            raise RequestHumanTakeover
+
+        else:
+            try:
+                logger.info(f'Detect emulator from {emulator.name}')
+                serials = emulator.id_and_serial
+                for cur_serial in serials:
+                    if cur_serial[1] == serial:
+                        logger.info('Find the only emulator, using it')
+                        return emulator, cur_serial[0]
+            except FileNotFoundError:
+                pass
+            logger.warning('The emulator corresponding to serial is not found, '
+                           'please check the setting or use custom command')
+            raise RequestHumanTakeover
+
+    @staticmethod
+    def execute(command):
+        """
+        Args:
+            command (str):
+
+        Returns:
+            subprocess.Popen:
+        """
+        command = command.replace(r"\\", "/").replace("\\", "/").replace('"', '"')
+        logger.info(f'Execute: {command}')
+        return subprocess.Popen(command, close_fds=True)  # only work on Windows
+
+    @staticmethod
+    def task_kill(pid=None, name=None):
+        """
+        Args:
+            pid (list, int):
+            name (list, str):
+
+        Returns:
+            subprocess.Popen:
+        """
+        command = 'taskkill '
+        if pid is not None:
+            if isinstance(pid, list):
+                for p in pid:
+                    command += f'/pid {p} '
+            else:
+                command += f'/pid {pid} '
+        elif name is not None:
+            if isinstance(name, list):
+                for n in name:
+                    command += f'/im {n} '
+            else:
+                command += f'/im {name} '
+        else:
+            raise RequestHumanTakeover
+        command += '/t /f'
+
+        return EmulatorManager.execute(command)
+
+    def adb_connect(self, serial):
+        try:
+            return super(EmulatorManager, self).adb_connect(serial)
+        except EmulatorNotRunningError:
+            raise RequestHumanTakeover
+
+    def detect_emulator_status(self, serial):
+        devices = self.list_device()
+        for device in devices:
+            if device.serial == serial:
+                return device.status
+        return 'offline'
+
+    def emulator_start(self, serial, emulator=None, multi_id=None, command=None):
+        """
+        Args:
+            serial (str): Expected serial after simulator starts successfully.
+            emulator (EmulatorInstance): Emulator to start.
+            multi_id (str): Emulator ID used by multi open emulator.
+            command (str): Customized path and parameters of the simulator to start.
+
+        Return:
+            bool: If start successful.
+        """
+        if command is None:
+            command = '\"' + os.path.abspath(os.path.join(emulator.root, emulator.emu_path)) + '\"'
+            if emulator.multi_para is not None and multi_id is not None:
+                command += " " + emulator.multi_para.replace("#id", multi_id)
+
+        logger.info('Start emulator')
+        pipe = self.execute(command)
+        self.pid = pipe.pid
+        self.sleep(10)
+
+        for _ in range(20):
+            if pipe.poll() is not None:
+                break
+            try:
+                if super().adb_connect(serial):
+                    # Wait until emulator start completely
+                    self.sleep(10)
+                    return True
+            except EmulatorNotRunningError:
+                pass
+            self.sleep(5)
+        return False
+
+    def emulator_kill(self, serial, emulator=None, multi_id=None, command=None):
+        """
+        Args:
+            serial (str): Expected serial after simulator starts successfully.
+            emulator (EmulatorInstance): Emulator to start.
+            multi_id (str): Emulator ID used by multi open emulator.
+            command (str): Customized path and parameters of the simulator to start.
+
+        Return:
+            bool: If kill successful.
+        """
+        if command is None and emulator.kill_para is not None:
+            command = '\"' + os.path.abspath(os.path.join(emulator.root, emulator.emu_path)) + '\"'
+            if emulator.multi_para is not None and multi_id is not None:
+                command += " " + emulator.multi_para.replace("#id", multi_id)
+            command += " " + emulator.kill_para
+
+        logger.info('Kill emulator')
+        if emulator == self.SUPPORTED_EMULATORS['bluestacks_5']:
+            try:
+                self.adb_command(['reboot', '-p'], timeout=20)
+                if self.detect_emulator_status(serial) == 'offline':
+                    self.pid = None
+                    return True
+            except AdbError:
+                return False
+
+        if emulator == self.SUPPORTED_EMULATORS['mumu_player']:
+            self.task_kill(pid=None, name=['NemuHeadless.exe', 'NemuPlayer.exe', 'NemuSvc.exe'])
+        elif command is not None:
+            self.execute(command)
+        else:
+            self.task_kill(pid=self.pid, name=os.path.basename(emulator.emu_path))
+        self.sleep(5)
+
+        for _ in range(10):
+            if self.detect_emulator_status(serial) == 'offline':
+                self.pid = None
+                return True
+            self.sleep(2)
+        return False
+
+    def emulator_restart(self):
+        serial, _ = get_serial_pair(self.serial)
+        if serial is None:
+            serial = self.serial
+
+        if os.name != 'nt':
+            logger.warning('Restart simulator only works under Windows platform')
+            return False
+
+        logger.hr('Emulator restart')
+        # if self.config.RestartEmulator_EmulatorType == 'auto':
+        if self.config.get_arg('Restart', '') == 'auto':
+            emulator, multi_id = self.detect_emulator(serial)
+        else:
+            emulator = self.SUPPORTED_EMULATORS[self.config.RestartEmulator_EmulatorType]
+            emulator, multi_id = self.detect_emulator(serial, emulator=emulator)
+
+        for _ in range(3):
+            if not self.emulator_kill(serial, emulator, multi_id):
+                continue
+            if self.emulator_start(serial, emulator, multi_id):
+                return True
+
+        logger.warning('Restart emulator failed for 3 times, please check your settings')
+        raise RequestHumanTakeover
+
+

+ 308 - 0
module/device/handle.py

@@ -0,0 +1,308 @@
+# This Python file uses the following encoding: utf-8
+# @author runhey
+# github https://github.com/runhey
+import re
+
+from cached_property import cached_property
+from anytree import NodeMixin, RenderTree, PreOrderIter
+from win32api import GetSystemMetrics, SendMessage, MAKELONG, PostMessage
+from win32print import GetDeviceCaps
+from win32process import GetWindowThreadProcessId
+from win32gui import (GetWindowText, EnumWindows, FindWindow, FindWindowEx,
+                      IsWindow, GetWindowRect, GetWindowDC, DeleteObject,
+                      SetForegroundWindow, IsWindowVisible, GetDC, GetParent,
+                      EnumChildWindows)
+from win32con import (SRCCOPY, DESKTOPHORZRES, DESKTOPVERTRES, WM_LBUTTONUP,
+                      WM_LBUTTONDOWN, WM_ACTIVATE, WA_ACTIVE, MK_LBUTTON,
+                      WM_NCHITTEST, WM_SETCURSOR, HTCLIENT, WM_MOUSEMOVE)
+from module.config.config import Config
+from module.exception import RequestHumanTakeover, EmulatorNotRunningError
+from module.logger import logger
+
+
+def handle_title2num(title: str) -> int:
+    """
+    从标题到句柄号
+    :param title:
+    :return:  如果没有找到就是返回零
+    """
+    return FindWindow(None, title)
+
+
+def handle_num2title(num: int) -> str:
+    """
+    通过句柄号返回窗口的标题,如果传入句柄号不合法则返回None
+    :param num:
+    :return:
+    """
+    return None if num is None or num == 0 or num == '' else GetWindowText(num)
+
+
+def is_handle_valid(num: int) -> bool:
+    """
+    输入一个句柄号,如果还在返回True
+    :param num:
+    :return:
+    """
+    return IsWindow(num)
+
+def handle_num2pid(num :int) -> int:
+    """
+    通过句柄号获取句柄进程id,如果句柄号非法则返回0
+    :param num:
+    :return:
+    """
+    return 0 if num is None or num == 0 or num == '' else GetWindowThreadProcessId(num)[1]
+
+def window_scale_rate() -> float:
+    """
+    获取window的系统缩放 一遍是1
+    :return:
+    """
+    hDC = GetDC(0)
+    # 物理上(真实的)的 横纵向分辨率
+    wReal = GetDeviceCaps(hDC, DESKTOPHORZRES)
+    hReal = GetDeviceCaps(hDC, DESKTOPVERTRES)
+    # 缩放后的 分辨率
+    wAfter = GetSystemMetrics(0)
+    hAfter = GetSystemMetrics(1)
+    # print(wReal, wAfter)
+    return round(wReal / wAfter, 2)
+
+class WindowNode(NodeMixin):
+    def __init__(self, name, num, parent=None):
+        super().__init__()
+        self.name = name
+        self.num = num
+        self.parent = parent
+
+
+class Handle:
+    emulator_list = ['MuMu',
+                     '雷电',
+                     '夜神',
+                     '蓝叠',
+                     '逍遥',
+                     '模拟器']  # 最后一个我又不知道还有哪些模拟器
+    emulator_handle = {
+        # 夜神
+        'nox_player': ['root_handle_title', 'Nox'],
+        'nox_player_64': ['root_handle_title', 'Nox'],
+        'nox_player_family':['root_handle_title', 'Nox'],
+        # 雷电
+        'ld_player': ['TheRender'],
+        'ld_player_4': ['TheRender'],
+        'ld_player_9': ['TheRender'],
+        'ld_player_family': ['TheRender'],
+        # 逍遥
+        'memu_player': ['root_handle_title'],
+        'memu_player_family': ['root_handle_title'],
+        # mumu
+        'mumu_player': ['root_handle_title', 'NemuPlayer'],
+        'mumu_player_12': ['root_handle_title', 'MuMuPlayer'],
+        'mumu_player_family': ['root_handle_title', 'MuMuPlayer'],
+        # 蓝叠
+        'bluestacks_5': ['root_handle_title'],
+        'bluestacks_family': ['root_handle_title']
+    }
+
+    def __init__(self, config) -> None:
+        """
+
+        :param config:
+        """
+        logger.hr('Handle')
+        if isinstance(config, str):
+            self.config = Config(config, task=None)
+        else:
+            self.config = config
+
+        # 获取根的句柄
+        self.root_handle_title = ''
+        self.root_handle_num = 0
+        self.root_handle = self.config.get_arg('Script', 'Device', 'Handle')
+        if self.root_handle == "auto":
+            logger.info('handle is auto. oas will find window emulator')
+            window_list = Handle.all_windows()
+            self.root_handle_title = self.auto_handle_title(window_list)
+            self.root_handle_num = handle_title2num(self.root_handle_title)
+        elif isinstance(self.root_handle, int):
+            logger.info('handle is handle num. oas use it as root handle num')
+            if is_handle_valid(self.root_handle):
+                logger.info(f'handle number {self.root_handle} is valid')
+                self.root_handle_title = handle_num2title(self.root_handle)
+                self.root_handle_num = self.root_handle
+        elif isinstance(self.root_handle, str):
+            logger.info('handle is handle string. oas use it as root handle title')
+            if handle_title2num(self.root_handle) !=0:
+                self.root_handle_num = handle_title2num(self.root_handle)
+                self.root_handle_title = self.root_handle
+        logger.info(f'the root handle title is {self.root_handle_title} and num is {self.root_handle_num}')
+
+        # 获取句柄树
+        self.root_node = WindowNode(name=self.root_handle_title, num=self.root_handle_num)
+        Handle.handle_tree(self.root_handle_num, self.root_node)
+        logger.info('the emulator handle structure is')
+        for pre, fill, node in RenderTree(self.root_node):
+            logger.info("%s%s" % (pre, node.name))
+        for pre, fill, node in RenderTree(self.root_node):
+            logger.info("%s%s" % (pre, node.num))
+
+        # 判断是哪一个模拟器
+        logger.info(f'the emulator family is {self.emulator_family}')
+
+        # window系统的缩放
+        logger.info(f'your window screen scale rate is {window_scale_rate()}')
+        _ = self.screenshot_handle_num
+        logger.info(f'screenshot handle num is {self.screenshot_handle_num}')
+        logger.info(f'the emulator screenshot size is {self.screenshot_size}')
+
+
+    @staticmethod
+    def all_windows() -> list:
+        """
+        获取桌面上的所有窗体
+
+        :return:  类似这样['MuMu模拟器']
+        """
+
+        def enum_windows_callback(hwnd, windows):
+            window_text = GetWindowText(hwnd)
+            windows.append(window_text)
+
+        windows = []
+        EnumWindows(enum_windows_callback, windows)
+        return windows
+
+    @classmethod
+    def auto_handle_title(cls, windows: list) -> str:
+        """
+        返回第一个找到的有模拟器的标题
+        :param windows:
+        :return:
+        """
+        if windows is None:
+            logger.error("handle_auto not get all wnidow")
+
+        for window_title in windows:
+            for item in Handle.emulator_list:
+                if window_title.find(item) != -1:
+                    logger.info(f'handle auto seclect to find {window_title} and use it as root_title')
+                    return window_title
+
+        return None
+
+    @staticmethod
+    def handle_tree(hwnd, node: WindowNode, level: int =0) -> None:
+        """
+        生成一个窗口的句柄树
+        :param hwnd:
+        :param node:
+        :param level:
+        :return:
+        """
+        child_windows = []
+        EnumChildWindows(hwnd, lambda hwnd, param: param.append(hwnd), child_windows)
+
+        if not child_windows:
+            return
+        for child_hwnd in child_windows:
+            if GetParent(child_hwnd) == hwnd:
+                child_text = GetWindowText(child_hwnd)
+                child_node = WindowNode(name=child_text, num=child_hwnd, parent=node)
+
+                # 递归遍历子窗体的子窗体
+                Handle.handle_tree(child_hwnd, child_node, level + 1)
+
+    @cached_property
+    def emulator_family(self) -> str:
+        """
+        通过句柄标题来判断这个是那个模拟器大类
+        :return:
+        """
+        for emu in Handle.emulator_list:
+            if self.root_handle_title.find(emu) != -1:
+                if emu == 'MuMu':
+                    return 'mumu_player_family'
+                elif emu == '雷电':
+                    return 'ld_player_family'
+                elif emu == '夜神':
+                    return 'nox_player_family'
+                elif emu == '蓝叠':
+                    return 'bluestacks_player_family'
+                elif emu == '逍遥':
+                    return 'memu_player_family'
+
+    @cached_property
+    def screenshot_handle_num(self) -> int:
+        """
+        截屏的句柄其实并不是根句柄
+        :return:  出错返回None
+        """
+        if self.emulator_family == 'mumu_player_family':
+            if re.match(string=self.root_handle_title, pattern=r".*12$"):
+                logger.info('is mumu12')
+                for node in PreOrderIter(self.root_node):
+                    if node.name == Handle.emulator_handle['mumu_player_12'][1]:
+                        return node.num
+            else:
+                logger.info('is mumu')
+                for node in PreOrderIter(self.root_node):
+                    if node.name == Handle.emulator_handle['mumu_player'][1]:
+                        return node.num
+        # 夜神
+        elif self.emulator_family == 'nox_player_family':
+            try:
+                return self.root_node.children[1].children[1].num
+            except:
+                return self.root_node.children[2].children[1].num
+
+        elif self.emulator_family == 'ld_player_family':
+            for node in PreOrderIter(self.root_node):
+                if node.name == Handle.emulator_handle['ld_player_family'][0]:
+                    return node.num
+
+        elif self.emulator_family == 'memu_player_family':
+            for node in PreOrderIter(self.root_node):
+                if node.name == Handle.emulator_handle['memu_player_family']:
+                    return node.num
+
+        elif self.emulator_family == 'bluestacks_family':
+            for node in PreOrderIter(self.root_node):
+                if node.name == Handle.emulator_handle['bluestacks_family']:
+                    return node.num
+        return None
+
+    @cached_property
+    def screenshot_size(self) -> tuple:
+        """
+        第一个是width 第二个是heigth
+        :return:
+        """
+        winRect = GetWindowRect(self.screenshot_handle_num)
+        width: int = winRect[2] - winRect[0]  # 右x-左x
+        height: int = winRect[3] - winRect[1]  # 下y - 上y 计算高度
+        return width, height
+
+    @cached_property
+    def window_scale_rate(self) -> float:
+        """
+        获取window的系统缩放 一般是1
+        :return:
+        """
+        hDC = GetDC(0)
+        # 物理上(真实的)的 横纵向分辨率
+        wReal = GetDeviceCaps(hDC, DESKTOPHORZRES)
+        hReal = GetDeviceCaps(hDC, DESKTOPVERTRES)
+        # 缩放后的 分辨率
+        wAfter = GetSystemMetrics(0)
+        hAfter = GetSystemMetrics(1)
+        # print(wReal, wAfter)
+        return round(wReal / wAfter, 2)
+
+if __name__ == '__main__':
+    h = Handle(config='oas1')
+    logger.info(h.auto_handle_title(h.all_windows()))
+    logger.info(h.root_handle_num)
+    logger.info(h.emulator_family)
+

+ 296 - 0
module/device/method/droidcast.py

@@ -0,0 +1,296 @@
+import typing as t
+from functools import wraps
+
+import cv2
+import numpy as np
+import requests
+from adbutils.errors import AdbError
+
+from module.base.decorator import Config, cached_property, del_cached_property
+from module.base.timer import Timer
+from module.device.method.uiautomator_2 import Uiautomator2, ProcessInfo
+from module.device.method.utils import (retry_sleep, RETRY_TRIES, handle_adb_error,
+                                        ImageTruncated, PackageNotInstalled)
+from module.exception import RequestHumanTakeover
+from module.logger import logger
+
+
+class DroidCastVersionIncompatible(Exception):
+    pass
+
+
+def retry(func):
+    @wraps(func)
+    def retry_wrapper(self, *args, **kwargs):
+        """
+        Args:
+            self (Adb):
+        """
+        init = None
+        for _ in range(RETRY_TRIES):
+            try:
+                if callable(init):
+                    retry_sleep(_)
+                    init()
+                return func(self, *args, **kwargs)
+            # Can't handle
+            except RequestHumanTakeover:
+                break
+            # When adb server was killed
+            except ConnectionResetError as e:
+                logger.error(e)
+
+                def init():
+                    self.adb_reconnect()
+            # AdbError
+            except AdbError as e:
+                if handle_adb_error(e):
+                    def init():
+                        self.adb_reconnect()
+                else:
+                    break
+            # Package not installed
+            except PackageNotInstalled as e:
+                logger.error(e)
+
+                def init():
+                    self.detect_package()
+            # DroidCast not running
+            # requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
+            # ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=20482): Read timed out. (read timeout=3)
+            except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
+                logger.error(e)
+
+                def init():
+                    self.droidcast_init()
+            # DroidCastVersionIncompatible
+            except DroidCastVersionIncompatible as e:
+                logger.error(e)
+
+                def init():
+                    self.droidcast_init()
+            # ImageTruncated
+            except ImageTruncated as e:
+                logger.error(e)
+
+                def init():
+                    pass
+            # Unknown
+            except Exception as e:
+                logger.exception(e)
+
+                def init():
+                    pass
+
+        logger.critical(f'Retry {func.__name__}() failed')
+        raise RequestHumanTakeover
+
+    return retry_wrapper
+
+
+class DroidCast(Uiautomator2):
+    """
+    DroidCast, another screenshot method, https://github.com/rayworks/DroidCast
+    DroidCast_raw, a modified version of DroidCast sending raw bitmap https://github.com/Torther/DroidCastS
+    """
+
+    _droidcast_port: int = 0
+
+    @cached_property
+    def droidcast_session(self):
+        session = requests.Session()
+        session.trust_env = False  # Ignore proxy
+        self._droidcast_port = self.adb_forward('tcp:53516')
+        return session
+
+    def droidcast_url(self, url='/screenshot?format=png'):
+        """
+        Check APIs from source code:
+        https://github.com/rayworks/DroidCast/blob/master/app/src/main/java/com/rayworks/droidcast/Main.java
+
+        Available APIs:
+        - /screenshot
+            To get JPG screenshots.
+        - /screenshot?format=png
+            To get PNG screenshots.
+        - /screenshot?format=webp
+            To get WEBP screenshots.
+        - /src
+            Websocket to get JPG screenshots.
+
+        Note that /screenshot?format=jpg is unavailable.
+        """
+        return f'http://127.0.0.1:{self._droidcast_port}{url}'
+
+    @Config.when(DROIDCAST_VERSION='DroidCast')
+    def droidcast_init(self):
+        logger.hr('Droidcast init')
+        self.droidcast_stop()
+
+        logger.info('Pushing DroidCast apk')
+        self.adb_push(self.config.DROIDCAST_FILEPATH_LOCAL, self.config.DROIDCAST_FILEPATH_REMOTE)
+
+        logger.info('Starting DroidCast apk')
+        # CLASSPATH=/data/local/tmp/DroidCast.apk app_process / com.rayworks.droidcast.Main > /dev/null
+        resp = self.u2_shell_background([
+            'CLASSPATH=/data/local/tmp/DroidCast.apk',
+            'app_process',
+            '/',
+            'com.rayworks.droidcast.Main',
+            '>',
+            '/dev/null'
+        ])
+        logger.info(resp)
+
+        del_cached_property(self, 'droidcast_session')
+        _ = self.droidcast_session
+        logger.attr('DroidCast', self.droidcast_url())
+        self.droidcast_wait_startup()
+
+    @Config.when(DROIDCAST_VERSION='DroidCast_raw')
+    def droidcast_init(self):
+        logger.hr('Droidcast init')
+        self.resolution_check_uiautomator2()
+        self.droidcast_stop()
+
+        logger.info('Pushing DroidCast apk')
+        self.adb_push(self.config.DROIDCAST_RAW_FILEPATH_LOCAL, self.config.DROIDCAST_RAW_FILEPATH_REMOTE)
+
+        logger.info('Starting DroidCast apk')
+        # DroidCastS-release-1.1.5.apk
+        # CLASSPATH=/data/local/tmp/DroidCastS-release-1.1.5.apk app_process / com.torther.droidcasts.Main > /dev/null
+        resp = self.u2_shell_background([
+            'CLASSPATH=/data/local/tmp/DroidCastS.apk',
+            'app_process',
+            '/',
+            'com.torther.droidcasts.Main',
+            '>',
+            '/dev/null'
+        ])
+        logger.info(resp)
+
+        del_cached_property(self, 'droidcast_session')
+        _ = self.droidcast_session
+        logger.attr('DroidCast', self.droidcast_url())
+        self.droidcast_wait_startup()
+
+    @retry
+    def screenshot_droidcast(self):
+        self.config.DROIDCAST_VERSION = 'DroidCast'
+        image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
+        image = np.frombuffer(image, np.uint8)
+        if image is None:
+            raise ImageTruncated('Empty image after reading from buffer')
+        if image.shape == (1843200,):
+            raise DroidCastVersionIncompatible('Requesting screenshots from `DroidCast` but server is `DroidCast_raw`')
+
+        image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+        if image is None:
+            raise ImageTruncated('Empty image after cv2.imdecode')
+
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        if image is None:
+            raise ImageTruncated('Empty image after cv2.cvtColor')
+
+        return image
+
+    @retry
+    def screenshot_droidcast_raw(self):
+        self.config.DROIDCAST_VERSION = 'DroidCast_raw'
+        image = self.droidcast_session.get(self.droidcast_url(), timeout=3).content
+        # DroidCast_raw returns a RGB565 bitmap
+
+        try:
+            arr = np.frombuffer(image, dtype=np.uint16).reshape((720, 1280))
+        except ValueError as e:
+            # Try to load as `DroidCast`
+            image = np.frombuffer(image, np.uint8)
+            if image is not None:
+                image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+                if image is not None:
+                    raise DroidCastVersionIncompatible(
+                        'Requesting screenshots from `DroidCast_raw` but server is `DroidCast`')
+            # ValueError: cannot reshape array of size 0 into shape (720,1280)
+            raise ImageTruncated(str(e))
+
+        # Convert RGB565 to RGB888
+        # https://blog.csdn.net/happy08god/article/details/10516871
+
+        # r = (arr & 0b1111100000000000) >> (11 - 3)
+        # g = (arr & 0b0000011111100000) >> (5 - 2)
+        # b = (arr & 0b0000000000011111) << 3
+        # r |= (r & 0b11100000) >> 5
+        # g |= (g & 0b11000000) >> 6
+        # b |= (b & 0b11100000) >> 5
+        # r = r.astype(np.uint8)
+        # g = g.astype(np.uint8)
+        # b = b.astype(np.uint8)
+        # image = cv2.merge([r, g, b])
+
+        # The same as the code above but costs about 5ms instead of 10ms.
+        r = cv2.multiply(arr & 0b1111100000000000, 0.00390625).astype(np.uint8)
+        g = cv2.multiply(arr & 0b0000011111100000, 0.125).astype(np.uint8)
+        b = cv2.multiply(arr & 0b0000000000011111, 8).astype(np.uint8)
+        r = cv2.add(r, cv2.multiply(r, 0.03125))
+        g = cv2.add(g, cv2.multiply(g, 0.015625))
+        b = cv2.add(b, cv2.multiply(b, 0.03125))
+        image = cv2.merge([r, g, b])
+
+        return image
+
+    def droidcast_wait_startup(self):
+        """
+        Wait until DroidCast startup completed.
+        """
+        timeout = Timer(10).start()
+        while 1:
+            self.sleep(0.25)
+            if timeout.reached():
+                break
+
+            try:
+                resp = self.droidcast_session.get(self.droidcast_url('/'), timeout=3)
+                # Route `/` is unavailable, but 404 means startup completed
+                if resp.status_code == 404:
+                    logger.attr('DroidCast', 'online')
+                    return True
+            except requests.exceptions.ConnectionError:
+                logger.attr('DroidCast', 'offline')
+
+        logger.warning('Wait DroidCast startup timeout, assume started')
+        return False
+
+    def droidcast_uninstall(self):
+        """
+        Stop all DroidCast processes and remove DroidCast APK.
+        DroidCast has't been installed but a JAVA class call, uninstall is a file delete.
+        """
+        self.droidcast_stop()
+        logger.info('Removing DroidCast')
+        self.adb_shell(["rm", self.config.DROIDCAST_FILEPATH_REMOTE])
+        self.adb_shell(["rm", self.config.DROIDCAST_RAW_FILEPATH_REMOTE])
+
+    def _iter_droidcast_proc(self) -> t.Iterable[ProcessInfo]:
+        """
+        List all DroidCast processes.
+        """
+        processes = self.proc_list_uiautomator2()
+        for proc in processes:
+            if 'com.rayworks.droidcast.Main' in proc.cmdline:
+                yield proc
+            if 'com.torther.droidcasts.Main' in proc.cmdline:
+                yield proc
+
+    def droidcast_stop(self):
+        """
+        Stop all DroidCast processes.
+        """
+        logger.info('Stopping DroidCast')
+        for proc in self._iter_droidcast_proc():
+            logger.info(f'Kill pid={proc.pid}')
+            self.adb_shell(['kill', '-s', 9, proc.pid])
+
+if __name__ == '__main__':
+    dc = DroidCast(config='oas1')
+    cv2.imshow("iiii", dc.screenshot_uiautomator2())
+    cv2.waitKey(0)

+ 6 - 1
module/device/method/minitouch.py

@@ -3,6 +3,7 @@ import json
 import re
 import re
 import socket
 import socket
 import time
 import time
+import numpy as np
 from functools import wraps
 from functools import wraps
 from typing import List
 from typing import List
 
 
@@ -12,7 +13,7 @@ from uiautomator2 import _Service
 
 
 from module.base.decorator import Config, cached_property, del_cached_property
 from module.base.decorator import Config, cached_property, del_cached_property
 from module.base.timer import Timer
 from module.base.timer import Timer
-from module.base.utils import *
+from module.base.utils import random_rectangle_point
 from module.device.connection import Connection
 from module.device.connection import Connection
 from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
 from module.device.method.utils import RETRY_TRIES, retry_sleep, handle_adb_error
 from module.exception import RequestHumanTakeover, ScriptError
 from module.exception import RequestHumanTakeover, ScriptError
@@ -568,3 +569,7 @@ class Minitouch(Connection):
 
 
         builder.up().commit()
         builder.up().commit()
         self.minitouch_send()
         self.minitouch_send()
+
+if __name__ == '__main__':
+    mm = Minitouch(config='oas1')
+    mm.click_minitouch(200, 150)

+ 0 - 1
module/device/method/uiautomator_2.py

@@ -328,6 +328,5 @@ class Uiautomator2(Connection):
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     ui2 = Uiautomator2(config='oas1')
     ui2 = Uiautomator2(config='oas1')
-    ui2.app_start_uiautomator2()
     cv2.imshow("iiii", ui2.screenshot_uiautomator2())
     cv2.imshow("iiii", ui2.screenshot_uiautomator2())
     cv2.waitKey(0)
     cv2.waitKey(0)

+ 279 - 0
module/device/method/window.py

@@ -0,0 +1,279 @@
+# This Python file uses the following encoding: utf-8
+# @author runhey
+# github https://github.com/runhey
+import cv2
+from numpy import frombuffer, uint8, array, random
+import numpy as np
+import time
+
+from math import dist
+from cached_property import cached_property
+from win32gui import (GetWindowText, EnumWindows, FindWindow, FindWindowEx,
+                      IsWindow, GetWindowRect, GetWindowDC, DeleteObject,
+                      SetForegroundWindow, IsWindowVisible, GetDC, GetParent,
+                      EnumChildWindows)
+from win32con import (SRCCOPY, DESKTOPHORZRES, DESKTOPVERTRES, WM_LBUTTONUP,
+                      WM_LBUTTONDOWN, WM_ACTIVATE, WA_ACTIVE, MK_LBUTTON,
+                      WM_NCHITTEST, WM_SETCURSOR, HTCLIENT, WM_MOUSEMOVE)
+from win32ui import CreateDCFromHandle, CreateBitmap
+from win32api import GetSystemMetrics, SendMessage, MAKELONG, PostMessage
+from win32con import SRCCOPY
+
+from module.base.cBezier import BezierTrajectory
+from module.exception import RequestHumanTakeover, ScriptError
+from module.base.decorator import Config
+from module.base.timer import timer
+from module.logger import logger
+from module.device.handle import Handle, window_scale_rate
+
+
+class Window(Handle):
+
+    def screenshot_window_background(self):
+        """
+        后台截屏
+        :return:
+        """
+        win_size = self.screenshot_size
+        scale_rate = window_scale_rate()
+        widthScreen = int(win_size[0] * scale_rate)
+        heightScreen = int(win_size[1] * scale_rate)
+        # 返回句柄窗口的设备环境,覆盖整个窗口,包括非客户区,标题栏,菜单,边框
+        hwndDc = GetWindowDC(self.screenshot_handle_num)
+        # 创建设备描述表
+        mfcDc = CreateDCFromHandle(hwndDc)
+        # 创建内存设备描述表
+        saveDc = mfcDc.CreateCompatibleDC()
+        # 创建位图对象准备保存图片
+        saveBitMap = CreateBitmap()
+        # 为bitmap开辟存储空间
+        saveBitMap.CreateCompatibleBitmap(mfcDc, widthScreen, heightScreen)
+        # 将截图保存到saveBitMap中
+        saveDc.SelectObject(saveBitMap)
+        # 保存bitmap到内存设备描述表
+        saveDc.BitBlt((0, 0), (widthScreen, heightScreen), mfcDc, (0, 0), SRCCOPY)
+
+        # 保存图像
+        signedIntsArray = saveBitMap.GetBitmapBits(True)
+        imgSrceen = frombuffer(signedIntsArray, dtype='uint8')
+        imgSrceen.shape = (heightScreen, widthScreen, 4)
+        # imgSrceen = cv2.cvtColor(imgSrceen, cv2.COLOR_BGRA2GRAY)
+        imgSrceen = cv2.resize(imgSrceen, (win_size[0], win_size[1]))
+
+        # 测试显示截图图片
+        cv2.namedWindow('imgSrceen')  # 命名窗口
+        cv2.imshow("imgSrceen", imgSrceen)  # 显示
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
+        # 内存释放
+        DeleteObject(saveBitMap.GetHandle())
+        saveDc.DeleteDC()
+        mfcDc.DeleteDC()
+        return imgSrceen
+
+    @cached_property
+    def control_handle_list(self) -> list:
+        """
+        不同的模拟器需要的子句柄不同
+        mumu模拟器的控制需要 根窗口和第一个子窗口
+        雷电模拟器只需要TheRender窗口
+        夜神模拟器
+        :return:
+        """
+        result = []
+        if self.emulator_family == "mumu_player_family":
+            result.append(self.root_node.num)
+            result.append(self.root_node.children[0].num)
+            return result
+        elif self.emulator_family == 'nox_player_family':
+            result.append(self.root_node.num)
+            try:
+                result.append(self.root_node.children[1].num)
+                result.append(self.root_node.children[1].children[1].num)
+                result.append(self.root_node.children[1].children[1].children[0].num)
+            except:
+                result.append(self.root_node.children[2].num)
+                result.append(self.root_node.children[2].children[1].num)
+                result.append(self.root_node.children[2].children[1].children[0].num)
+            return result
+        elif self.emulator_family == 'ld_player_family':
+            result.append(self.root_node.children[0].num)
+            return result
+        elif self.emulator_family == 'memu_player_family':
+            pass
+        elif self.emulator_family == 'bluestacks_family':
+            pass
+
+    def click_window_message(self, x: int, y: int):
+        """
+
+        :param x:
+        :param y:
+        :return:
+        """
+        # if x >= 1280:
+        #     x = 1280
+        # if x <= 0:
+        #     x = 0
+        # if y >= 720:
+        #     y = 720
+        # if y <= 0:
+        #     y = 0
+        # 我不知道为什么的使用的pywin32==306的版本会导致获取的图片的是(1024, 576)
+        # 所有我在点击的时候会除以这个缩放比例
+        x = int(x / self.window_scale_rate)
+        y = int(y / self.window_scale_rate)
+        logger.info(self.control_handle_list)
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 2:  # mumu模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            time.sleep((random.randint(100, 200)) / 1000.0)  # 点击弹起改为随机,时间100ms-200ms
+            SendMessage(self.control_handle_list[1], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type > 2:  # 夜神模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[2], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[3], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            time.sleep((random.randint(100, 200)) / 1000.0)  # 点击弹起改为随机,时间100ms-200ms
+            SendMessage(self.control_handle_list[3], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type == 1:  # 雷电模拟器
+            clickPos = MAKELONG(x, y)
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, clickPos)  # 模拟鼠标按下
+            time.sleep((random.randint(100, 200)) / 1000.0)  # 点击弹起改为随机,时间100ms-200ms
+            SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos)  # 模拟鼠标弹起
+
+    def long_click_window_message(self, x: int, y: int, duration: float):
+        """
+
+        :param x:
+        :param y:
+        :param duration: 持续时间 单位秒
+        :return:
+        """
+        # 我不知道为什么的使用的pywin32==306的版本会导致获取的图片的是(1024, 576)
+        # 所有我在点击的时候会除以这个缩放比例
+        x = int(x / self.window_scale_rate)
+        y = int(y / self.window_scale_rate)
+        logger.info(self.control_handle_list)
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 2:  # mumu模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[1], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type > 2:  # 夜神模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[2], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[3], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[3], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type == 1:  # 雷电模拟器
+            clickPos = MAKELONG(x, y)
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, clickPos)  # 模拟鼠标按下
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos)  # 模拟鼠标弹起
+
+    # @timer
+    def swipe_window_message(self, startPos: list, endPos: list) -> None:
+        """
+        后台滑动
+        :param startPos:
+        :param endPos:
+        :return:
+        """
+        # 生成的坐标点列表
+        interval: int = 8  # 每次移动的间隔时间
+        numberList: int = int(dist(startPos, endPos) / (1.5 * interval))  # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
+        le = random.randint(2, 4)  #
+        deviation = random.randint(20, 40)  # 幅度
+        _type: int = 3
+        obbsType = random.random()  # 0.8的概率是先快中间慢后面快, 0.1概率是先快后慢, 0.1概率先慢后快
+        if 0 < obbsType <= 0.8:
+            _type = 3
+        elif obbsType < 0.9:
+            _type = 2
+        else:
+            _type = 1
+        trace: list = BezierTrajectory.trackArray(start=startPos, end=endPos, numberList=numberList, le=le,
+                                                  deviation=deviation, bias=0.5, type=_type, cbb=0, yhh=20)
+
+        # 使用生成的点列表进行拖拽
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 1:  # 雷电模拟器
+            handleNum = self.control_handle_list[0]
+        elif emulator_type == 2:  # mumu模拟器
+            handleNum = self.control_handle_list[1]
+        elif emulator_type > 2:  # 夜神模拟器
+            handleNum = self.control_handle_list[3]
+        PostMessage(handleNum, WM_ACTIVATE, WA_ACTIVE, 0)  # 激活窗口
+        # 先移动到第一个点
+        tmpPos = MAKELONG(trace[0][0], trace[0][1])
+        SendMessage(handleNum, WM_NCHITTEST, 0, tmpPos)
+        SendMessage(handleNum, WM_SETCURSOR, handleNum, MAKELONG(HTCLIENT, WM_LBUTTONDOWN))
+        PostMessage(handleNum, WM_LBUTTONDOWN, MK_LBUTTON, tmpPos)
+        # 一点一点移动鼠标
+        for pos in trace:
+            tmpPos = MAKELONG(pos[0], pos[1])
+            PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, tmpPos)
+            time.sleep((interval + random.randint(-2, 2)) / 1000.0)
+        # 最后释放鼠标
+        tmpPos = MAKELONG(trace[-1][0], trace[-1][1])
+        PostMessage(handleNum, WM_LBUTTONUP, 0, tmpPos)
+
+    def swipe_vector_window_message2(self, startPos: list, endPos: list) -> None:
+        """
+        后台滑动
+        :param startPos:
+        :param endPos:
+        :return:
+        """
+        # 生成的坐标点列表
+        interval: int = 8  # 每次移动的间隔时间
+        numberList: int = int(dist(startPos, endPos) / (1.5 * interval))  # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
+
+        def generate_points(start_pos: list, end_pos: list, number: int) -> list:
+            # 确定两点之间的步长
+            step_x = (end_pos[0] - start_pos[0]) / (number + 1)
+            step_y = (end_pos[1] - start_pos[1]) / (number + 1)
+            # 生成中间点坐标列表
+            points = []
+            for i in range(number):
+                x = start_pos[0] + step_x * (i + 1)
+                y = start_pos[1] + step_y * (i + 1)
+                points.append((x, y))
+            # 添加起始点和最终点到列表中
+            points.insert(0, tuple(start_pos))
+            points.append(tuple(end_pos))
+            return points
+
+        trace: list = generate_points(start_pos=startPos, end_pos=endPos, number=numberList)
+
+        # 使用生成的点列表进行拖拽
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 1:  # 雷电模拟器
+            handleNum = self.control_handle_list[0]
+        elif emulator_type == 2:  # mumu模拟器
+            handleNum = self.control_handle_list[1]
+        elif emulator_type > 2:  # 夜神模拟器
+            handleNum = self.control_handle_list[3]
+        PostMessage(handleNum, WM_ACTIVATE, WA_ACTIVE, 0)  # 激活窗口
+        # 先移动到第一个点
+        tmpPos = MAKELONG(trace[0][0], trace[0][1])
+        SendMessage(handleNum, WM_NCHITTEST, 0, tmpPos)
+        SendMessage(handleNum, WM_SETCURSOR, handleNum, MAKELONG(HTCLIENT, WM_LBUTTONDOWN))
+        PostMessage(handleNum, WM_LBUTTONDOWN, MK_LBUTTON, tmpPos)
+        # 一点一点移动鼠标
+        for pos in trace:
+            tmpPos = MAKELONG(pos[0], pos[1])
+            PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, tmpPos)
+            time.sleep((interval + random.randint(-2, 2)) / 1000.0)
+        # 最后释放鼠标
+        tmpPos = MAKELONG(trace[-1][0], trace[-1][1])
+        PostMessage(handleNum, WM_LBUTTONUP, 0, tmpPos)
+
+
+if __name__ == "__main__":
+    w = Window(config='oas1')
+    # w.click_window_message(x=350, y=220)
+    w.swipe_window_message(startPos=[200, 5], endPos=[300, 300])

+ 256 - 0
module/device/screenshot.py

@@ -0,0 +1,256 @@
+import os
+import time
+from collections import deque
+from datetime import datetime
+
+import cv2
+import numpy as np
+from PIL import Image
+
+from module.base.decorator import cached_property
+from module.base.timer import Timer
+from module.base.utils import get_color, image_size, limit_in, save_image
+from module.device.method.adb import Adb
+from module.device.method.window import Window
+from module.device.method.droidcast import DroidCast
+from module.device.method.scrcpy import Scrcpy
+from module.exception import RequestHumanTakeover, ScriptError
+from module.logger import logger
+
+
+class Screenshot(Adb, DroidCast, Scrcpy, Window):
+    _screen_size_checked = False
+    _screen_black_checked = False
+    _minicap_uninstalled = False
+    _screenshot_interval = Timer(0.1)
+    _last_save_time = {}
+    image: np.ndarray
+
+    @cached_property
+    def screenshot_methods(self):
+        return {
+            'ADB': self.screenshot_adb,
+            'ADB_nc': self.screenshot_adb_nc,
+            'uiautomator2': self.screenshot_uiautomator2,
+            # 'aScreenCap': self.screenshot_ascreencap,
+            # 'aScreenCap_nc': self.screenshot_ascreencap_nc,
+            'DroidCast': self.screenshot_droidcast,
+            'DroidCast_raw': self.screenshot_droidcast_raw,
+            'scrcpy': self.screenshot_scrcpy,
+            'window_background': self.screenshot_window_background
+        }
+
+    def screenshot(self):
+        """
+        Returns:
+            np.ndarray:
+        """
+        self._screenshot_interval.wait()
+        self._screenshot_interval.reset()
+
+        for _ in range(2):
+            method = self.screenshot_methods.get(
+                self.config.get_arg('Script', 'Device', 'ScreenshotMethod'),
+                self.screenshot_adb  # 第二个参数默认的是screenshot_adb
+            )
+            self.image = method()
+
+            # if self.config.Emulator_ScreenshotDedithering:
+            #     # This will take 40-60ms
+            #     cv2.fastNlMeansDenoising(self.image, self.image, h=17, templateWindowSize=1, searchWindowSize=2)
+            # self.image = self._handle_orientated_image(self.image)
+
+            # if self.config.Error_SaveError:
+            if self.config.get_arg('Script', 'Error', 'SaveError'):
+                self.screenshot_deque.append({'time': datetime.now(), 'image': self.image})
+
+            if self.check_screen_size() and self.check_screen_black():
+                break
+            else:
+                continue
+
+        return self.image
+
+    def _handle_orientated_image(self, image):
+        """
+        Args:
+            image (np.ndarray):
+
+        Returns:
+            np.ndarray:
+        """
+        width, height = image_size(self.image)
+        if width == 1280 and height == 720:
+            return image
+
+        # Rotate screenshots only when they're not 1280x720
+        if self.orientation == 0:
+            pass
+        elif self.orientation == 1:
+            image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
+        elif self.orientation == 2:
+            image = cv2.rotate(image, cv2.ROTATE_180)
+        elif self.orientation == 3:
+            image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
+        else:
+            raise ScriptError(f'Invalid device orientation: {self.orientation}')
+
+        return image
+
+    @cached_property
+    def screenshot_deque(self):
+        # return deque(maxlen=int(self.config.Error_ScreenshotLength))
+        return deque(maxlen=int(self.config.get_arg("Script", "Error", "ScreenshotLength")))
+
+    def save_screenshot(self, genre='items', interval=None, to_base_folder=False):
+        """Save a screenshot. Use millisecond timestamp as file name.
+
+        Args:
+            genre (str, optional): Screenshot type.
+            interval (int, float): Seconds between two save. Saves in the interval will be dropped.
+            to_base_folder (bool): If save to base folder.
+
+        Returns:
+            bool: True if save succeed.
+        """
+        now = time.time()
+        if interval is None:
+            interval = self.config.SCREEN_SHOT_SAVE_INTERVAL
+
+        if now - self._last_save_time.get(genre, 0) > interval:
+            fmt = 'png'
+            file = '%s.%s' % (int(now * 1000), fmt)
+
+            folder = self.config.SCREEN_SHOT_SAVE_FOLDER_BASE if to_base_folder else self.config.SCREEN_SHOT_SAVE_FOLDER
+            folder = os.path.join(folder, genre)
+            if not os.path.exists(folder):
+                os.mkdir(folder)
+
+            file = os.path.join(folder, file)
+            self.image_save(file)
+            self._last_save_time[genre] = now
+            return True
+        else:
+            self._last_save_time[genre] = now
+            return False
+
+    def screenshot_last_save_time_reset(self, genre):
+        self._last_save_time[genre] = 0
+
+    def screenshot_interval_set(self, interval=None):
+        """
+        Args:
+            interval (int, float, str):
+                Minimum interval between 2 screenshots in seconds.
+                Or None for Optimization_ScreenshotInterval, 'combat' for Optimization_CombatScreenshotInterval
+        """
+        if interval is None:
+            origin = self.config.Optimization_ScreenshotInterval
+            interval = limit_in(origin, 0.1, 0.3)
+            if interval != origin:
+                logger.warning(f'Optimization.ScreenshotInterval {origin} is revised to {interval}')
+                self.config.Optimization_ScreenshotInterval = interval
+        elif interval == 'combat':
+            origin = self.config.Optimization_CombatScreenshotInterval
+            interval = limit_in(origin, 0.3, 1.0)
+            if interval != origin:
+                logger.warning(f'Optimization.CombatScreenshotInterval {origin} is revised to {interval}')
+                self.config.Optimization_CombatScreenshotInterval = interval
+        elif isinstance(interval, (int, float)):
+            # No limitation for manual set in code
+            pass
+        else:
+            logger.warning(f'Unknown screenshot interval: {interval}')
+            raise ScriptError(f'Unknown screenshot interval: {interval}')
+        # Screenshot interval in scrcpy is meaningless,
+        # video stream is received continuously no matter you use it or not.
+        # if self.config.Emulator_ScreenshotMethod == 'scrcpy':
+        if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'scrcpy':
+            interval = 0.1
+
+        if interval != self._screenshot_interval.limit:
+            logger.info(f'Screenshot interval set to {interval}s')
+            self._screenshot_interval.limit = interval
+
+    def image_show(self, image=None):
+        if image is None:
+            image = self.image
+        Image.fromarray(image).show()
+
+    def image_save(self, file):
+        save_image(self.image, file)
+
+    def check_screen_size(self):
+        """
+        Screen size must be 1280x720.
+        Take a screenshot before call.
+        """
+        if self._screen_size_checked:
+            return True
+
+        orientated = False
+        for _ in range(2):
+            # Check screen size
+            width, height = image_size(self.image)
+            logger.attr('Screen_size', f'{width}x{height}')
+            if width == 1280 and height == 720:
+                self._screen_size_checked = True
+                return True
+            elif not orientated and (width == 720 and height == 1280):
+                logger.info('Received orientated screenshot, handling')
+                self.get_orientation()
+                self.image = self._handle_orientated_image(self.image)
+                orientated = True
+                width, height = image_size(self.image)
+                if width == 720 and height == 1280:
+                    logger.info('Unable to handle orientated screenshot, continue for now')
+                    return True
+                else:
+                    continue
+            # elif self.config.Emulator_Serial == 'wsa-0':
+            #     self.display_resize_wsa(0)
+            #     return False
+            elif hasattr(self, 'app_is_running') and not self.app_is_running():
+                logger.warning('Received orientated screenshot, game not running')
+                return True
+            else:
+                logger.critical(f'Resolution not supported: {width}x{height}')
+                logger.critical('Please set emulator resolution to 1280x720')
+                raise RequestHumanTakeover
+
+    def check_screen_black(self):
+        if self._screen_black_checked:
+            return True
+        # Check screen color
+        # May get a pure black screenshot on some emulators.
+        color = get_color(self.image, area=(0, 0, 1280, 720))
+        if sum(color) < 1:
+            # if self.config.Emulator_Serial == 'wsa-0':
+            #     for _ in range(2):
+            #         display = self.get_display_id()
+            #         if display == 0:
+            #             return True
+            #     logger.info(f'Game running on display {display}')
+            #     logger.warning('Game not running on display 0, will be restarted')
+            #     self.app_stop_uiautomator2()
+            #     return False
+            if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'uiautomator2':
+                logger.warning(f'Received pure black screenshots from emulator, color: {color}')
+                logger.warning('Uninstall minicap and retry')
+                self.uninstall_minicap()
+                self._screen_black_checked = False
+                return False
+            else:
+                logger.warning(f'Received pure black screenshots from emulator, color: {color}')
+                logger.warning(f'Screenshot method {self.config.get_arg("Script", "Device", "ScreenshotMethod")}'
+                               f'may not work on emulator `{self.serial}`, or the emulator is not fully started')
+                if self.is_mumu_family:
+                    if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'DroidCast':
+                        self.droidcast_stop()
+                    else:
+                        logger.warning('If you are using MuMu X, please upgrade to version >= 12.1.5.0')
+                self._screen_black_checked = False
+                return False
+        else:
+            self._screen_black_checked = True
+            return True

+ 2 - 0
requirements-in.txt

@@ -7,3 +7,5 @@ pyyaml==6.0
 numpy==1.24.3
 numpy==1.24.3
 opencv-python==4.7.0.72
 opencv-python==4.7.0.72
 websockets==11.0.3
 websockets==11.0.3
+anytree==2.8.0
+pywin32==306

+ 10 - 2
requirements.txt

@@ -7,7 +7,7 @@
 --index-url https://pypi.tuna.tsinghua.edu.cn/simple
 --index-url https://pypi.tuna.tsinghua.edu.cn/simple
 --trusted-host pypi.tuna.tsinghua.edu.cn
 --trusted-host pypi.tuna.tsinghua.edu.cn
 
 
-adbutils==1.2.9         # via -r requirements-in.txt, uiautomator2
+adbutils==0.11.0          # via -r requirements-in.txt, uiautomator2
 apkutils2==1.0.0          # via adbutils
 apkutils2==1.0.0          # via adbutils
 cached-property==1.5.2    # via uiautomator2
 cached-property==1.5.2    # via uiautomator2
 certifi==2023.5.7         # via requests
 certifi==2023.5.7         # via requests
@@ -21,21 +21,29 @@ filelock==3.12.0          # via uiautomator2
 idna==3.4                 # via requests
 idna==3.4                 # via requests
 logzero==1.7.0            # via uiautomator2
 logzero==1.7.0            # via uiautomator2
 lxml==4.9.2               # via uiautomator2
 lxml==4.9.2               # via uiautomator2
+markdown-it-py==2.2.0     # via rich
+mdurl==0.1.2              # via markdown-it-py
+numpy==1.24.3             # via -r requirements-in.txt, opencv-python
+opencv-python==4.7.0.72   # via -r requirements-in.txt
 packaging==20.9           # via deprecation, uiautomator2
 packaging==20.9           # via deprecation, uiautomator2
 pillow==9.5.0             # via uiautomator2
 pillow==9.5.0             # via uiautomator2
 progress==1.6             # via uiautomator2
 progress==1.6             # via uiautomator2
 py==1.11.0                # via retry
 py==1.11.0                # via retry
 pyelftools==0.29          # via apkutils2
 pyelftools==0.29          # via apkutils2
+pygments==2.15.1          # via rich
 pyparsing==3.0.9          # via packaging
 pyparsing==3.0.9          # via packaging
 pyside6==6.4.3            # via -r requirements-in.txt
 pyside6==6.4.3            # via -r requirements-in.txt
 pyside6-addons==6.4.3     # via pyside6
 pyside6-addons==6.4.3     # via pyside6
 pyside6-essentials==6.4.3  # via pyside6, pyside6-addons
 pyside6-essentials==6.4.3  # via pyside6, pyside6-addons
+pyyaml==6.0               # via -r requirements-in.txt
 requests==2.30.0          # via adbutils, uiautomator2
 requests==2.30.0          # via adbutils, uiautomator2
 retry==0.9.2              # via adbutils, uiautomator2
 retry==0.9.2              # via adbutils, uiautomator2
+rich==13.3.5              # via -r requirements-in.txt
 shiboken6==6.4.3          # via pyside6, pyside6-addons, pyside6-essentials
 shiboken6==6.4.3          # via pyside6, pyside6-addons, pyside6-essentials
 six==1.16.0               # via adbutils, uiautomator2
 six==1.16.0               # via adbutils, uiautomator2
-uiautomator2==2.16.23     # via -r requirements-in.txt
+uiautomator2==2.16.17     # via -r requirements-in.txt
 urllib3==2.0.2            # via requests
 urllib3==2.0.2            # via requests
+websockets==11.0.3        # via -r requirements-in.txt
 whichcraft==0.6.1         # via adbutils, uiautomator2
 whichcraft==0.6.1         # via adbutils, uiautomator2
 wrapt==1.15.0             # via deprecated
 wrapt==1.15.0             # via deprecated
 xmltodict==0.13.0         # via apkutils2
 xmltodict==0.13.0         # via apkutils2