runhey 3 лет назад
Родитель
Сommit
60549ef7cf

+ 808 - 18
module/base/utils.py

@@ -1,10 +1,12 @@
-# This Python file uses the following encoding: utf-8
 import re
 
 import cv2
 import numpy as np
 from PIL import Image
 
+REGEX_NODE = re.compile(r'(-?[A-Za-z]+)(-?\d+)')
+
+
 def random_normal_distribution_int(a, b, n=3):
     """Generate a normal distribution int within the interval. Use the average value of several random numbers to
     simulate normal distribution.
@@ -23,6 +25,119 @@ def random_normal_distribution_int(a, b, n=3):
     else:
         return b
 
+
+def random_rectangle_point(area, n=3):
+    """Choose a random point in an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        n (int): The amount of numbers in simulation. Default to 3.
+
+    Returns:
+        tuple(int): (x, y)
+    """
+    x = random_normal_distribution_int(area[0], area[2], n=n)
+    y = random_normal_distribution_int(area[1], area[3], n=n)
+    return x, y
+
+
+def random_rectangle_vector(vector, box, random_range=(0, 0, 0, 0), padding=15):
+    """Place a vector in a box randomly.
+
+    Args:
+        vector: (x, y)
+        box: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        random_range (tuple): Add a random_range to vector. (x_min, y_min, x_max, y_max).
+        padding (int):
+
+    Returns:
+        tuple(int), tuple(int): start_point, end_point.
+    """
+    vector = np.array(vector) + random_rectangle_point(random_range)
+    vector = np.round(vector).astype(np.int)
+    half_vector = np.round(vector / 2).astype(np.int)
+    box = np.array(box) + np.append(np.abs(half_vector) + padding, -np.abs(half_vector) - padding)
+    center = random_rectangle_point(box)
+    start_point = center - half_vector
+    end_point = start_point + vector
+    return tuple(start_point), tuple(end_point)
+
+
+def random_rectangle_vector_opted(
+        vector, box, random_range=(0, 0, 0, 0), padding=15, whitelist_area=None, blacklist_area=None):
+    """
+    Place a vector in a box randomly.
+
+    When emulator/game stuck, it treats a swipe as a click, clicking at the end of swipe path.
+    To prevent this, random results need to be filtered.
+
+    Args:
+        vector: (x, y)
+        box: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        random_range (tuple): Add a random_range to vector. (x_min, y_min, x_max, y_max).
+        padding (int):
+        whitelist_area: (list[tuple[int]]):
+            A list of area that safe to click. Swipe path will end there.
+        blacklist_area: (list[tuple[int]]):
+            If none of the whitelist_area satisfies current vector, blacklist_area will be used.
+            Delete random path that ends in any blacklist_area.
+
+    Returns:
+        tuple(int), tuple(int): start_point, end_point.
+    """
+    vector = np.array(vector) + random_rectangle_point(random_range)
+    vector = np.round(vector).astype(np.int)
+    half_vector = np.round(vector / 2).astype(np.int)
+    box_pad = np.array(box) + np.append(np.abs(half_vector) + padding, -np.abs(half_vector) - padding)
+    box_pad = area_offset(box_pad, half_vector)
+    segment = int(np.linalg.norm(vector) // 70) + 1
+
+    def in_blacklist(end):
+        if not blacklist_area:
+            return False
+        for x in range(segment + 1):
+            point = - vector * x / segment + end
+            for area in blacklist_area:
+                if point_in_area(point, area, threshold=0):
+                    return True
+        return False
+
+    if whitelist_area:
+        for area in whitelist_area:
+            area = area_limit(area, box_pad)
+            if all([x > 0 for x in area_size(area)]):
+                end_point = random_rectangle_point(area)
+                for _ in range(10):
+                    if in_blacklist(end_point):
+                        continue
+                    return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+    for _ in range(100):
+        end_point = random_rectangle_point(box_pad)
+        if in_blacklist(end_point):
+            continue
+        return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+    end_point = random_rectangle_point(box_pad)
+    return point_limit(end_point - vector, box), point_limit(end_point, box)
+
+
+def random_line_segments(p1, p2, n, random_range=(0, 0, 0, 0)):
+    """Cut a line into multiple segments.
+
+    Args:
+        p1: (x, y).
+        p2: (x, y).
+        n: Number of slice.
+        random_range: Add a random_range to points.
+
+    Returns:
+        list[tuple]: [(x0, y0), (x1, y1), (x2, y2)]
+    """
+    return [tuple((((n - index) * p1 + index * p2) / n).astype(int) + random_rectangle_point(random_range))
+            for index in range(0, n + 1)]
+
+
 def ensure_time(second, n=3, precision=3):
     """Ensure to be time.
 
@@ -53,6 +168,190 @@ def ensure_time(second, n=3, precision=3):
         return second
 
 
+def ensure_int(*args):
+    """
+    Convert all elements to int.
+    Return the same structure as nested objects.
+
+    Args:
+        *args:
+
+    Returns:
+        list:
+    """
+
+    def to_int(item):
+        try:
+            return int(item)
+        except TypeError:
+            result = [to_int(i) for i in item]
+            if len(result) == 1:
+                result = result[0]
+            return result
+
+    return to_int(args)
+
+
+def area_offset(area, offset):
+    """
+    Move an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        offset: (x, y).
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    upper_left_x, upper_left_y, bottom_right_x, bottom_right_y = area
+    x, y = offset
+    return upper_left_x + x, upper_left_y + y, bottom_right_x + x, bottom_right_y + y
+
+
+def area_pad(area, pad=10):
+    """
+    Inner offset an area.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        pad (int):
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    upper_left_x, upper_left_y, bottom_right_x, bottom_right_y = area
+    return upper_left_x + pad, upper_left_y + pad, bottom_right_x - pad, bottom_right_y - pad
+
+
+def limit_in(x, lower, upper):
+    """
+    Limit x within range (lower, upper)
+
+    Args:
+        x:
+        lower:
+        upper:
+
+    Returns:
+        int, float:
+    """
+    return max(min(x, upper), lower)
+
+
+def area_limit(area1, area2):
+    """
+    Limit an area in another area.
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+    """
+    x_lower, y_lower, x_upper, y_upper = area2
+    return (
+        limit_in(area1[0], x_lower, x_upper),
+        limit_in(area1[1], y_lower, y_upper),
+        limit_in(area1[2], x_lower, x_upper),
+        limit_in(area1[3], y_lower, y_upper),
+    )
+
+
+def area_size(area):
+    """
+    Area size or shape.
+
+    Args:
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (x, y).
+    """
+    return (
+        max(area[2] - area[0], 0),
+        max(area[3] - area[1], 0)
+    )
+
+
+def point_limit(point, area):
+    """
+    Limit point in an area.
+
+    Args:
+        point: (x, y).
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+
+    Returns:
+        tuple: (x, y).
+    """
+    return (
+        limit_in(point[0], area[0], area[2]),
+        limit_in(point[1], area[1], area[3])
+    )
+
+
+def point_in_area(point, area, threshold=5):
+    """
+
+    Args:
+        point: (x, y).
+        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    return area[0] - threshold < point[0] < area[2] + threshold and area[1] - threshold < point[1] < area[3] + threshold
+
+
+def area_in_area(area1, area2, threshold=5):
+    """
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    return area2[0] - threshold <= area1[0] \
+           and area2[1] - threshold <= area1[1] \
+           and area1[2] <= area2[2] + threshold \
+           and area1[3] <= area2[3] + threshold
+
+
+def area_cross_area(area1, area2, threshold=5):
+    """
+
+    Args:
+        area1: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        area2: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+        threshold: int
+
+    Returns:
+        bool:
+    """
+    # https://www.yiiven.cn/rect-is-intersection.html
+    xa1, ya1, xa2, ya2 = area1
+    xb1, yb1, xb2, yb2 = area2
+    return abs(xb2 + xb1 - xa2 - xa1) <= xa2 - xa1 + xb2 - xb1 + threshold * 2 \
+           and abs(yb2 + yb1 - ya2 - ya1) <= ya2 - ya1 + yb2 - yb1 + threshold * 2
+
+
+def float2str(n, decimal=3):
+    """
+    Args:
+        n (float):
+        decimal (int):
+
+    Returns:
+        str:
+    """
+    return str(round(n, decimal)).ljust(decimal + 2, "0")
+
+
 def point2str(x, y, length=4):
     """
     Args:
@@ -66,31 +365,522 @@ def point2str(x, y, length=4):
     return '(%s, %s)' % (str(int(x)).rjust(length), str(int(y)).rjust(length))
 
 
-def random_rectangle_point(area, n=3):
-    """Choose a random point in an area.
+def col2name(col):
+    """
+    Convert a zero indexed column cell reference to a string.
 
     Args:
-        area: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
-        n (int): The amount of numbers in simulation. Default to 3.
+       col: The cell column. Int.
 
     Returns:
-        tuple(int): (x, y)
+        Column style string.
+
+    Examples:
+        0 -> A, 3 -> D, 35 -> AJ, -1 -> -A
     """
-    x = random_normal_distribution_int(area[0], area[2], n=n)
-    y = random_normal_distribution_int(area[1], area[3], n=n)
-    return x, y
 
-def random_line_segments(p1, p2, n, random_range=(0, 0, 0, 0)):
-    """Cut a line into multiple segments.
+    col_neg = col < 0
+    if col_neg:
+        col_num = -col
+    else:
+        col_num = col + 1  # Change to 1-index.
+    col_str = ''
+
+    while col_num:
+        # Set remainder from 1 .. 26
+        remainder = col_num % 26
+
+        if remainder == 0:
+            remainder = 26
+
+        # Convert the remainder to a character.
+        col_letter = chr(remainder + 64)
+
+        # Accumulate the column letters, right to left.
+        col_str = col_letter + col_str
+
+        # Get the next order of magnitude.
+        col_num = int((col_num - 1) / 26)
+
+    if col_neg:
+        return '-' + col_str
+    else:
+        return col_str
+
+
+def name2col(col_str):
+    """
+    Convert a cell reference in A1 notation to a zero indexed row and column.
 
     Args:
-        p1: (x, y).
-        p2: (x, y).
-        n: Number of slice.
-        random_range: Add a random_range to points.
+       col_str:  A1 style string.
 
     Returns:
-        list[tuple]: [(x0, y0), (x1, y1), (x2, y2)]
+        row, col: Zero indexed cell row and column indices.
     """
-    return [tuple((((n - index) * p1 + index * p2) / n).astype(int) + random_rectangle_point(random_range))
-            for index in range(0, n + 1)]
+    # Convert base26 column string to number.
+    expn = 0
+    col = 0
+    col_neg = col_str.startswith('-')
+    col_str = col_str.strip('-').upper()
+
+    for char in reversed(col_str):
+        col += (ord(char) - 64) * (26 ** expn)
+        expn += 1
+
+    if col_neg:
+        return -col
+    else:
+        return col - 1  # Convert 1-index to zero-index
+
+
+def node2location(node):
+    """
+    See location2node()
+
+    Args:
+        node (str): Example: 'E3'
+
+    Returns:
+        tuple[int]: Example: (4, 2)
+    """
+    res = REGEX_NODE.search(node)
+    if res:
+        x, y = res.group(1), res.group(2)
+        y = int(y)
+        if y > 0:
+            y -= 1
+        return name2col(x), y
+    else:
+        # Whatever
+        return ord(node[0]) % 32 - 1, int(node[1:]) - 1
+
+
+def location2node(location):
+    """
+    Convert location tuple to an Excel-like cell.
+    Accept negative values also.
+
+         -2   -1    0    1    2    3
+    -2 -B-2 -A-2  A-2  B-2  C-2  D-2
+    -1 -B-1 -A-1  A-1  B-1  C-1  D-1
+     0  -B1  -A1   A1   B1   C1   D1
+     1  -B2  -A2   A2   B2   C2   D2
+     2  -B3  -A3   A3   B3   C3   D3
+     3  -B4  -A4   A4   B4   C4   D4
+
+    # To generate the table above
+    index = range(-2, 4)
+    row = '   ' + ' '.join([str(i).rjust(4) for i in index])
+    print(row)
+    for y in index:
+        row = str(y).rjust(2) + ' ' + ' '.join([location2node((x, y)).rjust(4) for x in index])
+        print(row)
+
+    def check(node):
+        return point2str(*node2location(location2node(node)), length=2)
+    row = '   ' + ' '.join([str(i).rjust(8) for i in index])
+    print(row)
+    for y in index:
+        row = str(y).rjust(2) + ' ' + ' '.join([check((x, y)).rjust(4) for x in index])
+        print(row)
+
+    Args:
+        location (tuple[int]):
+
+    Returns:
+        str:
+    """
+    x, y = location
+    if y >= 0:
+        y += 1
+    return col2name(x) + str(y)
+
+
+def load_image(file, area=None):
+    """
+    Load an image like pillow and drop alpha channel.
+
+    Args:
+        file (str):
+        area (tuple):
+
+    Returns:
+        np.ndarray:
+    """
+    image = Image.open(file)
+    if area is not None:
+        image = image.crop(area)
+    image = np.array(image)
+    channel = image.shape[2] if len(image.shape) > 2 else 1
+    if channel > 3:
+        image = image[:, :, :3].copy()
+    return image
+
+
+def save_image(image, file):
+    """
+    Save an image like pillow.
+
+    Args:
+        image (np.ndarray):
+        file (str):
+    """
+    # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    # cv2.imwrite(file, image)
+    Image.fromarray(image).save(file)
+
+
+def crop(image, area, copy=True):
+    """
+    Crop image like pillow, when using opencv / numpy.
+    Provides a black background if cropping outside of image.
+
+    Args:
+        image (np.ndarray):
+        area:
+        copy (bool):
+
+    Returns:
+        np.ndarray:
+    """
+    x1, y1, x2, y2 = map(int, map(round, area))
+    h, w = image.shape[:2]
+    border = np.maximum((0 - y1, y2 - h, 0 - x1, x2 - w), 0)
+    x1, y1, x2, y2 = np.maximum((x1, y1, x2, y2), 0)
+    image = image[y1:y2, x1:x2]
+    if sum(border) > 0:
+        image = cv2.copyMakeBorder(image, *border, borderType=cv2.BORDER_CONSTANT, value=(0, 0, 0))
+    if copy:
+        image = image.copy()
+    return image
+
+
+def resize(image, size):
+    """
+    Resize image like pillow image.resize(), but implement in opencv.
+    Pillow uses PIL.Image.NEAREST by default.
+
+    Args:
+        image (np.ndarray):
+        size: (x, y)
+
+    Returns:
+        np.ndarray:
+    """
+    return cv2.resize(image, size, interpolation=cv2.INTER_NEAREST)
+
+
+def image_channel(image):
+    """
+    Args:
+        image (np.ndarray):
+
+    Returns:
+        int: 0 for grayscale, 3 for RGB.
+    """
+    return image.shape[2] if len(image.shape) == 3 else 0
+
+
+def image_size(image):
+    """
+    Args:
+        image (np.ndarray):
+
+    Returns:
+        int, int: width, height
+    """
+    shape = image.shape
+    return shape[1], shape[0]
+
+
+def rgb2gray(image):
+    """
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(image)
+    return cv2.add(
+        cv2.multiply(cv2.max(cv2.max(r, g), b), 0.5),
+        cv2.multiply(cv2.min(cv2.min(r, g), b), 0.5)
+    )
+
+
+def rgb2hsv(image):
+    """
+    Convert RGB color space to HSV color space.
+    HSV is Hue Saturation Value.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Hue (0~360), Saturation (0~100), Value (0~100).
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV).astype(np.float)
+    image *= (360 / 180, 100 / 255, 100 / 255)
+    return image
+
+
+def rgb2yuv(image):
+    """
+    Convert RGB to YUV color space.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
+    return image
+
+
+def rgb2luma(image):
+    """
+    Convert RGB to the Y channel (Luminance) in YUV color space.
+
+    Args:
+        image (np.ndarray): Shape (height, width, channel)
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
+    luma, _, _ = cv2.split(image)
+    return luma
+
+
+def get_color(image, area):
+    """Calculate the average color of a particular area of the image.
+
+    Args:
+        image (np.ndarray): Screenshot.
+        area (tuple): (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y)
+
+    Returns:
+        tuple: (r, g, b)
+    """
+    temp = crop(image, area, copy=False)
+    color = cv2.mean(temp)
+    return color[:3]
+
+
+def get_bbox(image, threshold=0):
+    """
+    A numpy implementation of the getbbox() in pillow.
+
+    Args:
+        image (np.ndarray): Screenshot.
+        threshold (int): Color <= threshold will be considered black
+
+    Returns:
+        tuple: (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y)
+    """
+    if image_channel(image) == 3:
+        image = np.max(image, axis=2)
+    x = np.where(np.max(image, axis=0) > threshold)[0]
+    y = np.where(np.max(image, axis=1) > threshold)[0]
+    return x[0], y[0], x[-1] + 1, y[-1] + 1
+
+
+def color_similarity(color1, color2):
+    """
+    Args:
+        color1 (tuple): (r, g, b)
+        color2 (tuple): (r, g, b)
+
+    Returns:
+        int:
+    """
+    diff = np.array(color1).astype(int) - np.array(color2).astype(int)
+    diff = np.max(np.maximum(diff, 0)) - np.min(np.minimum(diff, 0))
+    return diff
+
+
+def color_similar(color1, color2, threshold=10):
+    """Consider two colors are similar, if tolerance lesser or equal threshold.
+    Tolerance = Max(Positive(difference_rgb)) + Max(- Negative(difference_rgb))
+    The same as the tolerance in Photoshop.
+
+    Args:
+        color1 (tuple): (r, g, b)
+        color2 (tuple): (r, g, b)
+        threshold (int): Default to 10.
+
+    Returns:
+        bool: True if two colors are similar.
+    """
+    # print(color1, color2)
+    diff = np.array(color1).astype(int) - np.array(color2).astype(int)
+    diff = np.max(np.maximum(diff, 0)) - np.min(np.minimum(diff, 0))
+    return diff <= threshold
+
+
+def color_similar_1d(image, color, threshold=10):
+    """
+    Args:
+        image (np.ndarray): 1D array.
+        color: (r, g, b)
+        threshold(int): Default to 10.
+
+    Returns:
+        np.ndarray: bool
+    """
+    diff = image.astype(int) - color
+    diff = np.max(np.maximum(diff, 0), axis=1) - np.min(np.minimum(diff, 0), axis=1)
+    return diff <= threshold
+
+
+def color_similarity_2d(image, color):
+    """
+    Args:
+        image: 2D array.
+        color: (r, g, b)
+
+    Returns:
+        np.ndarray: uint8
+    """
+    r, g, b = cv2.split(cv2.subtract(image, (*color, 0)))
+    positive = cv2.max(cv2.max(r, g), b)
+    r, g, b = cv2.split(cv2.subtract((*color, 0), image))
+    negative = cv2.max(cv2.max(r, g), b)
+    return cv2.subtract(255, cv2.add(positive, negative))
+
+
+def extract_letters(image, letter=(255, 255, 255), threshold=128):
+    """Set letter color to black, set background color to white.
+
+    Args:
+        image: Shape (height, width, channel)
+        letter (tuple): Letter RGB.
+        threshold (int):
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(cv2.subtract(image, (*letter, 0)))
+    positive = cv2.max(cv2.max(r, g), b)
+    r, g, b = cv2.split(cv2.subtract((*letter, 0), image))
+    negative = cv2.max(cv2.max(r, g), b)
+    return cv2.multiply(cv2.add(positive, negative), 255.0 / threshold)
+
+
+def extract_white_letters(image, threshold=128):
+    """Set letter color to black, set background color to white.
+    This function will discourage color pixels (Non-gray pixels)
+
+    Args:
+        image: Shape (height, width, channel)
+        threshold (int):
+
+    Returns:
+        np.ndarray: Shape (height, width)
+    """
+    r, g, b = cv2.split(cv2.subtract((255, 255, 255, 0), image))
+    minimum = cv2.min(cv2.min(r, g), b)
+    maximum = cv2.max(cv2.max(r, g), b)
+    return cv2.multiply(cv2.add(maximum, cv2.subtract(maximum, minimum)), 255.0 / threshold)
+
+
+def color_mapping(image, max_multiply=2):
+    """
+    Mapping color to 0-255.
+    Minimum color to 0, maximum color to 255, multiply colors by 2 at max.
+
+    Args:
+        image (np.ndarray):
+        max_multiply (int, float):
+
+    Returns:
+        np.ndarray:
+    """
+    image = image.astype(float)
+    low, high = np.min(image), np.max(image)
+    multiply = min(255 / (high - low), max_multiply)
+    add = (255 - multiply * (low + high)) / 2
+    image = cv2.add(cv2.multiply(image, multiply), add)
+    image[image > 255] = 255
+    image[image < 0] = 0
+    return image.astype(np.uint8)
+
+
+def image_left_strip(image, threshold, length):
+    """
+    In `DAILY:200/200` strip `DAILY:` and leave `200/200`
+
+    Args:
+        image (np.ndarray): (height, width)
+        threshold (int):
+            0-255
+            The first column with brightness lower than this
+            will be considered as left edge.
+        length (int):
+            Strip this length of image after the left edge
+
+    Returns:
+        np.ndarray:
+    """
+    brightness = np.mean(image, axis=0)
+    match = np.where(brightness < threshold)[0]
+
+    if len(match):
+        left = match[0] + length
+        total = image.shape[1]
+        if left < total:
+            image = image[:, left:]
+    return image
+
+
+def red_overlay_transparency(color1, color2, red=247):
+    """Calculate the transparency of red overlay.
+
+    Args:
+        color1: origin color.
+        color2: changed color.
+        red(int): red color 0-255. Default to 247.
+
+    Returns:
+        float: 0-1
+    """
+    return (color2[0] - color1[0]) / (red - color1[0])
+
+
+def color_bar_percentage(image, area, prev_color, reverse=False, starter=0, threshold=30):
+    """
+    Args:
+        image:
+        area:
+        prev_color:
+        reverse: True if bar goes from right to left.
+        starter:
+        threshold:
+
+    Returns:
+        float: 0 to 1.
+    """
+    image = crop(image, area)
+    image = image[:, ::-1, :] if reverse else image
+    length = image.shape[1]
+    prev_index = starter
+
+    for _ in range(1280):
+        bar = color_similarity_2d(image, color=prev_color)
+        index = np.where(np.any(bar > 255 - threshold, axis=0))[0]
+        if not index.size:
+            return prev_index / length
+        else:
+            index = index[-1]
+        if index <= prev_index:
+            return index / length
+        prev_index = index
+
+        prev_row = bar[:, prev_index] > 255 - threshold
+        if not prev_row.size:
+            return prev_index / length
+        prev_color = np.mean(image[:, prev_index], axis=0)
+
+    return 0.

+ 35 - 0
module/config/utils.py

@@ -120,3 +120,38 @@ def deep_iter(data, depth=0, current_depth=1):
                 yield [key] + child_path, child_value
     else:
         yield [], data
+
+def server_timezone() -> timedelta:
+    return timedelta(hours=8)
+
+def server_time_offset() -> timedelta:
+    """
+    To convert local time to server time:
+        server_time = local_time + server_time_offset()
+    To convert server time to local time:
+        local_time = server_time - server_time_offset()
+    """
+    return datetime.now(timezone.utc).astimezone().utcoffset() - server_timezone()
+
+def get_server_next_update(daily_trigger):
+    """
+    Args:
+        daily_trigger (list[str], str): [ "00:00", "12:00", "18:00",]
+
+    Returns:
+        datetime.datetime
+    """
+    if isinstance(daily_trigger, str):
+        daily_trigger = daily_trigger.replace(' ', '').split(',')
+
+    diff = server_time_offset()
+    local_now = datetime.now()
+    trigger = []
+    for t in daily_trigger:
+        h, m = [int(x) for x in t.split(':')]
+        future = local_now.replace(hour=h, minute=m, second=0, microsecond=0) + diff
+        s = (future - local_now).total_seconds() % 86400
+        future = local_now + timedelta(seconds=s)
+        trigger.append(future)
+    update = sorted(trigger)[0]
+    return update

+ 67 - 0
module/device/app_control.py

@@ -0,0 +1,67 @@
+from lxml import etree
+
+from module.device.method.adb import Adb
+from module.device.method.uiautomator_2 import Uiautomator2
+from module.device.method.utils import HierarchyButton
+# from module.device.method.wsa import WSA
+from module.logger import logger
+
+
+class AppControl(Adb, Uiautomator2):
+    hierarchy: etree._Element
+    _app_u2_family = ['uiautomator2', 'minitouch', 'scrcpy']
+
+    def app_is_running(self) -> bool:
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        # if self.is_wsa:
+        #     package = self.app_current_wsa()
+        if method in AppControl._app_u2_family:
+            package = self.app_current_uiautomator2()
+        else:
+            package = self.app_current_adb()
+
+        package = package.strip(' \t\r\n')
+        logger.attr('Package_name', package)
+        return package == self.package
+
+    def app_start(self):
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        logger.info(f'App start: {self.package}')
+        # if self.config.Emulator_Serial == 'wsa-0':
+        #     self.app_start_wsa(display=0)
+        if method in AppControl._app_u2_family:
+            self.app_start_uiautomator2()
+        else:
+            self.app_start_adb()
+
+    def app_stop(self):
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        logger.info(f'App stop: {self.package}')
+        if method in AppControl._app_u2_family:
+            self.app_stop_uiautomator2()
+        else:
+            self.app_stop_adb()
+
+    def dump_hierarchy(self) -> etree._Element:
+        """
+        Returns:
+            etree._Element: Select elements with `self.hierarchy.xpath('//*[@text="Hermit"]')` for example.
+        """
+        method = self.config.get_arg('Script', 'Device', 'ScreenshotMethod')
+        if method in AppControl._app_u2_family:
+            self.hierarchy = self.dump_hierarchy_uiautomator2()
+        else:
+            self.hierarchy = self.dump_hierarchy_adb()
+        return self.hierarchy
+
+    def xpath_to_button(self, xpath: str) -> HierarchyButton:
+        """
+        Args:
+            xpath (str):
+
+        Returns:
+            HierarchyButton:
+                An object with methods and properties similar to Button.
+                If element not found or multiple elements were found, return None.
+        """
+        return HierarchyButton(self.hierarchy, xpath)

+ 186 - 0
module/device/control.py

@@ -0,0 +1,186 @@
+# from module.base.button import Button
+from module.base.decorator import cached_property
+from module.base.timer import Timer
+from module.base.utils import *
+# from module.device.method.hermit import Hermit
+# from module.device.method.maatouch import MaaTouch
+from module.device.method.minitouch import Minitouch
+from module.device.method.adb import Adb
+from module.device.method.scrcpy import Scrcpy
+from module.device.method.window import Window
+from module.logger import logger
+
+
+class Control(Minitouch, Scrcpy, Window, Adb):
+    def handle_control_check(self, button):
+        # Will be overridden in Device
+        pass
+
+    @cached_property
+    def click_methods(self):
+        return {
+            'ADB': self.click_adb,
+            'uiautomator2': self.click_uiautomator2,
+            'minitouch': self.click_minitouch,
+            'window_message': self.click_window_message
+            # 'Hermit': self.click_hermit,
+            # 'MaaTouch': self.click_maatouch,
+        }
+
+    def click(self, button, control_check=True):
+        """
+        后面改一改  不用用button的逻辑
+        :param button:
+        :param control_check:
+        :return:
+        """
+        if control_check:
+            self.handle_control_check(button)
+        x, y = random_rectangle_point(button.button)
+        x, y = ensure_int(x, y)
+        logger.info(
+            'Click %s @ %s' % (point2str(x, y), button)
+        )
+        method = self.click_methods.get(
+            self.config.get_arg("Script", 'Emulator', 'ControlMethod'),
+            self.click_adb
+        )
+        method(x, y)
+
+    def multi_click(self, button, n, interval=(0.1, 0.2)):
+        """
+        也是不能用button的逻辑
+        :param button:
+        :param n:
+        :param interval:
+        :return:
+        """
+        self.handle_control_check(button)
+        click_timer = Timer(0.1)
+        for _ in range(n):
+            remain = ensure_time(interval) - click_timer.current()
+            if remain > 0:
+                self.sleep(remain)
+            click_timer.reset()
+
+            self.click(button, control_check=False)
+
+    def long_click(self, button, duration=(1, 1.2)):
+        """
+
+        :param button:
+        :param duration:
+        :return:
+        """
+        self.handle_control_check(button)
+        x, y = random_rectangle_point(button.button)
+        x, y = ensure_int(x, y)
+        duration = ensure_time(duration)
+        logger.info(
+            'Click %s @ %s, %s' % (point2str(x, y), button, duration)
+        )
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            self.long_click_minitouch(x, y, duration)
+        elif method == 'window_message':
+            self.long_click_window_message(x, y, duration)
+        elif method == 'uiautomator2':
+            self.long_click_uiautomator2(x, y, duration)
+        elif method == 'scrcpy':
+            self.long_click_scrcpy(x, y, duration)
+        # elif method == 'MaaTouch':
+        #     self.long_click_maatouch(x, y, duration)
+        else:
+            self.swipe_adb((x, y), (x, y), duration)
+
+    def swipe(self, p1, p2, duration=(0.1, 0.2), name='SWIPE', distance_check=True):
+        self.handle_control_check(name)
+        p1, p2 = ensure_int(p1, p2)
+        duration = ensure_time(duration)
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        elif method == 'window_message':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        elif method == 'uiautomator2':
+            logger.info('Swipe %s -> %s, %s' % (point2str(*p1), point2str(*p2), duration))
+        elif method == 'scrcpy':
+            logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        # elif method == 'MaaTouch':
+        #     logger.info('Swipe %s -> %s' % (point2str(*p1), point2str(*p2)))
+        else:
+            # ADB needs to be slow, or swipe doesn't work
+            duration *= 2.5
+            logger.info('Swipe %s -> %s, %s' % (point2str(*p1), point2str(*p2), duration))
+
+        if distance_check:
+            if np.linalg.norm(np.subtract(p1, p2)) < 10:
+                # Should swipe a certain distance, otherwise AL will treat it as click.
+                # uiautomator2 should >= 6px, minitouch should >= 5px
+                logger.info('Swipe distance < 10px, dropped')
+                return
+
+        if method == 'minitouch':
+            self.swipe_minitouch(p1, p2)
+        elif method == 'window_message':
+            self.swipe_window_message(p1, p2)
+        elif method == 'uiautomator2':
+            self.swipe_uiautomator2(p1, p2, duration=duration)
+        elif method == 'scrcpy':
+            self.swipe_scrcpy(p1, p2)
+        # elif method == 'MaaTouch':
+        #     self.swipe_maatouch(p1, p2)
+        else:
+            self.swipe_adb(p1, p2, duration=duration)
+
+    def swipe_vector(self, vector, box=(123, 159, 1175, 628), random_range=(0, 0, 0, 0), padding=15,
+                     duration=(0.1, 0.2), whitelist_area=None, blacklist_area=None, name='SWIPE', distance_check=True):
+        """Method to swipe.
+
+        Args:
+            box (tuple): Swipe in box (upper_left_x, upper_left_y, bottom_right_x, bottom_right_y).
+            vector (tuple): (x, y).
+            random_range (tuple): (x_min, y_min, x_max, y_max).
+            padding (int):
+            duration (int, float, tuple):
+            whitelist_area: (list[tuple[int]]):
+                A list of area that safe to click. Swipe path will end there.
+            blacklist_area: (list[tuple[int]]):
+                If none of the whitelist_area satisfies current vector, blacklist_area will be used.
+                Delete random path that ends in any blacklist_area.
+            name (str): Swipe name
+            distance_check: (bool):
+        """
+        p1, p2 = random_rectangle_vector_opted(
+            vector,
+            box=box,
+            random_range=random_range,
+            padding=padding,
+            whitelist_area=whitelist_area,
+            blacklist_area=blacklist_area
+        )
+        self.swipe(p1, p2, duration=duration, name=name, distance_check=distance_check)
+
+    def drag(self, p1, p2, segments=1, shake=(0, 15), point_random=(-10, -10, 10, 10), shake_random=(-5, -5, 5, 5),
+             swipe_duration=0.25, shake_duration=0.1, name='DRAG'):
+        self.handle_control_check(name)
+        p1, p2 = ensure_int(p1, p2)
+        logger.info(
+            'Drag %s -> %s' % (point2str(*p1), point2str(*p2))
+        )
+        method = self.config.get_arg("Script", 'Emulator', 'ControlMethod')
+        if method == 'minitouch':
+            self.drag_minitouch(p1, p2, point_random=point_random)
+        elif method == 'uiautomator2':
+            self.drag_uiautomator2(
+                p1, p2, segments=segments, shake=shake, point_random=point_random, shake_random=shake_random,
+                swipe_duration=swipe_duration, shake_duration=shake_duration)
+        elif method == 'scrcpy':
+            self.drag_scrcpy(p1, p2, point_random=point_random)
+        # elif method == 'MaaTouch':
+        #     self.drag_maatouch(p1, p2, point_random=point_random)
+        else:
+            logger.warning(f'Control method {method} does not support drag well, '
+                           f'falling back to ADB swipe may cause unexpected behaviour')
+            self.swipe_adb(p1, p2, duration=ensure_time(swipe_duration * 2))
+            # self.click(Button(area=(), color=(), button=area_offset(point_random, p2), name=name))

+ 214 - 5
module/device/device.py

@@ -1,6 +1,215 @@
-# This Python file uses the following encoding: utf-8
-# @author runhey
-# github https://github.com/runhey
+import sys
+from collections import deque
+from datetime import datetime
 
-class Device:
-    pass
+from module.base.timer import Timer
+from module.config.utils import get_server_next_update
+from module.device.app_control import AppControl
+from module.device.control import Control
+from module.device.screenshot import Screenshot
+from module.exception import (GameNotRunningError, GameStuckError,
+                              GameTooManyClickError, RequestHumanTakeover)
+from module.handler.assets import GET_MISSION
+from module.logger import logger
+
+if sys.platform == 'win32':
+    from module.device.emulator import EmulatorManager
+else:
+    class EmulatorManager:
+        pass
+
+
+class Device(Screenshot, Control, AppControl, EmulatorManager):
+    _screen_size_checked = False
+    detect_record = set()
+    click_record = deque(maxlen=15)
+    stuck_timer = Timer(60, count=60).start()
+    stuck_timer_long = Timer(180, count=180).start()
+    stuck_long_wait_list = ['BATTLE_STATUS_S', 'PAUSE', 'LOGIN_CHECK']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.screenshot_interval_set()
+
+        # Auto-select the fastest screenshot method
+        if not self.config.is_template_config and self.config.get_arg("Script", 'Device', 'ScreenshotMethod') == 'auto':
+            self.run_simple_screenshot_benchmark()
+
+    def run_simple_screenshot_benchmark(self):
+        """
+        Perform a screenshot method benchmark, test 3 times on each method.
+        The fastest one will be set into config.
+        """
+        logger.info('run_simple_screenshot_benchmark')
+        # Check resolution first
+        self.resolution_check_uiautomator2()
+        # Perform benchmark
+        # from module.daemon.benchmark import Benchmark
+        # bench = Benchmark(config=self.config, device=self)
+        # method = bench.run_simple_screenshot_benchmark()
+        # # Set
+        # self.config.Emulator_ScreenshotMethod = method
+
+    def handle_night_commission(self, daily_trigger='21:00', threshold=30):
+        """
+        Args:
+            daily_trigger (int): Time for commission refresh.
+            threshold (int): Seconds around refresh time.
+
+        Returns:
+            bool: If handled.
+        """
+        update = get_server_next_update(daily_trigger=daily_trigger)
+        now = datetime.now()
+        diff = (update.timestamp() - now.timestamp()) % 86400
+        if threshold < diff < 86400 - threshold:
+            return False
+
+        if GET_MISSION.match(self.image, offset=True):
+            logger.info('Night commission appear.')
+            self.click(GET_MISSION)
+            return True
+
+        return False
+
+    def screenshot(self):
+        """
+        Returns:
+            np.ndarray:
+        """
+        self.stuck_record_check()
+
+        try:
+            super().screenshot()
+        except RequestHumanTakeover:
+            if not self.ascreencap_available:
+                logger.error('aScreenCap unavailable on current device, fallback to auto')
+                self.run_simple_screenshot_benchmark()
+                super().screenshot()
+            else:
+                raise
+
+        if self.handle_night_commission():
+            super().screenshot()
+
+        return self.image
+
+    def release_during_wait(self):
+        # Scrcpy server is still sending video stream,
+        # stop it during wait
+        if self.config.Emulator_ScreenshotMethod == 'scrcpy':
+            self._scrcpy_server_stop()
+
+    def stuck_record_add(self, button):
+        self.detect_record.add(str(button))
+
+    def stuck_record_clear(self):
+        self.detect_record = set()
+        self.stuck_timer.reset()
+        self.stuck_timer_long.reset()
+
+    def stuck_record_check(self):
+        """
+        Raises:
+            GameStuckError:
+        """
+        reached = self.stuck_timer.reached()
+        reached_long = self.stuck_timer_long.reached()
+
+        if not reached:
+            return False
+        if not reached_long:
+            for button in self.stuck_long_wait_list:
+                if button in self.detect_record:
+                    return False
+
+        logger.warning('Wait too long')
+        logger.warning(f'Waiting for {self.detect_record}')
+        self.stuck_record_clear()
+
+        if self.app_is_running():
+            raise GameStuckError(f'Wait too long')
+        else:
+            raise GameNotRunningError('Game died')
+
+    def handle_control_check(self, button):
+        self.stuck_record_clear()
+        self.click_record_add(button)
+        self.click_record_check()
+
+    def click_record_add(self, button):
+        self.click_record.append(str(button))
+
+    def click_record_clear(self):
+        self.click_record.clear()
+
+    def click_record_remove(self, button):
+        """
+        Remove a button from `click_record`
+
+        Args:
+            button (Button):
+
+        Returns:
+            int: Number of button removed
+        """
+        removed = 0
+        for _ in range(self.click_record.maxlen):
+            try:
+                self.click_record.remove(str(button))
+                removed += 1
+            except ValueError:
+                # Value not in queue
+                break
+
+        return removed
+
+    def click_record_check(self):
+        """
+        Raises:
+            GameTooManyClickError:
+        """
+        count = {}
+        for key in self.click_record:
+            count[key] = count.get(key, 0) + 1
+        count = sorted(count.items(), key=lambda item: item[1])
+        if count[0][1] >= 12:
+            logger.warning(f'Too many click for a button: {count[0][0]}')
+            logger.warning(f'History click: {[str(prev) for prev in self.click_record]}')
+            self.click_record_clear()
+            raise GameTooManyClickError(f'Too many click for a button: {count[0][0]}')
+        if len(count) >= 2 and count[0][1] >= 6 and count[1][1] >= 6:
+            logger.warning(f'Too many click between 2 buttons: {count[0][0]}, {count[1][0]}')
+            logger.warning(f'History click: {[str(prev) for prev in self.click_record]}')
+            self.click_record_clear()
+            raise GameTooManyClickError(f'Too many click between 2 buttons: {count[0][0]}, {count[1][0]}')
+
+    def disable_stuck_detection(self):
+        """
+        Disable stuck detection and its handler. Usually uses in semi auto and debugging.
+        """
+        logger.info('Disable stuck detection')
+
+        def empty_function(*arg, **kwargs):
+            return False
+
+        self.click_record_check = empty_function
+        self.stuck_record_check = empty_function
+
+    def app_start(self):
+        if not self.config.Error_HandleError:
+            logger.critical('No app stop/start, because HandleError disabled')
+            logger.critical('Please enable Alas.Error.HandleError or manually login to AzurLane')
+            raise RequestHumanTakeover
+        super().app_start()
+        self.stuck_record_clear()
+        self.click_record_clear()
+
+    def app_stop(self):
+        if not self.config.Error_HandleError:
+            logger.critical('No app stop/start, because HandleError disabled')
+            logger.critical('Please enable Alas.Error.HandleError or manually login to AzurLane')
+            raise RequestHumanTakeover
+        super().app_stop()
+        self.stuck_record_clear()
+        self.click_record_clear()

+ 2 - 1
module/device/emulator.py

@@ -309,7 +309,8 @@ class EmulatorManager(Connection):
             return False
 
         logger.hr('Emulator restart')
-        if self.config.RestartEmulator_EmulatorType == 'auto':
+        # if self.config.RestartEmulator_EmulatorType == 'auto':
+        if self.config.get_arg('Restart', '') == 'auto':
             emulator, multi_id = self.detect_emulator(serial)
         else:
             emulator = self.SUPPORTED_EMULATORS[self.config.RestartEmulator_EmulatorType]

+ 90 - 7
module/device/method/window.py

@@ -26,6 +26,7 @@ from module.base.timer import timer
 from module.logger import logger
 from module.device.handle import Handle, window_scale_rate
 
+
 class Window(Handle):
 
     def screenshot_window_background(self):
@@ -142,8 +143,39 @@ class Window(Handle):
             time.sleep((random.randint(100, 200)) / 1000.0)  # 点击弹起改为随机,时间100ms-200ms
             SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos)  # 模拟鼠标弹起
 
+    def long_click_window_message(self, x: int, y: int, duration: float):
+        """
+
+        :param x:
+        :param y:
+        :param duration: 持续时间 单位秒
+        :return:
+        """
+        # 我不知道为什么的使用的pywin32==306的版本会导致获取的图片的是(1024, 576)
+        # 所有我在点击的时候会除以这个缩放比例
+        x = int(x / self.window_scale_rate)
+        y = int(y / self.window_scale_rate)
+        logger.info(self.control_handle_list)
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 2:  # mumu模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[1], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type > 2:  # 夜神模拟器
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, MAKELONG(x, y))  # 模拟鼠标按下 先是父窗口 上面的框高度是57
+            SendMessage(self.control_handle_list[1], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[2], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            SendMessage(self.control_handle_list[3], WM_LBUTTONDOWN, 0, MAKELONG(x, y))
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[3], WM_LBUTTONUP, 0, MAKELONG(x, y))  # 模拟鼠标弹起 后是子窗口
+        elif emulator_type == 1:  # 雷电模拟器
+            clickPos = MAKELONG(x, y)
+            SendMessage(self.control_handle_list[0], WM_LBUTTONDOWN, 0, clickPos)  # 模拟鼠标按下
+            time.sleep(duration)  # 长按时间1000ms-1500ms
+            SendMessage(self.control_handle_list[0], WM_LBUTTONUP, 0, clickPos)  # 模拟鼠标弹起
+
     # @timer
-    def swipe_window_message(self, startPos :list, endPos :list) -> None:
+    def swipe_window_message(self, startPos: list, endPos: list) -> None:
         """
         后台滑动
         :param startPos:
@@ -152,19 +184,70 @@ class Window(Handle):
         """
         # 生成的坐标点列表
         interval: int = 8  # 每次移动的间隔时间
-        numberList :int = int(dist(startPos, endPos)/(1.5*interval))  # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
+        numberList: int = int(dist(startPos, endPos) / (1.5 * interval))  # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
         le = random.randint(2, 4)  #
         deviation = random.randint(20, 40)  # 幅度
         _type: int = 3
-        obbsType = random.random()   # 0.8的概率是先快中间慢后面快, 0.1概率是先快后慢, 0.1概率先慢后快
+        obbsType = random.random()  # 0.8的概率是先快中间慢后面快, 0.1概率是先快后慢, 0.1概率先慢后快
         if 0 < obbsType <= 0.8:
             _type = 3
-        elif obbsType<0.9:
+        elif obbsType < 0.9:
             _type = 2
         else:
             _type = 1
-        trace :list = BezierTrajectory.trackArray(start=startPos, end=endPos, numberList=numberList, le=4,
-                     deviation=30, bias=0.5, type=_type, cbb=0, yhh=20)
+        trace: list = BezierTrajectory.trackArray(start=startPos, end=endPos, numberList=numberList, le=le,
+                                                  deviation=deviation, bias=0.5, type=_type, cbb=0, yhh=20)
+
+        # 使用生成的点列表进行拖拽
+        emulator_type = len(self.control_handle_list)
+        if emulator_type == 1:  # 雷电模拟器
+            handleNum = self.control_handle_list[0]
+        elif emulator_type == 2:  # mumu模拟器
+            handleNum = self.control_handle_list[1]
+        elif emulator_type > 2:  # 夜神模拟器
+            handleNum = self.control_handle_list[3]
+        PostMessage(handleNum, WM_ACTIVATE, WA_ACTIVE, 0)  # 激活窗口
+        # 先移动到第一个点
+        tmpPos = MAKELONG(trace[0][0], trace[0][1])
+        SendMessage(handleNum, WM_NCHITTEST, 0, tmpPos)
+        SendMessage(handleNum, WM_SETCURSOR, handleNum, MAKELONG(HTCLIENT, WM_LBUTTONDOWN))
+        PostMessage(handleNum, WM_LBUTTONDOWN, MK_LBUTTON, tmpPos)
+        # 一点一点移动鼠标
+        for pos in trace:
+            tmpPos = MAKELONG(pos[0], pos[1])
+            PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, tmpPos)
+            time.sleep((interval + random.randint(-2, 2)) / 1000.0)
+        # 最后释放鼠标
+        tmpPos = MAKELONG(trace[-1][0], trace[-1][1])
+        PostMessage(handleNum, WM_LBUTTONUP, 0, tmpPos)
+
+    def swipe_vector_window_message2(self, startPos: list, endPos: list) -> None:
+        """
+        后台滑动
+        :param startPos:
+        :param endPos:
+        :return:
+        """
+        # 生成的坐标点列表
+        interval: int = 8  # 每次移动的间隔时间
+        numberList: int = int(dist(startPos, endPos) / (1.5 * interval))  # 表示每毫秒移动1.5个像素点, 总的时间除以每个点10ms就得到总的点的个数
+
+        def generate_points(start_pos: list, end_pos: list, number: int) -> list:
+            # 确定两点之间的步长
+            step_x = (end_pos[0] - start_pos[0]) / (number + 1)
+            step_y = (end_pos[1] - start_pos[1]) / (number + 1)
+            # 生成中间点坐标列表
+            points = []
+            for i in range(number):
+                x = start_pos[0] + step_x * (i + 1)
+                y = start_pos[1] + step_y * (i + 1)
+                points.append((x, y))
+            # 添加起始点和最终点到列表中
+            points.insert(0, tuple(start_pos))
+            points.append(tuple(end_pos))
+            return points
+
+        trace: list = generate_points(start_pos=startPos, end_pos=endPos, number=numberList)
 
         # 使用生成的点列表进行拖拽
         emulator_type = len(self.control_handle_list)
@@ -184,7 +267,7 @@ class Window(Handle):
         for pos in trace:
             tmpPos = MAKELONG(pos[0], pos[1])
             PostMessage(handleNum, WM_MOUSEMOVE, MK_LBUTTON, tmpPos)
-            time.sleep((interval+random.randint(-2, 2))/1000.0)
+            time.sleep((interval + random.randint(-2, 2)) / 1000.0)
         # 最后释放鼠标
         tmpPos = MAKELONG(trace[-1][0], trace[-1][1])
         PostMessage(handleNum, WM_LBUTTONUP, 0, tmpPos)

+ 256 - 0
module/device/screenshot.py

@@ -0,0 +1,256 @@
+import os
+import time
+from collections import deque
+from datetime import datetime
+
+import cv2
+import numpy as np
+from PIL import Image
+
+from module.base.decorator import cached_property
+from module.base.timer import Timer
+from module.base.utils import get_color, image_size, limit_in, save_image
+from module.device.method.adb import Adb
+from module.device.method.window import Window
+from module.device.method.droidcast import DroidCast
+from module.device.method.scrcpy import Scrcpy
+from module.exception import RequestHumanTakeover, ScriptError
+from module.logger import logger
+
+
+class Screenshot(Adb, DroidCast, Scrcpy, Window):
+    _screen_size_checked = False
+    _screen_black_checked = False
+    _minicap_uninstalled = False
+    _screenshot_interval = Timer(0.1)
+    _last_save_time = {}
+    image: np.ndarray
+
+    @cached_property
+    def screenshot_methods(self):
+        return {
+            'ADB': self.screenshot_adb,
+            'ADB_nc': self.screenshot_adb_nc,
+            'uiautomator2': self.screenshot_uiautomator2,
+            # 'aScreenCap': self.screenshot_ascreencap,
+            # 'aScreenCap_nc': self.screenshot_ascreencap_nc,
+            'DroidCast': self.screenshot_droidcast,
+            'DroidCast_raw': self.screenshot_droidcast_raw,
+            'scrcpy': self.screenshot_scrcpy,
+            'window_background': self.screenshot_window_background
+        }
+
+    def screenshot(self):
+        """
+        Returns:
+            np.ndarray:
+        """
+        self._screenshot_interval.wait()
+        self._screenshot_interval.reset()
+
+        for _ in range(2):
+            method = self.screenshot_methods.get(
+                self.config.get_arg('Script', 'Device', 'ScreenshotMethod'),
+                self.screenshot_adb  # 第二个参数默认的是screenshot_adb
+            )
+            self.image = method()
+
+            # if self.config.Emulator_ScreenshotDedithering:
+            #     # This will take 40-60ms
+            #     cv2.fastNlMeansDenoising(self.image, self.image, h=17, templateWindowSize=1, searchWindowSize=2)
+            # self.image = self._handle_orientated_image(self.image)
+
+            # if self.config.Error_SaveError:
+            if self.config.get_arg('Script', 'Error', 'SaveError'):
+                self.screenshot_deque.append({'time': datetime.now(), 'image': self.image})
+
+            if self.check_screen_size() and self.check_screen_black():
+                break
+            else:
+                continue
+
+        return self.image
+
+    def _handle_orientated_image(self, image):
+        """
+        Args:
+            image (np.ndarray):
+
+        Returns:
+            np.ndarray:
+        """
+        width, height = image_size(self.image)
+        if width == 1280 and height == 720:
+            return image
+
+        # Rotate screenshots only when they're not 1280x720
+        if self.orientation == 0:
+            pass
+        elif self.orientation == 1:
+            image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
+        elif self.orientation == 2:
+            image = cv2.rotate(image, cv2.ROTATE_180)
+        elif self.orientation == 3:
+            image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
+        else:
+            raise ScriptError(f'Invalid device orientation: {self.orientation}')
+
+        return image
+
+    @cached_property
+    def screenshot_deque(self):
+        # return deque(maxlen=int(self.config.Error_ScreenshotLength))
+        return deque(maxlen=int(self.config.get_arg("Script", "Error", "ScreenshotLength")))
+
+    def save_screenshot(self, genre='items', interval=None, to_base_folder=False):
+        """Save a screenshot. Use millisecond timestamp as file name.
+
+        Args:
+            genre (str, optional): Screenshot type.
+            interval (int, float): Seconds between two save. Saves in the interval will be dropped.
+            to_base_folder (bool): If save to base folder.
+
+        Returns:
+            bool: True if save succeed.
+        """
+        now = time.time()
+        if interval is None:
+            interval = self.config.SCREEN_SHOT_SAVE_INTERVAL
+
+        if now - self._last_save_time.get(genre, 0) > interval:
+            fmt = 'png'
+            file = '%s.%s' % (int(now * 1000), fmt)
+
+            folder = self.config.SCREEN_SHOT_SAVE_FOLDER_BASE if to_base_folder else self.config.SCREEN_SHOT_SAVE_FOLDER
+            folder = os.path.join(folder, genre)
+            if not os.path.exists(folder):
+                os.mkdir(folder)
+
+            file = os.path.join(folder, file)
+            self.image_save(file)
+            self._last_save_time[genre] = now
+            return True
+        else:
+            self._last_save_time[genre] = now
+            return False
+
+    def screenshot_last_save_time_reset(self, genre):
+        self._last_save_time[genre] = 0
+
+    def screenshot_interval_set(self, interval=None):
+        """
+        Args:
+            interval (int, float, str):
+                Minimum interval between 2 screenshots in seconds.
+                Or None for Optimization_ScreenshotInterval, 'combat' for Optimization_CombatScreenshotInterval
+        """
+        if interval is None:
+            origin = self.config.Optimization_ScreenshotInterval
+            interval = limit_in(origin, 0.1, 0.3)
+            if interval != origin:
+                logger.warning(f'Optimization.ScreenshotInterval {origin} is revised to {interval}')
+                self.config.Optimization_ScreenshotInterval = interval
+        elif interval == 'combat':
+            origin = self.config.Optimization_CombatScreenshotInterval
+            interval = limit_in(origin, 0.3, 1.0)
+            if interval != origin:
+                logger.warning(f'Optimization.CombatScreenshotInterval {origin} is revised to {interval}')
+                self.config.Optimization_CombatScreenshotInterval = interval
+        elif isinstance(interval, (int, float)):
+            # No limitation for manual set in code
+            pass
+        else:
+            logger.warning(f'Unknown screenshot interval: {interval}')
+            raise ScriptError(f'Unknown screenshot interval: {interval}')
+        # Screenshot interval in scrcpy is meaningless,
+        # video stream is received continuously no matter you use it or not.
+        # if self.config.Emulator_ScreenshotMethod == 'scrcpy':
+        if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'scrcpy':
+            interval = 0.1
+
+        if interval != self._screenshot_interval.limit:
+            logger.info(f'Screenshot interval set to {interval}s')
+            self._screenshot_interval.limit = interval
+
+    def image_show(self, image=None):
+        if image is None:
+            image = self.image
+        Image.fromarray(image).show()
+
+    def image_save(self, file):
+        save_image(self.image, file)
+
+    def check_screen_size(self):
+        """
+        Screen size must be 1280x720.
+        Take a screenshot before call.
+        """
+        if self._screen_size_checked:
+            return True
+
+        orientated = False
+        for _ in range(2):
+            # Check screen size
+            width, height = image_size(self.image)
+            logger.attr('Screen_size', f'{width}x{height}')
+            if width == 1280 and height == 720:
+                self._screen_size_checked = True
+                return True
+            elif not orientated and (width == 720 and height == 1280):
+                logger.info('Received orientated screenshot, handling')
+                self.get_orientation()
+                self.image = self._handle_orientated_image(self.image)
+                orientated = True
+                width, height = image_size(self.image)
+                if width == 720 and height == 1280:
+                    logger.info('Unable to handle orientated screenshot, continue for now')
+                    return True
+                else:
+                    continue
+            # elif self.config.Emulator_Serial == 'wsa-0':
+            #     self.display_resize_wsa(0)
+            #     return False
+            elif hasattr(self, 'app_is_running') and not self.app_is_running():
+                logger.warning('Received orientated screenshot, game not running')
+                return True
+            else:
+                logger.critical(f'Resolution not supported: {width}x{height}')
+                logger.critical('Please set emulator resolution to 1280x720')
+                raise RequestHumanTakeover
+
+    def check_screen_black(self):
+        if self._screen_black_checked:
+            return True
+        # Check screen color
+        # May get a pure black screenshot on some emulators.
+        color = get_color(self.image, area=(0, 0, 1280, 720))
+        if sum(color) < 1:
+            # if self.config.Emulator_Serial == 'wsa-0':
+            #     for _ in range(2):
+            #         display = self.get_display_id()
+            #         if display == 0:
+            #             return True
+            #     logger.info(f'Game running on display {display}')
+            #     logger.warning('Game not running on display 0, will be restarted')
+            #     self.app_stop_uiautomator2()
+            #     return False
+            if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'uiautomator2':
+                logger.warning(f'Received pure black screenshots from emulator, color: {color}')
+                logger.warning('Uninstall minicap and retry')
+                self.uninstall_minicap()
+                self._screen_black_checked = False
+                return False
+            else:
+                logger.warning(f'Received pure black screenshots from emulator, color: {color}')
+                logger.warning(f'Screenshot method {self.config.get_arg("Script", "Device", "ScreenshotMethod")}'
+                               f'may not work on emulator `{self.serial}`, or the emulator is not fully started')
+                if self.is_mumu_family:
+                    if self.config.get_arg('Script', 'Device', 'ScreenshotMethod') == 'DroidCast':
+                        self.droidcast_stop()
+                    else:
+                        logger.warning('If you are using MuMu X, please upgrade to version >= 12.1.5.0')
+                self._screen_black_checked = False
+                return False
+        else:
+            self._screen_black_checked = True
+            return True