From b2ed1a3df17c7c080d156fa2dde11895481d2e97 Mon Sep 17 00:00:00 2001
From: wangfenglai1 <wangfenglai1@huawei.com>
Date: Fri, 17 Jan 2025 17:00:00 +0800
Subject: [PATCH] add local file test case

---
 tests/e2e_tests/test_local_file.py | 209 +++++++++++++++++++++++++++++
 1 file changed, 209 insertions(+)
 create mode 100644 tests/e2e_tests/test_local_file.py

diff --git a/tests/e2e_tests/test_local_file.py b/tests/e2e_tests/test_local_file.py
new file mode 100644
index 0000000..d16b271
--- /dev/null
+++ b/tests/e2e_tests/test_local_file.py
@@ -0,0 +1,209 @@
+#!/usr/bin/python3
+# ******************************************************************************
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# gala-anteater is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#          http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************/
+"""
+Time: 2025-01-17
+Author: wangfl
+Description: The main function of gala-anteater project.
+"""
+import os
+import argparse
+import logging
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+logging.basicConfig(level=logging.ERROR) 
+
+
+def read_table_data_pandas(file_path):
+    """
+    此函数使用 pandas 从文件读取表格数据。
+
+    参数:
+    file_path (str): CSV 文件的路径。
+
+    返回:
+    pandas.DataFrame: 包含表格数据的数据框。
+    """
+    try:
+        df = pd.read_csv(file_path)
+        return df
+    except FileNotFoundError:
+        logging.error(f"Error: The file {file_path} does not exist.")
+        return None
+    except Exception as e:
+        logging.error(f"Error: An error occurred while reading the file {file_path}: {e}")
+        return None
+
+
+def calculate_thresholds(window):
+    """
+    计算窗口内的上下阈值。
+
+    参数:
+    window (pandas.DataFrame): 滑动窗口的数据。
+
+    返回:
+    tuple: 包含上下阈值的元组。
+    """
+    mean_value = window['value'].mean()
+    std_value = window['value'].std()
+    threshold = 3 * std_value
+    upper_threshold = mean_value + threshold
+    lower_threshold = mean_value - threshold
+    return upper_threshold, lower_threshold
+
+
+def check_deviation(window, upper_threshold, lower_threshold):
+    """
+    检查窗口内的数据是否偏离阈值。
+
+    参数:
+    window (pandas.DataFrame): 滑动窗口的数据。
+    upper_threshold (float): 上阈值。
+    lower_threshold (float): 下阈值。
+
+    返回:
+    pandas.Series: 偏离状态的布尔序列。
+    """
+    return ((window['value'] > upper_threshold) | (window['value'] < lower_threshold))
+
+
+def check_consecutive_deviation(deviation_status_list, consecutive_count=5):
+    """
+    检查连续偏离点是否超过指定数量。
+
+    参数:
+    deviation_status_list (list): 偏离状态列表。
+    consecutive_count (int): 连续偏离点的阈值，默认为 5。
+
+    返回:
+    list: 标记为故障点的布尔列表。
+    """
+    corrected_status = []
+    consecutive_count_current = 0
+    for status in deviation_status_list:
+        if status:
+            consecutive_count_current += 1
+        else:
+            consecutive_count_current = 0
+        if consecutive_count_current >= consecutive_count:
+            corrected_status.append(True)
+        else:
+            corrected_status.append(False)
+    return corrected_status
+
+
+def analyze_table_data_pandas(df, window_size=600):
+    """
+    此函数使用 pandas 检查 value 是否偏离均值分布 3 倍以上，
+    并将结果保存在新列 deviation_status 中。
+    进一步检查滑窗内连续 5 个以上的偏离点，将其标记为故障点，其余为正常点。
+    滑窗间不重叠。
+
+    参数:
+    df (pandas.DataFrame): 包含表格数据的数据框。
+    window_size (int): 滑窗大小，默认为 600。
+
+    返回:
+    pandas.DataFrame: 包含 'timestamp', 'value', 'deviation_status' 列的数据框。
+    """
+    df['deviation_status'] = False
+    df['lower_threshold'] = np.nan
+    df['upper_threshold'] = np.nan
+    num_windows = len(df) // window_size
+    for i in range(num_windows):
+        start = i * window_size
+        end = (i + 1) * window_size
+        window = df.iloc[start:end]
+        upper_threshold, lower_threshold = calculate_thresholds(window)
+        deviation_status = check_deviation(window, upper_threshold, lower_threshold)
+        deviation_status_list = deviation_status.tolist()
+        corrected_status = check_consecutive_deviation(deviation_status_list)
+        df.loc[start:end - 1, 'deviation_status'] = corrected_status
+        df.loc[start:end - 1, 'lower_threshold'] = lower_threshold
+        df.loc[start:end - 1, 'upper_threshold'] = upper_threshold
+    return df
+
+
+def plot_table_data(df, output_path, window_size):
+    """
+    此函数使用 matplotlib 绘制表格数据，并标记出故障点，同时增加方格线，保存图像，并绘制上下阈值线。
+
+    参数:
+    df (pandas.DataFrame): 包含 'timestamp', 'value', 'deviation_status' 列的数据框。
+    output_path (str): 图像保存的路径。
+    window_size (int): 滑窗大小。
+    """
+    fig, ax = plt.subplots(figsize=(10, 6))
+    # 绘制正常点
+    normal_df = df[~df['deviation_status']]
+    ax.plot(normal_df['timestamp'], normal_df['value'], label='Normal', marker='o', linestyle='-', color='blue')
+    # 绘制故障点
+    fault_df = df[df['deviation_status']]
+    ax.plot(fault_df['timestamp'], fault_df['value'], label='Fault', marker='x', linestyle='None', color='red',
+            markersize=10)
+    # 绘制上下阈值线
+    ax.plot(df['timestamp'], df["upper_threshold"], label='Upper Threshold', linestyle='--', color='green')
+    ax.plot(df['timestamp'], df["lower_threshold"], label='Lower Threshold', linestyle='--', color='orange')
+    ax.set_xlabel('Timestamp')
+    ax.set_ylabel('Value')
+    ax.set_title('Table Data Analysis')
+    ax.legend()
+    # 添加方格线
+    ax.grid(True, which='both', linestyle='--', linewidth=0.5, color='gray')
+    plt.savefig(output_path)
+    plt.close(fig)
+
+
+def save_analysis_result(df, result_path):
+    """
+    此函数将分析结果保存到 CSV 文件中。
+
+    参数:
+    df (pandas.DataFrame): 包含分析结果的数据框。
+    result_path (str): 结果保存的路径。
+    """
+    try:
+        df.to_csv(result_path, index=False)
+    except Exception as e:
+        logging.error(f"Error: An error occurred while saving the file {result_path}: {e}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Table Data Analysis')
+    parser.add_argument('directory', type=str, nargs='?', default='./test',
+                        help='Path to the directory containing CSV files')
+    parser.add_argument('--window_size', type=int, default=600, help='Window size for analysis')
+    args = parser.parse_args()
+
+    # 遍历目录下的所有文件
+    for root, dirs, files in os.walk(args.directory):
+        for file in files:
+            if file.endswith('.csv'):
+                file_path = os.path.join(root, file)
+                # 获取输入文件的基本名称（不包含路径）
+                base_name = os.path.basename(file_path)
+                file_name = os.path.splitext(base_name)[0]
+                output_image_path = os.path.join(root, f'{file_name}.png')
+                output_result_path = os.path.join(root, f'{file_name}_result.csv')
+
+                df = read_table_data_pandas(file_path)
+                if df is not None:
+                    analysis_result = analyze_table_data_pandas(df, args.window_size)
+                    plot_table_data(analysis_result, output_image_path, args.window_size)
+                    save_analysis_result(analysis_result, output_result_path)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
-- 
Gitee
