From db97139c411e86d6dc07fe0e91ae38c1bef17a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> Date: Tue, 22 Oct 2024 16:37:52 +0800 Subject: [PATCH] ai_block_io support iodump --- config/plugins/ai_block_io.ini | 6 +- .../sentryPlugins/ai_block_io/ai_block_io.py | 75 ++++++++++++------- .../ai_block_io/config_parser.py | 30 ++++++++ .../ai_block_io/sliding_window.py | 4 +- 4 files changed, 84 insertions(+), 31 deletions(-) diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini index 422cfa3..040237d 100644 --- a/config/plugins/ai_block_io.ini +++ b/config/plugins/ai_block_io.ini @@ -29,4 +29,8 @@ write_tot_lim=500 [latency_sata_hdd] read_tot_lim=50000 -write_tot_lim=50000 \ No newline at end of file +write_tot_lim=50000 + +[iodump] +read_iodump_lim=0 +write_iodump_lim=0 \ No newline at end of file diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py index 4eecd43..f25e6d5 100644 --- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py +++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py @@ -15,7 +15,7 @@ import logging from collections import defaultdict from .detector import Detector, DiskDetector -from .threshold import ThresholdFactory +from .threshold import ThresholdFactory, ThresholdType from .sliding_window import SlidingWindowFactory from .utils import get_data_queue_size_and_update_size from .config_parser import ConfigParser @@ -91,9 +91,8 @@ class SlowIODetection: continue for stage in stages: for iotype in iotypes: - self._detector_name_list[disk].append( - MetricName(disk, disk_type, stage, iotype, "latency") - ) + self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "latency")) + self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "io_dump")) if disks: logging.warning( "disks: %s not in available disk list, so they will be ignored.", @@ -123,31 +122,51 @@ class SlowIODetection: for disk, metric_name_list in self._detector_name_list.items(): disk_detector = DiskDetector(disk) for metric_name in metric_name_list: - threshold = ThresholdFactory().get_threshold( - threshold_type, - boxplot_parameter=self._config_parser.boxplot_parameter, - n_sigma_paramter=self._config_parser.n_sigma_parameter, - data_queue_size=data_queue_size, - data_queue_update_size=update_size, - ) - abs_threshold = self._config_parser.get_tot_lim( - metric_name.disk_type, metric_name.io_access_type_name - ) - if abs_threshold is None: - logging.warning( - "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", - disk, - metric_name.disk_type, - metric_name.io_access_type_name, + + if metric_name.metric_name == 'latency': + threshold = ThresholdFactory().get_threshold( + threshold_type, + boxplot_parameter=self._config_parser.boxplot_parameter, + n_sigma_paramter=self._config_parser.n_sigma_parameter, + data_queue_size=data_queue_size, + data_queue_update_size=update_size, ) - sliding_window = SlidingWindowFactory().get_sliding_window( - sliding_window_type, - queue_length=window_size, - threshold=window_threshold, - abs_threshold=abs_threshold, - ) - detector = Detector(metric_name, threshold, sliding_window) - disk_detector.add_detector(detector) + abs_threshold = self._config_parser.get_tot_lim( + metric_name.disk_type, metric_name.io_access_type_name + ) + if abs_threshold is None: + logging.warning( + "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", + disk, + metric_name.disk_type, + metric_name.io_access_type_name, + ) + sliding_window = SlidingWindowFactory().get_sliding_window( + sliding_window_type, + queue_length=window_size, + threshold=window_threshold, + abs_threshold=abs_threshold, + ) + detector = Detector(metric_name, threshold, sliding_window) + disk_detector.add_detector(detector) + continue + + elif metric_name.metric_name == 'io_dump': + threshold = ThresholdFactory().get_threshold(ThresholdType.AbsoluteThreshold) + abs_threshold = None + if metric_name.io_access_type_name == 'read': + abs_threshold = self._config_parser.read_iodump_lim + elif metric_name.io_access_type_name == 'write': + abs_threshold = self._config_parser.write_iodump_lim + sliding_window = SlidingWindowFactory().get_sliding_window( + sliding_window_type, + queue_length=window_size, + threshold=window_threshold + ) + detector = Detector(metric_name, threshold, sliding_window) + threshold.set_threshold(abs_threshold) + disk_detector.add_detector(detector) + logging.info(f"disk: [{disk}] add detector:\n [{disk_detector}]") self._disk_detectors[disk] = disk_detector diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py index 274a31e..1117939 100644 --- a/src/python/sentryPlugins/ai_block_io/config_parser.py +++ b/src/python/sentryPlugins/ai_block_io/config_parser.py @@ -72,6 +72,7 @@ class ConfigParser: "latency_sata_ssd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, "latency_nvme_ssd": {"read_tot_lim": 500, "write_tot_lim": 500}, "latency_sata_hdd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, + "iodump": {"read_iodump_lim": 0, "write_iodump_lim": 0} } def __init__(self, config_file_name): @@ -497,6 +498,27 @@ class ConfigParser: logging.critical("not found latency_sata_hdd section. exiting...") exit(1) + if con.has_section("iodump"): + items_iodump = dict(con.items("iodump")) + self._conf["iodump"]["read_iodump_lim"] = self._get_config_value( + items_iodump, + "read_iodump_lim", + int, + self.DEFAULT_CONF["iodump"]["read_iodump_lim"], + ge=0 + ) + self._conf["iodump"]["write_iodump_lim"] = self._get_config_value( + items_iodump, + "write_iodump_lim", + int, + self.DEFAULT_CONF["iodump"]["write_iodump_lim"], + ge=0 + ) + else: + Report.report_pass("not found iodump section. exiting...") + logging.critical("not found iodump section. exiting...") + exit(1) + self.__print_all_config_value() def __repr__(self) -> str: @@ -587,3 +609,11 @@ class ConfigParser: @property def n_sigma_parameter(self): return self._conf["algorithm"]["n_sigma_parameter"] + + @property + def read_iodump_lim(self): + return self._conf["iodump"]["read_iodump_lim"] + + @property + def write_iodump_lim(self): + return self._conf["iodump"]["write_iodump_lim"] \ No newline at end of file diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py index d7c402a..cebe41f 100644 --- a/src/python/sentryPlugins/ai_block_io/sliding_window.py +++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py @@ -35,8 +35,8 @@ class SlidingWindow: self._io_data_queue_abnormal_tag.pop(0) self._io_data_queue.append(data) tag = False - if ((self._ai_threshold is not None and data >= self._ai_threshold) or - (self._abs_threshold is not None and data >= self._abs_threshold)): + if ((self._ai_threshold is not None and data > self._ai_threshold) or + (self._abs_threshold is not None and data > self._abs_threshold)): tag = True self._io_data_queue_abnormal_tag.append(tag) return tag -- 2.23.0