201 lines
9.2 KiB
Diff
201 lines
9.2 KiB
Diff
From db97139c411e86d6dc07fe0e91ae38c1bef17a8d Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com>
|
|
Date: Tue, 22 Oct 2024 16:37:52 +0800
|
|
Subject: [PATCH] ai_block_io support iodump
|
|
|
|
---
|
|
config/plugins/ai_block_io.ini | 6 +-
|
|
.../sentryPlugins/ai_block_io/ai_block_io.py | 75 ++++++++++++-------
|
|
.../ai_block_io/config_parser.py | 30 ++++++++
|
|
.../ai_block_io/sliding_window.py | 4 +-
|
|
4 files changed, 84 insertions(+), 31 deletions(-)
|
|
|
|
diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini
|
|
index 422cfa3..040237d 100644
|
|
--- a/config/plugins/ai_block_io.ini
|
|
+++ b/config/plugins/ai_block_io.ini
|
|
@@ -29,4 +29,8 @@ write_tot_lim=500
|
|
|
|
[latency_sata_hdd]
|
|
read_tot_lim=50000
|
|
-write_tot_lim=50000
|
|
\ No newline at end of file
|
|
+write_tot_lim=50000
|
|
+
|
|
+[iodump]
|
|
+read_iodump_lim=0
|
|
+write_iodump_lim=0
|
|
\ No newline at end of file
|
|
diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py
|
|
index 4eecd43..f25e6d5 100644
|
|
--- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py
|
|
+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py
|
|
@@ -15,7 +15,7 @@ import logging
|
|
from collections import defaultdict
|
|
|
|
from .detector import Detector, DiskDetector
|
|
-from .threshold import ThresholdFactory
|
|
+from .threshold import ThresholdFactory, ThresholdType
|
|
from .sliding_window import SlidingWindowFactory
|
|
from .utils import get_data_queue_size_and_update_size
|
|
from .config_parser import ConfigParser
|
|
@@ -91,9 +91,8 @@ class SlowIODetection:
|
|
continue
|
|
for stage in stages:
|
|
for iotype in iotypes:
|
|
- self._detector_name_list[disk].append(
|
|
- MetricName(disk, disk_type, stage, iotype, "latency")
|
|
- )
|
|
+ self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "latency"))
|
|
+ self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "io_dump"))
|
|
if disks:
|
|
logging.warning(
|
|
"disks: %s not in available disk list, so they will be ignored.",
|
|
@@ -123,31 +122,51 @@ class SlowIODetection:
|
|
for disk, metric_name_list in self._detector_name_list.items():
|
|
disk_detector = DiskDetector(disk)
|
|
for metric_name in metric_name_list:
|
|
- threshold = ThresholdFactory().get_threshold(
|
|
- threshold_type,
|
|
- boxplot_parameter=self._config_parser.boxplot_parameter,
|
|
- n_sigma_paramter=self._config_parser.n_sigma_parameter,
|
|
- data_queue_size=data_queue_size,
|
|
- data_queue_update_size=update_size,
|
|
- )
|
|
- abs_threshold = self._config_parser.get_tot_lim(
|
|
- metric_name.disk_type, metric_name.io_access_type_name
|
|
- )
|
|
- if abs_threshold is None:
|
|
- logging.warning(
|
|
- "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.",
|
|
- disk,
|
|
- metric_name.disk_type,
|
|
- metric_name.io_access_type_name,
|
|
+
|
|
+ if metric_name.metric_name == 'latency':
|
|
+ threshold = ThresholdFactory().get_threshold(
|
|
+ threshold_type,
|
|
+ boxplot_parameter=self._config_parser.boxplot_parameter,
|
|
+ n_sigma_paramter=self._config_parser.n_sigma_parameter,
|
|
+ data_queue_size=data_queue_size,
|
|
+ data_queue_update_size=update_size,
|
|
)
|
|
- sliding_window = SlidingWindowFactory().get_sliding_window(
|
|
- sliding_window_type,
|
|
- queue_length=window_size,
|
|
- threshold=window_threshold,
|
|
- abs_threshold=abs_threshold,
|
|
- )
|
|
- detector = Detector(metric_name, threshold, sliding_window)
|
|
- disk_detector.add_detector(detector)
|
|
+ abs_threshold = self._config_parser.get_tot_lim(
|
|
+ metric_name.disk_type, metric_name.io_access_type_name
|
|
+ )
|
|
+ if abs_threshold is None:
|
|
+ logging.warning(
|
|
+ "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.",
|
|
+ disk,
|
|
+ metric_name.disk_type,
|
|
+ metric_name.io_access_type_name,
|
|
+ )
|
|
+ sliding_window = SlidingWindowFactory().get_sliding_window(
|
|
+ sliding_window_type,
|
|
+ queue_length=window_size,
|
|
+ threshold=window_threshold,
|
|
+ abs_threshold=abs_threshold,
|
|
+ )
|
|
+ detector = Detector(metric_name, threshold, sliding_window)
|
|
+ disk_detector.add_detector(detector)
|
|
+ continue
|
|
+
|
|
+ elif metric_name.metric_name == 'io_dump':
|
|
+ threshold = ThresholdFactory().get_threshold(ThresholdType.AbsoluteThreshold)
|
|
+ abs_threshold = None
|
|
+ if metric_name.io_access_type_name == 'read':
|
|
+ abs_threshold = self._config_parser.read_iodump_lim
|
|
+ elif metric_name.io_access_type_name == 'write':
|
|
+ abs_threshold = self._config_parser.write_iodump_lim
|
|
+ sliding_window = SlidingWindowFactory().get_sliding_window(
|
|
+ sliding_window_type,
|
|
+ queue_length=window_size,
|
|
+ threshold=window_threshold
|
|
+ )
|
|
+ detector = Detector(metric_name, threshold, sliding_window)
|
|
+ threshold.set_threshold(abs_threshold)
|
|
+ disk_detector.add_detector(detector)
|
|
+
|
|
logging.info(f"disk: [{disk}] add detector:\n [{disk_detector}]")
|
|
self._disk_detectors[disk] = disk_detector
|
|
|
|
diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py
|
|
index 274a31e..1117939 100644
|
|
--- a/src/python/sentryPlugins/ai_block_io/config_parser.py
|
|
+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py
|
|
@@ -72,6 +72,7 @@ class ConfigParser:
|
|
"latency_sata_ssd": {"read_tot_lim": 50000, "write_tot_lim": 50000},
|
|
"latency_nvme_ssd": {"read_tot_lim": 500, "write_tot_lim": 500},
|
|
"latency_sata_hdd": {"read_tot_lim": 50000, "write_tot_lim": 50000},
|
|
+ "iodump": {"read_iodump_lim": 0, "write_iodump_lim": 0}
|
|
}
|
|
|
|
def __init__(self, config_file_name):
|
|
@@ -497,6 +498,27 @@ class ConfigParser:
|
|
logging.critical("not found latency_sata_hdd section. exiting...")
|
|
exit(1)
|
|
|
|
+ if con.has_section("iodump"):
|
|
+ items_iodump = dict(con.items("iodump"))
|
|
+ self._conf["iodump"]["read_iodump_lim"] = self._get_config_value(
|
|
+ items_iodump,
|
|
+ "read_iodump_lim",
|
|
+ int,
|
|
+ self.DEFAULT_CONF["iodump"]["read_iodump_lim"],
|
|
+ ge=0
|
|
+ )
|
|
+ self._conf["iodump"]["write_iodump_lim"] = self._get_config_value(
|
|
+ items_iodump,
|
|
+ "write_iodump_lim",
|
|
+ int,
|
|
+ self.DEFAULT_CONF["iodump"]["write_iodump_lim"],
|
|
+ ge=0
|
|
+ )
|
|
+ else:
|
|
+ Report.report_pass("not found iodump section. exiting...")
|
|
+ logging.critical("not found iodump section. exiting...")
|
|
+ exit(1)
|
|
+
|
|
self.__print_all_config_value()
|
|
|
|
def __repr__(self) -> str:
|
|
@@ -587,3 +609,11 @@ class ConfigParser:
|
|
@property
|
|
def n_sigma_parameter(self):
|
|
return self._conf["algorithm"]["n_sigma_parameter"]
|
|
+
|
|
+ @property
|
|
+ def read_iodump_lim(self):
|
|
+ return self._conf["iodump"]["read_iodump_lim"]
|
|
+
|
|
+ @property
|
|
+ def write_iodump_lim(self):
|
|
+ return self._conf["iodump"]["write_iodump_lim"]
|
|
\ No newline at end of file
|
|
diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py
|
|
index d7c402a..cebe41f 100644
|
|
--- a/src/python/sentryPlugins/ai_block_io/sliding_window.py
|
|
+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py
|
|
@@ -35,8 +35,8 @@ class SlidingWindow:
|
|
self._io_data_queue_abnormal_tag.pop(0)
|
|
self._io_data_queue.append(data)
|
|
tag = False
|
|
- if ((self._ai_threshold is not None and data >= self._ai_threshold) or
|
|
- (self._abs_threshold is not None and data >= self._abs_threshold)):
|
|
+ if ((self._ai_threshold is not None and data > self._ai_threshold) or
|
|
+ (self._abs_threshold is not None and data > self._abs_threshold)):
|
|
tag = True
|
|
self._io_data_queue_abnormal_tag.append(tag)
|
|
return tag
|
|
--
|
|
2.23.0
|
|
|