From c9f62e01f09a56743ccc3e470f273875ab22ac5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> Date: Wed, 9 Oct 2024 16:19:52 +0800 Subject: [PATCH] fix config relative some issues --- .../sentryPlugins/ai_block_io/README.md | 1 - .../sentryPlugins/ai_block_io/ai_block_io.py | 21 +++++----- .../ai_block_io/config_parser.py | 42 +++++++++---------- .../sentryPlugins/ai_block_io/detector.py | 2 +- .../ai_block_io/sliding_window.py | 8 ++-- .../sentryPlugins/ai_block_io/threshold.py | 6 +-- 6 files changed, 39 insertions(+), 41 deletions(-) diff --git a/src/python/sentryPlugins/ai_block_io/README.md b/src/python/sentryPlugins/ai_block_io/README.md index f9b8388..95c1111 100644 --- a/src/python/sentryPlugins/ai_block_io/README.md +++ b/src/python/sentryPlugins/ai_block_io/README.md @@ -1,2 +1 @@ # slow_io_detection - diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py index 31b8a97..3b00ef3 100644 --- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py +++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py @@ -16,8 +16,7 @@ import logging from .detector import Detector from .threshold import ThresholdFactory, AbsoluteThreshold from .sliding_window import SlidingWindowFactory -from .utils import (get_threshold_type_enum, get_sliding_window_type_enum, get_data_queue_size_and_update_size, - get_log_level) +from .utils import get_data_queue_size_and_update_size from .config_parser import ConfigParser from .data_access import get_io_data_from_collect_plug, check_collect_valid from .io_data import MetricName @@ -45,25 +44,25 @@ class SlowIODetection: def __init_detector_name_list(self): self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) + logging.info(f"ai_block_io plug has found disks: {self._disk_list}") disks_to_detection: list = self._config_parser.get_disks_to_detection() # 情况1：None，则启用所有磁盘检测 # 情况2：is not None and len = 0，则不启动任何磁盘检测 # 情况3：len ！= 0，则取交集 if disks_to_detection is None: + logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") for disk in self._disk_list: self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) elif len(disks_to_detection) == 0: - logging.warning('please attention: conf file not specify any disk to detection, ' - 'so it will not start ai block io.') + logging.warning('please attention: conf file not specify any disk to detection, so it will not start ai block io.') else: - disks_name_to_detection = [] - for disk_name_to_detection in disks_to_detection: - disks_name_to_detection.append(disk_name_to_detection.get_disk_name()) - disk_intersection = [disk for disk in self._disk_list if disk in disks_name_to_detection] - for disk in disk_intersection: - self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) - self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) + for disk_to_detection in disks_to_detection: + if disk_to_detection in self._disk_list: + self._detector_name_list.append(MetricName(disk_to_detection, "bio", "read", "latency")) + self._detector_name_list.append(MetricName(disk_to_detection, "bio", "write", "latency")) + else: + logging.warning(f"disk：[{disk_to_detection}] not in available disk list, so it will be ignored.") logging.info(f'start to detection follow disk and it\'s metric: {self._detector_name_list}') def __init_detector(self): diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py index 632391d..354c122 100644 --- a/src/python/sentryPlugins/ai_block_io/config_parser.py +++ b/src/python/sentryPlugins/ai_block_io/config_parser.py @@ -10,18 +10,19 @@ # See the Mulan PSL v2 for more details. import configparser -import json import logging -from .io_data import MetricName from .threshold import ThresholdType from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level + LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" def init_log_format(log_level: str): - logging.basicConfig(level=get_log_level(log_level), format=LOG_FORMAT) + logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) + if log_level.lower() not in ('info', 'warning', 'error', 'debug'): + logging.warning(f'the log_level: {log_level} you set is invalid, use default value: info.') class ConfigParser: @@ -43,7 +44,7 @@ class ConfigParser: self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL - self.__disks_to_detection: list = [] + self.__disks_to_detection = None self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION @@ -83,26 +84,20 @@ class ConfigParser: logging.warning(f'slow_io_detect_frequency type conversion has error, use default value: {self.__slow_io_detect_frequency}.') def __read__disks_to_detect(self, items_common: dict): - disks_to_detection = items_common.get('disks_to_detect') + disks_to_detection = items_common.get('disk') if disks_to_detection is None: - logging.warning(f'config of disks_to_detect not found, the default value be used.') + logging.warning(f'config of disk not found, the default value will be used.') self.__disks_to_detection = None return - try: - disks_to_detection_list = json.loads(disks_to_detection) - for disk_to_detection in disks_to_detection_list: - disk_name = disk_to_detection.get('disk_name', None) - stage_name = disk_to_detection.get('stage_name', None) - io_access_type_name = disk_to_detection.get('io_access_type_name', None) - metric_name = disk_to_detection.get('metric_name', None) - if not (disk_name is None or stage_name is None or io_access_type_name is None or metric_name is None): - metric_name_object = MetricName(disk_name, stage_name, io_access_type_name, metric_name) - self.__disks_to_detection.append(metric_name_object) - else: - logging.warning(f'config of disks_to_detect\'s some part has some error: {disk_to_detection}, it will be ignored.') - except json.decoder.JSONDecodeError as e: - logging.warning(f'config of disks_to_detect is error: {e}, it will be ignored and default value be used.') + disk_list = disks_to_detection.split(',') + if len(disk_list) == 0 or (len(disk_list) == 1 and disk_list[0] == ''): + logging.warning("you don't specify any disk.") + self.__disks_to_detection = [] + return + if len(disk_list) == 1 and disk_list[0] == 'default': self.__disks_to_detection = None + return + self.__disks_to_detection = disk_list def __read__train_data_duration(self, items_algorithm: dict): try: @@ -189,7 +184,12 @@ class ConfigParser: def read_config_from_file(self): con = configparser.ConfigParser() - con.read(self.__config_file_name, encoding='utf-8') + try: + con.read(self.__config_file_name, encoding='utf-8') + except configparser.Error as e: + init_log_format(self.__log_level) + logging.critical(f'config file read error: {e}, ai_block_io plug will exit.') + exit(1) if con.has_section('common'): items_common = dict(con.items('common')) diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py index bcf62cb..a48144f 100644 --- a/src/python/sentryPlugins/ai_block_io/detector.py +++ b/src/python/sentryPlugins/ai_block_io/detector.py @@ -50,6 +50,6 @@ class Detector: def __repr__(self): return (f'disk_name: {self._metric_name.get_disk_name()}, stage_name: {self._metric_name.get_stage_name()},' - f' access_type_name: {self._metric_name.get_io_access_type_name()},' + f' io_type_name: {self._metric_name.get_io_access_type_name()},' f' metric_name: {self._metric_name.get_metric_name()}, threshold_type: {self._threshold},' f' sliding_window_type: {self._slidingWindow}') diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py index d395d48..89191e5 100644 --- a/src/python/sentryPlugins/ai_block_io/sliding_window.py +++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py @@ -52,7 +52,7 @@ class SlidingWindow: return False, None, None def __repr__(self): - return "SlidingWindow" + return "[SlidingWindow]" class NotContinuousSlidingWindow(SlidingWindow): @@ -65,7 +65,7 @@ class NotContinuousSlidingWindow(SlidingWindow): return False, self._io_data_queue, self._ai_threshold def __repr__(self): - return "NotContinuousSlidingWindow" + return f"[NotContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" class ContinuousSlidingWindow(SlidingWindow): @@ -84,7 +84,7 @@ class ContinuousSlidingWindow(SlidingWindow): return False, self._io_data_queue, self._ai_threshold def __repr__(self): - return "ContinuousSlidingWindow" + return f"[ContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" class MedianSlidingWindow(SlidingWindow): @@ -98,7 +98,7 @@ class MedianSlidingWindow(SlidingWindow): return False, self._io_data_queue, self._ai_threshold def __repr__(self): - return "MedianSlidingWindow" + return f"[MedianSlidingWindow, window size: {self._queue_length}]" class SlidingWindowFactory: diff --git a/src/python/sentryPlugins/ai_block_io/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py index ff85d85..3b7a5a8 100644 --- a/src/python/sentryPlugins/ai_block_io/threshold.py +++ b/src/python/sentryPlugins/ai_block_io/threshold.py @@ -75,7 +75,7 @@ class AbsoluteThreshold(Threshold): pass def __repr__(self): - return "AbsoluteThreshold" + return "[AbsoluteThreshold]" class BoxplotThreshold(Threshold): @@ -109,7 +109,7 @@ class BoxplotThreshold(Threshold): self.new_data_size = 0 def __repr__(self): - return "BoxplotThreshold" + return f"[BoxplotThreshold, param is: {self.parameter}]" class NSigmaThreshold(Threshold): @@ -142,7 +142,7 @@ class NSigmaThreshold(Threshold): self.new_data_size = 0 def __repr__(self): - return "NSigmaThreshold" + return f"[NSigmaThreshold, param is: {self.parameter}]" class ThresholdType(Enum): -- 2.23.0