From c1ab550a3f817826ac6f279de97e6d3820901275 Mon Sep 17 00:00:00 2001 From: gaoruoshu Date: Fri, 27 Sep 2024 14:10:18 +0800 Subject: [PATCH] add log level and change log format --- config/collector.conf | 5 ++- config/inspect.conf | 5 ++- config/plugins/avg_block_io.ini | 5 ++- config/xalarm.conf | 3 ++ src/python/sentryCollector/collect_config.py | 29 ++++++++++++++++ src/python/sentryCollector/collect_io.py | 15 ++------- src/python/sentryCollector/collect_plugin.py | 32 +++++++++--------- src/python/sentryCollector/collectd.py | 6 ++-- .../avg_block_io/avg_block_io.py | 7 ++-- .../sentryPlugins/avg_block_io/utils.py | 32 ++++++++++++++++++ src/python/syssentry/sentry_config.py | 28 ++++++++++++++++ src/python/syssentry/syssentry.py | 7 ++-- src/python/xalarm/xalarm_config.py | 33 +++++++++++++++++-- src/python/xalarm/xalarm_daemon.py | 7 ++-- 14 files changed, 172 insertions(+), 42 deletions(-) diff --git a/config/collector.conf b/config/collector.conf index 9baa086..56b0ed1 100644 --- a/config/collector.conf +++ b/config/collector.conf @@ -4,4 +4,7 @@ modules=io [io] period_time=1 max_save=10 -disk=default \ No newline at end of file +disk=default + +[log] +level=info \ No newline at end of file diff --git a/config/inspect.conf b/config/inspect.conf index 071cca1..f451d9e 100644 --- a/config/inspect.conf +++ b/config/inspect.conf @@ -1,2 +1,5 @@ [inspect] -Interval=3 \ No newline at end of file +Interval=3 + +[log] +level=info diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini index bc33dde..858db18 100644 --- a/config/plugins/avg_block_io.ini +++ b/config/plugins/avg_block_io.ini @@ -1,8 +1,11 @@ +[log] +level=info + [common] disk=default stage=default iotype=read,write -period_time=1 +period_time=1 [algorithm] win_size=30 diff --git a/config/xalarm.conf b/config/xalarm.conf index 14c6d39..323d2dd 100644 --- a/config/xalarm.conf +++ b/config/xalarm.conf @@ -1,2 +1,5 @@ [filter] id_mask = 1001-1128 + +[log] +level=info diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py index 0fdd9f0..5aa38ec 100644 --- a/src/python/sentryCollector/collect_config.py +++ b/src/python/sentryCollector/collect_config.py @@ -32,6 +32,35 @@ CONF_IO_PERIOD_TIME_DEFAULT = 1 CONF_IO_MAX_SAVE_DEFAULT = 10 CONF_IO_DISK_DEFAULT = "default" +# log +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=COLLECT_CONF_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + + class CollectConfig: def __init__(self, filename=COLLECT_CONF_PATH): diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py index 9c8dae7..019d174 100644 --- a/src/python/sentryCollector/collect_io.py +++ b/src/python/sentryCollector/collect_io.py @@ -163,18 +163,6 @@ class CollectIo(): logging.error("An error occurred2: %s", e) return column_names - def task_loop(self): - if self.stop_event.is_set(): - logging.info("collect io thread exit") - return - - for disk_name, stage_list in self.disk_map_stage.items(): - if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: - continue - self.append_period_lat(disk_name, stage_list) - - threading.Timer(self.period_time, self.task_loop).start() - def is_kernel_avaliable(self): base_path = '/sys/kernel/debug/block' all_disk = [] @@ -191,6 +179,9 @@ class CollectIo(): if file_name == 'stats': all_disk.append(disk_name) + if self.loop_all: + self.disk_list = all_disk + for disk_name in self.disk_list: if not self.loop_all and disk_name not in all_disk: logging.warning("the %s disk not exist!", disk_name) diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py index 1faa5e3..3e2cf4c 100644 --- a/src/python/sentryCollector/collect_plugin.py +++ b/src/python/sentryCollector/collect_plugin.py @@ -75,14 +75,14 @@ def client_send_and_recv(request_data, data_str_len, protocol): try: client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) except socket.error: - print("collect_plugin: client create socket error") + logging.error("collect_plugin: client create socket error") return None try: client_socket.connect(COLLECT_SOCKET_PATH) except OSError: client_socket.close() - print("collect_plugin: client connect error") + logging.error("collect_plugin: client connect error") return None req_data_len = len(request_data) @@ -94,23 +94,23 @@ def client_send_and_recv(request_data, data_str_len, protocol): res_data = res_data.decode() except (OSError, UnicodeError): client_socket.close() - print("collect_plugin: client communicate error") + logging.error("collect_plugin: client communicate error") return None res_magic = res_data[:CLT_MSG_MAGIC_LEN] if res_magic != "RES": - print("res msg format error") + logging.error("res msg format error") return None protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] try: protocol_id = int(protocol_str) except ValueError: - print("recv msg protocol id is invalid %s", protocol_str) + logging.error("recv msg protocol id is invalid %s", protocol_str) return None if protocol_id >= ClientProtocol.PRO_END: - print("protocol id is invalid") + logging.error("protocol id is invalid") return None try: @@ -119,7 +119,7 @@ def client_send_and_recv(request_data, data_str_len, protocol): res_msg_data = res_msg_data.decode() return res_msg_data except (OSError, ValueError, UnicodeError): - print("collect_plugin: client recv res msg error") + logging.error("collect_plugin: client recv res msg error") finally: client_socket.close() @@ -128,30 +128,30 @@ def client_send_and_recv(request_data, data_str_len, protocol): def validate_parameters(param, len_limit, char_limit): ret = ResultMessage.RESULT_SUCCEED if not param: - print("param is invalid") + logging.error("param is invalid, param = %s", param) ret = ResultMessage.RESULT_NOT_PARAM return [False, ret] if not isinstance(param, list): - print(f"{param} is not list type.") + logging.error("%s is not list type.", param) ret = ResultMessage.RESULT_NOT_PARAM return [False, ret] if len(param) <= 0: - print(f"{param} length is 0.") + logging.error("%s length is 0.", param) ret = ResultMessage.RESULT_INVALID_LENGTH return [False, ret] pattern = r'^[a-zA-Z0-9_-]+$' for info in param: if not re.match(pattern, info): - print(f"{info} is invalid char") + logging.error("%s is invalid char", info) ret = ResultMessage.RESULT_INVALID_CHAR return [False, ret] # length of len_limit is exceeded, keep len_limit if len(param) > len_limit: - print(f"{param} length more than {len_limit}, keep the first {len_limit}") + logging.error("%s length more than %d, keep the first %d", param, len_limit, len_limit) param[:] = param[0:len_limit] # only keep elements under the char_limit length @@ -202,13 +202,13 @@ def inter_is_iocollect_valid(period, disk_list=None, stage=None): request_message = json.dumps(req_msg_struct) result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.IS_IOCOLLECT_VALID) if not result_message: - print("collect_plugin: client_send_and_recv failed") + logging.error("collect_plugin: client_send_and_recv failed") return result try: json.loads(result_message) except json.JSONDecodeError: - print("is_iocollect_valid: json decode error") + logging.error("is_iocollect_valid: json decode error") result['ret'] = ResultMessage.RESULT_PARSE_FAILED return result @@ -260,12 +260,12 @@ def inter_get_io_data(period, disk_list, stage, iotype): request_message = json.dumps(req_msg_struct) result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.GET_IO_DATA) if not result_message: - print("collect_plugin: client_send_and_recv failed") + logging.error("collect_plugin: client_send_and_recv failed") return result try: json.loads(result_message) except json.JSONDecodeError: - print("get_io_data: json decode error") + logging.error("get_io_data: json decode error") result['ret'] = ResultMessage.RESULT_PARSE_FAILED return result diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py index d9d8862..33f4b04 100644 --- a/src/python/sentryCollector/collectd.py +++ b/src/python/sentryCollector/collectd.py @@ -26,7 +26,7 @@ import threading from .collect_io import CollectIo from .collect_server import CollectServer -from .collect_config import CollectConfig +from .collect_config import CollectConfig, get_log_level SENTRY_RUN_DIR = "/var/run/sysSentry" COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" @@ -57,7 +57,9 @@ def main(): os.mkdir(SENTRY_RUN_DIR) os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) - logging.basicConfig(filename=COLLECT_LOG_FILE, level=logging.INFO) + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + logging.basicConfig(filename=COLLECT_LOG_FILE, level=log_level, format=log_format) os.chmod(COLLECT_LOG_FILE, 0o600) try: diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py index ac35be2..b6b3b28 100644 --- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py +++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py @@ -15,7 +15,7 @@ import time from .stage_window import IoWindow, IoDumpWindow from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler -from .utils import update_avg_and_check_abnormal +from .utils import update_avg_and_check_abnormal, get_log_level CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" @@ -283,7 +283,10 @@ def main(): signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGTERM, sig_handler) - logging.basicConfig(level=logging.INFO) + log_level = get_log_level(CONFIG_FILE) + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + + logging.basicConfig(level=log_level, format=log_format) # 初始化配置读取 config = configparser.ConfigParser(comment_prefixes=('#', ';')) diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py index 54ed080..2de9a46 100644 --- a/src/python/sentryPlugins/avg_block_io/utils.py +++ b/src/python/sentryPlugins/avg_block_io/utils.py @@ -8,9 +8,41 @@ # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR # PURPOSE. # See the Mulan PSL v2 for more details. +import configparser +import logging +import os + AVG_VALUE = 0 AVG_COUNT = 1 +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + def get_nested_value(data, keys): """get data from nested dict""" diff --git a/src/python/syssentry/sentry_config.py b/src/python/syssentry/sentry_config.py index a0e7b79..1169887 100644 --- a/src/python/syssentry/sentry_config.py +++ b/src/python/syssentry/sentry_config.py @@ -21,6 +21,34 @@ import sys DEFAULT_INSPECT_DELAY = 3 INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf" +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=INSPECT_CONF_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + class SentryConfig: """ diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py index 776971f..9ef0203 100644 --- a/src/python/syssentry/syssentry.py +++ b/src/python/syssentry/syssentry.py @@ -23,7 +23,7 @@ import fcntl import select -from .sentry_config import SentryConfig +from .sentry_config import SentryConfig, get_log_level from .task_map import TasksMap from .global_values import SENTRY_RUN_DIR, CTL_SOCKET_PATH, SENTRY_RUN_DIR_PERM @@ -563,7 +563,10 @@ def main(): os.mkdir(SENTRY_RUN_DIR) os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) - logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=logging.INFO) + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" + + logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=log_level, format=log_format) os.chmod(SYSSENTRY_LOG_FILE, 0o600) if not chk_and_set_pidfile(): diff --git a/src/python/xalarm/xalarm_config.py b/src/python/xalarm/xalarm_config.py index 8e56d10..754a816 100644 --- a/src/python/xalarm/xalarm_config.py +++ b/src/python/xalarm/xalarm_config.py @@ -15,9 +15,10 @@ Create: 2023-11-02 """ import re +import os import dataclasses import logging -from configparser import ConfigParser +import configparser MAIN_CONFIG_PATH = '/etc/sysSentry/xalarm.conf' @@ -27,6 +28,34 @@ MIN_ID_NUMBER = 1001 MAX_ID_NUMBER = 1128 MAX_ID_MASK_CAPACITY = 128 +# log +CONF_LOG = 'log' +CONF_LOG_LEVEL = 'level' +LogLevel = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL +} + + +def get_log_level(filename=MAIN_CONFIG_PATH): + if not os.path.exists(filename): + return logging.INFO + + try: + config = configparser.ConfigParser() + config.read(filename) + if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): + return logging.INFO + log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) + if log_level.lower() in LogLevel: + return LogLevel.get(log_level.lower()) + return logging.INFO + except configparser.Error: + return logging.INFO + @dataclasses.dataclass class AlarmConfig: @@ -106,7 +135,7 @@ def config_init(): """ alarm_config = AlarmConfig() - cfg = ConfigParser() + cfg = configparser.ConfigParser() cfg.read(MAIN_CONFIG_PATH) id_mask = parse_id_mask(cfg) diff --git a/src/python/xalarm/xalarm_daemon.py b/src/python/xalarm/xalarm_daemon.py index 00e8886..3ab211c 100644 --- a/src/python/xalarm/xalarm_daemon.py +++ b/src/python/xalarm/xalarm_daemon.py @@ -21,7 +21,7 @@ import signal import fcntl import socket -from .xalarm_config import config_init +from .xalarm_config import config_init, get_log_level from .xalarm_server import server_loop, SOCK_FILE ALARM_DIR = "/var/run/xalarm" @@ -120,9 +120,10 @@ def alarm_process_create(): os.mkdir(ALARM_DIR) os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION) + log_level = get_log_level() + log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - logging.basicConfig(filename=ALARM_LOGFILE, level=logging.INFO, - format='%(asctime)s|%(levelname)s| %(message)s') + logging.basicConfig(filename=ALARM_LOGFILE, level=log_level, format=log_format) signal.signal(signal.SIGTERM, signal_handler) -- 2.23.0