packages/sysSentry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

相关日志格式以及日志打印修改

Browse Source 
相关冗余代码删除
当磁盘disk字段设置为default的时候，采集不生效的情况
This commit is contained in:
zhuofeng 2024-09-27 14:16:24 +08:00
parent d6e572e746
commit e35e45b1e0
2 changed files with 530 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

522
add-log-level-and-change-log-format.patch Normal file
View File

@ -0,0 +1,522 @@
From c1ab550a3f817826ac6f279de97e6d3820901275 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Fri, 27 Sep 2024 14:10:18 +0800
Subject: [PATCH] add log level and change log format
---
config/collector.conf | 5 ++-
config/inspect.conf | 5 ++-
config/plugins/avg_block_io.ini | 5 ++-
config/xalarm.conf | 3 ++
src/python/sentryCollector/collect_config.py | 29 ++++++++++++++++
src/python/sentryCollector/collect_io.py | 15 ++-------
src/python/sentryCollector/collect_plugin.py | 32 +++++++++---------
src/python/sentryCollector/collectd.py | 6 ++--
.../avg_block_io/avg_block_io.py | 7 ++--
.../sentryPlugins/avg_block_io/utils.py | 32 ++++++++++++++++++
src/python/syssentry/sentry_config.py | 28 ++++++++++++++++
src/python/syssentry/syssentry.py | 7 ++--
src/python/xalarm/xalarm_config.py | 33 +++++++++++++++++--
src/python/xalarm/xalarm_daemon.py | 7 ++--
14 files changed, 172 insertions(+), 42 deletions(-)
diff --git a/config/collector.conf b/config/collector.conf
index 9baa086..56b0ed1 100644
--- a/config/collector.conf
+++ b/config/collector.conf
@@ -4,4 +4,7 @@ modules=io
[io]
period_time=1
max_save=10
-disk=default
\ No newline at end of file
+disk=default
+
+[log]
+level=info
\ No newline at end of file
diff --git a/config/inspect.conf b/config/inspect.conf
index 071cca1..f451d9e 100644
--- a/config/inspect.conf
+++ b/config/inspect.conf
@@ -1,2 +1,5 @@
[inspect]
-Interval=3
\ No newline at end of file
+Interval=3
+
+[log]
+level=info
diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini
index bc33dde..858db18 100644
--- a/config/plugins/avg_block_io.ini
+++ b/config/plugins/avg_block_io.ini
@@ -1,8 +1,11 @@
+[log]
+level=info
+
[common]
disk=default
stage=default
iotype=read,write
-period_time=1
+period_time=1
[algorithm]
win_size=30
diff --git a/config/xalarm.conf b/config/xalarm.conf
index 14c6d39..323d2dd 100644
--- a/config/xalarm.conf
+++ b/config/xalarm.conf
@@ -1,2 +1,5 @@
[filter]
id_mask = 1001-1128
+
+[log]
+level=info
diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py
index 0fdd9f0..5aa38ec 100644
--- a/src/python/sentryCollector/collect_config.py
+++ b/src/python/sentryCollector/collect_config.py
@@ -32,6 +32,35 @@ CONF_IO_PERIOD_TIME_DEFAULT = 1
CONF_IO_MAX_SAVE_DEFAULT = 10
CONF_IO_DISK_DEFAULT = "default"
+# log
+CONF_LOG = 'log'
+CONF_LOG_LEVEL = 'level'
+LogLevel = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL
+}
+
+
+def get_log_level(filename=COLLECT_CONF_PATH):
+ if not os.path.exists(filename):
+ return logging.INFO
+
+ try:
+ config = configparser.ConfigParser()
+ config.read(filename)
+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL):
+ return logging.INFO
+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL)
+ if log_level.lower() in LogLevel:
+ return LogLevel.get(log_level.lower())
+ return logging.INFO
+ except configparser.Error:
+ return logging.INFO
+
+
class CollectConfig:
def __init__(self, filename=COLLECT_CONF_PATH):
diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py
index 9c8dae7..019d174 100644
--- a/src/python/sentryCollector/collect_io.py
+++ b/src/python/sentryCollector/collect_io.py
@@ -163,18 +163,6 @@ class CollectIo():
logging.error("An error occurred2: %s", e)
return column_names
- def task_loop(self):
- if self.stop_event.is_set():
- logging.info("collect io thread exit")
- return
-
- for disk_name, stage_list in self.disk_map_stage.items():
- if self.get_blk_io_hierarchy(disk_name, stage_list) < 0:
- continue
- self.append_period_lat(disk_name, stage_list)
-
- threading.Timer(self.period_time, self.task_loop).start()
-
def is_kernel_avaliable(self):
base_path = '/sys/kernel/debug/block'
all_disk = []
@@ -191,6 +179,9 @@ class CollectIo():
if file_name == 'stats':
all_disk.append(disk_name)
+ if self.loop_all:
+ self.disk_list = all_disk
+
for disk_name in self.disk_list:
if not self.loop_all and disk_name not in all_disk:
logging.warning("the %s disk not exist!", disk_name)
diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py
index 1faa5e3..3e2cf4c 100644
--- a/src/python/sentryCollector/collect_plugin.py
+++ b/src/python/sentryCollector/collect_plugin.py
@@ -75,14 +75,14 @@ def client_send_and_recv(request_data, data_str_len, protocol):
try:
client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
except socket.error:
- print("collect_plugin: client create socket error")
+ logging.error("collect_plugin: client create socket error")
return None
try:
client_socket.connect(COLLECT_SOCKET_PATH)
except OSError:
client_socket.close()
- print("collect_plugin: client connect error")
+ logging.error("collect_plugin: client connect error")
return None
req_data_len = len(request_data)
@@ -94,23 +94,23 @@ def client_send_and_recv(request_data, data_str_len, protocol):
res_data = res_data.decode()
except (OSError, UnicodeError):
client_socket.close()
- print("collect_plugin: client communicate error")
+ logging.error("collect_plugin: client communicate error")
return None
res_magic = res_data[:CLT_MSG_MAGIC_LEN]
if res_magic != "RES":
- print("res msg format error")
+ logging.error("res msg format error")
return None
protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN]
try:
protocol_id = int(protocol_str)
except ValueError:
- print("recv msg protocol id is invalid %s", protocol_str)
+ logging.error("recv msg protocol id is invalid %s", protocol_str)
return None
if protocol_id >= ClientProtocol.PRO_END:
- print("protocol id is invalid")
+ logging.error("protocol id is invalid")
return None
try:
@@ -119,7 +119,7 @@ def client_send_and_recv(request_data, data_str_len, protocol):
res_msg_data = res_msg_data.decode()
return res_msg_data
except (OSError, ValueError, UnicodeError):
- print("collect_plugin: client recv res msg error")
+ logging.error("collect_plugin: client recv res msg error")
finally:
client_socket.close()
@@ -128,30 +128,30 @@ def client_send_and_recv(request_data, data_str_len, protocol):
def validate_parameters(param, len_limit, char_limit):
ret = ResultMessage.RESULT_SUCCEED
if not param:
- print("param is invalid")
+ logging.error("param is invalid, param = %s", param)
ret = ResultMessage.RESULT_NOT_PARAM
return [False, ret]
if not isinstance(param, list):
- print(f"{param} is not list type.")
+ logging.error("%s is not list type.", param)
ret = ResultMessage.RESULT_NOT_PARAM
return [False, ret]
if len(param) <= 0:
- print(f"{param} length is 0.")
+ logging.error("%s length is 0.", param)
ret = ResultMessage.RESULT_INVALID_LENGTH
return [False, ret]
pattern = r'^[a-zA-Z0-9_-]+$'
for info in param:
if not re.match(pattern, info):
- print(f"{info} is invalid char")
+ logging.error("%s is invalid char", info)
ret = ResultMessage.RESULT_INVALID_CHAR
return [False, ret]
# length of len_limit is exceeded, keep len_limit
if len(param) > len_limit:
- print(f"{param} length more than {len_limit}, keep the first {len_limit}")
+ logging.error("%s length more than %d, keep the first %d", param, len_limit, len_limit)
param[:] = param[0:len_limit]
# only keep elements under the char_limit length
@@ -202,13 +202,13 @@ def inter_is_iocollect_valid(period, disk_list=None, stage=None):
request_message = json.dumps(req_msg_struct)
result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.IS_IOCOLLECT_VALID)
if not result_message:
- print("collect_plugin: client_send_and_recv failed")
+ logging.error("collect_plugin: client_send_and_recv failed")
return result
try:
json.loads(result_message)
except json.JSONDecodeError:
- print("is_iocollect_valid: json decode error")
+ logging.error("is_iocollect_valid: json decode error")
result['ret'] = ResultMessage.RESULT_PARSE_FAILED
return result
@@ -260,12 +260,12 @@ def inter_get_io_data(period, disk_list, stage, iotype):
request_message = json.dumps(req_msg_struct)
result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.GET_IO_DATA)
if not result_message:
- print("collect_plugin: client_send_and_recv failed")
+ logging.error("collect_plugin: client_send_and_recv failed")
return result
try:
json.loads(result_message)
except json.JSONDecodeError:
- print("get_io_data: json decode error")
+ logging.error("get_io_data: json decode error")
result['ret'] = ResultMessage.RESULT_PARSE_FAILED
return result
diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py
index d9d8862..33f4b04 100644
--- a/src/python/sentryCollector/collectd.py
+++ b/src/python/sentryCollector/collectd.py
@@ -26,7 +26,7 @@ import threading
from .collect_io import CollectIo
from .collect_server import CollectServer
-from .collect_config import CollectConfig
+from .collect_config import CollectConfig, get_log_level
SENTRY_RUN_DIR = "/var/run/sysSentry"
COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock"
@@ -57,7 +57,9 @@ def main():
os.mkdir(SENTRY_RUN_DIR)
os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM)
- logging.basicConfig(filename=COLLECT_LOG_FILE, level=logging.INFO)
+ log_level = get_log_level()
+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s"
+ logging.basicConfig(filename=COLLECT_LOG_FILE, level=log_level, format=log_format)
os.chmod(COLLECT_LOG_FILE, 0o600)
try:
diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py
index ac35be2..b6b3b28 100644
--- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py
+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py
@@ -15,7 +15,7 @@ import time
from .stage_window import IoWindow, IoDumpWindow
from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler
-from .utils import update_avg_and_check_abnormal
+from .utils import update_avg_and_check_abnormal, get_log_level
CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini"
@@ -283,7 +283,10 @@ def main():
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
- logging.basicConfig(level=logging.INFO)
+ log_level = get_log_level(CONFIG_FILE)
+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s"
+
+ logging.basicConfig(level=log_level, format=log_format)
# 初始化配置读取
config = configparser.ConfigParser(comment_prefixes=('#', ';'))
diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py
index 54ed080..2de9a46 100644
--- a/src/python/sentryPlugins/avg_block_io/utils.py
+++ b/src/python/sentryPlugins/avg_block_io/utils.py
@@ -8,9 +8,41 @@
# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
# PURPOSE.
# See the Mulan PSL v2 for more details.
+import configparser
+import logging
+import os
+
AVG_VALUE = 0
AVG_COUNT = 1
+CONF_LOG = 'log'
+CONF_LOG_LEVEL = 'level'
+LogLevel = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL
+}
+
+
+def get_log_level(filename):
+ if not os.path.exists(filename):
+ return logging.INFO
+
+ try:
+ config = configparser.ConfigParser()
+ config.read(filename)
+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL):
+ return logging.INFO
+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL)
+
+ if log_level.lower() in LogLevel:
+ return LogLevel.get(log_level.lower())
+ return logging.INFO
+ except configparser.Error:
+ return logging.INFO
+
def get_nested_value(data, keys):
"""get data from nested dict"""
diff --git a/src/python/syssentry/sentry_config.py b/src/python/syssentry/sentry_config.py
index a0e7b79..1169887 100644
--- a/src/python/syssentry/sentry_config.py
+++ b/src/python/syssentry/sentry_config.py
@@ -21,6 +21,34 @@ import sys
DEFAULT_INSPECT_DELAY = 3
INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf"
+CONF_LOG = 'log'
+CONF_LOG_LEVEL = 'level'
+LogLevel = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL
+}
+
+
+def get_log_level(filename=INSPECT_CONF_PATH):
+ if not os.path.exists(filename):
+ return logging.INFO
+
+ try:
+ config = configparser.ConfigParser()
+ config.read(filename)
+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL):
+ return logging.INFO
+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL)
+
+ if log_level.lower() in LogLevel:
+ return LogLevel.get(log_level.lower())
+ return logging.INFO
+ except configparser.Error:
+ return logging.INFO
+
class SentryConfig:
"""
diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py
index 776971f..9ef0203 100644
--- a/src/python/syssentry/syssentry.py
+++ b/src/python/syssentry/syssentry.py
@@ -23,7 +23,7 @@ import fcntl
import select
-from .sentry_config import SentryConfig
+from .sentry_config import SentryConfig, get_log_level
from .task_map import TasksMap
from .global_values import SENTRY_RUN_DIR, CTL_SOCKET_PATH, SENTRY_RUN_DIR_PERM
@@ -563,7 +563,10 @@ def main():
os.mkdir(SENTRY_RUN_DIR)
os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM)
- logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=logging.INFO)
+ log_level = get_log_level()
+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s"
+
+ logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=log_level, format=log_format)
os.chmod(SYSSENTRY_LOG_FILE, 0o600)
if not chk_and_set_pidfile():
diff --git a/src/python/xalarm/xalarm_config.py b/src/python/xalarm/xalarm_config.py
index 8e56d10..754a816 100644
--- a/src/python/xalarm/xalarm_config.py
+++ b/src/python/xalarm/xalarm_config.py
@@ -15,9 +15,10 @@ Create: 2023-11-02
"""
import re
+import os
import dataclasses
import logging
-from configparser import ConfigParser
+import configparser
MAIN_CONFIG_PATH = '/etc/sysSentry/xalarm.conf'
@@ -27,6 +28,34 @@ MIN_ID_NUMBER = 1001
MAX_ID_NUMBER = 1128
MAX_ID_MASK_CAPACITY = 128
+# log
+CONF_LOG = 'log'
+CONF_LOG_LEVEL = 'level'
+LogLevel = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL
+}
+
+
+def get_log_level(filename=MAIN_CONFIG_PATH):
+ if not os.path.exists(filename):
+ return logging.INFO
+
+ try:
+ config = configparser.ConfigParser()
+ config.read(filename)
+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL):
+ return logging.INFO
+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL)
+ if log_level.lower() in LogLevel:
+ return LogLevel.get(log_level.lower())
+ return logging.INFO
+ except configparser.Error:
+ return logging.INFO
+
@dataclasses.dataclass
class AlarmConfig:
@@ -106,7 +135,7 @@ def config_init():
"""
alarm_config = AlarmConfig()
- cfg = ConfigParser()
+ cfg = configparser.ConfigParser()
cfg.read(MAIN_CONFIG_PATH)
id_mask = parse_id_mask(cfg)
diff --git a/src/python/xalarm/xalarm_daemon.py b/src/python/xalarm/xalarm_daemon.py
index 00e8886..3ab211c 100644
--- a/src/python/xalarm/xalarm_daemon.py
+++ b/src/python/xalarm/xalarm_daemon.py
@@ -21,7 +21,7 @@ import signal
import fcntl
import socket
-from .xalarm_config import config_init
+from .xalarm_config import config_init, get_log_level
from .xalarm_server import server_loop, SOCK_FILE
ALARM_DIR = "/var/run/xalarm"
@@ -120,9 +120,10 @@ def alarm_process_create():
os.mkdir(ALARM_DIR)
os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION)
+ log_level = get_log_level()
+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s"
- logging.basicConfig(filename=ALARM_LOGFILE, level=logging.INFO,
- format='%(asctime)s|%(levelname)s| %(message)s')
+ logging.basicConfig(filename=ALARM_LOGFILE, level=log_level, format=log_format)
signal.signal(signal.SIGTERM, signal_handler)
--
2.23.0

9
sysSentry.spec
View File

@ -4,7 +4,7 @@
Summary: System Inspection Framework
Name: sysSentry
Version: 1.0.2
Release: 17
Release: 18
License: Mulan PSL v2
Group: System Environment/Daemons
Source0: https://gitee.com/openeuler/sysSentry/releases/download/v%{version}/%{name}-%{version}.tar.gz
@ -29,6 +29,7 @@ Patch16: add-ai-threshold-slow-io-detection-plugin.patch
Patch17: optimize-the-handing-of-cat-cli-error-msg-in-cpu_sentry.patch
Patch18: over-threshold-should-be-warn-level-log-in-cat-cli.patch
Patch19: fix-bug-step-2-about-collect-module-and-avg-block-io.patch
Patch20: add-log-level-and-change-log-format.patch
BuildRequires: cmake gcc-c++
BuildRequires: python3 python3-setuptools
@ -247,6 +248,12 @@ rm -rf %{buildroot}
%attr(0550,root,root) %{python3_sitelib}/sentryPlugins/ai_threshold_slow_io_detection
%changelog
* Fri Sep 27 2024 zhuofeng <zhuofeng2@huawei.com> - 1.0.2-18
- Type:bugfix
- CVE:NA
- SUG:NA
- DESC:add log level and change log format
* Wed Sep 25 2024 zhuofeng <zhuofeng2@huawei.com> - 1.0.2-17
- Type:bugfix
- CVE:NA