packages/A-Tune-Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!42 update v1.3.0

Browse Source 
From: @zhoupengcheng11 
Reviewed-by: @gaoruoshu 
Signed-off-by: @gaoruoshu
This commit is contained in:
openeuler-ci-bot 2024-02-07 06:17:32 +00:00 committed by Gitee
commit fed1820343
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
20 changed files with 6 additions and 2223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
README-add-data-collecting-output-description.patch
View File

@ -1,47 +0,0 @@
From c8d0e4c3030808fe0a575e595d8bd9a51d30124a Mon Sep 17 00:00:00 2001
From: HuBin95 <hubin73@huawei.com>
Date: Mon, 7 Mar 2022 02:10:24 +0000
Subject: [PATCH 02/11] README: add data collecting output description
---
README.en.md | 4 +++-
README.md | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/README.en.md b/README.en.md
index 2f11270..8a54fa4 100644
--- a/README.en.md
+++ b/README.en.md
@@ -52,9 +52,11 @@ Table 1 **collect_data.json** file
| sample_num | Sample number to be collected. | Integer | > 0 |
| interval | Interval for collecting data, in seconds. | Integer | > 0 |
| output_dir | Path for storing collected data. | Character string | - |
-| workload_type | Application load type of the collection environment. The default value is **default**. | Character string | - |
+| workload_type | Application load type of the collection environment, used as output file name. The default value is **default**. | Character string | - |
| collection_items | Table 2 lists the system parameters to be collected. | List | - |
+When data collecting is finished, the data will be saved as: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
+
Table 2 Description of the **collection_items** configuration
| Parameter | Description | Type | Value Range |
diff --git a/README.md b/README.md
index 36d5954..c121c8d 100644
--- a/README.md
+++ b/README.md
@@ -53,9 +53,11 @@ python3 collect_data.py [OPTIONS]
| sample_num | 待采集的次数 | 整型 | >0 |
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
-| workload_type | 采集环境的应用负载类型，默认为default | 字符串 | - |
+| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
+最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
+
表2 collection_items项配置说明
| **配置名称** | **配置说明** | **参数类型** | **取值范围** |
--
2.27.0

39
Ubuntu-20.04.3-LTS.patch
View File

@ -1,39 +0,0 @@
From 2fd1148dcec21a8a99fbb59240ba2e2abfe7c31c Mon Sep 17 00:00:00 2001
From: Masterwater-y <yin12162000@outlook.com>
Date: Fri, 28 Jan 2022 18:07:52 +0800
Subject: [PATCH 01/11] adapt Ubuntu 20.04.3 LTS
---
atune_collector/plugin/monitor/common.py | 2 +-
atune_collector/plugin/monitor/memory/topo.py | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
index a646b32..12a07f2 100755
--- a/atune_collector/plugin/monitor/common.py
+++ b/atune_collector/plugin/monitor/common.py
@@ -230,7 +230,7 @@ class Monitor(object):
def walk_class_type(father, class_type, desc, datas):
"""get key field"""
if "class" in father and father["class"] == class_type:
- if "description" in father and (desc is None or father["description"] == desc):
+ if "description" in father and (desc is None or father["description"].lower() == desc.lower()):
datas.append(father)
return
if "children" in father:
diff --git a/atune_collector/plugin/monitor/memory/topo.py b/atune_collector/plugin/monitor/memory/topo.py
index 7e828f3..2c0c39e 100755
--- a/atune_collector/plugin/monitor/memory/topo.py
+++ b/atune_collector/plugin/monitor/memory/topo.py
@@ -95,6 +95,8 @@ class MemTopo(Monitor):
cmd=self.__cmd).split(), stderr=no_print)
info = o_json.decode()
json_content = json.loads(info)
+ if (isinstance(json_content, list)):
+ json_content = {'children': json_content}
dict_datas = get_class_type(json_content, "memory", "System Memory")
if fmt == "json":
--
2.27.0

427
add-new-dims.patch
View File

@ -1,427 +0,0 @@
From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001
From: hamster <hujing@isrc.iscas.ac.cn>
Date: Mon, 14 Nov 2022 03:24:49 +0000
Subject: [PATCH 04/11] add new dimensions
From: @hujing2
Reviewed-by: @gaoruoshu
Signed-off-by: @gaoruoshu
---
README.md | 16 +++
atune_collector/collect_data.json | 14 ++
atune_collector/collect_data.py | 97 ++++++++------
atune_collector/plugin/monitor/__init__.py | 1 +
atune_collector/plugin/monitor/common.py | 1 +
.../plugin/monitor/process/__init__.py | 20 +++
.../plugin/monitor/process/sched.py | 124 ++++++++++++++++++
7 files changed, 233 insertions(+), 40 deletions(-)
create mode 100644 atune_collector/plugin/monitor/process/__init__.py
create mode 100644 atune_collector/plugin/monitor/process/sched.py
diff --git a/README.md b/README.md
index c121c8d..242f345 100644
--- a/README.md
+++ b/README.md
@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS]
| ---------------- | ------------------------------------- | ------------ | ------------ |
| network | 待采集的指定网卡 | 字符串 | - |
| block | 待采集的指定磁盘 | 字符串 | - |
+| application | 需要采集的应用进程 | 字符串 | - |
| sample_num | 待采集的次数 | 整型 | >0 |
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
+
最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
表2 collection_items项配置说明
@@ -76,6 +78,7 @@ collect_data.json文件配置示例：
{
"network": "eth0",
"block": "sda",
+ "application": "mysqld",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -203,6 +206,19 @@ collect_data.json文件配置示例：
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json
index db96501..af286bd 100755
--- a/atune_collector/collect_data.json
+++ b/atune_collector/collect_data.json
@@ -1,6 +1,7 @@
{
"network": "eth0",
"block": "sda",
+ "application": "firewalld,dockerd",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -140,6 +141,19 @@
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
\ No newline at end of file
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
index 1764304..3593db6 100755
--- a/atune_collector/collect_data.py
+++ b/atune_collector/collect_data.py
@@ -18,6 +18,7 @@ import argparse
import json
import os
import time
+import csv
from plugin.plugin import MPI
@@ -30,66 +31,80 @@ class Collector:
self.field_name = []
self.support_multi_block = ['storage']
self.support_multi_nic = ['network', 'network-err']
+ self.support_multi_app = ['process']
def parse_json(self):
"""parse json data"""
monitors = []
for item in self.data["collection_items"]:
- parameters = ["--interval=%s;" % self.data["interval"]]
- for metric in item["metrics"]:
- nics = self.data["network"].split(',')
- blocks = self.data["block"].split(',')
- if item["name"] in self.support_multi_nic and len(nics) > 1:
- for net in nics:
+ if item["name"] in self.support_multi_app and ('application' not in self.data or
+ self.data["application"] == ""):
+ continue
+ if item["name"] in self.support_multi_app:
+ applications = self.data["application"].split(',')
+ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])]
+ for application in applications:
+ for metric in item["metrics"]:
self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
- elif item["name"] in self.support_multi_block and len(blocks) > 1:
- for block in blocks:
- self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
- else:
- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
- parameters.append("--fields=%s" % metric)
- if "threshold" in item:
- parameters.append("--threshold=%s" % item["threshold"])
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application))
+ parameters.append("--fields=%s" % metric)
+ else:
+ parameters = ["--interval=%s;" % self.data["interval"]]
+ for metric in item["metrics"]:
+ nics = self.data["network"].split(',')
+ blocks = self.data["block"].split(',')
+
+ if item["name"] in self.support_multi_nic and len(nics) > 1:
+ for net in nics:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
+ elif item["name"] in self.support_multi_block and len(blocks) > 1:
+ for block in blocks:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
+ else:
+ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
+ parameters.append("--fields=%s" % metric)
+ if "threshold" in item:
+ parameters.append("--threshold=%s" % item["threshold"])
+
parameters.append("--nic=%s" % self.data["network"])
parameters.append("--device=%s" % self.data["block"])
monitors.append([item["module"], item["purpose"], " ".join(parameters)])
return monitors
- def save_csv(self, field_data):
- """save data to csv file"""
+ def collect_data(self):
+ """collect data"""
+ collect_num = self.data["sample_num"]
+ if int(collect_num) < 1:
+ os.abort("sample_num must be greater than 0")
+
+ mpi = MPI()
+ monitors = self.parse_json()
path = self.data["output_dir"]
if not os.path.exists(path):
os.makedirs(path, 0o750)
file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"),
int(round(time.time() * 1000)))
- import csv
+
+ print("start to collect data, csv path is %s" % os.path.join(path, file_name))
+ print(" ".join(self.field_name))
with open(os.path.join(path, file_name), "w") as csvfile:
writer = csv.writer(csvfile)
self.field_name.insert(0, "TimeStamp")
writer.writerow(self.field_name)
- writer.writerows(field_data)
+ csvfile.flush()
+ for _ in range(collect_num):
+ raw_data = mpi.get_monitors_data(monitors)
+ float_data = [float(num) for num in raw_data]
+ str_data = [str(round(value, 3)) for value in float_data]
+ print(" ".join(str_data))
+ float_data.insert(0, time.strftime("%H:%M:%S"))
+ # field_data.append(float_data)
+ writer.writerow(float_data)
+ csvfile.flush()
print("finish to collect data, csv path is %s" % os.path.join(path, file_name))
- def collect_data(self):
- """collect data"""
- collect_num = self.data["sample_num"]
- if int(collect_num) < 1:
- os.abort("sample_num must be greater than 0")
- field_data = []
- mpi = MPI()
- monitors = self.parse_json()
- print(" ".join(self.field_name))
- for _ in range(collect_num):
- raw_data = mpi.get_monitors_data(monitors)
- float_data = [float(num) for num in raw_data]
- str_data = [str(round(value, 3)) for value in float_data]
- print(" ".join(str_data))
- float_data.insert(0, time.strftime("%H:%M:%S"))
- field_data.append(float_data)
- return field_data
-
if __name__ == "__main__":
default_json_path = "/etc/atune_collector/collect_data.json"
@@ -100,5 +115,7 @@ if __name__ == "__main__":
with open(ARGS.config, 'r') as file:
json_data = json.load(file)
collector = Collector(json_data)
- dataset = collector.collect_data()
- collector.save_csv(dataset)
+ try:
+ collector.collect_data()
+ except KeyboardInterrupt:
+ print("user stop collect data")
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
index 6291401..9292071 100755
--- a/atune_collector/plugin/monitor/__init__.py
+++ b/atune_collector/plugin/monitor/__init__.py
@@ -19,6 +19,7 @@ __all__ = [
"memory",
"network",
"performance",
+ "process",
"processor",
"storage",
"common",
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
index 12a07f2..d0aa60c 100755
--- a/atune_collector/plugin/monitor/common.py
+++ b/atune_collector/plugin/monitor/common.py
@@ -134,6 +134,7 @@ class Monitor(object):
"--interval=" to specify period of time
"--cpu=" to select which cpu
"--event=" to select which event
+ "--app" to select which applications
:returns value: Success, collected info string
:raises Exceptions: Fail, with info
"""
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
new file mode 100644
index 0000000..4c4ceb3
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/__init__.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+Init file.
+"""
+
+__all__ = ["sched"]
+
+from . import *
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
new file mode 100644
index 0000000..5289d84
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/sched.py
@@ -0,0 +1,124 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+The sub class of the monitor, used to collect the process sched info
+"""
+import inspect
+import logging
+import subprocess
+import getopt
+import re
+from ..common import Monitor
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ProcSched(Monitor):
+ """To collect the process sched info"""
+ _module = "PROCESS"
+ _purpose = "SCHED"
+ _option = "/proc/{}/sched"
+
+ def __init__(self, user=None):
+ Monitor.__init__(self, user)
+ self.__cmd = "cat"
+ self.__interval = 1
+ self.__applications = []
+ self.__pids = []
+
+ def _get(self, para=None):
+ output = ""
+ pids = []
+ if para is not None:
+ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app='])
+ for opt, val in opts:
+ if opt in '--interval':
+ if val.isdigit():
+ self.__interval = int(val)
+ else:
+ err = ValueError(
+ "Invalid parameter: {opt}={val}".format(
+ opt=opt, val=val))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+ continue
+ elif opt in '--app':
+ if val is not None:
+ self.__applications = val.split(',')
+ else:
+ err = ValueError(
+ "{opt} parameter is none".format(
+ opt=opt))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ for app in self.__applications:
+ pid = subprocess.getoutput(
+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0]
+ pids.append(pid)
+ self.__pids = pids
+
+ for pid in self.__pids:
+ out = subprocess.check_output(
+ "{cmd} {opt}".format(
+ cmd=self.__cmd,
+ opt=self._option.format(pid)).split())
+ output = output + "" + out.decode()
+ return output
+
+ def decode(self, info, para):
+ """
+ decode the result of the operation
+ :param info: content that needs to be decoded
+ :param para: command line argument
+ :returns ret: operation result
+ """
+
+ if para is None:
+ return info
+
+ start = 0
+ keys = []
+ ret = ""
+
+ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device='])
+ for opt, val in opts:
+ if opt in '--fields':
+ keys.append(val)
+ continue
+
+ pattern = re.compile(
+ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)",
+ re.I | re.UNICODE | re.MULTILINE)
+ search_obj = pattern.findall(info)
+ search_list = []
+ for obj in search_obj:
+ if obj[0][:3] == "nr_":
+ search_list.append(obj[0][3:])
+ else:
+ search_list.append(obj[0])
+ search_list.append(obj[1])
+ if len(search_obj) == 0:
+ err = LookupError("Fail to find data")
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ while start <= len(self.__applications) * len(keys):
+ for key in keys:
+ ret = ret + " " + search_list[search_list.index(key, start)+1]
+ start = search_list.index(key, start) + 1
+ return ret
\ No newline at end of file
--
2.27.0

56
atune-collector-add-backup-for-apps.patch
View File

@ -1,56 +0,0 @@
From bfe9d212c518944678bbf526d61a8e778f378b41 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Wed, 16 Aug 2023 12:36:46 +0800
Subject: [PATCH] [atune-collector]add backup for apps
---
atune_collector/plugin/configurator/mysql/mysql.py | 3 +++
atune_collector/plugin/configurator/nginx/nginx.py | 3 +++
atune_collector/plugin/configurator/redis/redis.py | 3 +++
3 files changed, 9 insertions(+)
diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py
index e472a1c..9badd21 100644
--- a/atune_collector/plugin/configurator/mysql/mysql.py
+++ b/atune_collector/plugin/configurator/mysql/mysql.py
@@ -81,6 +81,9 @@ class Mysql(Configurator):
def _get(self, key, _):
pass
+ def _backup(self, config, _):
+ return str(config)
+
@staticmethod
def check(config1, config2):
return True
diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py
index 9d38dd0..7f7eaad 100644
--- a/atune_collector/plugin/configurator/nginx/nginx.py
+++ b/atune_collector/plugin/configurator/nginx/nginx.py
@@ -198,6 +198,9 @@ class Nginx(Configurator):
def _get(self, key, _):
pass
+ def _backup(self, config, _):
+ return str(config)
+
@staticmethod
def check(config1, config2):
return True
diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py
index aa14bcd..44da99e 100644
--- a/atune_collector/plugin/configurator/redis/redis.py
+++ b/atune_collector/plugin/configurator/redis/redis.py
@@ -68,6 +68,9 @@ class Redis(Configurator):
def _get(self, key, _):
pass
+ def _backup(self, config, _):
+ return str(config)
+
@staticmethod
def check(config1, config2):
return True
--
2.27.0

26
atune-collector.spec
View File

@ -1,31 +1,13 @@
%define debug_package %{nil} %define debug_package %{nil}
Name: atune-collector Name: atune-collector
Version: 1.1.0 Version: 1.3.0
Release: 7 Release: 1
Summary: A-Tune-Collector is used to collect various system resources. Summary: A-Tune-Collector is used to collect various system resources.
License: Mulan PSL v2 License: Mulan PSL v2
URL: https://gitee.com/openeuler/A-Tune-Collector URL: https://gitee.com/openeuler/A-Tune-Collector
Source: https://gitee.com/openeuler/A-Tune-Collector/repository/archive/v%{version}.tar.gz Source: https://gitee.com/openeuler/A-Tune-Collector/repository/archive/v%{version}.tar.gz
Patch1: Ubuntu-20.04.3-LTS.patch
Patch2: README-add-data-collecting-output-description.patch
Patch3: fix-bug-list-index-out-of-range-in-virtualbox.patch
Patch4: add-new-dims.patch
Patch5: fix-CWE-23.patch
Patch6: memory-bandwidth-dimm-slot-atuned.patch
Patch7: feature-set-mysql.patch
Patch8: feature-set-redis-value.patch
Patch9: feature-set-nginx.patch
Patch10: bugfix-create-file-only-if-setting-params.patch
Patch11: change-import-behavior.patch
Patch12: atune-collector-add-backup-for-apps.patch
Patch13: feature-add-network-CPI.patch
Patch14: feature-add-multi-for-rps-xps.patch
Patch15: feature-add-rfs-to-network.patch
Patch16: fix-bug-procsched-report-list-index-out-of-range.patch
Patch17: fix-bug-procsched-data-collection-issue.patch
BuildRequires: python3-setuptools BuildRequires: python3-setuptools
Requires: python3-dict2xml python3-werkzeug Requires: python3-dict2xml python3-werkzeug
@ -46,9 +28,13 @@ The A-Tune-Collector is used to collect various system resources and can also be
%files %files
%license LICENSE %license LICENSE
%{python3_sitelib}/atune_* %{python3_sitelib}/atune_*
%exclude %{python3_sitelib}/atune_collector/ui/*
%attr(0600,root,root) %{_sysconfdir}/atune_collector/* %attr(0600,root,root) %{_sysconfdir}/atune_collector/*
%changelog %changelog
* Sat Jan 27 2024 zhoupengcheng <zhoupengcheng11@huawei.com> - 1.3.0-1
- update v1.3.0
* Fri Dec 15 2023 weiyaping <weiyaping@xfusion.com> - 1.1.0-7 * Fri Dec 15 2023 weiyaping <weiyaping@xfusion.com> - 1.1.0-7
- fix bug: ProcSched.report: list index out of range - fix bug: ProcSched.report: list index out of range

113
bugfix-create-file-only-if-setting-params.patch
View File

@ -1,113 +0,0 @@
From 1c513970f8f5e81268e998cc33ab8a93802bb722 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Fri, 14 Jul 2023 18:47:25 +0800
Subject: [PATCH 10/11] bugfix: create file only if setting params
---
atune_collector/plugin/configurator/mysql/mysql.py | 9 ++++++---
atune_collector/plugin/configurator/nginx/nginx.py | 7 ++++---
atune_collector/plugin/configurator/redis/redis.py | 7 +++++--
3 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py
index cfc533d..e472a1c 100644
--- a/atune_collector/plugin/configurator/mysql/mysql.py
+++ b/atune_collector/plugin/configurator/mysql/mysql.py
@@ -31,8 +31,10 @@ class Mysql(Configurator):
self.__cmd = ""
self.__file_path = "/etc/my.cnf"
self.__mysqld_ind = -1
+
+ def _init_file(self):
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w'):
+ with open(self.__file_path, 'w', 0o644):
pass
os.chmod(self.__file_path, 0o644)
self.__check_mysqld()
@@ -61,7 +63,8 @@ class Mysql(Configurator):
return False, ind
def _set(self, key, value):
- with open(self.__file_path, 'r') as f:
+ self._init_file()
+ with open(self.__file_path, 'r', 0o400) as f:
lines = f.readlines()
key_exist, ind = self.__check_file_exists(lines, key)
@@ -71,7 +74,7 @@ class Mysql(Configurator):
else:
lines[ind] = new_line
- with open(self.__file_path, 'w') as f:
+ with open(self.__file_path, 'w', 0o644) as f:
f.writelines(lines)
return 0
diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py
index 7ab619d..9d38dd0 100644
--- a/atune_collector/plugin/configurator/nginx/nginx.py
+++ b/atune_collector/plugin/configurator/nginx/nginx.py
@@ -42,18 +42,18 @@ class Nginx(Configurator):
os.mkdir(self.__file_dir)
os.chmod(self.__file_dir, 0o755)
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w'):
+ with open(self.__file_path, 'w', 0o644):
pass
os.chmod(self.__file_path, 0o644)
self._set_index()
def _get_lines(self):
- with open(self.__file_path, 'r') as f:
+ with open(self.__file_path, 'r', 0o444) as f:
lines = f.readlines()
return lines
def _set_lines(self, lines):
- with open(self.__file_path, 'w') as f:
+ with open(self.__file_path, 'w', 0o644) as f:
f.writelines(lines)
def _set_index(self):
@@ -209,3 +209,4 @@ def rewrite_value(value):
raise SetConfigError("Failed to get cpu number")
return output.stdout.decode().count("\n")
+
diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py
index c56f38e..aa14bcd 100644
--- a/atune_collector/plugin/configurator/redis/redis.py
+++ b/atune_collector/plugin/configurator/redis/redis.py
@@ -34,15 +34,18 @@ class Redis(Configurator):
self.__re = r"^#\?\s*{key}\s* "
self.__file_dir = "/etc/redis/"
self.__file_path = self.__file_dir + "redis.conf"
+
+ def _init_file(self):
if not os.path.isdir(self.__file_dir):
os.mkdir(self.__file_dir)
os.chmod(self.__file_dir, 0o755)
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w', 0o200):
+ with open(self.__file_path, 'w', 0o600):
pass
os.chmod(self.__file_path, 0o644)
def _set(self, key, value):
+ self._init_file()
re_cmd = self.__re.format(key=key)
grep_cmd = [r"grep", re_cmd, self.__file_path]
out, err = self.execute_cmd(grep_cmd)
@@ -51,7 +54,7 @@ class Redis(Configurator):
num_lines = out.count("\n")
new_line = self.__lines.format(key=key, value=value)
if num_lines == 0:
- with open(self.__file_path, 'a', 0o600) as f:
+ with open(self.__file_path, 'a', 0o644) as f:
f.write(new_line + '\n')
elif num_lines == 1:
sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path]
--
2.27.0

355
change-import-behavior.patch
View File

@ -1,355 +0,0 @@
From c02c86fbd6927a3d1968d5a1e3030058bbb943a3 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Sat, 15 Jul 2023 15:50:06 +0800
Subject: [PATCH 11/11] change import behavior
---
.../plugin/configurator/__init__.py | 47 +++++++++++--------
.../plugin/configurator/affinity/__init__.py | 2 +-
.../plugin/configurator/bios/__init__.py | 2 +-
.../configurator/bootloader/__init__.py | 2 +-
atune_collector/plugin/configurator/common.py | 2 +-
.../configurator/file_config/__init__.py | 2 +-
.../configurator/kernel_config/__init__.py | 2 +-
.../plugin/configurator/mysql/__init__.py | 3 +-
.../plugin/configurator/nginx/__init__.py | 3 +-
.../plugin/configurator/redis/__init__.py | 3 +-
.../plugin/configurator/script/__init__.py | 2 +-
.../plugin/configurator/sysctl/__init__.py | 2 +-
.../plugin/configurator/sysfs/__init__.py | 2 +-
.../plugin/configurator/systemctl/__init__.py | 2 +-
.../plugin/configurator/ulimit/__init__.py | 2 +-
atune_collector/plugin/monitor/__init__.py | 11 ++++-
.../plugin/monitor/memory/__init__.py | 2 +-
.../plugin/monitor/network/__init__.py | 2 +-
.../plugin/monitor/performance/__init__.py | 2 +-
.../plugin/monitor/process/__init__.py | 2 +-
.../plugin/monitor/processor/__init__.py | 2 +-
.../plugin/monitor/storage/__init__.py | 2 +-
.../plugin/monitor/system/__init__.py | 2 +-
atune_collector/plugin/plugin.py | 4 +-
24 files changed, 60 insertions(+), 47 deletions(-)
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 31520f1..fd7ea63 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -17,24 +17,33 @@ Init file.
import sys
import os
-sys.path.append(os.path.dirname(__file__))
+from .affinity import *
+from .bios import *
+from .bootloader import *
+from .file_config import *
+from .kernel_config import *
+from .script import *
+from .sysctl import *
+from .sysfs import *
+from .systemctl import *
+from .ulimit import *
+from .mysql import *
+from .redis import *
+from .nginx import *
-__all__ = [
- "exceptions",
- "affinity",
- "bios",
- "bootloader",
- "file_config",
- "kernel_config",
- "script",
- "sysctl",
- "sysfs",
- "systemctl",
- "ulimit",
- "mysql",
- "redis",
- "nginx",
- "common"]
-
-from . import *
+__all__ = ["exceptions",
+ "affinity",
+ "bios",
+ "bootloader",
+ "file_config",
+ "kernel_config",
+ "script",
+ "sysctl",
+ "sysfs",
+ "systemctl",
+ "ulimit",
+ "mysql",
+ "redis",
+ "nginx",
+ "common"]
diff --git a/atune_collector/plugin/configurator/affinity/__init__.py b/atune_collector/plugin/configurator/affinity/__init__.py
index e29377f..1610cc3 100755
--- a/atune_collector/plugin/configurator/affinity/__init__.py
+++ b/atune_collector/plugin/configurator/affinity/__init__.py
@@ -19,6 +19,6 @@ import os
__all__ = ["irq", "task", "processid"]
-from . import *
+from . import irq, task, processid
sys.path.insert(0, os.path.dirname(__file__))
diff --git a/atune_collector/plugin/configurator/bios/__init__.py b/atune_collector/plugin/configurator/bios/__init__.py
index 1e193cf..1f9df24 100755
--- a/atune_collector/plugin/configurator/bios/__init__.py
+++ b/atune_collector/plugin/configurator/bios/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["bios"]
-from . import *
+from . import bios
diff --git a/atune_collector/plugin/configurator/bootloader/__init__.py b/atune_collector/plugin/configurator/bootloader/__init__.py
index 2412c54..d6cf156 100755
--- a/atune_collector/plugin/configurator/bootloader/__init__.py
+++ b/atune_collector/plugin/configurator/bootloader/__init__.py
@@ -20,6 +20,6 @@ import os
__all__ = ["grub2", "cmdline"]
-from . import *
+from . import grub2, cmdline
sys.path.insert(0, os.path.dirname(__file__))
diff --git a/atune_collector/plugin/configurator/common.py b/atune_collector/plugin/configurator/common.py
index 0fbc2f3..4ab459a 100755
--- a/atune_collector/plugin/configurator/common.py
+++ b/atune_collector/plugin/configurator/common.py
@@ -20,7 +20,7 @@ import logging
import json
from functools import wraps
-from exceptions import SetConfigError
+from .exceptions import SetConfigError
LOGGER = logging.getLogger(__name__)
diff --git a/atune_collector/plugin/configurator/file_config/__init__.py b/atune_collector/plugin/configurator/file_config/__init__.py
index 48aeb30..023de44 100755
--- a/atune_collector/plugin/configurator/file_config/__init__.py
+++ b/atune_collector/plugin/configurator/file_config/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["fconfig"]
-from . import *
+from . import fconfig
diff --git a/atune_collector/plugin/configurator/kernel_config/__init__.py b/atune_collector/plugin/configurator/kernel_config/__init__.py
index 5905327..58dd2b0 100755
--- a/atune_collector/plugin/configurator/kernel_config/__init__.py
+++ b/atune_collector/plugin/configurator/kernel_config/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["kconfig"]
-from . import *
+from . import kconfig
diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py
index 52bb8c3..bfdb807 100644
--- a/atune_collector/plugin/configurator/mysql/__init__.py
+++ b/atune_collector/plugin/configurator/mysql/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["mysql"]
-from . import *
-
+from . import mysql
diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py
index e28a310..07c82b0 100644
--- a/atune_collector/plugin/configurator/nginx/__init__.py
+++ b/atune_collector/plugin/configurator/nginx/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["nginx"]
-from . import *
-
+from . import nginx
diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py
index d386f1f..7d21825 100644
--- a/atune_collector/plugin/configurator/redis/__init__.py
+++ b/atune_collector/plugin/configurator/redis/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["redis"]
-from . import *
-
+from . import redis
diff --git a/atune_collector/plugin/configurator/script/__init__.py b/atune_collector/plugin/configurator/script/__init__.py
index f743a20..ec6aa17 100755
--- a/atune_collector/plugin/configurator/script/__init__.py
+++ b/atune_collector/plugin/configurator/script/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["script"]
-from . import *
+from . import script
diff --git a/atune_collector/plugin/configurator/sysctl/__init__.py b/atune_collector/plugin/configurator/sysctl/__init__.py
index d6b3f61..3d28df7 100755
--- a/atune_collector/plugin/configurator/sysctl/__init__.py
+++ b/atune_collector/plugin/configurator/sysctl/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sysctl"]
-from . import *
+from . import sysctl
diff --git a/atune_collector/plugin/configurator/sysfs/__init__.py b/atune_collector/plugin/configurator/sysfs/__init__.py
index d220f40..9d5afb6 100755
--- a/atune_collector/plugin/configurator/sysfs/__init__.py
+++ b/atune_collector/plugin/configurator/sysfs/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sysfs"]
-from . import *
+from . import sysfs
diff --git a/atune_collector/plugin/configurator/systemctl/__init__.py b/atune_collector/plugin/configurator/systemctl/__init__.py
index 94762b6..c51a07f 100755
--- a/atune_collector/plugin/configurator/systemctl/__init__.py
+++ b/atune_collector/plugin/configurator/systemctl/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["systemctl"]
-from . import *
+from . import systemctl
diff --git a/atune_collector/plugin/configurator/ulimit/__init__.py b/atune_collector/plugin/configurator/ulimit/__init__.py
index 2582b7c..c28a601 100755
--- a/atune_collector/plugin/configurator/ulimit/__init__.py
+++ b/atune_collector/plugin/configurator/ulimit/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["ulimit"]
-from . import *
+from . import ulimit
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
index 9292071..948f869 100755
--- a/atune_collector/plugin/monitor/__init__.py
+++ b/atune_collector/plugin/monitor/__init__.py
@@ -15,6 +15,15 @@
Init file.
"""
+from .memory import *
+from .network import *
+from .performance import *
+from .process import *
+from .processor import *
+from .storage import *
+from .system import *
+from . import common
+
__all__ = [
"memory",
"network",
@@ -24,5 +33,3 @@ __all__ = [
"storage",
"common",
"system"]
-
-from . import *
diff --git a/atune_collector/plugin/monitor/memory/__init__.py b/atune_collector/plugin/monitor/memory/__init__.py
index 0a5a356..5002d6e 100755
--- a/atune_collector/plugin/monitor/memory/__init__.py
+++ b/atune_collector/plugin/monitor/memory/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["numainfo", "topo", "bandwidth", "vmstat", "utilstat", "meminfo"]
-from . import *
+from . import numainfo, topo, bandwidth, vmstat, utilstat, meminfo
diff --git a/atune_collector/plugin/monitor/network/__init__.py b/atune_collector/plugin/monitor/network/__init__.py
index 5d40ab8..388611a 100755
--- a/atune_collector/plugin/monitor/network/__init__.py
+++ b/atune_collector/plugin/monitor/network/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["info", "netstat", "netestat", "topo"]
-from . import *
+from . import info, netstat, netestat, topo
diff --git a/atune_collector/plugin/monitor/performance/__init__.py b/atune_collector/plugin/monitor/performance/__init__.py
index 707f911..93fc6c4 100755
--- a/atune_collector/plugin/monitor/performance/__init__.py
+++ b/atune_collector/plugin/monitor/performance/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["stat", "top"]
-from . import *
+from . import stat, top
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
index 4c4ceb3..5a34c10 100644
--- a/atune_collector/plugin/monitor/process/__init__.py
+++ b/atune_collector/plugin/monitor/process/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sched"]
-from . import *
\ No newline at end of file
+from . import sched
diff --git a/atune_collector/plugin/monitor/processor/__init__.py b/atune_collector/plugin/monitor/processor/__init__.py
index fc88e1b..f46d8fd 100755
--- a/atune_collector/plugin/monitor/processor/__init__.py
+++ b/atune_collector/plugin/monitor/processor/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["info", "stat", "topo"]
-from . import *
+from . import info, stat, topo
diff --git a/atune_collector/plugin/monitor/storage/__init__.py b/atune_collector/plugin/monitor/storage/__init__.py
index 670294d..9e5e152 100755
--- a/atune_collector/plugin/monitor/storage/__init__.py
+++ b/atune_collector/plugin/monitor/storage/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["iostat", "topo"]
-from . import *
+from . import iostat, topo
diff --git a/atune_collector/plugin/monitor/system/__init__.py b/atune_collector/plugin/monitor/system/__init__.py
index 5c6a00c..f66c1ef 100755
--- a/atune_collector/plugin/monitor/system/__init__.py
+++ b/atune_collector/plugin/monitor/system/__init__.py
@@ -17,4 +17,4 @@ The import content of the package.
__all__ = ["bios", "ldavg", "tasks", "filed", "interrupts"]
-from . import *
+from . import bios, ldavg, tasks, filed, interrupts
diff --git a/atune_collector/plugin/plugin.py b/atune_collector/plugin/plugin.py
index b83d3aa..3f84f2f 100755
--- a/atune_collector/plugin/plugin.py
+++ b/atune_collector/plugin/plugin.py
@@ -19,9 +19,9 @@ import logging
import threading
import time
-from configurator.common import Configurator
+from .configurator.common import Configurator
-from monitor.common import Monitor
+from .monitor.common import Monitor
LOGGER = logging.getLogger(__name__)
--
2.27.0

100
feature-add-multi-for-rps-xps.patch
View File

@ -1,100 +0,0 @@
From c2df28764f159163233cdab4fb34f62efe888c4e Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Mon, 27 Nov 2023 12:56:09 +0000
Subject: [PATCH 1/2] feature: add 'multi' for rps/xps
Signed-off-by: gaoruoshu <gaoruoshu@huawei.com>
---
.../plugin/configurator/network/network.py | 55 ++++++++++++++++++-
1 file changed, 52 insertions(+), 3 deletions(-)
diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py
index 977f29c..afc31f2 100644
--- a/atune_collector/plugin/configurator/network/network.py
+++ b/atune_collector/plugin/configurator/network/network.py
@@ -35,8 +35,8 @@ class Network(Configurator):
def __init__(self, user=None):
Configurator.__init__(self, user)
- self._cmd_map = {'xps': ['all', 'off', 'half', 'separate'],
- 'rps': ['all', 'off', 'half', 'separate']}
+ self._cmd_map = {'xps': ['all', 'off', 'half', 'separate', 'multi'],
+ 'rps': ['all', 'off', 'half', 'separate', 'multi']}
self._nic = self._get_nic()
self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic)
@@ -52,6 +52,12 @@ class Network(Configurator):
'Failed to get dir under {}'.format(self._queue_dir))
dir_list = re.findall(pattern, dir_strs)
core_num = os.cpu_count()
+
+ multi_qs = len(dir_list)
+ stride = 1 if core_num // multi_qs == 0 else core_num // multi_qs
+ stragglers = 0 if core_num <= multi_qs else core_num % multi_qs
+ cur_cpu = 0
+ dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1]))
for index, dir_name in enumerate(dir_list):
file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key)
shell_cmd(['cat', file_path],
@@ -75,14 +81,57 @@ class Network(Configurator):
offset = index % half_num
val_format = 1 << (offset + half_num + core_num % 2)
set_value = f"{val_format:x}"
- else: # value == 'separate'
+ elif value == 'separate':
num = 1 << (index % core_num)
set_value = f"{num:x}"
+ else: # value = multi:
+ str_value = ''
+ gsize = stride if index >= stragglers else stride + 1
+ for _ in range(gsize):
+ str_value = str(cur_cpu) + ' ' + str_value
+ cur_cpu += 1
+ set_value = self._u32list(str_value)
shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)],
'Failed to set {} to {}'.format(key, file_path))
return 0
+ @staticmethod
+ def _u32list(cpulist):
+ max_cpu = 0
+ ii = 0
+
+ for _cpu in cpulist.split():
+ if max_cpu < int(_cpu):
+ max_cpu = int(_cpu)
+
+ #init a bitmap
+ map = [0] * (max_cpu + 1)
+
+ # set bit map according to cpulist
+ for _cpu in cpulist.split():
+ map[int(_cpu)] = 1
+
+ #format a u32list
+ seg = 0
+ mask = ''
+ for ii in range(max_cpu + 1):
+ if ii % 4 == 0 and ii != 0:
+ seg = format(seg, 'x')
+ mask = seg + mask
+ seg = 0
+ if ii % 32 == 0 and ii != 0:
+ mask = ',' + mask
+ cur = map[ii]
+ if cur == 1:
+ val = 1 << (ii % 4)
+ seg += val
+
+ if seg != 0:
+ seg = format(seg, 'x')
+ mask = seg + mask
+ return mask
+
def _set(self, key, value):
if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]:
raise SetConfigError("Invalid value {}={}".format(key, value))
--
2.27.0

182
feature-add-network-CPI.patch
View File

@ -1,182 +0,0 @@
From 148c57c3695080667e138425b1295e89197a241b Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 22 Aug 2023 15:07:50 +0800
Subject: [PATCH] feature: add [network] CPI
---
.../plugin/configurator/__init__.py | 2 +
.../plugin/configurator/network/__init__.py | 21 ++++
.../plugin/configurator/network/network.py | 113 ++++++++++++++++++
3 files changed, 136 insertions(+)
create mode 100644 atune_collector/plugin/configurator/network/__init__.py
create mode 100644 atune_collector/plugin/configurator/network/network.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index fd7ea63..46cff84 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -22,6 +22,7 @@ from .bios import *
from .bootloader import *
from .file_config import *
from .kernel_config import *
+from .network import *
from .script import *
from .sysctl import *
from .sysfs import *
@@ -37,6 +38,7 @@ __all__ = ["exceptions",
"bootloader",
"file_config",
"kernel_config",
+ "network",
"script",
"sysctl",
"sysfs",
diff --git a/atune_collector/plugin/configurator/network/__init__.py b/atune_collector/plugin/configurator/network/__init__.py
new file mode 100644
index 0000000..088e459
--- /dev/null
+++ b/atune_collector/plugin/configurator/network/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-08-22
+
+"""
+Init file.
+"""
+
+__all__ = ["network"]
+
+from . import network
+
diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py
new file mode 100644
index 0000000..977f29c
--- /dev/null
+++ b/atune_collector/plugin/configurator/network/network.py
@@ -0,0 +1,113 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-08-22
+
+"""
+The sub class of the Configurator, used to extend script for network for CPI.
+"""
+
+import re
+import logging
+import os
+import math
+import configparser
+import subprocess
+
+from ..exceptions import SetConfigError
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Network(Configurator):
+ _module = "NETWORK"
+ _submod = "NETWORK"
+
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self._cmd_map = {'xps': ['all', 'off', 'half', 'separate'],
+ 'rps': ['all', 'off', 'half', 'separate']}
+ self._nic = self._get_nic()
+ self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic)
+
+ def _get_nic(self):
+ if not os.path.isfile('/etc/atuned/atuned.cnf'):
+ raise SetConfigError('Cannot get network name')
+ config = configparser.ConfigParser()
+ config.read('/etc/atuned/atuned.cnf')
+ return config.get('system', 'network')
+
+ def _set_cpus(self, key, value, pattern):
+ dir_strs = shell_cmd(['ls', self._queue_dir],
+ 'Failed to get dir under {}'.format(self._queue_dir))
+ dir_list = re.findall(pattern, dir_strs)
+ core_num = os.cpu_count()
+ for index, dir_name in enumerate(dir_list):
+ file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key)
+ shell_cmd(['cat', file_path],
+ 'Failed to set {}={}: does not support for {}'.format(key, value, key))
+
+ if value == 'off':
+ set_value = '0'
+ elif value == 'all':
+ set_value = 'f' * (core_num // 4)
+ if core_num % 4 != 0:
+ bin_num = int('0b' + '1' * (core_num % 4), 2)
+ set_value = f"{bin_num:x}" + set_value
+ elif value == 'half':
+ half_num = core_num // 2
+ if core_num == 1:
+ val_format = 1
+ elif key == 'rps':
+ offset = index % (half_num + core_num % 2)
+ val_format = 1 << offset
+ else:
+ offset = index % half_num
+ val_format = 1 << (offset + half_num + core_num % 2)
+ set_value = f"{val_format:x}"
+ else: # value == 'separate'
+ num = 1 << (index % core_num)
+ set_value = f"{num:x}"
+
+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)],
+ 'Failed to set {} to {}'.format(key, file_path))
+ return 0
+
+ def _set(self, key, value):
+ if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]:
+ raise SetConfigError("Invalid value {}={}".format(key, value))
+
+ if key == 'xps':
+ self._set_cpus(key.lower(), value.lower(), r'tx-\d+')
+ elif key == 'rps':
+ self._set_cpus(key.lower(), value.lower(), r'rx-\d+')
+
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ def _backup(self, config, _):
+ return str(config)
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
+
+def shell_cmd(cmd, error_message):
+ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ raise SetConfigError(error_message)
+ return output.stdout.decode()
+
--
2.27.0

132
feature-add-rfs-to-network.patch
View File

@ -1,132 +0,0 @@
From abc01d60cc574b27b943f2d619f0b3cca8f6e1c6 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 28 Nov 2023 03:35:26 +0000
Subject: [PATCH 2/2] feature: add rfs to network
Signed-off-by: gaoruoshu <gaoruoshu@huawei.com>
---
.../plugin/configurator/network/network.py | 56 +++++++++++++++----
1 file changed, 44 insertions(+), 12 deletions(-)
diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py
index afc31f2..01e5640 100644
--- a/atune_collector/plugin/configurator/network/network.py
+++ b/atune_collector/plugin/configurator/network/network.py
@@ -36,7 +36,8 @@ class Network(Configurator):
def __init__(self, user=None):
Configurator.__init__(self, user)
self._cmd_map = {'xps': ['all', 'off', 'half', 'separate', 'multi'],
- 'rps': ['all', 'off', 'half', 'separate', 'multi']}
+ 'rps': ['all', 'off', 'half', 'separate', 'multi'],
+ 'rfs': ['on', 'off']}
self._nic = self._get_nic()
self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic)
@@ -47,17 +48,21 @@ class Network(Configurator):
config.read('/etc/atuned/atuned.cnf')
return config.get('system', 'network')
- def _set_cpus(self, key, value, pattern):
+ def _get_init(self, pattern):
dir_strs = shell_cmd(['ls', self._queue_dir],
'Failed to get dir under {}'.format(self._queue_dir))
dir_list = re.findall(pattern, dir_strs)
+ dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1]))
core_num = os.cpu_count()
+ return dir_list, core_num
+
+ def _set_cpus(self, key, value, pattern):
+ dir_list, core_num = self._get_init(pattern)
multi_qs = len(dir_list)
stride = 1 if core_num // multi_qs == 0 else core_num // multi_qs
stragglers = 0 if core_num <= multi_qs else core_num % multi_qs
cur_cpu = 0
- dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1]))
for index, dir_name in enumerate(dir_list):
file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key)
shell_cmd(['cat', file_path],
@@ -84,13 +89,15 @@ class Network(Configurator):
elif value == 'separate':
num = 1 << (index % core_num)
set_value = f"{num:x}"
- else: # value = multi:
- str_value = ''
+ elif value == 'multi': # value = multi:
+ list_cpu = []
gsize = stride if index >= stragglers else stride + 1
for _ in range(gsize):
- str_value = str(cur_cpu) + ' ' + str_value
+ list_cpu.append(cur_cpu)
cur_cpu += 1
- set_value = self._u32list(str_value)
+ set_value = self._u32list(list_cpu)
+ else:
+ raise SetConfigError("cannot set {} to {}".format(key, value))
shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)],
'Failed to set {} to {}'.format(key, file_path))
@@ -101,16 +108,15 @@ class Network(Configurator):
max_cpu = 0
ii = 0
- for _cpu in cpulist.split():
- if max_cpu < int(_cpu):
- max_cpu = int(_cpu)
+ for _cpu in cpulist:
+ max_cpu = max(max_cpu, _cpu)
#init a bitmap
map = [0] * (max_cpu + 1)
# set bit map according to cpulist
- for _cpu in cpulist.split():
- map[int(_cpu)] = 1
+ for _cpu in cpulist:
+ map[_cpu] = 1
#format a u32list
seg = 0
@@ -131,6 +137,30 @@ class Network(Configurator):
seg = format(seg, 'x')
mask = seg + mask
return mask
+
+ def _set_rfs(self, value, pattern):
+ dir_list, _ = self._get_init(pattern)
+ for dir_name in dir_list:
+ file_path = '{}/{}/rps_flow_cnt'.format(self._queue_dir, dir_name)
+ shell_cmd(['cat', file_path],
+ 'Failed to set rfs={}: does not support for rfs'.format(value))
+ if value == 'off':
+ set_value = 0
+ elif value == 'on':
+ set_value = 4096
+ else:
+ raise SetConfigError("cannot set rfs to {}".format(value))
+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)],
+ 'Failed to set rfs to {}'.format(file_path))
+
+ if value == 'off':
+ entries = 0
+ elif value == 'on':
+ entries = 4096 * len(dir_list)
+ entries_path = '/proc/sys/net/core/rps_sock_flow_entries'
+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(entries, entries_path)],
+ 'Failed to set rfs to {}'.format(entries_path))
+ return 0
def _set(self, key, value):
if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]:
@@ -140,6 +170,8 @@ class Network(Configurator):
self._set_cpus(key.lower(), value.lower(), r'tx-\d+')
elif key == 'rps':
self._set_cpus(key.lower(), value.lower(), r'rx-\d+')
+ elif key == 'rfs':
+ self._set_rfs(value.lower(), r'rx-\d+')
return 0
--
2.27.0

145
feature-set-mysql.patch
View File

@ -1,145 +0,0 @@
From 16ef5dc208cd79b9f9d4c7c9d66ce46dc9d013a1 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 11 Jul 2023 20:53:28 +0800
Subject: [PATCH 07/11] feature: set mysql
---
.../plugin/configurator/__init__.py | 1 +
.../plugin/configurator/mysql/__init__.py | 21 +++++
.../plugin/configurator/mysql/mysql.py | 84 +++++++++++++++++++
3 files changed, 106 insertions(+)
create mode 100644 atune_collector/plugin/configurator/mysql/__init__.py
create mode 100644 atune_collector/plugin/configurator/mysql/mysql.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 29c674a..677d208 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -31,6 +31,7 @@ __all__ = [
"sysfs",
"systemctl",
"ulimit",
+ "mysql",
"common"]
from . import *
diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py
new file mode 100644
index 0000000..52bb8c3
--- /dev/null
+++ b/atune_collector/plugin/configurator/mysql/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-11
+
+"""
+Init file.
+"""
+
+__all__ = ["mysql"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py
new file mode 100644
index 0000000..cfc533d
--- /dev/null
+++ b/atune_collector/plugin/configurator/mysql/mysql.py
@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-11
+
+"""
+The sub class of the Configurator, used to change the /etc/my.cnf config.
+"""
+
+
+import logging
+import os
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+class Mysql(Configurator):
+ _module = "MYSQL"
+ _submod = "MYSQL"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__cmd = ""
+ self.__file_path = "/etc/my.cnf"
+ self.__mysqld_ind = -1
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w'):
+ pass
+ os.chmod(self.__file_path, 0o644)
+ self.__check_mysqld()
+
+ def __check_mysqld(self):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+
+ for line in lines:
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ if line[:-1] == '[mysqld]':
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ return
+
+ lines.insert(self.__mysqld_ind, "[mysqld]\n")
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+
+ def __check_file_exists(self, lines, start_with):
+ ind = -1
+ for line in lines:
+ ind = ind + 1
+ if line.split('=')[0].rstrip() == start_with:
+ return True, ind
+ return False, ind
+
+ def _set(self, key, value):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+
+ key_exist, ind = self.__check_file_exists(lines, key)
+ new_line = key + " = " + value + "\n"
+ if not key_exist:
+ lines.insert(self.__mysqld_ind, new_line)
+ else:
+ lines[ind] = new_line
+
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
--
2.27.0

272
feature-set-nginx.patch
View File

@ -1,272 +0,0 @@
From b723f4cc584e6df195fbd8b8d0213a3b53aeceea Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Fri, 14 Jul 2023 16:50:53 +0800
Subject: [PATCH 09/11] feature: set nginx
---
.../plugin/configurator/__init__.py | 1 +
.../plugin/configurator/nginx/__init__.py | 21 ++
.../plugin/configurator/nginx/nginx.py | 211 ++++++++++++++++++
3 files changed, 233 insertions(+)
create mode 100644 atune_collector/plugin/configurator/nginx/__init__.py
create mode 100644 atune_collector/plugin/configurator/nginx/nginx.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 9597692..31520f1 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -33,6 +33,7 @@ __all__ = [
"ulimit",
"mysql",
"redis",
+ "nginx",
"common"]
from . import *
diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py
new file mode 100644
index 0000000..e28a310
--- /dev/null
+++ b/atune_collector/plugin/configurator/nginx/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-14
+
+"""
+Init file.
+"""
+
+__all__ = ["nginx"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py
new file mode 100644
index 0000000..7ab619d
--- /dev/null
+++ b/atune_collector/plugin/configurator/nginx/nginx.py
@@ -0,0 +1,211 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-14
+
+"""
+The sub class of the Configurator, used to extend script for /etc/nginx/nginx.conf for CPI.
+"""
+
+import logging
+import os
+import subprocess
+
+from ..exceptions import SetConfigError
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Nginx(Configurator):
+ _module = "NGINX"
+ _submod = "NGINX"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__file_dir = "/etc/nginx/"
+ self.__file_path = self.__file_dir + "nginx.conf"
+ self.__line = "{key} {value};\n"
+ self.__key_match = r"^([a-zA-Z]+(_[a-zA-Z]+)*.?[a-zA-Z]+)*$"
+ self.__ind = {"main": [0, 0]}
+
+ def _init_file(self):
+ if not os.path.isdir(self.__file_dir):
+ os.mkdir(self.__file_dir)
+ os.chmod(self.__file_dir, 0o755)
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w'):
+ pass
+ os.chmod(self.__file_path, 0o644)
+ self._set_index()
+
+ def _get_lines(self):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+ return lines
+
+ def _set_lines(self, lines):
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+
+ def _set_index(self):
+ lines = self._get_lines()
+ filo = []
+ ind = []
+ filo.append("start")
+ ind.append(0)
+ is_https_server = False
+ for i, line in enumerate(lines):
+ if i == len(lines) - 1:
+ self.__ind["main"] = [ind.pop(), i]
+ del filo, ind
+ if line.strip().startswith("}"):
+ close = filo.pop()
+ close_ind = ind.pop()
+ if close.startswith("events"):
+ self.__ind["events"] = [close_ind, i]
+ if close.startswith("http"):
+ self.__ind["http"] = [close_ind, i]
+ if close.startswith("server"):
+ if is_https_server:
+ self.__ind["https_server"] = [close_ind, i]
+ else:
+ self.__ind["http_server"] = [close_ind, i]
+ is_https_server = False
+ if "{" in line.strip().split("#")[0]:
+ filo.append(line.strip())
+ ind.append(i)
+ if line.strip().startswith("ssl_"):
+ is_https_server = True
+
+ def _check_key(self, key):
+ if len(key) == 0:
+ return False, 0
+ count = 1
+ if not ('a' <= key[0] <= 'z' or 'A' <= key[0] <= 'Z'):
+ return False, count
+ for i in range(1, len(key)):
+ if key[i] == '_' or key[i] == '.':
+ if not ('a' <= key[i - 1] <= 'z' or 'A' <= key[i - 1] <= 'Z'):
+ return False, count
+ if i == len(key) - 1:
+ return False, count
+ if key[i] == '.':
+ count = count + 1
+ elif 'a' <= key[i] <= 'z' or 'A' <= key[i] <= 'Z':
+ continue
+ else:
+ return False, count
+ return True, count
+
+ def _update_index(self, after_ind):
+ for key in self.__ind:
+ for i, val in enumerate(self.__ind[key]):
+ if val > after_ind:
+ self.__ind[key][i] = val + 1
+
+ def _append_lines(self, section, key, value):
+ lines = self._get_lines()
+ filo = []
+ for i in range(self.__ind[section][0], self.__ind[section][1]):
+ curr_line = lines[i].strip()
+ if "{" in curr_line.split("#")[0]:
+ filo.append(lines[i].strip())
+ elif curr_line.startswith("}"):
+ filo.pop()
+ elif section == 'main' and len(filo) > 0:
+ continue
+ elif len(filo) > 1:
+ continue
+ elif curr_line.startswith(key + " ") or curr_line.startswith(key + "\t"):
+ new_line = lines[i].split(key)[0]
+ new_line = new_line + self.__line.format(key=key, value=value)
+ lines[i] = new_line
+ return lines
+ new_line = ""
+ if section != "main" and not "server" in section:
+ new_line = lines[self.__ind[section][0]].split(section)[0] + " "
+ elif "server" in section:
+ new_line = lines[self.__ind[section][0]].split("server")[0] + " "
+ new_line = new_line + self.__line.format(key=key, value=value)
+ lines.insert(self.__ind[section][0] + 1, new_line)
+ self._update_index(self.__ind[section][0])
+ return lines
+
+ def _set_main_section(self, key, value):
+ return self._append_lines("main", key, value)
+
+ def _set_event_section(self, key, value):
+ if "events" not in self.__ind:
+ raise SetConfigError("No events section")
+ return self._append_lines("events", key, value)
+
+ def _set_http_section(self, key, value):
+ if "http" not in self.__ind:
+ raise SetConfigError("No http section")
+ return self._append_lines("http", key, value)
+
+ def _set_http_server(self, key, value):
+ if "http_server" not in self.__ind:
+ raise SetConfigError("No http server in http section")
+ return self._append_lines("http_server", key, value)
+
+ def _set_https_server(self, key, value):
+ if "https_server" not in self.__ind:
+ raise SetConfigError("No https server in http section")
+ return self._append_lines("https_server", key, value)
+
+ def _set(self, key, value):
+ self._init_file()
+ valid, count = self._check_key(key)
+ if not valid or count < 1 or count > 3:
+ raise SetConfigError("Invalid value {}".format(key))
+ if value == "!CPU_CORE!":
+ value = rewrite_value(value)
+ lines = []
+ try:
+ if count == 1:
+ lines = self._set_main_section(key, value)
+ elif count == 2 and key.split('.')[0] == "events":
+ lines = self._set_event_section(key.split('.')[1], value)
+ elif key.split('.')[0] == "http":
+ if count == 2:
+ lines = self._set_http_section(key.split('.')[1], value)
+ elif count == 3:
+ if key.split('.')[1] == "http":
+ lines = self._set_http_server(key.split('.')[2], value)
+ elif key.split('.')[1] == "https":
+ lines = self._set_https_server(key.split('.')[2], value)
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ except SetConfigError as err:
+ raise err
+ self._set_lines(lines)
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
+def rewrite_value(value):
+ command = ["grep", "processor", "/proc/cpuinfo"]
+ output = subprocess.run(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ raise SetConfigError("Failed to get cpu number")
+ return output.stdout.decode().count("\n")
+
--
2.27.0

139
feature-set-redis-value.patch
View File

@ -1,139 +0,0 @@
From 70a1f8bc884b6a1b2772fd7199e103fa5351ca3d Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Thu, 13 Jul 2023 14:40:57 +0800
Subject: [PATCH 08/11] feature: set redis value
---
.../plugin/configurator/__init__.py | 2 +
.../plugin/configurator/redis/__init__.py | 21 +++++
.../plugin/configurator/redis/redis.py | 77 +++++++++++++++++++
3 files changed, 100 insertions(+)
create mode 100644 atune_collector/plugin/configurator/redis/__init__.py
create mode 100644 atune_collector/plugin/configurator/redis/redis.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 677d208..9597692 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -32,6 +32,8 @@ __all__ = [
"systemctl",
"ulimit",
"mysql",
+ "redis",
"common"]
from . import *
+
diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py
new file mode 100644
index 0000000..d386f1f
--- /dev/null
+++ b/atune_collector/plugin/configurator/redis/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-13
+
+"""
+Init file.
+"""
+
+__all__ = ["redis"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py
new file mode 100644
index 0000000..c56f38e
--- /dev/null
+++ b/atune_collector/plugin/configurator/redis/redis.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-13
+
+"""
+The sub class of the Configurator, used to change the redis config.
+"""
+
+import logging
+import os
+import subprocess
+from ..exceptions import GetConfigError, SetConfigError
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Redis(Configurator):
+ _module = "REDIS"
+ _submod = "REDIS"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__lines = "{key} {value}"
+ self.__re = r"^#\?\s*{key}\s* "
+ self.__file_dir = "/etc/redis/"
+ self.__file_path = self.__file_dir + "redis.conf"
+ if not os.path.isdir(self.__file_dir):
+ os.mkdir(self.__file_dir)
+ os.chmod(self.__file_dir, 0o755)
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w', 0o200):
+ pass
+ os.chmod(self.__file_path, 0o644)
+
+ def _set(self, key, value):
+ re_cmd = self.__re.format(key=key)
+ grep_cmd = [r"grep", re_cmd, self.__file_path]
+ out, err = self.execute_cmd(grep_cmd)
+ if len(err) != 0:
+ raise SetConfigError("Failed to set {}: {}".format(key, err))
+ num_lines = out.count("\n")
+ new_line = self.__lines.format(key=key, value=value)
+ if num_lines == 0:
+ with open(self.__file_path, 'a', 0o600) as f:
+ f.write(new_line + '\n')
+ elif num_lines == 1:
+ sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path]
+ _, err_rep = self.execute_cmd(sed_cmd)
+ if len(err_rep) > 0:
+ raise SetConfigError("Failed to set {}: {}".format(key, err_rep))
+ else:
+ raise SetConfigError("Failed to set {}: more than 1 key has same name".format(key))
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
+ @staticmethod
+ def execute_cmd(cmd):
+ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ return output.stdout.decode(), output.stderr.decode()[:-1]
+
+
--
2.27.0

33
fix-CWE-23.patch
View File

@ -1,33 +0,0 @@
From 121a1bcbd68ef9b18ec0c0cdcc8ca0748fe08bdd Mon Sep 17 00:00:00 2001
From: unknown <u202012145@hust.edu.cn>
Date: Wed, 26 Apr 2023 21:44:16 +0800
Subject: [PATCH 05/11] fix CWE-23
collect_data.py直接使用命令行参数作为文件路径，可以被攻击者输入../访问上级目录，从而获取系统的敏感信息，或者写入任意文件（使用保存路径遍历文件名的恶意zip存档）。为了修复这一漏洞，我们采用了werzeug库中的secure_filename函数，这一函数会过滤掉文件路径中的所有危险字符，防范这一攻击方式。
---
atune_collector/collect_data.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
index 3593db6..167141e 100755
--- a/atune_collector/collect_data.py
+++ b/atune_collector/collect_data.py
@@ -21,6 +21,7 @@ import time
import csv
from plugin.plugin import MPI
+from werkzeug.utils import secure_filename
class Collector:
@@ -112,6 +113,7 @@ if __name__ == "__main__":
ARG_PARSER.add_argument('-c', '--config', metavar='json',
default=default_json_path, help='input json path')
ARGS = ARG_PARSER.parse_args()
+ filename=secure_filename(ARGS.config)
with open(ARGS.config, 'r') as file:
json_data = json.load(file)
collector = Collector(json_data)
--
2.27.0

25
fix-bug-list-index-out-of-range-in-virtualbox.patch
View File

@ -1,25 +0,0 @@
From 4cb49adb623f36590f254b7e11c8aa5b0777bd00 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 26 Apr 2022 02:01:27 +0000
Subject: [PATCH 03/11] fix bug: list index out of range in virtualbox
---
atune_collector/plugin/monitor/memory/bandwidth.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py
index 593a4a8..f44925f 100755
--- a/atune_collector/plugin/monitor/memory/bandwidth.py
+++ b/atune_collector/plugin/monitor/memory/bandwidth.py
@@ -153,7 +153,7 @@ class MemBandwidth(Monitor):
info = json.loads(info_json)
dimms = [[0 for i in range(8)] for i in range(8)]
- if "children" not in info["memorys"][0]:
+ if info["memorys"] is None or len(info["memorys"]) == 0 or "children" not in info["memorys"][0]:
return 0
for dimm in info["memorys"][0]["children"]:
if dimm.get("size") is None:
--
2.27.0

56
fix-bug-procsched-data-collection-issue.patch
View File

@ -1,56 +0,0 @@
From 43c7e8eb00268570abbaeacbb2689079dddec63d Mon Sep 17 00:00:00 2001
From: weiyaping <weiyaping@xfusion.com>
Date: Fri, 15 Dec 2023 03:50:06 -0500
Subject: [PATCH] fix-bug-procsched-data-collection-issue
---
atune_collector/plugin/monitor/process/sched.py | 27 ++++++++++++++++---------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
index f184957..0fadeba 100644
--- a/atune_collector/plugin/monitor/process/sched.py
+++ b/atune_collector/plugin/monitor/process/sched.py
@@ -107,7 +107,7 @@ class ProcSched(Monitor):
continue
pattern = re.compile(
- r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)",
+ r"(\w+)\ {1,}\:\ {1,}(\d+\.?\d*)",
re.I | re.UNICODE | re.MULTILINE)
search_obj = pattern.findall(info)
search_list = []
@@ -118,13 +118,20 @@ class ProcSched(Monitor):
search_list.append(obj[0])
search_list.append(obj[1])
if len(search_obj) == 0:
- err = LookupError("Fail to find data")
- LOGGER.error("%s.%s: %s", self.__class__.__name__,
- inspect.stack()[0][3], str(err))
- raise err
+ return " " + " ".join(['0'] * len(keys))
+ proc_data = []
+ proc_keys = 0
+ proc_step = len(set(keys))
- while start <= len(self.__applications) * len(keys):
- for key in keys:
- ret = ret + " " + search_list[search_list.index(key, start)+1]
- start = search_list.index(key, start) + 1
- return ret
\ No newline at end of file
+ for key in keys:
+ if len(proc_data) >= self.__proc_flag.count(True) * proc_step:
+ break
+ proc_data.append(search_list[search_list.index(key, start) + 1])
+ start = search_list.index(key, start) + 1
+ for pid_flag in self.__proc_flag:
+ if not pid_flag:
+ ret = ret + " " + " ".join(['0'] * proc_step)
+ else:
+ ret = ret + " " + " ".join(proc_data[proc_keys:(proc_keys + proc_step)])
+ proc_keys += proc_step
+ return ret
--
1.8.3.1

45
fix-bug-procsched-report-list-index-out-of-range.patch
View File

@ -1,45 +0,0 @@
From 009428bbf8f38bf9754999ec77403c4b181cf8ed Mon Sep 17 00:00:00 2001
From: weiyaping <weiyaping@xfusion.com>
Date: Fri, 15 Dec 2023 03:37:02 -0500
Subject: [PATCH] fix bug:ProcSched.report: list index out of range
---
atune_collector/plugin/monitor/process/sched.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
index 5289d84..f184957 100644
--- a/atune_collector/plugin/monitor/process/sched.py
+++ b/atune_collector/plugin/monitor/process/sched.py
@@ -36,10 +36,12 @@ class ProcSched(Monitor):
self.__interval = 1
self.__applications = []
self.__pids = []
+ self.__proc_flag = []
def _get(self, para=None):
output = ""
pids = []
+ proc_flag = []
if para is not None:
opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app='])
for opt, val in opts:
@@ -67,9 +69,13 @@ class ProcSched(Monitor):
for app in self.__applications:
pid = subprocess.getoutput(
- "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0]
- pids.append(pid)
+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()
+ app_pid_flag = True if pid else False
+ proc_flag.append(app_pid_flag)
+ if pid:
+ pids.append(pid[0])
self.__pids = pids
+ self.__proc_flag = proc_flag
for pid in self.__pids:
out = subprocess.check_output(
--
1.8.3.1

37
memory-bandwidth-dimm-slot-atuned.patch
View File

@ -1,37 +0,0 @@
From 7c21a2da2c931089e4ec2b86ae35a510f2f7793d Mon Sep 17 00:00:00 2001
From: SWWBF <1152719547@qq.com>
Date: Mon, 10 Jul 2023 05:52:54 +0800
Subject: [PATCH 06/11] some kvm has no slot which may cause panic of
atune
---
atune_collector/plugin/monitor/memory/bandwidth.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py
index f44925f..6d59ab6 100755
--- a/atune_collector/plugin/monitor/memory/bandwidth.py
+++ b/atune_collector/plugin/monitor/memory/bandwidth.py
@@ -158,12 +158,13 @@ class MemBandwidth(Monitor):
for dimm in info["memorys"][0]["children"]:
if dimm.get("size") is None:
continue
- locator = memtopo.table_get_locator(dimm["slot"])
- if locator is None:
- continue
- if dimms[locator[0]][locator[1]] == 0:
- dimms[locator[0]][locator[1]] = dimm["width"] * \
- memtopo.table_get_freq(dimm["description"]) / 8
+ if "slot" in dimm:
+ locator = memtopo.table_get_locator(dimm["slot"])
+ if locator is None:
+ continue
+ if dimms[locator[0]][locator[1]] == 0:
+ dimms[locator[0]][locator[1]] = dimm["width"] * \
+ memtopo.table_get_freq(dimm["description"]) / 8
ret = 0
for channel in dimms[socket]:
ret += channel
--
2.27.0

BIN
v1.1.0.tar.gz
View File

Binary file not shown.

BIN
v1.3.0.tar.gz Normal file
View File

Binary file not shown.