diff --git a/README-add-data-collecting-output-description.patch b/README-add-data-collecting-output-description.patch deleted file mode 100644 index 277c72f..0000000 --- a/README-add-data-collecting-output-description.patch +++ /dev/null @@ -1,47 +0,0 @@ -From c8d0e4c3030808fe0a575e595d8bd9a51d30124a Mon Sep 17 00:00:00 2001 -From: HuBin95 -Date: Mon, 7 Mar 2022 02:10:24 +0000 -Subject: [PATCH 02/11] README: add data collecting output description - ---- - README.en.md | 4 +++- - README.md | 4 +++- - 2 files changed, 6 insertions(+), 2 deletions(-) - -diff --git a/README.en.md b/README.en.md -index 2f11270..8a54fa4 100644 ---- a/README.en.md -+++ b/README.en.md -@@ -52,9 +52,11 @@ Table 1 **collect_data.json** file - | sample_num | Sample number to be collected. | Integer | > 0 | - | interval | Interval for collecting data, in seconds. | Integer | > 0 | - | output_dir | Path for storing collected data. | Character string | - | --| workload_type | Application load type of the collection environment. The default value is **default**. | Character string | - | -+| workload_type | Application load type of the collection environment, used as output file name. The default value is **default**. | Character string | - | - | collection_items | Table 2 lists the system parameters to be collected. | List | - | - -+When data collecting is finished, the data will be saved as: `${output_dir}/${workload_type}-${finish_timestamp}.csv` -+ - Table 2 Description of the **collection_items** configuration - - | Parameter | Description | Type | Value Range | -diff --git a/README.md b/README.md -index 36d5954..c121c8d 100644 ---- a/README.md -+++ b/README.md -@@ -53,9 +53,11 @@ python3 collect_data.py [OPTIONS] - | sample_num | 待采集的次数 | 整型 | >0 | - | interval | 待采集的间隔时间，单位为秒 | 整型 | >0 | - | output_dir | 采集完后数据存储的文件路径 | 字符串 | - | --| workload_type | 采集环境的应用负载类型，默认为default | 字符串 | - | -+| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - | - | collection_items | 需要采集的系统参数项，参见表2 | 列表 | - | - -+最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv` -+ - 表2 collection_items项配置说明 - - | **配置名称** | **配置说明** | **参数类型** | **取值范围** | --- -2.27.0 - diff --git a/Ubuntu-20.04.3-LTS.patch b/Ubuntu-20.04.3-LTS.patch deleted file mode 100644 index e5b1f4d..0000000 --- a/Ubuntu-20.04.3-LTS.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 2fd1148dcec21a8a99fbb59240ba2e2abfe7c31c Mon Sep 17 00:00:00 2001 -From: Masterwater-y -Date: Fri, 28 Jan 2022 18:07:52 +0800 -Subject: [PATCH 01/11] adapt Ubuntu 20.04.3 LTS - ---- - atune_collector/plugin/monitor/common.py | 2 +- - atune_collector/plugin/monitor/memory/topo.py | 2 ++ - 2 files changed, 3 insertions(+), 1 deletion(-) - -diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py -index a646b32..12a07f2 100755 ---- a/atune_collector/plugin/monitor/common.py -+++ b/atune_collector/plugin/monitor/common.py -@@ -230,7 +230,7 @@ class Monitor(object): - def walk_class_type(father, class_type, desc, datas): - """get key field""" - if "class" in father and father["class"] == class_type: -- if "description" in father and (desc is None or father["description"] == desc): -+ if "description" in father and (desc is None or father["description"].lower() == desc.lower()): - datas.append(father) - return - if "children" in father: -diff --git a/atune_collector/plugin/monitor/memory/topo.py b/atune_collector/plugin/monitor/memory/topo.py -index 7e828f3..2c0c39e 100755 ---- a/atune_collector/plugin/monitor/memory/topo.py -+++ b/atune_collector/plugin/monitor/memory/topo.py -@@ -95,6 +95,8 @@ class MemTopo(Monitor): - cmd=self.__cmd).split(), stderr=no_print) - info = o_json.decode() - json_content = json.loads(info) -+ if (isinstance(json_content, list)): -+ json_content = {'children': json_content} - - dict_datas = get_class_type(json_content, "memory", "System Memory") - if fmt == "json": --- -2.27.0 - diff --git a/add-new-dims.patch b/add-new-dims.patch deleted file mode 100644 index 54825a2..0000000 --- a/add-new-dims.patch +++ /dev/null @@ -1,427 +0,0 @@ -From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001 -From: hamster -Date: Mon, 14 Nov 2022 03:24:49 +0000 -Subject: [PATCH 04/11] add new dimensions - -From: @hujing2 -Reviewed-by: @gaoruoshu -Signed-off-by: @gaoruoshu ---- - README.md | 16 +++ - atune_collector/collect_data.json | 14 ++ - atune_collector/collect_data.py | 97 ++++++++------ - atune_collector/plugin/monitor/__init__.py | 1 + - atune_collector/plugin/monitor/common.py | 1 + - .../plugin/monitor/process/__init__.py | 20 +++ - .../plugin/monitor/process/sched.py | 124 ++++++++++++++++++ - 7 files changed, 233 insertions(+), 40 deletions(-) - create mode 100644 atune_collector/plugin/monitor/process/__init__.py - create mode 100644 atune_collector/plugin/monitor/process/sched.py - -diff --git a/README.md b/README.md -index c121c8d..242f345 100644 ---- a/README.md -+++ b/README.md -@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS] - | ---------------- | ------------------------------------- | ------------ | ------------ | - | network | 待采集的指定网卡 | 字符串 | - | - | block | 待采集的指定磁盘 | 字符串 | - | -+| application | 需要采集的应用进程 | 字符串 | - | - | sample_num | 待采集的次数 | 整型 | >0 | - | interval | 待采集的间隔时间，单位为秒 | 整型 | >0 | - | output_dir | 采集完后数据存储的文件路径 | 字符串 | - | - | workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - | - | collection_items | 需要采集的系统参数项，参见表2 | 列表 | - | - -+ - 最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv` - - 表2 collection_items项配置说明 -@@ -76,6 +78,7 @@ collect_data.json文件配置示例： - { - "network": "eth0", - "block": "sda", -+ "application": "mysqld", - "sample_num": 20, - "interval": 5, - "output_dir": "/var/atuned/collect_data", -@@ -203,6 +206,19 @@ collect_data.json文件配置示例： - "metrics": [ - "fd-util" - ] -+ }, -+ { -+ "name": "process", -+ "module": "PROCESS", -+ "purpose": "SCHED", -+ "metrics": [ -+ "exec_start", -+ "vruntime", -+ "sum_exec_runtime", -+ "switches", -+ "voluntary_switches", -+ "involuntary_switches" -+ ] - } - ] - } -diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json -index db96501..af286bd 100755 ---- a/atune_collector/collect_data.json -+++ b/atune_collector/collect_data.json -@@ -1,6 +1,7 @@ - { - "network": "eth0", - "block": "sda", -+ "application": "firewalld,dockerd", - "sample_num": 20, - "interval": 5, - "output_dir": "/var/atuned/collect_data", -@@ -140,6 +141,19 @@ - "metrics": [ - "fd-util" - ] -+ }, -+ { -+ "name": "process", -+ "module": "PROCESS", -+ "purpose": "SCHED", -+ "metrics": [ -+ "exec_start", -+ "vruntime", -+ "sum_exec_runtime", -+ "switches", -+ "voluntary_switches", -+ "involuntary_switches" -+ ] - } - ] - } -\ No newline at end of file -diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py -index 1764304..3593db6 100755 ---- a/atune_collector/collect_data.py -+++ b/atune_collector/collect_data.py -@@ -18,6 +18,7 @@ import argparse - import json - import os - import time -+import csv - - from plugin.plugin import MPI - -@@ -30,66 +31,80 @@ class Collector: - self.field_name = [] - self.support_multi_block = ['storage'] - self.support_multi_nic = ['network', 'network-err'] -+ self.support_multi_app = ['process'] - - def parse_json(self): - """parse json data""" - monitors = [] - for item in self.data["collection_items"]: -- parameters = ["--interval=%s;" % self.data["interval"]] -- for metric in item["metrics"]: -- nics = self.data["network"].split(',') -- blocks = self.data["block"].split(',') -- if item["name"] in self.support_multi_nic and len(nics) > 1: -- for net in nics: -+ if item["name"] in self.support_multi_app and ('application' not in self.data or -+ self.data["application"] == ""): -+ continue -+ if item["name"] in self.support_multi_app: -+ applications = self.data["application"].split(',') -+ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])] -+ for application in applications: -+ for metric in item["metrics"]: - self.field_name.append( -- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net)) -- elif item["name"] in self.support_multi_block and len(blocks) > 1: -- for block in blocks: -- self.field_name.append( -- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block)) -- else: -- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric)) -- parameters.append("--fields=%s" % metric) -- if "threshold" in item: -- parameters.append("--threshold=%s" % item["threshold"]) -+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application)) -+ parameters.append("--fields=%s" % metric) -+ else: -+ parameters = ["--interval=%s;" % self.data["interval"]] -+ for metric in item["metrics"]: -+ nics = self.data["network"].split(',') -+ blocks = self.data["block"].split(',') -+ -+ if item["name"] in self.support_multi_nic and len(nics) > 1: -+ for net in nics: -+ self.field_name.append( -+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net)) -+ elif item["name"] in self.support_multi_block and len(blocks) > 1: -+ for block in blocks: -+ self.field_name.append( -+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block)) -+ else: -+ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric)) -+ parameters.append("--fields=%s" % metric) -+ if "threshold" in item: -+ parameters.append("--threshold=%s" % item["threshold"]) -+ - parameters.append("--nic=%s" % self.data["network"]) - parameters.append("--device=%s" % self.data["block"]) - monitors.append([item["module"], item["purpose"], " ".join(parameters)]) - return monitors - -- def save_csv(self, field_data): -- """save data to csv file""" -+ def collect_data(self): -+ """collect data""" -+ collect_num = self.data["sample_num"] -+ if int(collect_num) < 1: -+ os.abort("sample_num must be greater than 0") -+ -+ mpi = MPI() -+ monitors = self.parse_json() - path = self.data["output_dir"] - if not os.path.exists(path): - os.makedirs(path, 0o750) - file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"), - int(round(time.time() * 1000))) -- import csv -+ -+ print("start to collect data, csv path is %s" % os.path.join(path, file_name)) -+ print(" ".join(self.field_name)) - with open(os.path.join(path, file_name), "w") as csvfile: - writer = csv.writer(csvfile) - self.field_name.insert(0, "TimeStamp") - writer.writerow(self.field_name) -- writer.writerows(field_data) -+ csvfile.flush() -+ for _ in range(collect_num): -+ raw_data = mpi.get_monitors_data(monitors) -+ float_data = [float(num) for num in raw_data] -+ str_data = [str(round(value, 3)) for value in float_data] -+ print(" ".join(str_data)) -+ float_data.insert(0, time.strftime("%H:%M:%S")) -+ # field_data.append(float_data) -+ writer.writerow(float_data) -+ csvfile.flush() - print("finish to collect data, csv path is %s" % os.path.join(path, file_name)) - -- def collect_data(self): -- """collect data""" -- collect_num = self.data["sample_num"] -- if int(collect_num) < 1: -- os.abort("sample_num must be greater than 0") -- field_data = [] -- mpi = MPI() -- monitors = self.parse_json() -- print(" ".join(self.field_name)) -- for _ in range(collect_num): -- raw_data = mpi.get_monitors_data(monitors) -- float_data = [float(num) for num in raw_data] -- str_data = [str(round(value, 3)) for value in float_data] -- print(" ".join(str_data)) -- float_data.insert(0, time.strftime("%H:%M:%S")) -- field_data.append(float_data) -- return field_data -- - - if __name__ == "__main__": - default_json_path = "/etc/atune_collector/collect_data.json" -@@ -100,5 +115,7 @@ if __name__ == "__main__": - with open(ARGS.config, 'r') as file: - json_data = json.load(file) - collector = Collector(json_data) -- dataset = collector.collect_data() -- collector.save_csv(dataset) -+ try: -+ collector.collect_data() -+ except KeyboardInterrupt: -+ print("user stop collect data") -\ No newline at end of file -diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py -index 6291401..9292071 100755 ---- a/atune_collector/plugin/monitor/__init__.py -+++ b/atune_collector/plugin/monitor/__init__.py -@@ -19,6 +19,7 @@ __all__ = [ - "memory", - "network", - "performance", -+ "process", - "processor", - "storage", - "common", -diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py -index 12a07f2..d0aa60c 100755 ---- a/atune_collector/plugin/monitor/common.py -+++ b/atune_collector/plugin/monitor/common.py -@@ -134,6 +134,7 @@ class Monitor(object): - "--interval=" to specify period of time - "--cpu=" to select which cpu - "--event=" to select which event -+ "--app" to select which applications - :returns value: Success, collected info string - :raises Exceptions: Fail, with info - """ -diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py -new file mode 100644 -index 0000000..4c4ceb3 ---- /dev/null -+++ b/atune_collector/plugin/monitor/process/__init__.py -@@ -0,0 +1,20 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2022 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2022-10-14 -+ -+""" -+Init file. -+""" -+ -+__all__ = ["sched"] -+ -+from . import * -\ No newline at end of file -diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py -new file mode 100644 -index 0000000..5289d84 ---- /dev/null -+++ b/atune_collector/plugin/monitor/process/sched.py -@@ -0,0 +1,124 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2022 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2022-10-14 -+ -+""" -+The sub class of the monitor, used to collect the process sched info -+""" -+import inspect -+import logging -+import subprocess -+import getopt -+import re -+from ..common import Monitor -+ -+LOGGER = logging.getLogger(__name__) -+ -+ -+class ProcSched(Monitor): -+ """To collect the process sched info""" -+ _module = "PROCESS" -+ _purpose = "SCHED" -+ _option = "/proc/{}/sched" -+ -+ def __init__(self, user=None): -+ Monitor.__init__(self, user) -+ self.__cmd = "cat" -+ self.__interval = 1 -+ self.__applications = [] -+ self.__pids = [] -+ -+ def _get(self, para=None): -+ output = "" -+ pids = [] -+ if para is not None: -+ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app=']) -+ for opt, val in opts: -+ if opt in '--interval': -+ if val.isdigit(): -+ self.__interval = int(val) -+ else: -+ err = ValueError( -+ "Invalid parameter: {opt}={val}".format( -+ opt=opt, val=val)) -+ LOGGER.error("%s.%s: %s", self.__class__.__name__, -+ inspect.stack()[0][3], str(err)) -+ raise err -+ continue -+ elif opt in '--app': -+ if val is not None: -+ self.__applications = val.split(',') -+ else: -+ err = ValueError( -+ "{opt} parameter is none".format( -+ opt=opt)) -+ LOGGER.error("%s.%s: %s", self.__class__.__name__, -+ inspect.stack()[0][3], str(err)) -+ raise err -+ -+ for app in self.__applications: -+ pid = subprocess.getoutput( -+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0] -+ pids.append(pid) -+ self.__pids = pids -+ -+ for pid in self.__pids: -+ out = subprocess.check_output( -+ "{cmd} {opt}".format( -+ cmd=self.__cmd, -+ opt=self._option.format(pid)).split()) -+ output = output + "" + out.decode() -+ return output -+ -+ def decode(self, info, para): -+ """ -+ decode the result of the operation -+ :param info: content that needs to be decoded -+ :param para: command line argument -+ :returns ret: operation result -+ """ -+ -+ if para is None: -+ return info -+ -+ start = 0 -+ keys = [] -+ ret = "" -+ -+ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device=']) -+ for opt, val in opts: -+ if opt in '--fields': -+ keys.append(val) -+ continue -+ -+ pattern = re.compile( -+ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)", -+ re.I | re.UNICODE | re.MULTILINE) -+ search_obj = pattern.findall(info) -+ search_list = [] -+ for obj in search_obj: -+ if obj[0][:3] == "nr_": -+ search_list.append(obj[0][3:]) -+ else: -+ search_list.append(obj[0]) -+ search_list.append(obj[1]) -+ if len(search_obj) == 0: -+ err = LookupError("Fail to find data") -+ LOGGER.error("%s.%s: %s", self.__class__.__name__, -+ inspect.stack()[0][3], str(err)) -+ raise err -+ -+ while start <= len(self.__applications) * len(keys): -+ for key in keys: -+ ret = ret + " " + search_list[search_list.index(key, start)+1] -+ start = search_list.index(key, start) + 1 -+ return ret -\ No newline at end of file --- -2.27.0 - diff --git a/atune-collector-add-backup-for-apps.patch b/atune-collector-add-backup-for-apps.patch deleted file mode 100644 index 9aa0392..0000000 --- a/atune-collector-add-backup-for-apps.patch +++ /dev/null @@ -1,56 +0,0 @@ -From bfe9d212c518944678bbf526d61a8e778f378b41 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Wed, 16 Aug 2023 12:36:46 +0800 -Subject: [PATCH] [atune-collector]add backup for apps - ---- - atune_collector/plugin/configurator/mysql/mysql.py | 3 +++ - atune_collector/plugin/configurator/nginx/nginx.py | 3 +++ - atune_collector/plugin/configurator/redis/redis.py | 3 +++ - 3 files changed, 9 insertions(+) - -diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py -index e472a1c..9badd21 100644 ---- a/atune_collector/plugin/configurator/mysql/mysql.py -+++ b/atune_collector/plugin/configurator/mysql/mysql.py -@@ -81,6 +81,9 @@ class Mysql(Configurator): - def _get(self, key, _): - pass - -+ def _backup(self, config, _): -+ return str(config) -+ - @staticmethod - def check(config1, config2): - return True -diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py -index 9d38dd0..7f7eaad 100644 ---- a/atune_collector/plugin/configurator/nginx/nginx.py -+++ b/atune_collector/plugin/configurator/nginx/nginx.py -@@ -198,6 +198,9 @@ class Nginx(Configurator): - def _get(self, key, _): - pass - -+ def _backup(self, config, _): -+ return str(config) -+ - @staticmethod - def check(config1, config2): - return True -diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py -index aa14bcd..44da99e 100644 ---- a/atune_collector/plugin/configurator/redis/redis.py -+++ b/atune_collector/plugin/configurator/redis/redis.py -@@ -68,6 +68,9 @@ class Redis(Configurator): - def _get(self, key, _): - pass - -+ def _backup(self, config, _): -+ return str(config) -+ - @staticmethod - def check(config1, config2): - return True --- -2.27.0 - diff --git a/atune-collector.spec b/atune-collector.spec index f65cdd7..3e1fe0d 100644 --- a/atune-collector.spec +++ b/atune-collector.spec @@ -1,31 +1,13 @@ %define debug_package %{nil} Name: atune-collector -Version: 1.1.0 -Release: 7 +Version: 1.3.0 +Release: 1 Summary: A-Tune-Collector is used to collect various system resources. License: Mulan PSL v2 URL: https://gitee.com/openeuler/A-Tune-Collector Source: https://gitee.com/openeuler/A-Tune-Collector/repository/archive/v%{version}.tar.gz -Patch1: Ubuntu-20.04.3-LTS.patch -Patch2: README-add-data-collecting-output-description.patch -Patch3: fix-bug-list-index-out-of-range-in-virtualbox.patch -Patch4: add-new-dims.patch -Patch5: fix-CWE-23.patch -Patch6: memory-bandwidth-dimm-slot-atuned.patch -Patch7: feature-set-mysql.patch -Patch8: feature-set-redis-value.patch -Patch9: feature-set-nginx.patch -Patch10: bugfix-create-file-only-if-setting-params.patch -Patch11: change-import-behavior.patch -Patch12: atune-collector-add-backup-for-apps.patch -Patch13: feature-add-network-CPI.patch -Patch14: feature-add-multi-for-rps-xps.patch -Patch15: feature-add-rfs-to-network.patch -Patch16: fix-bug-procsched-report-list-index-out-of-range.patch -Patch17: fix-bug-procsched-data-collection-issue.patch - BuildRequires: python3-setuptools Requires: python3-dict2xml python3-werkzeug @@ -46,9 +28,13 @@ The A-Tune-Collector is used to collect various system resources and can also be %files %license LICENSE %{python3_sitelib}/atune_* +%exclude %{python3_sitelib}/atune_collector/ui/* %attr(0600,root,root) %{_sysconfdir}/atune_collector/* %changelog +* Sat Jan 27 2024 zhoupengcheng - 1.3.0-1 +- update v1.3.0 + * Fri Dec 15 2023 weiyaping - 1.1.0-7 - fix bug: ProcSched.report: list index out of range diff --git a/bugfix-create-file-only-if-setting-params.patch b/bugfix-create-file-only-if-setting-params.patch deleted file mode 100644 index d927561..0000000 --- a/bugfix-create-file-only-if-setting-params.patch +++ /dev/null @@ -1,113 +0,0 @@ -From 1c513970f8f5e81268e998cc33ab8a93802bb722 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Fri, 14 Jul 2023 18:47:25 +0800 -Subject: [PATCH 10/11] bugfix: create file only if setting params - ---- - atune_collector/plugin/configurator/mysql/mysql.py | 9 ++++++--- - atune_collector/plugin/configurator/nginx/nginx.py | 7 ++++--- - atune_collector/plugin/configurator/redis/redis.py | 7 +++++-- - 3 files changed, 15 insertions(+), 8 deletions(-) - -diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py -index cfc533d..e472a1c 100644 ---- a/atune_collector/plugin/configurator/mysql/mysql.py -+++ b/atune_collector/plugin/configurator/mysql/mysql.py -@@ -31,8 +31,10 @@ class Mysql(Configurator): - self.__cmd = "" - self.__file_path = "/etc/my.cnf" - self.__mysqld_ind = -1 -+ -+ def _init_file(self): - if not os.path.isfile(self.__file_path): -- with open(self.__file_path, 'w'): -+ with open(self.__file_path, 'w', 0o644): - pass - os.chmod(self.__file_path, 0o644) - self.__check_mysqld() -@@ -61,7 +63,8 @@ class Mysql(Configurator): - return False, ind - - def _set(self, key, value): -- with open(self.__file_path, 'r') as f: -+ self._init_file() -+ with open(self.__file_path, 'r', 0o400) as f: - lines = f.readlines() - - key_exist, ind = self.__check_file_exists(lines, key) -@@ -71,7 +74,7 @@ class Mysql(Configurator): - else: - lines[ind] = new_line - -- with open(self.__file_path, 'w') as f: -+ with open(self.__file_path, 'w', 0o644) as f: - f.writelines(lines) - return 0 - -diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py -index 7ab619d..9d38dd0 100644 ---- a/atune_collector/plugin/configurator/nginx/nginx.py -+++ b/atune_collector/plugin/configurator/nginx/nginx.py -@@ -42,18 +42,18 @@ class Nginx(Configurator): - os.mkdir(self.__file_dir) - os.chmod(self.__file_dir, 0o755) - if not os.path.isfile(self.__file_path): -- with open(self.__file_path, 'w'): -+ with open(self.__file_path, 'w', 0o644): - pass - os.chmod(self.__file_path, 0o644) - self._set_index() - - def _get_lines(self): -- with open(self.__file_path, 'r') as f: -+ with open(self.__file_path, 'r', 0o444) as f: - lines = f.readlines() - return lines - - def _set_lines(self, lines): -- with open(self.__file_path, 'w') as f: -+ with open(self.__file_path, 'w', 0o644) as f: - f.writelines(lines) - - def _set_index(self): -@@ -209,3 +209,4 @@ def rewrite_value(value): - raise SetConfigError("Failed to get cpu number") - return output.stdout.decode().count("\n") - -+ -diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py -index c56f38e..aa14bcd 100644 ---- a/atune_collector/plugin/configurator/redis/redis.py -+++ b/atune_collector/plugin/configurator/redis/redis.py -@@ -34,15 +34,18 @@ class Redis(Configurator): - self.__re = r"^#\?\s*{key}\s* " - self.__file_dir = "/etc/redis/" - self.__file_path = self.__file_dir + "redis.conf" -+ -+ def _init_file(self): - if not os.path.isdir(self.__file_dir): - os.mkdir(self.__file_dir) - os.chmod(self.__file_dir, 0o755) - if not os.path.isfile(self.__file_path): -- with open(self.__file_path, 'w', 0o200): -+ with open(self.__file_path, 'w', 0o600): - pass - os.chmod(self.__file_path, 0o644) - - def _set(self, key, value): -+ self._init_file() - re_cmd = self.__re.format(key=key) - grep_cmd = [r"grep", re_cmd, self.__file_path] - out, err = self.execute_cmd(grep_cmd) -@@ -51,7 +54,7 @@ class Redis(Configurator): - num_lines = out.count("\n") - new_line = self.__lines.format(key=key, value=value) - if num_lines == 0: -- with open(self.__file_path, 'a', 0o600) as f: -+ with open(self.__file_path, 'a', 0o644) as f: - f.write(new_line + '\n') - elif num_lines == 1: - sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path] --- -2.27.0 - diff --git a/change-import-behavior.patch b/change-import-behavior.patch deleted file mode 100644 index af2dd98..0000000 --- a/change-import-behavior.patch +++ /dev/null @@ -1,355 +0,0 @@ -From c02c86fbd6927a3d1968d5a1e3030058bbb943a3 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Sat, 15 Jul 2023 15:50:06 +0800 -Subject: [PATCH 11/11] change import behavior - ---- - .../plugin/configurator/__init__.py | 47 +++++++++++-------- - .../plugin/configurator/affinity/__init__.py | 2 +- - .../plugin/configurator/bios/__init__.py | 2 +- - .../configurator/bootloader/__init__.py | 2 +- - atune_collector/plugin/configurator/common.py | 2 +- - .../configurator/file_config/__init__.py | 2 +- - .../configurator/kernel_config/__init__.py | 2 +- - .../plugin/configurator/mysql/__init__.py | 3 +- - .../plugin/configurator/nginx/__init__.py | 3 +- - .../plugin/configurator/redis/__init__.py | 3 +- - .../plugin/configurator/script/__init__.py | 2 +- - .../plugin/configurator/sysctl/__init__.py | 2 +- - .../plugin/configurator/sysfs/__init__.py | 2 +- - .../plugin/configurator/systemctl/__init__.py | 2 +- - .../plugin/configurator/ulimit/__init__.py | 2 +- - atune_collector/plugin/monitor/__init__.py | 11 ++++- - .../plugin/monitor/memory/__init__.py | 2 +- - .../plugin/monitor/network/__init__.py | 2 +- - .../plugin/monitor/performance/__init__.py | 2 +- - .../plugin/monitor/process/__init__.py | 2 +- - .../plugin/monitor/processor/__init__.py | 2 +- - .../plugin/monitor/storage/__init__.py | 2 +- - .../plugin/monitor/system/__init__.py | 2 +- - atune_collector/plugin/plugin.py | 4 +- - 24 files changed, 60 insertions(+), 47 deletions(-) - -diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py -index 31520f1..fd7ea63 100755 ---- a/atune_collector/plugin/configurator/__init__.py -+++ b/atune_collector/plugin/configurator/__init__.py -@@ -17,24 +17,33 @@ Init file. - import sys - import os - --sys.path.append(os.path.dirname(__file__)) -+from .affinity import * -+from .bios import * -+from .bootloader import * -+from .file_config import * -+from .kernel_config import * -+from .script import * -+from .sysctl import * -+from .sysfs import * -+from .systemctl import * -+from .ulimit import * -+from .mysql import * -+from .redis import * -+from .nginx import * - --__all__ = [ -- "exceptions", -- "affinity", -- "bios", -- "bootloader", -- "file_config", -- "kernel_config", -- "script", -- "sysctl", -- "sysfs", -- "systemctl", -- "ulimit", -- "mysql", -- "redis", -- "nginx", -- "common"] -- --from . import * -+__all__ = ["exceptions", -+ "affinity", -+ "bios", -+ "bootloader", -+ "file_config", -+ "kernel_config", -+ "script", -+ "sysctl", -+ "sysfs", -+ "systemctl", -+ "ulimit", -+ "mysql", -+ "redis", -+ "nginx", -+ "common"] - -diff --git a/atune_collector/plugin/configurator/affinity/__init__.py b/atune_collector/plugin/configurator/affinity/__init__.py -index e29377f..1610cc3 100755 ---- a/atune_collector/plugin/configurator/affinity/__init__.py -+++ b/atune_collector/plugin/configurator/affinity/__init__.py -@@ -19,6 +19,6 @@ import os - - __all__ = ["irq", "task", "processid"] - --from . import * -+from . import irq, task, processid - - sys.path.insert(0, os.path.dirname(__file__)) -diff --git a/atune_collector/plugin/configurator/bios/__init__.py b/atune_collector/plugin/configurator/bios/__init__.py -index 1e193cf..1f9df24 100755 ---- a/atune_collector/plugin/configurator/bios/__init__.py -+++ b/atune_collector/plugin/configurator/bios/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["bios"] - --from . import * -+from . import bios -diff --git a/atune_collector/plugin/configurator/bootloader/__init__.py b/atune_collector/plugin/configurator/bootloader/__init__.py -index 2412c54..d6cf156 100755 ---- a/atune_collector/plugin/configurator/bootloader/__init__.py -+++ b/atune_collector/plugin/configurator/bootloader/__init__.py -@@ -20,6 +20,6 @@ import os - - __all__ = ["grub2", "cmdline"] - --from . import * -+from . import grub2, cmdline - - sys.path.insert(0, os.path.dirname(__file__)) -diff --git a/atune_collector/plugin/configurator/common.py b/atune_collector/plugin/configurator/common.py -index 0fbc2f3..4ab459a 100755 ---- a/atune_collector/plugin/configurator/common.py -+++ b/atune_collector/plugin/configurator/common.py -@@ -20,7 +20,7 @@ import logging - import json - from functools import wraps - --from exceptions import SetConfigError -+from .exceptions import SetConfigError - - LOGGER = logging.getLogger(__name__) - -diff --git a/atune_collector/plugin/configurator/file_config/__init__.py b/atune_collector/plugin/configurator/file_config/__init__.py -index 48aeb30..023de44 100755 ---- a/atune_collector/plugin/configurator/file_config/__init__.py -+++ b/atune_collector/plugin/configurator/file_config/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["fconfig"] - --from . import * -+from . import fconfig -diff --git a/atune_collector/plugin/configurator/kernel_config/__init__.py b/atune_collector/plugin/configurator/kernel_config/__init__.py -index 5905327..58dd2b0 100755 ---- a/atune_collector/plugin/configurator/kernel_config/__init__.py -+++ b/atune_collector/plugin/configurator/kernel_config/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["kconfig"] - --from . import * -+from . import kconfig -diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py -index 52bb8c3..bfdb807 100644 ---- a/atune_collector/plugin/configurator/mysql/__init__.py -+++ b/atune_collector/plugin/configurator/mysql/__init__.py -@@ -17,5 +17,4 @@ Init file. - - __all__ = ["mysql"] - --from . import * -- -+from . import mysql -diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py -index e28a310..07c82b0 100644 ---- a/atune_collector/plugin/configurator/nginx/__init__.py -+++ b/atune_collector/plugin/configurator/nginx/__init__.py -@@ -17,5 +17,4 @@ Init file. - - __all__ = ["nginx"] - --from . import * -- -+from . import nginx -diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py -index d386f1f..7d21825 100644 ---- a/atune_collector/plugin/configurator/redis/__init__.py -+++ b/atune_collector/plugin/configurator/redis/__init__.py -@@ -17,5 +17,4 @@ Init file. - - __all__ = ["redis"] - --from . import * -- -+from . import redis -diff --git a/atune_collector/plugin/configurator/script/__init__.py b/atune_collector/plugin/configurator/script/__init__.py -index f743a20..ec6aa17 100755 ---- a/atune_collector/plugin/configurator/script/__init__.py -+++ b/atune_collector/plugin/configurator/script/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["script"] - --from . import * -+from . import script -diff --git a/atune_collector/plugin/configurator/sysctl/__init__.py b/atune_collector/plugin/configurator/sysctl/__init__.py -index d6b3f61..3d28df7 100755 ---- a/atune_collector/plugin/configurator/sysctl/__init__.py -+++ b/atune_collector/plugin/configurator/sysctl/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["sysctl"] - --from . import * -+from . import sysctl -diff --git a/atune_collector/plugin/configurator/sysfs/__init__.py b/atune_collector/plugin/configurator/sysfs/__init__.py -index d220f40..9d5afb6 100755 ---- a/atune_collector/plugin/configurator/sysfs/__init__.py -+++ b/atune_collector/plugin/configurator/sysfs/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["sysfs"] - --from . import * -+from . import sysfs -diff --git a/atune_collector/plugin/configurator/systemctl/__init__.py b/atune_collector/plugin/configurator/systemctl/__init__.py -index 94762b6..c51a07f 100755 ---- a/atune_collector/plugin/configurator/systemctl/__init__.py -+++ b/atune_collector/plugin/configurator/systemctl/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["systemctl"] - --from . import * -+from . import systemctl -diff --git a/atune_collector/plugin/configurator/ulimit/__init__.py b/atune_collector/plugin/configurator/ulimit/__init__.py -index 2582b7c..c28a601 100755 ---- a/atune_collector/plugin/configurator/ulimit/__init__.py -+++ b/atune_collector/plugin/configurator/ulimit/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["ulimit"] - --from . import * -+from . import ulimit -diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py -index 9292071..948f869 100755 ---- a/atune_collector/plugin/monitor/__init__.py -+++ b/atune_collector/plugin/monitor/__init__.py -@@ -15,6 +15,15 @@ - Init file. - """ - -+from .memory import * -+from .network import * -+from .performance import * -+from .process import * -+from .processor import * -+from .storage import * -+from .system import * -+from . import common -+ - __all__ = [ - "memory", - "network", -@@ -24,5 +33,3 @@ __all__ = [ - "storage", - "common", - "system"] -- --from . import * -diff --git a/atune_collector/plugin/monitor/memory/__init__.py b/atune_collector/plugin/monitor/memory/__init__.py -index 0a5a356..5002d6e 100755 ---- a/atune_collector/plugin/monitor/memory/__init__.py -+++ b/atune_collector/plugin/monitor/memory/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["numainfo", "topo", "bandwidth", "vmstat", "utilstat", "meminfo"] - --from . import * -+from . import numainfo, topo, bandwidth, vmstat, utilstat, meminfo -diff --git a/atune_collector/plugin/monitor/network/__init__.py b/atune_collector/plugin/monitor/network/__init__.py -index 5d40ab8..388611a 100755 ---- a/atune_collector/plugin/monitor/network/__init__.py -+++ b/atune_collector/plugin/monitor/network/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["info", "netstat", "netestat", "topo"] - --from . import * -+from . import info, netstat, netestat, topo -diff --git a/atune_collector/plugin/monitor/performance/__init__.py b/atune_collector/plugin/monitor/performance/__init__.py -index 707f911..93fc6c4 100755 ---- a/atune_collector/plugin/monitor/performance/__init__.py -+++ b/atune_collector/plugin/monitor/performance/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["stat", "top"] - --from . import * -+from . import stat, top -diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py -index 4c4ceb3..5a34c10 100644 ---- a/atune_collector/plugin/monitor/process/__init__.py -+++ b/atune_collector/plugin/monitor/process/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["sched"] - --from . import * -\ No newline at end of file -+from . import sched -diff --git a/atune_collector/plugin/monitor/processor/__init__.py b/atune_collector/plugin/monitor/processor/__init__.py -index fc88e1b..f46d8fd 100755 ---- a/atune_collector/plugin/monitor/processor/__init__.py -+++ b/atune_collector/plugin/monitor/processor/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["info", "stat", "topo"] - --from . import * -+from . import info, stat, topo -diff --git a/atune_collector/plugin/monitor/storage/__init__.py b/atune_collector/plugin/monitor/storage/__init__.py -index 670294d..9e5e152 100755 ---- a/atune_collector/plugin/monitor/storage/__init__.py -+++ b/atune_collector/plugin/monitor/storage/__init__.py -@@ -17,4 +17,4 @@ Init file. - - __all__ = ["iostat", "topo"] - --from . import * -+from . import iostat, topo -diff --git a/atune_collector/plugin/monitor/system/__init__.py b/atune_collector/plugin/monitor/system/__init__.py -index 5c6a00c..f66c1ef 100755 ---- a/atune_collector/plugin/monitor/system/__init__.py -+++ b/atune_collector/plugin/monitor/system/__init__.py -@@ -17,4 +17,4 @@ The import content of the package. - - __all__ = ["bios", "ldavg", "tasks", "filed", "interrupts"] - --from . import * -+from . import bios, ldavg, tasks, filed, interrupts -diff --git a/atune_collector/plugin/plugin.py b/atune_collector/plugin/plugin.py -index b83d3aa..3f84f2f 100755 ---- a/atune_collector/plugin/plugin.py -+++ b/atune_collector/plugin/plugin.py -@@ -19,9 +19,9 @@ import logging - import threading - import time - --from configurator.common import Configurator -+from .configurator.common import Configurator - --from monitor.common import Monitor -+from .monitor.common import Monitor - - LOGGER = logging.getLogger(__name__) - --- -2.27.0 - diff --git a/feature-add-multi-for-rps-xps.patch b/feature-add-multi-for-rps-xps.patch deleted file mode 100644 index a7fe4b0..0000000 --- a/feature-add-multi-for-rps-xps.patch +++ /dev/null @@ -1,100 +0,0 @@ -From c2df28764f159163233cdab4fb34f62efe888c4e Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Mon, 27 Nov 2023 12:56:09 +0000 -Subject: [PATCH 1/2] feature: add 'multi' for rps/xps - -Signed-off-by: gaoruoshu ---- - .../plugin/configurator/network/network.py | 55 ++++++++++++++++++- - 1 file changed, 52 insertions(+), 3 deletions(-) - -diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py -index 977f29c..afc31f2 100644 ---- a/atune_collector/plugin/configurator/network/network.py -+++ b/atune_collector/plugin/configurator/network/network.py -@@ -35,8 +35,8 @@ class Network(Configurator): - - def __init__(self, user=None): - Configurator.__init__(self, user) -- self._cmd_map = {'xps': ['all', 'off', 'half', 'separate'], -- 'rps': ['all', 'off', 'half', 'separate']} -+ self._cmd_map = {'xps': ['all', 'off', 'half', 'separate', 'multi'], -+ 'rps': ['all', 'off', 'half', 'separate', 'multi']} - self._nic = self._get_nic() - self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic) - -@@ -52,6 +52,12 @@ class Network(Configurator): - 'Failed to get dir under {}'.format(self._queue_dir)) - dir_list = re.findall(pattern, dir_strs) - core_num = os.cpu_count() -+ -+ multi_qs = len(dir_list) -+ stride = 1 if core_num // multi_qs == 0 else core_num // multi_qs -+ stragglers = 0 if core_num <= multi_qs else core_num % multi_qs -+ cur_cpu = 0 -+ dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1])) - for index, dir_name in enumerate(dir_list): - file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key) - shell_cmd(['cat', file_path], -@@ -75,14 +81,57 @@ class Network(Configurator): - offset = index % half_num - val_format = 1 << (offset + half_num + core_num % 2) - set_value = f"{val_format:x}" -- else: # value == 'separate' -+ elif value == 'separate': - num = 1 << (index % core_num) - set_value = f"{num:x}" -+ else: # value = multi: -+ str_value = '' -+ gsize = stride if index >= stragglers else stride + 1 -+ for _ in range(gsize): -+ str_value = str(cur_cpu) + ' ' + str_value -+ cur_cpu += 1 -+ set_value = self._u32list(str_value) - - shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)], - 'Failed to set {} to {}'.format(key, file_path)) - return 0 - -+ @staticmethod -+ def _u32list(cpulist): -+ max_cpu = 0 -+ ii = 0 -+ -+ for _cpu in cpulist.split(): -+ if max_cpu < int(_cpu): -+ max_cpu = int(_cpu) -+ -+ #init a bitmap -+ map = [0] * (max_cpu + 1) -+ -+ # set bit map according to cpulist -+ for _cpu in cpulist.split(): -+ map[int(_cpu)] = 1 -+ -+ #format a u32list -+ seg = 0 -+ mask = '' -+ for ii in range(max_cpu + 1): -+ if ii % 4 == 0 and ii != 0: -+ seg = format(seg, 'x') -+ mask = seg + mask -+ seg = 0 -+ if ii % 32 == 0 and ii != 0: -+ mask = ',' + mask -+ cur = map[ii] -+ if cur == 1: -+ val = 1 << (ii % 4) -+ seg += val -+ -+ if seg != 0: -+ seg = format(seg, 'x') -+ mask = seg + mask -+ return mask -+ - def _set(self, key, value): - if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]: - raise SetConfigError("Invalid value {}={}".format(key, value)) --- -2.27.0 - diff --git a/feature-add-network-CPI.patch b/feature-add-network-CPI.patch deleted file mode 100644 index c366ffc..0000000 --- a/feature-add-network-CPI.patch +++ /dev/null @@ -1,182 +0,0 @@ -From 148c57c3695080667e138425b1295e89197a241b Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 22 Aug 2023 15:07:50 +0800 -Subject: [PATCH] feature: add [network] CPI - ---- - .../plugin/configurator/__init__.py | 2 + - .../plugin/configurator/network/__init__.py | 21 ++++ - .../plugin/configurator/network/network.py | 113 ++++++++++++++++++ - 3 files changed, 136 insertions(+) - create mode 100644 atune_collector/plugin/configurator/network/__init__.py - create mode 100644 atune_collector/plugin/configurator/network/network.py - -diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py -index fd7ea63..46cff84 100755 ---- a/atune_collector/plugin/configurator/__init__.py -+++ b/atune_collector/plugin/configurator/__init__.py -@@ -22,6 +22,7 @@ from .bios import * - from .bootloader import * - from .file_config import * - from .kernel_config import * -+from .network import * - from .script import * - from .sysctl import * - from .sysfs import * -@@ -37,6 +38,7 @@ __all__ = ["exceptions", - "bootloader", - "file_config", - "kernel_config", -+ "network", - "script", - "sysctl", - "sysfs", -diff --git a/atune_collector/plugin/configurator/network/__init__.py b/atune_collector/plugin/configurator/network/__init__.py -new file mode 100644 -index 0000000..088e459 ---- /dev/null -+++ b/atune_collector/plugin/configurator/network/__init__.py -@@ -0,0 +1,21 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2019 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-08-22 -+ -+""" -+Init file. -+""" -+ -+__all__ = ["network"] -+ -+from . import network -+ -diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py -new file mode 100644 -index 0000000..977f29c ---- /dev/null -+++ b/atune_collector/plugin/configurator/network/network.py -@@ -0,0 +1,113 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2019 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-08-22 -+ -+""" -+The sub class of the Configurator, used to extend script for network for CPI. -+""" -+ -+import re -+import logging -+import os -+import math -+import configparser -+import subprocess -+ -+from ..exceptions import SetConfigError -+from ..common import Configurator -+ -+LOGGER = logging.getLogger(__name__) -+ -+ -+class Network(Configurator): -+ _module = "NETWORK" -+ _submod = "NETWORK" -+ -+ -+ def __init__(self, user=None): -+ Configurator.__init__(self, user) -+ self._cmd_map = {'xps': ['all', 'off', 'half', 'separate'], -+ 'rps': ['all', 'off', 'half', 'separate']} -+ self._nic = self._get_nic() -+ self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic) -+ -+ def _get_nic(self): -+ if not os.path.isfile('/etc/atuned/atuned.cnf'): -+ raise SetConfigError('Cannot get network name') -+ config = configparser.ConfigParser() -+ config.read('/etc/atuned/atuned.cnf') -+ return config.get('system', 'network') -+ -+ def _set_cpus(self, key, value, pattern): -+ dir_strs = shell_cmd(['ls', self._queue_dir], -+ 'Failed to get dir under {}'.format(self._queue_dir)) -+ dir_list = re.findall(pattern, dir_strs) -+ core_num = os.cpu_count() -+ for index, dir_name in enumerate(dir_list): -+ file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key) -+ shell_cmd(['cat', file_path], -+ 'Failed to set {}={}: does not support for {}'.format(key, value, key)) -+ -+ if value == 'off': -+ set_value = '0' -+ elif value == 'all': -+ set_value = 'f' * (core_num // 4) -+ if core_num % 4 != 0: -+ bin_num = int('0b' + '1' * (core_num % 4), 2) -+ set_value = f"{bin_num:x}" + set_value -+ elif value == 'half': -+ half_num = core_num // 2 -+ if core_num == 1: -+ val_format = 1 -+ elif key == 'rps': -+ offset = index % (half_num + core_num % 2) -+ val_format = 1 << offset -+ else: -+ offset = index % half_num -+ val_format = 1 << (offset + half_num + core_num % 2) -+ set_value = f"{val_format:x}" -+ else: # value == 'separate' -+ num = 1 << (index % core_num) -+ set_value = f"{num:x}" -+ -+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)], -+ 'Failed to set {} to {}'.format(key, file_path)) -+ return 0 -+ -+ def _set(self, key, value): -+ if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]: -+ raise SetConfigError("Invalid value {}={}".format(key, value)) -+ -+ if key == 'xps': -+ self._set_cpus(key.lower(), value.lower(), r'tx-\d+') -+ elif key == 'rps': -+ self._set_cpus(key.lower(), value.lower(), r'rx-\d+') -+ -+ return 0 -+ -+ def _get(self, key, _): -+ pass -+ -+ def _backup(self, config, _): -+ return str(config) -+ -+ @staticmethod -+ def check(config1, config2): -+ return True -+ -+ -+def shell_cmd(cmd, error_message): -+ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -+ if output.returncode != 0: -+ raise SetConfigError(error_message) -+ return output.stdout.decode() -+ --- -2.27.0 - diff --git a/feature-add-rfs-to-network.patch b/feature-add-rfs-to-network.patch deleted file mode 100644 index a228589..0000000 --- a/feature-add-rfs-to-network.patch +++ /dev/null @@ -1,132 +0,0 @@ -From abc01d60cc574b27b943f2d619f0b3cca8f6e1c6 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 28 Nov 2023 03:35:26 +0000 -Subject: [PATCH 2/2] feature: add rfs to network - -Signed-off-by: gaoruoshu ---- - .../plugin/configurator/network/network.py | 56 +++++++++++++++---- - 1 file changed, 44 insertions(+), 12 deletions(-) - -diff --git a/atune_collector/plugin/configurator/network/network.py b/atune_collector/plugin/configurator/network/network.py -index afc31f2..01e5640 100644 ---- a/atune_collector/plugin/configurator/network/network.py -+++ b/atune_collector/plugin/configurator/network/network.py -@@ -36,7 +36,8 @@ class Network(Configurator): - def __init__(self, user=None): - Configurator.__init__(self, user) - self._cmd_map = {'xps': ['all', 'off', 'half', 'separate', 'multi'], -- 'rps': ['all', 'off', 'half', 'separate', 'multi']} -+ 'rps': ['all', 'off', 'half', 'separate', 'multi'], -+ 'rfs': ['on', 'off']} - self._nic = self._get_nic() - self._queue_dir = '/sys/class/net/{}/queues'.format(self._nic) - -@@ -47,17 +48,21 @@ class Network(Configurator): - config.read('/etc/atuned/atuned.cnf') - return config.get('system', 'network') - -- def _set_cpus(self, key, value, pattern): -+ def _get_init(self, pattern): - dir_strs = shell_cmd(['ls', self._queue_dir], - 'Failed to get dir under {}'.format(self._queue_dir)) - dir_list = re.findall(pattern, dir_strs) -+ dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1])) - core_num = os.cpu_count() -+ return dir_list, core_num -+ -+ def _set_cpus(self, key, value, pattern): -+ dir_list, core_num = self._get_init(pattern) - - multi_qs = len(dir_list) - stride = 1 if core_num // multi_qs == 0 else core_num // multi_qs - stragglers = 0 if core_num <= multi_qs else core_num % multi_qs - cur_cpu = 0 -- dir_list = sorted(dir_list, key=lambda x: int(x.split('-')[1])) - for index, dir_name in enumerate(dir_list): - file_path = '{}/{}/{}_cpus'.format(self._queue_dir, dir_name, key) - shell_cmd(['cat', file_path], -@@ -84,13 +89,15 @@ class Network(Configurator): - elif value == 'separate': - num = 1 << (index % core_num) - set_value = f"{num:x}" -- else: # value = multi: -- str_value = '' -+ elif value == 'multi': # value = multi: -+ list_cpu = [] - gsize = stride if index >= stragglers else stride + 1 - for _ in range(gsize): -- str_value = str(cur_cpu) + ' ' + str_value -+ list_cpu.append(cur_cpu) - cur_cpu += 1 -- set_value = self._u32list(str_value) -+ set_value = self._u32list(list_cpu) -+ else: -+ raise SetConfigError("cannot set {} to {}".format(key, value)) - - shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)], - 'Failed to set {} to {}'.format(key, file_path)) -@@ -101,16 +108,15 @@ class Network(Configurator): - max_cpu = 0 - ii = 0 - -- for _cpu in cpulist.split(): -- if max_cpu < int(_cpu): -- max_cpu = int(_cpu) -+ for _cpu in cpulist: -+ max_cpu = max(max_cpu, _cpu) - - #init a bitmap - map = [0] * (max_cpu + 1) - - # set bit map according to cpulist -- for _cpu in cpulist.split(): -- map[int(_cpu)] = 1 -+ for _cpu in cpulist: -+ map[_cpu] = 1 - - #format a u32list - seg = 0 -@@ -131,6 +137,30 @@ class Network(Configurator): - seg = format(seg, 'x') - mask = seg + mask - return mask -+ -+ def _set_rfs(self, value, pattern): -+ dir_list, _ = self._get_init(pattern) -+ for dir_name in dir_list: -+ file_path = '{}/{}/rps_flow_cnt'.format(self._queue_dir, dir_name) -+ shell_cmd(['cat', file_path], -+ 'Failed to set rfs={}: does not support for rfs'.format(value)) -+ if value == 'off': -+ set_value = 0 -+ elif value == 'on': -+ set_value = 4096 -+ else: -+ raise SetConfigError("cannot set rfs to {}".format(value)) -+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(set_value, file_path)], -+ 'Failed to set rfs to {}'.format(file_path)) -+ -+ if value == 'off': -+ entries = 0 -+ elif value == 'on': -+ entries = 4096 * len(dir_list) -+ entries_path = '/proc/sys/net/core/rps_sock_flow_entries' -+ shell_cmd(['sh', '-c', 'echo {} > {}'.format(entries, entries_path)], -+ 'Failed to set rfs to {}'.format(entries_path)) -+ return 0 - - def _set(self, key, value): - if not key.lower() in self._cmd_map or not value.lower() in self._cmd_map[key.lower()]: -@@ -140,6 +170,8 @@ class Network(Configurator): - self._set_cpus(key.lower(), value.lower(), r'tx-\d+') - elif key == 'rps': - self._set_cpus(key.lower(), value.lower(), r'rx-\d+') -+ elif key == 'rfs': -+ self._set_rfs(value.lower(), r'rx-\d+') - - return 0 - --- -2.27.0 - diff --git a/feature-set-mysql.patch b/feature-set-mysql.patch deleted file mode 100644 index 41cac5c..0000000 --- a/feature-set-mysql.patch +++ /dev/null @@ -1,145 +0,0 @@ -From 16ef5dc208cd79b9f9d4c7c9d66ce46dc9d013a1 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 11 Jul 2023 20:53:28 +0800 -Subject: [PATCH 07/11] feature: set mysql - ---- - .../plugin/configurator/__init__.py | 1 + - .../plugin/configurator/mysql/__init__.py | 21 +++++ - .../plugin/configurator/mysql/mysql.py | 84 +++++++++++++++++++ - 3 files changed, 106 insertions(+) - create mode 100644 atune_collector/plugin/configurator/mysql/__init__.py - create mode 100644 atune_collector/plugin/configurator/mysql/mysql.py - -diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py -index 29c674a..677d208 100755 ---- a/atune_collector/plugin/configurator/__init__.py -+++ b/atune_collector/plugin/configurator/__init__.py -@@ -31,6 +31,7 @@ __all__ = [ - "sysfs", - "systemctl", - "ulimit", -+ "mysql", - "common"] - - from . import * -diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py -new file mode 100644 -index 0000000..52bb8c3 ---- /dev/null -+++ b/atune_collector/plugin/configurator/mysql/__init__.py -@@ -0,0 +1,21 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2023 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-11 -+ -+""" -+Init file. -+""" -+ -+__all__ = ["mysql"] -+ -+from . import * -+ -diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py -new file mode 100644 -index 0000000..cfc533d ---- /dev/null -+++ b/atune_collector/plugin/configurator/mysql/mysql.py -@@ -0,0 +1,84 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2023 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-11 -+ -+""" -+The sub class of the Configurator, used to change the /etc/my.cnf config. -+""" -+ -+ -+import logging -+import os -+from ..common import Configurator -+ -+LOGGER = logging.getLogger(__name__) -+ -+class Mysql(Configurator): -+ _module = "MYSQL" -+ _submod = "MYSQL" -+ -+ def __init__(self, user=None): -+ Configurator.__init__(self, user) -+ self.__cmd = "" -+ self.__file_path = "/etc/my.cnf" -+ self.__mysqld_ind = -1 -+ if not os.path.isfile(self.__file_path): -+ with open(self.__file_path, 'w'): -+ pass -+ os.chmod(self.__file_path, 0o644) -+ self.__check_mysqld() -+ -+ def __check_mysqld(self): -+ with open(self.__file_path, 'r') as f: -+ lines = f.readlines() -+ -+ for line in lines: -+ self.__mysqld_ind = self.__mysqld_ind + 1 -+ if line[:-1] == '[mysqld]': -+ self.__mysqld_ind = self.__mysqld_ind + 1 -+ return -+ -+ lines.insert(self.__mysqld_ind, "[mysqld]\n") -+ self.__mysqld_ind = self.__mysqld_ind + 1 -+ with open(self.__file_path, 'w') as f: -+ f.writelines(lines) -+ -+ def __check_file_exists(self, lines, start_with): -+ ind = -1 -+ for line in lines: -+ ind = ind + 1 -+ if line.split('=')[0].rstrip() == start_with: -+ return True, ind -+ return False, ind -+ -+ def _set(self, key, value): -+ with open(self.__file_path, 'r') as f: -+ lines = f.readlines() -+ -+ key_exist, ind = self.__check_file_exists(lines, key) -+ new_line = key + " = " + value + "\n" -+ if not key_exist: -+ lines.insert(self.__mysqld_ind, new_line) -+ else: -+ lines[ind] = new_line -+ -+ with open(self.__file_path, 'w') as f: -+ f.writelines(lines) -+ return 0 -+ -+ def _get(self, key, _): -+ pass -+ -+ @staticmethod -+ def check(config1, config2): -+ return True -+ --- -2.27.0 - diff --git a/feature-set-nginx.patch b/feature-set-nginx.patch deleted file mode 100644 index 2cf2474..0000000 --- a/feature-set-nginx.patch +++ /dev/null @@ -1,272 +0,0 @@ -From b723f4cc584e6df195fbd8b8d0213a3b53aeceea Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Fri, 14 Jul 2023 16:50:53 +0800 -Subject: [PATCH 09/11] feature: set nginx - ---- - .../plugin/configurator/__init__.py | 1 + - .../plugin/configurator/nginx/__init__.py | 21 ++ - .../plugin/configurator/nginx/nginx.py | 211 ++++++++++++++++++ - 3 files changed, 233 insertions(+) - create mode 100644 atune_collector/plugin/configurator/nginx/__init__.py - create mode 100644 atune_collector/plugin/configurator/nginx/nginx.py - -diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py -index 9597692..31520f1 100755 ---- a/atune_collector/plugin/configurator/__init__.py -+++ b/atune_collector/plugin/configurator/__init__.py -@@ -33,6 +33,7 @@ __all__ = [ - "ulimit", - "mysql", - "redis", -+ "nginx", - "common"] - - from . import * -diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py -new file mode 100644 -index 0000000..e28a310 ---- /dev/null -+++ b/atune_collector/plugin/configurator/nginx/__init__.py -@@ -0,0 +1,21 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2019 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-14 -+ -+""" -+Init file. -+""" -+ -+__all__ = ["nginx"] -+ -+from . import * -+ -diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py -new file mode 100644 -index 0000000..7ab619d ---- /dev/null -+++ b/atune_collector/plugin/configurator/nginx/nginx.py -@@ -0,0 +1,211 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2019 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-14 -+ -+""" -+The sub class of the Configurator, used to extend script for /etc/nginx/nginx.conf for CPI. -+""" -+ -+import logging -+import os -+import subprocess -+ -+from ..exceptions import SetConfigError -+from ..common import Configurator -+ -+LOGGER = logging.getLogger(__name__) -+ -+ -+class Nginx(Configurator): -+ _module = "NGINX" -+ _submod = "NGINX" -+ -+ def __init__(self, user=None): -+ Configurator.__init__(self, user) -+ self.__file_dir = "/etc/nginx/" -+ self.__file_path = self.__file_dir + "nginx.conf" -+ self.__line = "{key} {value};\n" -+ self.__key_match = r"^([a-zA-Z]+(_[a-zA-Z]+)*.?[a-zA-Z]+)*$" -+ self.__ind = {"main": [0, 0]} -+ -+ def _init_file(self): -+ if not os.path.isdir(self.__file_dir): -+ os.mkdir(self.__file_dir) -+ os.chmod(self.__file_dir, 0o755) -+ if not os.path.isfile(self.__file_path): -+ with open(self.__file_path, 'w'): -+ pass -+ os.chmod(self.__file_path, 0o644) -+ self._set_index() -+ -+ def _get_lines(self): -+ with open(self.__file_path, 'r') as f: -+ lines = f.readlines() -+ return lines -+ -+ def _set_lines(self, lines): -+ with open(self.__file_path, 'w') as f: -+ f.writelines(lines) -+ -+ def _set_index(self): -+ lines = self._get_lines() -+ filo = [] -+ ind = [] -+ filo.append("start") -+ ind.append(0) -+ is_https_server = False -+ for i, line in enumerate(lines): -+ if i == len(lines) - 1: -+ self.__ind["main"] = [ind.pop(), i] -+ del filo, ind -+ if line.strip().startswith("}"): -+ close = filo.pop() -+ close_ind = ind.pop() -+ if close.startswith("events"): -+ self.__ind["events"] = [close_ind, i] -+ if close.startswith("http"): -+ self.__ind["http"] = [close_ind, i] -+ if close.startswith("server"): -+ if is_https_server: -+ self.__ind["https_server"] = [close_ind, i] -+ else: -+ self.__ind["http_server"] = [close_ind, i] -+ is_https_server = False -+ if "{" in line.strip().split("#")[0]: -+ filo.append(line.strip()) -+ ind.append(i) -+ if line.strip().startswith("ssl_"): -+ is_https_server = True -+ -+ def _check_key(self, key): -+ if len(key) == 0: -+ return False, 0 -+ count = 1 -+ if not ('a' <= key[0] <= 'z' or 'A' <= key[0] <= 'Z'): -+ return False, count -+ for i in range(1, len(key)): -+ if key[i] == '_' or key[i] == '.': -+ if not ('a' <= key[i - 1] <= 'z' or 'A' <= key[i - 1] <= 'Z'): -+ return False, count -+ if i == len(key) - 1: -+ return False, count -+ if key[i] == '.': -+ count = count + 1 -+ elif 'a' <= key[i] <= 'z' or 'A' <= key[i] <= 'Z': -+ continue -+ else: -+ return False, count -+ return True, count -+ -+ def _update_index(self, after_ind): -+ for key in self.__ind: -+ for i, val in enumerate(self.__ind[key]): -+ if val > after_ind: -+ self.__ind[key][i] = val + 1 -+ -+ def _append_lines(self, section, key, value): -+ lines = self._get_lines() -+ filo = [] -+ for i in range(self.__ind[section][0], self.__ind[section][1]): -+ curr_line = lines[i].strip() -+ if "{" in curr_line.split("#")[0]: -+ filo.append(lines[i].strip()) -+ elif curr_line.startswith("}"): -+ filo.pop() -+ elif section == 'main' and len(filo) > 0: -+ continue -+ elif len(filo) > 1: -+ continue -+ elif curr_line.startswith(key + " ") or curr_line.startswith(key + "\t"): -+ new_line = lines[i].split(key)[0] -+ new_line = new_line + self.__line.format(key=key, value=value) -+ lines[i] = new_line -+ return lines -+ new_line = "" -+ if section != "main" and not "server" in section: -+ new_line = lines[self.__ind[section][0]].split(section)[0] + " " -+ elif "server" in section: -+ new_line = lines[self.__ind[section][0]].split("server")[0] + " " -+ new_line = new_line + self.__line.format(key=key, value=value) -+ lines.insert(self.__ind[section][0] + 1, new_line) -+ self._update_index(self.__ind[section][0]) -+ return lines -+ -+ def _set_main_section(self, key, value): -+ return self._append_lines("main", key, value) -+ -+ def _set_event_section(self, key, value): -+ if "events" not in self.__ind: -+ raise SetConfigError("No events section") -+ return self._append_lines("events", key, value) -+ -+ def _set_http_section(self, key, value): -+ if "http" not in self.__ind: -+ raise SetConfigError("No http section") -+ return self._append_lines("http", key, value) -+ -+ def _set_http_server(self, key, value): -+ if "http_server" not in self.__ind: -+ raise SetConfigError("No http server in http section") -+ return self._append_lines("http_server", key, value) -+ -+ def _set_https_server(self, key, value): -+ if "https_server" not in self.__ind: -+ raise SetConfigError("No https server in http section") -+ return self._append_lines("https_server", key, value) -+ -+ def _set(self, key, value): -+ self._init_file() -+ valid, count = self._check_key(key) -+ if not valid or count < 1 or count > 3: -+ raise SetConfigError("Invalid value {}".format(key)) -+ if value == "!CPU_CORE!": -+ value = rewrite_value(value) -+ lines = [] -+ try: -+ if count == 1: -+ lines = self._set_main_section(key, value) -+ elif count == 2 and key.split('.')[0] == "events": -+ lines = self._set_event_section(key.split('.')[1], value) -+ elif key.split('.')[0] == "http": -+ if count == 2: -+ lines = self._set_http_section(key.split('.')[1], value) -+ elif count == 3: -+ if key.split('.')[1] == "http": -+ lines = self._set_http_server(key.split('.')[2], value) -+ elif key.split('.')[1] == "https": -+ lines = self._set_https_server(key.split('.')[2], value) -+ else: -+ raise SetConfigError("Invalid value {}".format(key)) -+ else: -+ raise SetConfigError("Invalid value {}".format(key)) -+ else: -+ raise SetConfigError("Invalid value {}".format(key)) -+ except SetConfigError as err: -+ raise err -+ self._set_lines(lines) -+ return 0 -+ -+ def _get(self, key, _): -+ pass -+ -+ @staticmethod -+ def check(config1, config2): -+ return True -+ -+def rewrite_value(value): -+ command = ["grep", "processor", "/proc/cpuinfo"] -+ output = subprocess.run(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -+ if output.returncode != 0: -+ raise SetConfigError("Failed to get cpu number") -+ return output.stdout.decode().count("\n") -+ --- -2.27.0 - diff --git a/feature-set-redis-value.patch b/feature-set-redis-value.patch deleted file mode 100644 index 421a46a..0000000 --- a/feature-set-redis-value.patch +++ /dev/null @@ -1,139 +0,0 @@ -From 70a1f8bc884b6a1b2772fd7199e103fa5351ca3d Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Thu, 13 Jul 2023 14:40:57 +0800 -Subject: [PATCH 08/11] feature: set redis value - ---- - .../plugin/configurator/__init__.py | 2 + - .../plugin/configurator/redis/__init__.py | 21 +++++ - .../plugin/configurator/redis/redis.py | 77 +++++++++++++++++++ - 3 files changed, 100 insertions(+) - create mode 100644 atune_collector/plugin/configurator/redis/__init__.py - create mode 100644 atune_collector/plugin/configurator/redis/redis.py - -diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py -index 677d208..9597692 100755 ---- a/atune_collector/plugin/configurator/__init__.py -+++ b/atune_collector/plugin/configurator/__init__.py -@@ -32,6 +32,8 @@ __all__ = [ - "systemctl", - "ulimit", - "mysql", -+ "redis", - "common"] - - from . import * -+ -diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py -new file mode 100644 -index 0000000..d386f1f ---- /dev/null -+++ b/atune_collector/plugin/configurator/redis/__init__.py -@@ -0,0 +1,21 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2023 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-13 -+ -+""" -+Init file. -+""" -+ -+__all__ = ["redis"] -+ -+from . import * -+ -diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py -new file mode 100644 -index 0000000..c56f38e ---- /dev/null -+++ b/atune_collector/plugin/configurator/redis/redis.py -@@ -0,0 +1,77 @@ -+#!/usr/bin/python3 -+# -*- coding: utf-8 -*- -+# Copyright (c) 2023 Huawei Technologies Co., Ltd. -+# A-Tune is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+# Create: 2023-07-13 -+ -+""" -+The sub class of the Configurator, used to change the redis config. -+""" -+ -+import logging -+import os -+import subprocess -+from ..exceptions import GetConfigError, SetConfigError -+from ..common import Configurator -+ -+LOGGER = logging.getLogger(__name__) -+ -+ -+class Redis(Configurator): -+ _module = "REDIS" -+ _submod = "REDIS" -+ -+ def __init__(self, user=None): -+ Configurator.__init__(self, user) -+ self.__lines = "{key} {value}" -+ self.__re = r"^#\?\s*{key}\s* " -+ self.__file_dir = "/etc/redis/" -+ self.__file_path = self.__file_dir + "redis.conf" -+ if not os.path.isdir(self.__file_dir): -+ os.mkdir(self.__file_dir) -+ os.chmod(self.__file_dir, 0o755) -+ if not os.path.isfile(self.__file_path): -+ with open(self.__file_path, 'w', 0o200): -+ pass -+ os.chmod(self.__file_path, 0o644) -+ -+ def _set(self, key, value): -+ re_cmd = self.__re.format(key=key) -+ grep_cmd = [r"grep", re_cmd, self.__file_path] -+ out, err = self.execute_cmd(grep_cmd) -+ if len(err) != 0: -+ raise SetConfigError("Failed to set {}: {}".format(key, err)) -+ num_lines = out.count("\n") -+ new_line = self.__lines.format(key=key, value=value) -+ if num_lines == 0: -+ with open(self.__file_path, 'a', 0o600) as f: -+ f.write(new_line + '\n') -+ elif num_lines == 1: -+ sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path] -+ _, err_rep = self.execute_cmd(sed_cmd) -+ if len(err_rep) > 0: -+ raise SetConfigError("Failed to set {}: {}".format(key, err_rep)) -+ else: -+ raise SetConfigError("Failed to set {}: more than 1 key has same name".format(key)) -+ return 0 -+ -+ def _get(self, key, _): -+ pass -+ -+ @staticmethod -+ def check(config1, config2): -+ return True -+ -+ @staticmethod -+ def execute_cmd(cmd): -+ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -+ return output.stdout.decode(), output.stderr.decode()[:-1] -+ -+ --- -2.27.0 - diff --git a/fix-CWE-23.patch b/fix-CWE-23.patch deleted file mode 100644 index 4afb3b4..0000000 --- a/fix-CWE-23.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 121a1bcbd68ef9b18ec0c0cdcc8ca0748fe08bdd Mon Sep 17 00:00:00 2001 -From: unknown -Date: Wed, 26 Apr 2023 21:44:16 +0800 -Subject: [PATCH 05/11] fix CWE-23 - -collect_data.py直接使用命令行参数作为文件路径，可以被攻击者输入../访问上级目录，从而获取系统的敏感信息，或者写入任意文件（使用保存路径遍历文件名的恶意zip存档）。为了修复这一漏洞，我们采用了werzeug库中的secure_filename函数，这一函数会过滤掉文件路径中的所有危险字符，防范这一攻击方式。 ---- - atune_collector/collect_data.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py -index 3593db6..167141e 100755 ---- a/atune_collector/collect_data.py -+++ b/atune_collector/collect_data.py -@@ -21,6 +21,7 @@ import time - import csv - - from plugin.plugin import MPI -+from werkzeug.utils import secure_filename - - - class Collector: -@@ -112,6 +113,7 @@ if __name__ == "__main__": - ARG_PARSER.add_argument('-c', '--config', metavar='json', - default=default_json_path, help='input json path') - ARGS = ARG_PARSER.parse_args() -+ filename=secure_filename(ARGS.config) - with open(ARGS.config, 'r') as file: - json_data = json.load(file) - collector = Collector(json_data) --- -2.27.0 - diff --git a/fix-bug-list-index-out-of-range-in-virtualbox.patch b/fix-bug-list-index-out-of-range-in-virtualbox.patch deleted file mode 100644 index 9ed762a..0000000 --- a/fix-bug-list-index-out-of-range-in-virtualbox.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 4cb49adb623f36590f254b7e11c8aa5b0777bd00 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 26 Apr 2022 02:01:27 +0000 -Subject: [PATCH 03/11] fix bug: list index out of range in virtualbox - ---- - atune_collector/plugin/monitor/memory/bandwidth.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py -index 593a4a8..f44925f 100755 ---- a/atune_collector/plugin/monitor/memory/bandwidth.py -+++ b/atune_collector/plugin/monitor/memory/bandwidth.py -@@ -153,7 +153,7 @@ class MemBandwidth(Monitor): - info = json.loads(info_json) - - dimms = [[0 for i in range(8)] for i in range(8)] -- if "children" not in info["memorys"][0]: -+ if info["memorys"] is None or len(info["memorys"]) == 0 or "children" not in info["memorys"][0]: - return 0 - for dimm in info["memorys"][0]["children"]: - if dimm.get("size") is None: --- -2.27.0 - diff --git a/fix-bug-procsched-data-collection-issue.patch b/fix-bug-procsched-data-collection-issue.patch deleted file mode 100644 index c5fc8aa..0000000 --- a/fix-bug-procsched-data-collection-issue.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 43c7e8eb00268570abbaeacbb2689079dddec63d Mon Sep 17 00:00:00 2001 -From: weiyaping -Date: Fri, 15 Dec 2023 03:50:06 -0500 -Subject: [PATCH] fix-bug-procsched-data-collection-issue - ---- - atune_collector/plugin/monitor/process/sched.py | 27 ++++++++++++++++--------- - 1 file changed, 17 insertions(+), 10 deletions(-) - -diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py -index f184957..0fadeba 100644 ---- a/atune_collector/plugin/monitor/process/sched.py -+++ b/atune_collector/plugin/monitor/process/sched.py -@@ -107,7 +107,7 @@ class ProcSched(Monitor): - continue - - pattern = re.compile( -- r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)", -+ r"(\w+)\ {1,}\:\ {1,}(\d+\.?\d*)", - re.I | re.UNICODE | re.MULTILINE) - search_obj = pattern.findall(info) - search_list = [] -@@ -118,13 +118,20 @@ class ProcSched(Monitor): - search_list.append(obj[0]) - search_list.append(obj[1]) - if len(search_obj) == 0: -- err = LookupError("Fail to find data") -- LOGGER.error("%s.%s: %s", self.__class__.__name__, -- inspect.stack()[0][3], str(err)) -- raise err -+ return " " + " ".join(['0'] * len(keys)) -+ proc_data = [] -+ proc_keys = 0 -+ proc_step = len(set(keys)) - -- while start <= len(self.__applications) * len(keys): -- for key in keys: -- ret = ret + " " + search_list[search_list.index(key, start)+1] -- start = search_list.index(key, start) + 1 -- return ret -\ No newline at end of file -+ for key in keys: -+ if len(proc_data) >= self.__proc_flag.count(True) * proc_step: -+ break -+ proc_data.append(search_list[search_list.index(key, start) + 1]) -+ start = search_list.index(key, start) + 1 -+ for pid_flag in self.__proc_flag: -+ if not pid_flag: -+ ret = ret + " " + " ".join(['0'] * proc_step) -+ else: -+ ret = ret + " " + " ".join(proc_data[proc_keys:(proc_keys + proc_step)]) -+ proc_keys += proc_step -+ return ret --- -1.8.3.1 - diff --git a/fix-bug-procsched-report-list-index-out-of-range.patch b/fix-bug-procsched-report-list-index-out-of-range.patch deleted file mode 100644 index 5136b1b..0000000 --- a/fix-bug-procsched-report-list-index-out-of-range.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 009428bbf8f38bf9754999ec77403c4b181cf8ed Mon Sep 17 00:00:00 2001 -From: weiyaping -Date: Fri, 15 Dec 2023 03:37:02 -0500 -Subject: [PATCH] fix bug:ProcSched.report: list index out of range - ---- - atune_collector/plugin/monitor/process/sched.py | 10 ++++++++-- - 1 file changed, 8 insertions(+), 2 deletions(-) - -diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py -index 5289d84..f184957 100644 ---- a/atune_collector/plugin/monitor/process/sched.py -+++ b/atune_collector/plugin/monitor/process/sched.py -@@ -36,10 +36,12 @@ class ProcSched(Monitor): - self.__interval = 1 - self.__applications = [] - self.__pids = [] -+ self.__proc_flag = [] - - def _get(self, para=None): - output = "" - pids = [] -+ proc_flag = [] - if para is not None: - opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app=']) - for opt, val in opts: -@@ -67,9 +69,13 @@ class ProcSched(Monitor): - - for app in self.__applications: - pid = subprocess.getoutput( -- "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0] -- pids.append(pid) -+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split() -+ app_pid_flag = True if pid else False -+ proc_flag.append(app_pid_flag) -+ if pid: -+ pids.append(pid[0]) - self.__pids = pids -+ self.__proc_flag = proc_flag - - for pid in self.__pids: - out = subprocess.check_output( --- -1.8.3.1 - diff --git a/memory-bandwidth-dimm-slot-atuned.patch b/memory-bandwidth-dimm-slot-atuned.patch deleted file mode 100644 index c9845e0..0000000 --- a/memory-bandwidth-dimm-slot-atuned.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 7c21a2da2c931089e4ec2b86ae35a510f2f7793d Mon Sep 17 00:00:00 2001 -From: SWWBF <1152719547@qq.com> -Date: Mon, 10 Jul 2023 05:52:54 +0800 -Subject: [PATCH 06/11] some kvm has no slot which may cause panic of -atune - ---- - atune_collector/plugin/monitor/memory/bandwidth.py | 13 +++++++------ - 1 file changed, 7 insertions(+), 6 deletions(-) - -diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py -index f44925f..6d59ab6 100755 ---- a/atune_collector/plugin/monitor/memory/bandwidth.py -+++ b/atune_collector/plugin/monitor/memory/bandwidth.py -@@ -158,12 +158,13 @@ class MemBandwidth(Monitor): - for dimm in info["memorys"][0]["children"]: - if dimm.get("size") is None: - continue -- locator = memtopo.table_get_locator(dimm["slot"]) -- if locator is None: -- continue -- if dimms[locator[0]][locator[1]] == 0: -- dimms[locator[0]][locator[1]] = dimm["width"] * \ -- memtopo.table_get_freq(dimm["description"]) / 8 -+ if "slot" in dimm: -+ locator = memtopo.table_get_locator(dimm["slot"]) -+ if locator is None: -+ continue -+ if dimms[locator[0]][locator[1]] == 0: -+ dimms[locator[0]][locator[1]] = dimm["width"] * \ -+ memtopo.table_get_freq(dimm["description"]) / 8 - ret = 0 - for channel in dimms[socket]: - ret += channel --- -2.27.0 - diff --git a/v1.1.0.tar.gz b/v1.1.0.tar.gz deleted file mode 100644 index 9c50d9f..0000000 Binary files a/v1.1.0.tar.gz and /dev/null differ diff --git a/v1.3.0.tar.gz b/v1.3.0.tar.gz new file mode 100644 index 0000000..18e7b7d Binary files /dev/null and b/v1.3.0.tar.gz differ