packages/A-Tune-Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
A-Tune-Collector/add-new-dims.patch

428 lines
16 KiB
Diff
Raw Normal View History

feature: enable application configs
2023-08-01 09:43:39 +08:00
From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001
From: hamster <hujing@isrc.iscas.ac.cn>
Date: Mon, 14 Nov 2022 03:24:49 +0000
Subject: [PATCH 04/11] add new dimensions
From: @hujing2
Reviewed-by: @gaoruoshu
Signed-off-by: @gaoruoshu
---
README.md | 16 +++
atune_collector/collect_data.json | 14 ++
atune_collector/collect_data.py | 97 ++++++++------
atune_collector/plugin/monitor/__init__.py | 1 +
atune_collector/plugin/monitor/common.py | 1 +
.../plugin/monitor/process/__init__.py | 20 +++
.../plugin/monitor/process/sched.py | 124 ++++++++++++++++++
7 files changed, 233 insertions(+), 40 deletions(-)
create mode 100644 atune_collector/plugin/monitor/process/__init__.py
create mode 100644 atune_collector/plugin/monitor/process/sched.py
diff --git a/README.md b/README.md
index c121c8d..242f345 100644
--- a/README.md
+++ b/README.md
@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS]
| ---------------- | ------------------------------------- | ------------ | ------------ |
| network | 待采集的指定网卡 | 字符串 | - |
| block | 待采集的指定磁盘 | 字符串 | - |
+| application | 需要采集的应用进程 | 字符串 | - |
| sample_num | 待采集的次数 | 整型 | >0 |
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
+
最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
表2 collection_items项配置说明
@@ -76,6 +78,7 @@ collect_data.json文件配置示例：
{
"network": "eth0",
"block": "sda",
+ "application": "mysqld",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -203,6 +206,19 @@ collect_data.json文件配置示例：
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json
index db96501..af286bd 100755
--- a/atune_collector/collect_data.json
+++ b/atune_collector/collect_data.json
@@ -1,6 +1,7 @@
{
"network": "eth0",
"block": "sda",
+ "application": "firewalld,dockerd",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -140,6 +141,19 @@
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
\ No newline at end of file
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
index 1764304..3593db6 100755
--- a/atune_collector/collect_data.py
+++ b/atune_collector/collect_data.py
@@ -18,6 +18,7 @@ import argparse
import json
import os
import time
+import csv
from plugin.plugin import MPI
@@ -30,66 +31,80 @@ class Collector:
self.field_name = []
self.support_multi_block = ['storage']
self.support_multi_nic = ['network', 'network-err']
+ self.support_multi_app = ['process']
def parse_json(self):
"""parse json data"""
monitors = []
for item in self.data["collection_items"]:
- parameters = ["--interval=%s;" % self.data["interval"]]
- for metric in item["metrics"]:
- nics = self.data["network"].split(',')
- blocks = self.data["block"].split(',')
- if item["name"] in self.support_multi_nic and len(nics) > 1:
- for net in nics:
+ if item["name"] in self.support_multi_app and ('application' not in self.data or
+ self.data["application"] == ""):
+ continue
+ if item["name"] in self.support_multi_app:
+ applications = self.data["application"].split(',')
+ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])]
+ for application in applications:
+ for metric in item["metrics"]:
self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
- elif item["name"] in self.support_multi_block and len(blocks) > 1:
- for block in blocks:
- self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
- else:
- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
- parameters.append("--fields=%s" % metric)
- if "threshold" in item:
- parameters.append("--threshold=%s" % item["threshold"])
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application))
+ parameters.append("--fields=%s" % metric)
+ else:
+ parameters = ["--interval=%s;" % self.data["interval"]]
+ for metric in item["metrics"]:
+ nics = self.data["network"].split(',')
+ blocks = self.data["block"].split(',')
+
+ if item["name"] in self.support_multi_nic and len(nics) > 1:
+ for net in nics:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
+ elif item["name"] in self.support_multi_block and len(blocks) > 1:
+ for block in blocks:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
+ else:
+ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
+ parameters.append("--fields=%s" % metric)
+ if "threshold" in item:
+ parameters.append("--threshold=%s" % item["threshold"])
+
parameters.append("--nic=%s" % self.data["network"])
parameters.append("--device=%s" % self.data["block"])
monitors.append([item["module"], item["purpose"], " ".join(parameters)])
return monitors
- def save_csv(self, field_data):
- """save data to csv file"""
+ def collect_data(self):
+ """collect data"""
+ collect_num = self.data["sample_num"]
+ if int(collect_num) < 1:
+ os.abort("sample_num must be greater than 0")
+
+ mpi = MPI()
+ monitors = self.parse_json()
path = self.data["output_dir"]
if not os.path.exists(path):
os.makedirs(path, 0o750)
file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"),
int(round(time.time() * 1000)))
- import csv
+
+ print("start to collect data, csv path is %s" % os.path.join(path, file_name))
+ print(" ".join(self.field_name))
with open(os.path.join(path, file_name), "w") as csvfile:
writer = csv.writer(csvfile)
self.field_name.insert(0, "TimeStamp")
writer.writerow(self.field_name)
- writer.writerows(field_data)
+ csvfile.flush()
+ for _ in range(collect_num):
+ raw_data = mpi.get_monitors_data(monitors)
+ float_data = [float(num) for num in raw_data]
+ str_data = [str(round(value, 3)) for value in float_data]
+ print(" ".join(str_data))
+ float_data.insert(0, time.strftime("%H:%M:%S"))
+ # field_data.append(float_data)
+ writer.writerow(float_data)
+ csvfile.flush()
print("finish to collect data, csv path is %s" % os.path.join(path, file_name))
- def collect_data(self):
- """collect data"""
- collect_num = self.data["sample_num"]
- if int(collect_num) < 1:
- os.abort("sample_num must be greater than 0")
- field_data = []
- mpi = MPI()
- monitors = self.parse_json()
- print(" ".join(self.field_name))
- for _ in range(collect_num):
- raw_data = mpi.get_monitors_data(monitors)
- float_data = [float(num) for num in raw_data]
- str_data = [str(round(value, 3)) for value in float_data]
- print(" ".join(str_data))
- float_data.insert(0, time.strftime("%H:%M:%S"))
- field_data.append(float_data)
- return field_data
-
if __name__ == "__main__":
default_json_path = "/etc/atune_collector/collect_data.json"
@@ -100,5 +115,7 @@ if __name__ == "__main__":
with open(ARGS.config, 'r') as file:
json_data = json.load(file)
collector = Collector(json_data)
- dataset = collector.collect_data()
- collector.save_csv(dataset)
+ try:
+ collector.collect_data()
+ except KeyboardInterrupt:
+ print("user stop collect data")
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
index 6291401..9292071 100755
--- a/atune_collector/plugin/monitor/__init__.py
+++ b/atune_collector/plugin/monitor/__init__.py
@@ -19,6 +19,7 @@ __all__ = [
"memory",
"network",
"performance",
+ "process",
"processor",
"storage",
"common",
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
index 12a07f2..d0aa60c 100755
--- a/atune_collector/plugin/monitor/common.py
+++ b/atune_collector/plugin/monitor/common.py
@@ -134,6 +134,7 @@ class Monitor(object):
"--interval=" to specify period of time
"--cpu=" to select which cpu
"--event=" to select which event
+ "--app" to select which applications
:returns value: Success, collected info string
:raises Exceptions: Fail, with info
"""
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
new file mode 100644
index 0000000..4c4ceb3
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/__init__.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+Init file.
+"""
+
+__all__ = ["sched"]
+
+from . import *
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
new file mode 100644
index 0000000..5289d84
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/sched.py
@@ -0,0 +1,124 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+The sub class of the monitor, used to collect the process sched info
+"""
+import inspect
+import logging
+import subprocess
+import getopt
+import re
+from ..common import Monitor
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ProcSched(Monitor):
+ """To collect the process sched info"""
+ _module = "PROCESS"
+ _purpose = "SCHED"
+ _option = "/proc/{}/sched"
+
+ def __init__(self, user=None):
+ Monitor.__init__(self, user)
+ self.__cmd = "cat"
+ self.__interval = 1
+ self.__applications = []
+ self.__pids = []
+
+ def _get(self, para=None):
+ output = ""
+ pids = []
+ if para is not None:
+ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app='])
+ for opt, val in opts:
+ if opt in '--interval':
+ if val.isdigit():
+ self.__interval = int(val)
+ else:
+ err = ValueError(
+ "Invalid parameter: {opt}={val}".format(
+ opt=opt, val=val))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+ continue
+ elif opt in '--app':
+ if val is not None:
+ self.__applications = val.split(',')
+ else:
+ err = ValueError(
+ "{opt} parameter is none".format(
+ opt=opt))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ for app in self.__applications:
+ pid = subprocess.getoutput(
+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0]
+ pids.append(pid)
+ self.__pids = pids
+
+ for pid in self.__pids:
+ out = subprocess.check_output(
+ "{cmd} {opt}".format(
+ cmd=self.__cmd,
+ opt=self._option.format(pid)).split())
+ output = output + "" + out.decode()
+ return output
+
+ def decode(self, info, para):
+ """
+ decode the result of the operation
+ :param info: content that needs to be decoded
+ :param para: command line argument
+ :returns ret: operation result
+ """
+
+ if para is None:
+ return info
+
+ start = 0
+ keys = []
+ ret = ""
+
+ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device='])
+ for opt, val in opts:
+ if opt in '--fields':
+ keys.append(val)
+ continue
+
+ pattern = re.compile(
+ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)",
+ re.I | re.UNICODE | re.MULTILINE)
+ search_obj = pattern.findall(info)
+ search_list = []
+ for obj in search_obj:
+ if obj[0][:3] == "nr_":
+ search_list.append(obj[0][3:])
+ else:
+ search_list.append(obj[0])
+ search_list.append(obj[1])
+ if len(search_obj) == 0:
+ err = LookupError("Fail to find data")
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ while start <= len(self.__applications) * len(keys):
+ for key in keys:
+ ret = ret + " " + search_list[search_list.index(key, start)+1]
+ start = search_list.index(key, start) + 1
+ return ret
\ No newline at end of file
--
2.27.0
Reference in New Issue Copy Permalink