122 lines
5.4 KiB
Diff
122 lines
5.4 KiB
Diff
From acefcbdbb4891aa0b3f1b7afe500b8fdef440806 Mon Sep 17 00:00:00 2001
|
||
From: huangbin <huangbin58@huawei.com>
|
||
Date: Fri, 8 Nov 2024 16:42:52 +0800
|
||
Subject: [PATCH] configure-group-in-json
|
||
|
||
---
|
||
anteater/model/detector/slow_node_detector.py | 28 +++++++++++--------
|
||
config/module/slow_node_detection.job.json | 9 ++++--
|
||
2 files changed, 24 insertions(+), 13 deletions(-)
|
||
|
||
diff --git a/anteater/model/detector/slow_node_detector.py b/anteater/model/detector/slow_node_detector.py
|
||
index 15a6cee..d5d10e1 100644
|
||
--- a/anteater/model/detector/slow_node_detector.py
|
||
+++ b/anteater/model/detector/slow_node_detector.py
|
||
@@ -55,6 +55,9 @@ class SlowNodeDetector(Detector):
|
||
hccl_domain = json.load(f_out)
|
||
except Exception:
|
||
logger.error(f"Read hccl domain info fail!")
|
||
+ if not hccl_domain:
|
||
+ # 增加手动设置hccl_domain
|
||
+ hccl_domain = params.get("hccl_domain", {})
|
||
if os.path.exists(rank_table_path):
|
||
try:
|
||
with open(rank_table_path, 'r', encoding='utf-8') as f_out:
|
||
@@ -106,15 +109,15 @@ class SlowNodeDetector(Detector):
|
||
# 获取machine_ids,
|
||
machines_to_devices = self.get_machines_to_devices(start, end, kpis)
|
||
npu_id2host_id, hosts_ids = self.npu_id2host_id(machines_to_devices)
|
||
-
|
||
group_dataloader = GroupDataLoader(self.hccl_domain, self.rank_table, machines_to_devices)
|
||
group_ranks: list = group_dataloader.get_group_ranks()
|
||
all_results = []
|
||
for kpi in kpis:
|
||
- for ranks in group_ranks:
|
||
+ for index, ranks in enumerate(group_ranks):
|
||
+ logger.info(f"Groups-{index}, metric: {kpi.metric}, start detection.")
|
||
machine_ids: dict = group_dataloader.rank_table_loader.get_group_nodes_by_ranks(ranks)
|
||
host_ids: list = self.get_host_ids_by_npu_ids(machine_ids, npu_id2host_id, hosts_ids)
|
||
- group_result = self.group_detect_single_kpi(kpi, machine_ids, host_ids)
|
||
+ group_result = self.group_detect_single_kpi(kpi, machine_ids, host_ids, ranks)
|
||
all_results.extend(group_result)
|
||
|
||
response, all_anomaly_nodes = self.gen_final_alarm(kpis, all_results)
|
||
@@ -148,19 +151,26 @@ class SlowNodeDetector(Detector):
|
||
|
||
return response, all_anomaly_nodes
|
||
|
||
- def group_detect_single_kpi(self, kpi: KPI, machine_ids: dict, host_ids: list) -> list:
|
||
+ def group_detect_single_kpi(self, kpi: KPI, machine_ids: dict, host_ids: list, ranks) -> list:
|
||
"""Detects kpi based on signal time series anomaly detection model"""
|
||
# 普罗会一次性抓到所有的数据，需要根据machine_id, device_id去对数据作分组
|
||
- # get数据
|
||
metric_name: str = kpi.metric
|
||
|
||
all_machines_ts = []
|
||
for machine_id in machine_ids:
|
||
single_machine_ts_list = self.get_kpi_ts_list(metric_name, machine_id, kpi.params)
|
||
- all_machines_ts.extend(single_machine_ts_list)
|
||
+ if single_machine_ts_list:
|
||
+ # 根据ranks匹配组内device的指标
|
||
+ local_ranks = [int(rank) % 8 for rank in ranks]
|
||
+ for single_machine_ts in single_machine_ts_list:
|
||
+ ts_id = int(single_machine_ts.labels.get("id", -1))
|
||
+ if ts_id in local_ranks:
|
||
+ all_machines_ts.append(single_machine_ts)
|
||
+
|
||
for host_id in host_ids:
|
||
single_machine_ts_list = self.get_kpi_ts_list(metric_name, host_id, kpi.params)
|
||
all_machines_ts.extend(single_machine_ts_list)
|
||
+ logger.info(f"Metric-{metric_name} single group has data {len(all_machines_ts)}. ranks: {ranks}")
|
||
|
||
anomaly_devices = []
|
||
anomaly_locations = {}
|
||
@@ -195,7 +205,7 @@ class SlowNodeDetector(Detector):
|
||
logger.info(
|
||
f"space_nodes_compare result: {self.output_anomaly_devices(metric_name, space_anomaly_locations)}.")
|
||
else:
|
||
- logger.info(f"Skip space nodes compare, due to nodes number{len(all_machines_ts)} is smaller than 4.")
|
||
+ logger.info(f"Skip space nodes compare, due to nodes number {len(all_machines_ts)} is smaller than 4.")
|
||
else:
|
||
logger.info(f"Skip space nodes compare.")
|
||
|
||
@@ -287,10 +297,6 @@ class SlowNodeDetector(Detector):
|
||
look_back = self.config.params.get("look_back", 10)
|
||
metric_type = kpi_params.get("metric_type", "device")
|
||
start, end = dt.last(minutes=look_back)
|
||
- # from datetime import timedelta
|
||
- # # 6:40
|
||
- # start = start - timedelta(hours=4.5)
|
||
- # end = end - timedelta(hours=4.5)
|
||
|
||
if metric_type == "device":
|
||
# npu device
|
||
diff --git a/config/module/slow_node_detection.job.json b/config/module/slow_node_detection.job.json
|
||
index 91ff621..27a6d53 100644
|
||
--- a/config/module/slow_node_detection.job.json
|
||
+++ b/config/module/slow_node_detection.job.json
|
||
@@ -17,7 +17,12 @@
|
||
"obs_size": 5,
|
||
"outlier_ratio_th": 0.6,
|
||
"hccl_domain_json": "./hccl_domain.json",
|
||
- "rank_table_json": "./hccl_domain.json"
|
||
+ "hccl_domain":{
|
||
+ "pp": 2,
|
||
+ "dp": 1,
|
||
+ "tp": 1
|
||
+ },
|
||
+ "rank_table_json": "./rank_table.json"
|
||
}
|
||
},
|
||
"kpis": [
|
||
@@ -349,4 +354,4 @@
|
||
"metric": "gala_gopher_container_cpu_usage_seconds_total"
|
||
}
|
||
]
|
||
-}
|
||
\ No newline at end of file
|
||
+}
|
||
--
|
||
2.33.0
|
||
|