packages/aops-zeus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init 24.03-lts-sp1

Browse Source
This commit is contained in:
rearcher 2024-11-19 15:07:56 +08:00
parent 39171c9105
commit 54775bba07
14 changed files with 164 additions and 979 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
0001-add-interface-for-detecting-host-status.patch
View File

@ -1,251 +0,0 @@
From ee8e8cb1bbc3bb1ba27ee6f0e8acfc663cf10c12 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Tue, 12 Dec 2023 09:47:12 +0800
Subject: [PATCH] Add interface for detecting host status
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/conf/constant.py | 2 +
zeus/database/proxy/host.py | 73 +++++++++++++++++++++++++++++++++++-
zeus/function/verify/host.py | 8 ++++
zeus/host_manager/view.py | 63 +++++++++++++++++++++++++++++++
zeus/url.py | 2 +
5 files changed, 147 insertions(+), 1 deletion(-)
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index 9305604..994dd90 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -42,6 +42,7 @@ ADD_HOST_BATCH = "/manage/host/add/batch"
GET_HOST_TEMPLATE_FILE = "/manage/host/file/template"
DELETE_HOST = "/manage/host/delete"
QUERY_HOST = "/manage/host/get"
+GET_HOST_STATUS = "/manage/host/status/get"
GET_HOST_COUNT = "/manage/host/count"
AUTH_REDIRECT_URL = "/manage/account/authredirecturl"
BIND_AUTH_ACCOUNT = "/manage/account/bindaccount"
@@ -116,3 +117,4 @@ class HostStatus:
ONLINE = 0
OFFLINE = 1
UNESTABLISHED = 2
+ SCANNING = 3
diff --git a/zeus/database/proxy/host.py b/zeus/database/proxy/host.py
index 1656c56..477c482 100644
--- a/zeus/database/proxy/host.py
+++ b/zeus/database/proxy/host.py
@@ -268,7 +268,6 @@ class HostProxy(MysqlProxy):
"host_group_name": host.host_group_name,
"host_ip": host.host_ip,
"management": host.management,
- "status": host.status,
"scene": host.scene,
"os_version": host.os_version,
"ssh_port": host.ssh_port,
@@ -340,6 +339,52 @@ class HostProxy(MysqlProxy):
LOGGER.error("query host %s basic info fail", host_list)
return DATABASE_QUERY_ERROR, result
+ def get_host_ssh_info(self, data):
+ """
+ Get host ssh info according to host id from table
+
+ Args:
+ data(dict): parameter, e.g.
+ {
+ "username": "admin"
+ "host_list": ["id1", "id2"]
+ }
+
+ Returns:
+ int: status code
+ dict: query result
+ """
+ username = data.get('username')
+ host_list = data.get('host_list')
+ result = []
+ query_fields = [
+ Host.host_id,
+ Host.host_ip,
+ Host.ssh_port,
+ Host.pkey,
+ Host.ssh_user,
+ ]
+ filters = {Host.user == username}
+ if host_list:
+ filters.add(Host.host_id.in_(host_list))
+ try:
+ hosts = self.session.query(*query_fields).filter(*filters).all()
+ for host in hosts:
+ host_info = {
+ "host_id": host.host_id,
+ "host_ip": host.host_ip,
+ "ssh_port": host.ssh_port,
+ "pkey": host.pkey,
+ "ssh_user": host.ssh_user,
+ }
+ result.append(host_info)
+ LOGGER.debug("query host %s ssh info succeed", host_list)
+ return SUCCEED, result
+ except sqlalchemy.exc.SQLAlchemyError as error:
+ LOGGER.error(error)
+ LOGGER.error("query host %s ssh info fail", host_list)
+ return DATABASE_QUERY_ERROR, result
+
def get_total_host_info_by_user(self, data):
"""
Get host basic info according to user from table
@@ -775,3 +820,29 @@ class HostProxy(MysqlProxy):
LOGGER.error(error)
self.session.rollback()
return DATABASE_UPDATE_ERROR
+
+
+ def update_host_status(self, host_info: list) -> str:
+ """
+ update host status to host table
+
+ Args:
+ host_info(list): e.g
+ {
+ "host_id": host_id,
+ "status": status
+ }
+
+ Returns:
+ str: SUCCEED or DATABASE_UPDATE_ERROR
+ """
+ try:
+ for host in host_info:
+ self.session.query(Host).filter(Host.host_id == host.get('host_id')).update(
+ {"status": host.get('status')})
+ self.session.commit()
+ return SUCCEED
+ except sqlalchemy.exc.SQLAlchemyError as error:
+ LOGGER.error(error)
+ self.session.rollback()
+ return DATABASE_UPDATE_ERROR
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index d09eedd..f746968 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -60,6 +60,14 @@ class GetHostSchema(Schema):
per_page = fields.Integer(required=False, validate=lambda s: 50 > s > 0)
+class GetHostStatusSchema(Schema):
+ """
+ validators for parameter of /manage/host/getstatus
+ """
+
+ host_list = fields.List(fields.Integer(), required=True)
+
+
class AddHostGroupSchema(Schema):
"""
validators for parameter of /manage/host/add_host_group
diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py
index 10418d1..7ad133d 100644
--- a/zeus/host_manager/view.py
+++ b/zeus/host_manager/view.py
@@ -46,6 +46,7 @@ from zeus.function.verify.host import (
GetHostGroupSchema,
GetHostInfoSchema,
GetHostSchema,
+ GetHostStatusSchema,
UpdateHostSchema,
)
from zeus.host_manager.ssh import SSH, execute_command_and_parse_its_result, generate_key
@@ -118,6 +119,68 @@ class GetHostCount(BaseResponse):
return self.response(code=status_code, data=result)
+class GetHostStatus(BaseResponse):
+ """
+ Interface for get host status.
+ Restful API: POST
+ """
+
+ @BaseResponse.handle(schema=GetHostStatusSchema, proxy=HostProxy)
+ def post(self, callback: HostProxy, **params):
+ """
+ get host status
+
+ Args:
+ host_list (list): host id list
+ username: "admin"
+
+ Returns:
+ list: response body
+ """
+ status_code, host_infos = callback.get_host_ssh_info(params)
+
+ multi_thread_handler = MultiThreadHandler(lambda p: self.get_host_status(p), host_infos, None)
+ multi_thread_handler.create_thread()
+ result_list = multi_thread_handler.get_result()
+
+ callback.update_host_status(result_list)
+
+ return self.response(code=status_code, data=result_list)
+
+ @staticmethod
+ def get_host_status(host: dict) -> dict:
+ """
+ Get host status
+
+ Args:
+ host (dict): e.g
+ {
+ "host_id":"host id",
+ "ssh_user":"root",
+ "pkey":"pkey",
+ "host_ip":"host_ip",
+ "ssh_port":"port"
+ }
+
+ Returns:
+ """
+ status = verify_ssh_login_info(
+ ClientConnectArgs(
+ host.get("host_ip"), host.get("ssh_port"), host.get("ssh_user"), host.get("pkey")
+ )
+ )
+ if status == state.SUCCEED:
+ if status != HostStatus.SCANNING:
+ host['status'] = HostStatus.ONLINE
+ elif status == state.SSH_AUTHENTICATION_ERROR:
+ host['status'] = HostStatus.UNESTABLISHED
+ else:
+ host['status'] = HostStatus.OFFLINE
+
+ result = {"host_id": host.get("host_id"), "status": host.get("status")}
+ return result
+
+
class AddHostGroup(BaseResponse):
"""
Interface for add host group.
diff --git a/zeus/url.py b/zeus/url.py
index eb8a189..ad8cec9 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -52,6 +52,7 @@ from zeus.conf.constant import (
USER_LOGIN,
SYNC_CONFIG,
OBJECT_FILE_CONFIG,
+ GET_HOST_STATUS,
)
from zeus.config_manager import view as config_view
from zeus.host_manager import view as host_view
@@ -77,6 +78,7 @@ SPECIFIC_URLS = {
(host_view.DeleteHost, DELETE_HOST),
(host_view.UpdateHost, UPDATE_HOST),
(host_view.GetHost, QUERY_HOST),
+ (host_view.GetHostStatus, GET_HOST_STATUS),
(host_view.GetHostInfo, QUERY_HOST_DETAIL),
(host_view.GetHostCount, GET_HOST_COUNT),
(host_view.GetHostTemplateFile, GET_HOST_TEMPLATE_FILE),
--
2.33.0

77
0002-update-the-query-host-list-api.patch
View File

@ -1,77 +0,0 @@
From c10d1ff7ad3b74886911b719f50f4775120db789 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Thu, 14 Dec 2023 19:58:19 +0800
Subject: [PATCH] add a new query method based on host name for the host list query interface.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
...6\216\245\345\217\243\346\226\207\346\241\243.yaml" | 4 ++++
zeus/database/proxy/host.py | 10 +++++++++-
zeus/function/verify/host.py | 1 +
3 files changed, 14 insertions(+), 1 deletion(-)
diff --git "a/doc/design/aops-zeus\346\216\245\345\217\243\346\226\207\346\241\243.yaml" "b/doc/design/aops-zeus\346\216\245\345\217\243\346\226\207\346\241\243.yaml"
index efadcc6..87dfe68 100644
--- "a/doc/design/aops-zeus\346\216\245\345\217\243\346\226\207\346\241\243.yaml"
+++ "b/doc/design/aops-zeus\346\216\245\345\217\243\346\226\207\346\241\243.yaml"
@@ -998,6 +998,10 @@ definitions:
type: string
description: 获取指定主机组里的主机信息，为空表示所有
example: '[]'
+ search_key:
+ type: string
+ description: 输入主机名称或主机host_ip获取指定主机信息
+ example: search_key
management:
type: boolean
description: 管理节点or监控节点，不传表示所有
diff --git a/zeus/database/proxy/host.py b/zeus/database/proxy/host.py
index 477c482..441ef21 100644
--- a/zeus/database/proxy/host.py
+++ b/zeus/database/proxy/host.py
@@ -19,7 +19,7 @@ import math
from typing import Dict, List, Tuple
import sqlalchemy
-from sqlalchemy import func
+from sqlalchemy import func, or_
from sqlalchemy.sql.expression import asc, desc
from sqlalchemy.orm.collections import InstrumentedList
@@ -210,11 +210,19 @@ class HostProxy(MysqlProxy):
username = data['username']
host_group_list = data.get('host_group_list')
management = data.get('management')
+ search_key = data.get('search_key')
filters = {Host.user == username}
if host_group_list:
filters.add(Host.host_group_name.in_(host_group_list))
if management is not None:
filters.add(Host.management == management)
+ if search_key:
+ filters.add(
+ or_(
+ Host.host_name.like("%" + search_key + "%"),
+ Host.host_ip.like("%" + search_key + "%"),
+ )
+ )
if data.get('status'):
filters.add(Host.status.in_(data.get('status')))
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index f746968..3f8bab9 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -52,6 +52,7 @@ class GetHostSchema(Schema):
"""
host_group_list = fields.List(fields.String(), required=True)
+ search_key = fields.String(required=False, validate=lambda s: 50 > len(s) > 0)
management = fields.Boolean(required=False)
status = fields.List(fields.Integer(validate=lambda s: s >= 0), required=False)
sort = fields.String(required=False, validate=validate.OneOf(["host_name", "host_group_name", ""]))
--
2.33.0

25
0003-fix-search_key-validate.patch
View File

@ -1,25 +0,0 @@
From 06c0ffd136892bca685dfa036905ebc0ef46cf27 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Fri, 15 Dec 2023 14:53:40 +0800
Subject: [PATCH] fix search_key validate
---
zeus/function/verify/host.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index 3f8bab9..461fc12 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -52,7 +52,7 @@ class GetHostSchema(Schema):
"""
host_group_list = fields.List(fields.String(), required=True)
- search_key = fields.String(required=False, validate=lambda s: 50 > len(s) > 0)
+ search_key = fields.String(required=False, validate=lambda s: 50 >= len(s) > 0)
management = fields.Boolean(required=False)
status = fields.List(fields.Integer(validate=lambda s: s >= 0), required=False)
sort = fields.String(required=False, validate=validate.OneOf(["host_name", "host_group_name", ""]))
--
2.33.0

250
0004-add-rollback-task-execution-method.patch
View File

@ -1,250 +0,0 @@
From e7e9871111a67d1aee5b7a7d68029b13894f8fae Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Wed, 13 Dec 2023 10:11:22 +0800
Subject: [PATCH] add rollback task execution method and fix cve scan
callback error
---
zeus/conf/constant.py | 1 +
zeus/function/verify/vulnerability.py | 18 ++--
zeus/url.py | 1 +
zeus/vulnerability_manage/view.py | 135 +++++++++++++++++++++++++-
4 files changed, 143 insertions(+), 12 deletions(-)
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index 994dd90..1370d6e 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -32,6 +32,7 @@ CERES_HOST_INFO = "aops-ceres collect --host '%s'"
CERES_CVE_REPO_SET = "aops-ceres apollo --set-repo '%s'"
CERES_CVE_SCAN = "aops-ceres apollo --scan '%s'"
CERES_CVE_FIX = "aops-ceres apollo --fix '%s'"
+CERES_CVE_ROLLBACK = "aops-ceres apollo --rollback '%s'"
CERES_HOTPATCH_REMOVE = "aops-ceres apollo --remove-hotpatch '%s'"
CERES_SYNC_CONF = "aops-ceres sync --conf '%s'"
CERES_OBJECT_FILE_CONF = "aops-ceres ragdoll --list '%s'"
diff --git a/zeus/function/verify/vulnerability.py b/zeus/function/verify/vulnerability.py
index 07875e0..ff25c8d 100644
--- a/zeus/function/verify/vulnerability.py
+++ b/zeus/function/verify/vulnerability.py
@@ -86,14 +86,12 @@ class CveFixSchema(TaskGeneralSchema):
fix_type = fields.String(validate=validate.OneOf(["hotpatch", "coldpatch"]), required=True)
-class CveRollbackSingleInfoSchema(Schema):
- cve_id = fields.String(validate=lambda s: len(s) > 0)
- hotpatch = fields.Boolean(validate=validate.OneOf([True, False]))
-
-
-class CveRollbackTask(Schema):
+class CveRollbackTaskSchema(Schema):
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
- cves = fields.List(fields.Nested(CveRollbackSingleInfoSchema()), required=True)
+ installed_rpm = fields.String(required=True, validate=lambda s: 100 >= len(s) > 0)
+ target_rpm = fields.String(required=True, validate=lambda s: 100 >= len(s) > 0)
+ dnf_event_start = fields.Integer(allow_none=True, required=True, validate=lambda s: s > 0)
+ dnf_event_end = fields.Integer(allow_none=True, required=True, validate=lambda s: s > 0)
class CveRollbackSchema(TaskGeneralSchema):
@@ -101,10 +99,12 @@ class CveRollbackSchema(TaskGeneralSchema):
validator for cve rollback
"""
- tasks = fields.List(fields.Nested(CveRollbackTask()), required=True, validate=lambda s: len(s) > 0)
+ tasks = fields.List(fields.Nested(CveRollbackTaskSchema()), required=True, validate=lambda s: len(s) > 0)
+ fix_task_id = fields.String(required=True, validate=lambda s: len(s) > 0)
+ rollback_type = fields.String(validate=validate.OneOf(["hotpatch", "coldpatch"]), required=True)
class Meta:
- fields = ("tasks", "task_id", "task_name", "total_hosts", "task_type", "callback")
+ exclude = ("total_hosts",)
class HotpatchRemoveTask(Schema):
diff --git a/zeus/url.py b/zeus/url.py
index ad8cec9..5f00ef9 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -101,6 +101,7 @@ SPECIFIC_URLS = {
],
'CVE_URLS': [
(vulnerability_view.ExecuteRepoSetTask, EXECUTE_REPO_SET),
+ (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK),
(vulnerability_view.ExecuteCveScanTask, EXECUTE_CVE_SCAN),
(vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX),
(vulnerability_view.ExecuteHotpatchRemoveTask, EXECUTE_HOTPATCH_REMOVE),
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index be52e23..37ab633 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -26,6 +26,7 @@ from zeus.conf import configuration
from zeus.conf.constant import (
CERES_CVE_FIX,
CERES_CVE_REPO_SET,
+ CERES_CVE_ROLLBACK,
CERES_HOTPATCH_REMOVE,
CERES_CVE_SCAN,
CveTaskStatus,
@@ -34,7 +35,13 @@ from zeus.conf.constant import (
from zeus.database.proxy.host import HostProxy
from zeus.database.table import Host
from zeus.function.model import ClientConnectArgs
-from zeus.function.verify.vulnerability import CveFixSchema, CveScanSchema, HotpatchRemoveSchema, RepoSetSchema
+from zeus.function.verify.vulnerability import (
+ CveFixSchema,
+ CveRollbackSchema,
+ CveScanSchema,
+ HotpatchRemoveSchema,
+ RepoSetSchema,
+)
from zeus.host_manager.ssh import execute_command_and_parse_its_result
@@ -283,7 +290,7 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask):
CERES_CVE_SCAN % json.dumps({"check_items": self._check_items}),
)
if status != state.SUCCEED:
- request_body["status"] = CveTaskStatus.FAIL
+ request_body.update({"status":CveTaskStatus.FAIL, "reboot":False})
else:
request_body.update(json.loads(cve_scan_result))
@@ -500,7 +507,7 @@ class ExecuteHotpatchRemoveTask(BaseResponse, BaseExcuteTask):
),
command,
)
-
+
if status == state.SUCCEED:
request_body.update(json.loads(hotpatch_remove_result))
else:
@@ -552,3 +559,125 @@ class ExecuteHotpatchRemoveTask(BaseResponse, BaseExcuteTask):
]
threading.Thread(target=lambda: gevent.joinall(wait_execute_tasks)).start()
return self.response(code=state.SUCCEED)
+
+
+class ExecuteCveRollbackTask(BaseResponse, BaseExcuteTask):
+ """
+ Interface for cve rollback.
+ Restful API: POST
+ """
+
+ def _execute_task(self, host_info: dict, task_info: dict) -> None:
+ """
+ Execute cve rollback task
+
+ Args:
+ host_info(dict): e.g
+ {
+ "host_id": 1,
+ "host_ip": "127.0.0.1",
+ "host_name": "test_host",
+ "ssh_port": 22,
+ "ssh_user": "root",
+ "pkey": "RSA-KEY-string",
+ }
+ task_info (dict): e.g
+ {
+ "host_id": "id1",
+ "check_items":[],
+ "rollback_type": "hotpatch",
+ "installed_kernel": "kernel-5.1.10",
+ "target_kernel": "kernel-5.1.9",
+ "dnf_event_start": 1,
+ "dnf_event_end": 2,
+ }
+ Returns:
+ None
+ """
+ request_body = {
+ "execution_time": int(time.time()),
+ "task_id": self._task_id,
+ "host_id": host_info.get("host_id"),
+ "host_ip": host_info.get("host_ip"),
+ "host_name": host_info.get("host_name"),
+ }
+
+ task_info.pop("host_id")
+ command = CERES_CVE_ROLLBACK % json.dumps(task_info)
+ status, cve_rollback_result = execute_command_and_parse_its_result(
+ ClientConnectArgs(
+ host_info.get("host_ip"),
+ host_info.get("ssh_port"),
+ host_info.get("ssh_user"),
+ host_info.get("pkey"),
+ 60 * 10,
+ ),
+ command,
+ )
+ if status != state.SUCCEED:
+ request_body.update(
+ {
+ "status": CveTaskStatus.FAIL,
+ "log": cve_rollback_result,
+ "check_items": [
+ {"item": item, "result": CveTaskStatus.FAIL} for item in task_info.get("check_items")
+ ],
+ }
+ )
+ else:
+ request_body.update(json.loads(cve_rollback_result))
+
+ url = f'http://{configuration.apollo.get("IP")}:{ configuration.apollo.get("PORT")}{self._callback_url}'
+ self.get_response("post", url, request_body, self._header, timeout=10)
+
+ @BaseResponse.handle(schema=CveRollbackSchema)
+ def post(self, **params) -> Response:
+ """
+ execute cve rollback task
+
+ Args:
+ params (dict): e.g
+ {
+ "task_id": "c6714973c9b342a380fd01fdf7f90ef5",
+ "task_name": "cve rollback task",
+ "fix_task_id": "string",
+ "task_type": "cve rollback",
+ "rollback_type": "coldpatch",
+ "check_items": ["network"],
+ "tasks": [
+ {
+ "host_id": 74,
+ "installed_rpm": "kernel-5.1.10",
+ "target_rpm": "kernel-5.1.9",
+ "dnf_event_start": 1,
+ "dnf_event_end": 2
+ }
+ ],
+ "callback": "/vulnerability/task/callback/cve/rollback"
+ }
+ Returns:
+ response body
+ """
+ total_host = [task_info["host_id"] for task_info in params.get("tasks")]
+ status_code, host_infos = query_host_basic_info(total_host, params.get('username'))
+ if status_code != state.SUCCEED:
+ return self.response(code=status_code)
+ # parse args
+ self._task_id = params.get("task_id")
+ self._task_name = params.get("task_name")
+ self._task_type = params.get("task_type")
+ self._header["local_account"] = params.get("username")
+ self._callback_url = params.get('callback')
+ # Execute task
+ tasks = generate_tasks(
+ params.get('tasks'),
+ host_infos,
+ **{
+ "check_items": params.get('check_items'),
+ "rollback_type": params.get('rollback_type'),
+ },
+ )
+ threading.Thread(
+ target=lambda: gevent.joinall([gevent.spawn(self._execute_task, *task) for task in tasks])
+ ).start()
+ return self.response(code=state.SUCCEED)
--
2.33.0

27
0005-fix-apollo-TimedCorrectTask.patch
View File

@ -1,27 +0,0 @@
From cb3af79a8237c6b7e083dc8ba7d324bddf395e08 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Tue, 19 Dec 2023 10:59:30 +0800
Subject: [PATCH] fix apollo TimedCorrectTask
---
zeus/database/proxy/host.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/zeus/database/proxy/host.py b/zeus/database/proxy/host.py
index 441ef21..471390e 100644
--- a/zeus/database/proxy/host.py
+++ b/zeus/database/proxy/host.py
@@ -372,7 +372,9 @@ class HostProxy(MysqlProxy):
Host.pkey,
Host.ssh_user,
]
- filters = {Host.user == username}
+ filters = set()
+ if username:
+ filters = {Host.user == username}
if host_list:
filters.add(Host.host_id.in_(host_list))
try:
--
2.33.0

70
0006-update-verification-method-for-adding-host.patch
View File

@ -1,70 +0,0 @@
From 82cd9883bbf5fc95ca1bd38c36a8a2066aeaa4a1 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Tue, 19 Dec 2023 11:02:31 +0800
Subject: [PATCH] update verification method for adding host or updating host
info
---
zeus/function/verify/host.py | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index 461fc12..310373c 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -16,9 +16,7 @@ Author:
Description: For host related interfaces
"""
from vulcanus.restful.serialize.validate import ValidateRules
-from marshmallow import Schema
-from marshmallow import fields
-from marshmallow import validate
+from marshmallow import fields, Schema, validate, validates_schema, ValidationError
class HostSchema(Schema):
@@ -111,7 +109,7 @@ class AddHostSchema(Schema):
validators for parameter of /manage/host/add
"""
- ssh_user = fields.String(required=True, validate=lambda s: len(s) > 0)
+ ssh_user = fields.String(required=True, validate=lambda s: 32 >= len(s) > 0)
password = fields.String(required=True, allow_none=True, validate=lambda s: len(s) >= 0)
host_name = fields.String(
required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
@@ -119,8 +117,13 @@ class AddHostSchema(Schema):
host_ip = fields.IP(required=True)
ssh_pkey = fields.String(required=True, allow_none=True, validate=lambda s: 4096 >= len(s) >= 0)
ssh_port = fields.Integer(required=True, validate=lambda s: 65535 >= s > 0)
- host_group_name = fields.String(required=True, validate=lambda s: len(s) > 0)
- management = fields.Boolean(required=True)
+ host_group_name = fields.String(required=True, validate=lambda s: 20 >= len(s) > 0)
+ management = fields.Boolean(required=True, truthy={True}, falsy={False})
+
+ @validates_schema
+ def check_authentication_info(self, data, **kwargs):
+ if not data.get("ssh_pkey") and not data.get("password"):
+ raise ValidationError("At least one of the password and key needs to be provided")
class AddHostBatchSchema(Schema):
@@ -137,10 +140,12 @@ class UpdateHostSchema(Schema):
"""
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
- ssh_user = fields.String(required=False, validate=lambda s: len(s) > 0)
+ ssh_user = fields.String(required=False, validate=lambda s: 32 >= len(s) > 0)
password = fields.String(required=False, validate=lambda s: len(s) > 0)
ssh_port = fields.Integer(required=False, validate=lambda s: 65535 >= s > 0)
- host_name = fields.String(required=False, validate=lambda s: len(s) > 0)
- host_group_name = fields.String(required=False, validate=lambda s: len(s) > 0)
- management = fields.Boolean(required=False)
+ host_name = fields.String(
+ required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
+ )
+ host_group_name = fields.String(required=False, validate=lambda s: 20 >= len(s) > 0)
+ management = fields.Boolean(required=False, truthy={True}, falsy={False})
ssh_pkey = fields.String(required=False, validate=lambda s: 4096 >= len(s) >= 0)
--
2.33.0

34
0007-update-verification-method-for-host-ip-field.patch
View File

@ -1,34 +0,0 @@
From 4dcbd5294f781e71d609036b75922fcb09b469c9 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Wed, 20 Dec 2023 15:09:46 +0800
Subject: [PATCH] update verification method for host ip field
---
zeus/function/verify/host.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index 310373c..7dedfee 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -114,7 +114,7 @@ class AddHostSchema(Schema):
host_name = fields.String(
required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
)
- host_ip = fields.IP(required=True)
+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check)
ssh_pkey = fields.String(required=True, allow_none=True, validate=lambda s: 4096 >= len(s) >= 0)
ssh_port = fields.Integer(required=True, validate=lambda s: 65535 >= s > 0)
host_group_name = fields.String(required=True, validate=lambda s: 20 >= len(s) > 0)
@@ -144,7 +144,7 @@ class UpdateHostSchema(Schema):
password = fields.String(required=False, validate=lambda s: len(s) > 0)
ssh_port = fields.Integer(required=False, validate=lambda s: 65535 >= s > 0)
host_name = fields.String(
- required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
+ required=False, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
)
host_group_name = fields.String(required=False, validate=lambda s: 20 >= len(s) > 0)
management = fields.Boolean(required=False, truthy={True}, falsy={False})
--
2.33.0

111
0008-check-host-status-when-query-host-detail.patch
View File

@ -1,111 +0,0 @@
From 62e90ee407ab0f28c47fcd51fe8f1078810e7c94 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Thu, 21 Dec 2023 10:15:07 +0800
Subject: [PATCH] check host status when query host detail
---
zeus/host_manager/view.py | 69 ++++++++-------------------------------
1 file changed, 13 insertions(+), 56 deletions(-)
diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py
index 6b31d35..30d05a3 100644
--- a/zeus/host_manager/view.py
+++ b/zeus/host_manager/view.py
@@ -265,6 +265,15 @@ class GetHostInfo(BaseResponse):
)
if status == state.SUCCEED:
res["host_info"] = json.loads(host_info)
+
+ # check host status
+ if status == state.SSH_AUTHENTICATION_ERROR:
+ res['status'] = HostStatus.UNESTABLISHED
+ elif status == state.SSH_CONNECTION_ERROR:
+ res['status'] = HostStatus.OFFLINE
+ elif host['status'] != HostStatus.SCANNING:
+ res['status'] = HostStatus.ONLINE
+
return res
@staticmethod
@@ -282,63 +291,12 @@ class GetHostInfo(BaseResponse):
{
"host_id": host_id,
"host_info":{}
+ "status": null
}
...
]
"""
- return [{"host_id": host_id, "host_info": {}} for host_id in host_list]
-
- def analyse_query_result(self, all_host: List[str], multithreading_execute_result: List) -> List:
- """
- Analyze multi-threaded execution results,
- find out the data which fails to execute,
- and generate the final execution result.
- Args:
- all_host(list): e.g
- [host_id1, host_id2... ]
- multithreading_execute_result(list): e.g
- [
- {
- "host_id":"success host id",
- "host_info": {
- "cpu": {...},
- "os":" {...},
- "memory": {...}.
- "disk": [{...}]
- },
- }
- ]
-
- Returns:
- list: e.g
- [
- {
- "host_id":"success host id",
- "host_info": {
- "cpu": {...},
- "os":" {...},
- "memory": {...}.
- "disk": [{...}]
- },
- }.
- {
- "host_id":"fail host id",
- "host_info": {}
- }.
- ]
-
-
- """
- host_infos = []
- success_host = set()
- for result in multithreading_execute_result:
- if result.get('host_info'):
- host_infos.append(result)
- success_host.add(result.get('host_id'))
-
- fail_host = set(all_host) - success_host
- host_infos.extend(self.generate_fail_data(fail_host))
- return host_infos
+ return [{"host_id": host_id, "host_info": {}, "status": None} for host_id in host_list]
@BaseResponse.handle(schema=GetHostInfoSchema, proxy=HostProxy)
def post(self, callback: HostProxy, **params):
@@ -369,10 +327,9 @@ class GetHostInfo(BaseResponse):
# execute multi threading
multi_thread_handler = MultiThreadHandler(lambda p: self.get_host_info(*p), tasks, None)
multi_thread_handler.create_thread()
- result_list = multi_thread_handler.get_result()
+ host_infos = multi_thread_handler.get_result()
- # analyse execute result and generate target data format
- host_infos = self.analyse_query_result(params.get('host_list'), result_list)
+ callback.update_host_status(host_infos)
return self.response(code=state.SUCCEED, data={"host_infos": host_infos})
--
2.33.0

47
0009-fix-error-log-when-query-host-status.patch
View File

@ -1,47 +0,0 @@
From eaf05c0588e595d2f635c6bae867db5f15c3b034 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Sun, 24 Dec 2023 21:01:19 +0800
Subject: [PATCH] fix log error
---
zeus/host_manager/view.py | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py
index 30d05a3..d13868c 100644
--- a/zeus/host_manager/view.py
+++ b/zeus/host_manager/view.py
@@ -139,6 +139,10 @@ class GetHostStatus(BaseResponse):
"""
status_code, host_infos = callback.get_host_ssh_info(params)
+ result_list = []
+ if len(host_infos) == 0:
+ return self.response(code=status_code, data=result_list)
+
multi_thread_handler = MultiThreadHandler(lambda p: self.get_host_status(p), host_infos, None)
multi_thread_handler.create_thread()
result_list = multi_thread_handler.get_result()
@@ -457,13 +461,16 @@ def verify_ssh_login_info(ssh_login_info: ClientConnectArgs) -> str:
)
client.close()
except socket.error as error:
- LOGGER.error(error)
+ LOGGER.info(f"Failed to connect to host %s: %s", ssh_login_info.host_ip, error)
return state.SSH_CONNECTION_ERROR
except SSHException as error:
- LOGGER.error(error)
+ LOGGER.info(f"Failed to connect to host %s: %s", ssh_login_info.host_ip, error)
+ return state.SSH_AUTHENTICATION_ERROR
+ except IndexError:
+ LOGGER.error(f"Failed to connect to host %s because the pkey of the host are missing", ssh_login_info.host_ip)
return state.SSH_AUTHENTICATION_ERROR
except Exception as error:
- LOGGER.error(error)
+ LOGGER.error(f"Failed to connect to host %s: %s", ssh_login_info.host_ip, error)
return state.SSH_CONNECTION_ERROR
return state.SUCCEED
--
Gitee

33
0010-update-the-exception-catching-type-of-the-function.patch
View File

@ -1,33 +0,0 @@
From 90076e3a777576a482f37db6f67331ffcd2783fb Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Wed, 27 Dec 2023 10:35:35 +0800
Subject: [PATCH] update the exception catching type of the function
---
zeus/vulnerability_manage/view.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index 37ab633..0a3537a 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -19,6 +19,7 @@ from flask import Response, request
import sqlalchemy
import gevent
+from vulcanus.exceptions import DatabaseConnectionFailed
from vulcanus.log.log import LOGGER
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
@@ -70,7 +71,7 @@ def query_host_basic_info(host_list: list, username: str) -> Tuple[str, Dict]:
try:
with HostProxy() as proxy:
status_code, host_infos = proxy.get_host_info({"host_list": host_list, "username": username})
- except sqlalchemy.exc.SQLAlchemyError:
+ except DatabaseConnectionFailed:
LOGGER.error("Connect to database error")
return state.DATABASE_CONNECT_ERROR, dict()
--
2.33.0

27
0011-fix-command-injection-vulnerabilities.patch
View File

@ -1,27 +0,0 @@
From 1b2b79f2f3027be1a6d9280b5c091f3a18c5be18 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Thu, 7 Mar 2024 09:19:00 +0800
Subject: [PATCH 1/1] fix command injection vulnerabilities
---
zeus/conf/constant.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index 1370d6e..167d6c0 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -22,8 +22,8 @@ from vulcanus.conf.constant import BASE_CONFIG_PATH
MANAGER_CONFIG_PATH = os.path.join(BASE_CONFIG_PATH, 'zeus.ini')
# ceres
-CERES_PLUGIN_START = "aops-ceres plugin --start %s"
-CERES_PLUGIN_STOP = "aops-ceres plugin --stop %s"
+CERES_PLUGIN_START = "aops-ceres plugin --start '%s'"
+CERES_PLUGIN_STOP = "aops-ceres plugin --stop '%s'"
CERES_COLLECT_ITEMS_CHANGE = "aops-ceres plugin --change-collect-items '%s'"
CERES_PLUGIN_INFO = "aops-ceres plugin --info"
CERES_APPLICATION_INFO = "aops-ceres collect --application"
--
2.33.0

BIN
aops-zeus-v1.4.0.tar.gz
View File

Binary file not shown.

BIN
aops-zeus-v2.1.0.tar.gz Normal file
View File

Binary file not shown.

191
aops-zeus.spec
View File

@ -1,61 +1,198 @@
%define vulcanus_version v2.0.0
Name: aops-zeus
Version: v1.4.0
Release: 7
Summary: A host and user manager service which is the foundation of aops.
Version: v2.1.0
Release: 2
Summary: A service which is the foundation of aops.
License: MulanPSL2
URL: https://gitee.com/openeuler/%{name}
Source0: %{name}-%{version}.tar.gz
Patch0001: 0001-add-interface-for-detecting-host-status.patch
Patch0002: 0002-update-the-query-host-list-api.patch
Patch0003: 0003-fix-search_key-validate.patch
Patch0004: 0004-add-rollback-task-execution-method.patch
Patch0005: 0005-fix-apollo-TimedCorrectTask.patch
Patch0006: 0006-update-verification-method-for-adding-host.patch
Patch0007: 0007-update-verification-method-for-host-ip-field.patch
Patch0008: 0008-check-host-status-when-query-host-detail.patch
Patch0009: 0009-fix-error-log-when-query-host-status.patch
Patch0010: 0010-update-the-exception-catching-type-of-the-function.patch
Patch0011: 0011-fix-command-injection-vulnerabilities.patch
BuildRequires: python3-setuptools
Requires: aops-vulcanus >= v1.3.0
Requires: python3-marshmallow >= 3.13.0 python3-flask python3-flask-restful python3-gevent
Requires: python3-requests python3-uWSGI python3-sqlalchemy python3-werkzeug python3-PyMySQL
Requires: python3-paramiko >= 2.11.0 python3-redis python3-prometheus-api-client
Provides: aops-zeus
Conflicts: aops-manager
Requires: python3-pyyaml python3-PyMySQL python3-kazoo python3-click
%description
A host and user manager service which is the foundation of aops.
Provide one-click aops deployment, service start and stop, hot loading of
configuration files, and database initialization.
Provides: aops-zeus
%package -n zeus-host-information
Summary: A host manager service which is the foundation of aops.
Requires: aops-vulcanus >= %{vulcanus_version} aops-zeus >= %{version}
Requires: python3-gevent python3-uWSGI python3-paramiko
%description -n zeus-host-information
A host manager service which is the foundation of aops.
%package -n zeus-user-access
Summary: A user manager service which is the foundation of aops.
Requires: aops-vulcanus >= %{vulcanus_version} aops-zeus >= %{version}
Requires: python3-celery python3-uWSGI
%description -n zeus-user-access
A user manager service which is the foundation of aops.
%package -n async-task
Summary: A async task of aops.
Requires: aops-vulcanus >= %{vulcanus_version} python3-celery python3-paramiko
%description -n async-task
A async task of aops.
%package -n zeus-operation
Summary: A operation manager service which is the foundation of aops.
Requires: aops-vulcanus >= %{vulcanus_version} aops-zeus >= %{version}
Requires: python3-gevent python3-uWSGI python3-paramiko
%description -n zeus-operation
A operation manager of aops.
%package -n zeus-distribute
Summary: A distributed service of aops.
Requires: aops-vulcanus >= %{vulcanus_version} aops-zeus >= %{version}
Requires: python3-uWSGI python3-gevent
%description -n zeus-distribute
A distributed service of aops.
%prep
%autosetup -n %{name}-%{version} -p1
%autosetup -n %{name}-%{version}
# build for aops-zeus
%py3_build
# build for zeus-host-information
pushd host-information-service
%py3_build
popd
# build for zeus-operation
pushd operation-service
%py3_build
popd
# build for zeus-user-access
pushd user-access-service
%py3_build
popd
# build for async-task
pushd async-task
%py3_build
popd
# build for zeus-distribute
pushd distribute-service
%py3_build
popd
# install for aops-zeus
%py3_install
mkdir -p %{buildroot}/opt/aops/
cp -r database %{buildroot}/opt/aops/
# install for zeus-host-information
pushd host-information-service
%py3_install
mkdir -p %{buildroot}/opt/aops/database/
cp zeus/host_information_service/database/*.sql %{buildroot}/opt/aops/database/
popd
# install for zeus-operation
pushd operation-service
%py3_install
mkdir -p %{buildroot}/opt/aops/database/
cp zeus/operation_service/database/*.sql %{buildroot}/opt/aops/database/
popd
# install for zeus-user-access
pushd user-access-service
%py3_install
mkdir -p %{buildroot}/opt/aops/database/
cp zeus/user_access_service/database/*.sql %{buildroot}/opt/aops/database/
popd
# install for async-task
pushd async-task
%py3_install
mkdir -p %{buildroot}/opt/aops/celery
popd
# install for zeus-distribute
pushd distribute-service
%py3_install
popd
%files
%doc README.*
%attr(0644,root,root) %{_sysconfdir}/aops/zeus.ini
%attr(0755,root,root) %{_bindir}/aops-zeus
%attr(0755,root,root) /usr/lib/systemd/system/aops-zeus.service
%{python3_sitelib}/aops_zeus*.egg-info
%{python3_sitelib}/zeus/*
%attr(0755,root,root) %{_bindir}/aops-cli
%files -n zeus-host-information
%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/zeus-host-information.yml
%attr(0755,root,root) %{_unitdir}/zeus-host-information.service
%{python3_sitelib}/zeus_host_information*.egg-info/*
%{python3_sitelib}/zeus/host_information_service/*
%attr(0755, root, root) /opt/aops/database/*
%files -n zeus-operation
%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/zeus-operation.yml
%attr(0755,root,root) %{_unitdir}/zeus-operation.service
%{python3_sitelib}/zeus_operation*.egg-info/*
%{python3_sitelib}/zeus/operation_service/*
%attr(0755, root, root) /opt/aops/database/*
%files -n zeus-user-access
%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/zeus-user-access.yml
%attr(0755,root,root) %{_unitdir}/zeus-user-access.service
%{python3_sitelib}/zeus_user_access*.egg-info/*
%{python3_sitelib}/zeus/user_access_service/*
%attr(0755, root, root) /opt/aops/database/*
%files -n async-task
%attr(0644,root,root) %{_sysconfdir}/aops/crontab.yml
%attr(0644,root,root) %{_sysconfdir}/aops/sync-conf.d/instance.properties
%attr(0644,root,root) %{_sysconfdir}/aops/sync-conf.d/rdb/*
%attr(0755,root,root) %{_unitdir}/async-task.service
%{python3_sitelib}/async_task*.egg-info/*
%{python3_sitelib}/async_task/*
%attr(0755,root,root) %{_bindir}/async-task
%dir %attr(0644,root,root) /opt/aops/celery
%files -n zeus-distribute
%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/zeus-distribute.yml
%attr(0755,root,root) %{_unitdir}/zeus-distribute.service
%{python3_sitelib}/zeus_distribute*.egg-info/*
%{python3_sitelib}/zeus/distribute_service/*
%changelog
* Tue Nov 19 2024 luxuexian<luxuexian@huawei.com> - v2.1.0-2
- support osmind operation
* Mon Sep 9 2024 gongzhengtang<gong_zhengtang@163.com> - v2.1.0-1
- Support authhub local authentication
* Fri Aug 23 2024 wenxin<wenxin32@foxmail.com> - v2.0.0-2
- updated download host template api to support english
* Fri Aug 16 2024 wenxin<wenxin32@foxmail.com> - v2.0.0-1
- Split the existing service module into distinct modules:
- Task Module: Handles task management and execution.
- Host Module: Manages host configurations and interactions.
- User Module: Manages user accounts and permissions.
- Interface Dispatch Module: Manages the dispatching of requests to appropriate interfaces.
- Added a new command line tool for enhanced functionality and management.
- Introduced support for clustering, allowing for more robust and scalable deployments.
- Reorganized the service architecture to improve modularity and maintainability.
- Fixed various issues to enhance stability and performance.
* Mon Jul 01 2024 smjiao<smjiao@isoftstone.com> - v1.4.0-9
- file trace interface
* Mon Jul 01 2024 smjiao<smjiao@isoftstone.com> - v1.4.0-8
- conf trace sync interface optimization
* Thu Mar 07 2024 wenxin<wenxin32@foxmail.com> - v1.4.0-7
- fix command injection vulnerabilities