packages/aops-apollo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init 24.03-lts-sp1,set-uwsgi-buffer-size to 32k

Browse Source
This commit is contained in:
rearcher 2024-11-19 18:45:40 +08:00
parent 3e88459a83
commit e474859496
13 changed files with 406 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
0001-fix-cve-list-sort.patch
View File

@ -1,30 +0,0 @@
From b87c51761b3143aabd39a3e9c91596e540325172 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Mon, 18 Dec 2023 22:24:51 +0800
Subject: [PATCH] fix cve_list sort
---
apollo/database/proxy/cve.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index e92cb14..6210156 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -669,10 +669,11 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
sort_page['limt_size'] = int(per_page)
# sort by host num by default
+ sort_page["order_by"] = data.get("direction", "asc")
sort_page["order_by_filed"] = data.get('sort', "cve_id")
if sort_page["order_by_filed"] == "host_num":
- sort_page["order_by_filed"] = "cve_id, host_num"
- sort_page["order_by"] = data.get("direction", "asc")
+ sort_page["order_by_filed"] = f"host_num {sort_page['order_by']},cve_id "
+
return sort_page
def _query_cve_list(self, data):
--
Gitee

85
0001-fix-security-advisory-parsing-error.patch Normal file
View File

@ -0,0 +1,85 @@
From 25e79499a5c578579c1112bfcbdfb7137748fa84 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Fri, 19 Jul 2024 10:28:52 +0800
Subject: [PATCH 1/1] fix security advisory parsing error
---
apollo/cron/download_advisory.py | 6 ++++--
apollo/handler/cve_handler/view.py | 16 ++++++++--------
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/apollo/cron/download_advisory.py b/apollo/cron/download_advisory.py
index 6ba2f01..5bec2a8 100644
--- a/apollo/cron/download_advisory.py
+++ b/apollo/cron/download_advisory.py
@@ -115,14 +115,16 @@ class DownloadSATask:
file_path = os.path.join(advisory_dir, file_name)
advisory_year, advisory_serial_number = re.findall("\d+", file_name)
try:
- cve_rows, cve_pkg_rows, cve_pkg_docs, _, _ = parse_security_advisory(file_path)
+ security_cvrf_info = parse_security_advisory(file_path)
+ security_cvrf_info.sa_year = None
+ security_cvrf_info.sa_number = None
except (KeyError, ParseAdvisoryError) as error:
LOGGER.error(error)
LOGGER.error("Some error occurred when parse advisory '%s'." % file_name)
self._record_download_result(advisory_year, advisory_serial_number, False)
continue
- save_status_code = proxy.save_security_advisory(file_name, cve_rows, cve_pkg_rows, cve_pkg_docs)
+ save_status_code = proxy.save_security_advisory(file_name, security_cvrf_info)
status = True if save_status_code == SUCCEED else False
self._record_download_result(advisory_year, advisory_serial_number, status)
diff --git a/apollo/handler/cve_handler/view.py b/apollo/handler/cve_handler/view.py
index 58d3bb1..200cc0d 100644
--- a/apollo/handler/cve_handler/view.py
+++ b/apollo/handler/cve_handler/view.py
@@ -466,9 +466,9 @@ class VulUploadAdvisory(BaseResponse):
def _save_single_advisory(proxy, file_path):
file_name = os.path.basename(file_path)
try:
- cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number = parse_security_advisory(file_path)
+ security_cvrf_info = parse_security_advisory(file_path)
os.remove(file_path)
- if not all([cve_rows, cve_pkg_rows, cve_pkg_docs]):
+ if not all([security_cvrf_info.cve_rows, security_cvrf_info.cve_pkg_rows, security_cvrf_info.cve_pkg_docs]):
return WRONG_FILE_FORMAT
except (KeyError, ParseAdvisoryError) as error:
os.remove(file_path)
@@ -476,7 +476,7 @@ class VulUploadAdvisory(BaseResponse):
LOGGER.error(error)
return WRONG_FILE_FORMAT
- status_code = proxy.save_security_advisory(file_name, cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number)
+ status_code = proxy.save_security_advisory(file_name, security_cvrf_info)
return status_code
@@ -504,8 +504,10 @@ class VulUploadAdvisory(BaseResponse):
shutil.rmtree(folder_path)
return WRONG_FILE_FORMAT
try:
- cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number = parse_security_advisory(file_path)
- if not all([cve_rows, cve_pkg_rows, cve_pkg_docs]):
+ security_cvrf_info = parse_security_advisory(file_path)
+ if not all(
+ [security_cvrf_info.cve_rows, security_cvrf_info.cve_pkg_rows, security_cvrf_info.cve_pkg_docs]
+ ):
shutil.rmtree(folder_path)
return WRONG_FILE_FORMAT
except (KeyError, ParseAdvisoryError) as error:
@@ -519,9 +521,7 @@ class VulUploadAdvisory(BaseResponse):
LOGGER.error(error)
continue
# elasticsearch need 1 second to update doc
- status_code = proxy.save_security_advisory(
- file_name, cve_rows, cve_pkg_rows, cve_pkg_docs, sa_year, sa_number
- )
+ status_code = proxy.save_security_advisory(file_name, security_cvrf_info)
if status_code != SUCCEED:
fail_list.append(file_name)
else:
--
2.33.0

13
0006-fix-bug-with-host-count-in-cve-fix-task.patch → 0002-fix-bug-with-host-count-in-cve-fix-task.patch
View File

@ -1,6 +1,6 @@
From dffeb775d77e1c608eed8dc9553fcc46ebbfc450 Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Wed, 24 Jul 2024 16:46:56 +0800
From a82cceada0df66cf48d646a3cd6a55556ebf9962 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Tue, 23 Jul 2024 18:55:50 +0800
Subject: [PATCH 1/1] fix bug with host count in cve fix task
---
@ -8,11 +8,11 @@ Subject: [PATCH 1/1] fix bug with host count in cve fix task
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/apollo/database/proxy/task/cve_fix.py b/apollo/database/proxy/task/cve_fix.py
index efdc91e..468c03a 100644
index fa3f2a9..1aa5546 100644
--- a/apollo/database/proxy/task/cve_fix.py
+++ b/apollo/database/proxy/task/cve_fix.py
@@ -133,7 +133,8 @@ class CveFixTaskProxy(TaskProxy):
return PARAM_ERROR, dict()
@@ -121,7 +121,8 @@ class CveFixTaskProxy(TaskProxy):
wait_fix_rpms = dict()
for task_info in fix_host_rpm_info:
- wait_fix_rpms[task_info["cve_id"]] = dict(rpms=task_info.get("rpms", []), hosts=list(host_dict.keys()))
@ -23,3 +23,4 @@ index efdc91e..468c03a 100644
fix_tasks = []
--
2.33.0

27
0002-fix-the-query-error-of-cve-associated-host.patch
View File

@ -1,27 +0,0 @@
From f8cab408f39bb8f6b793021cdfbf93338fb6ed0a Mon Sep 17 00:00:00 2001
From: gongzt <gong_zhengtang@163.com>
Date: Wed, 20 Dec 2023 15:13:07 +0800
Subject: [PATCH] Fix the query error of cve associated host
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/database/proxy/cve.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index 6210156..b5a61b9 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -1532,6 +1532,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
def _get_processed_cve_packages_host(self, data):
result = {"total_count": 0, "total_page": 0, "result": []}
filters = {
+ CveHostAssociation.host_user == data["username"],
CveHostAssociation.cve_id == data["cve_id"],
CveHostAssociation.installed_rpm == data["installed_rpm"],
CveHostAssociation.fixed == data["fixed"],
--
2.33.0

149
0003-fix-issue-with-language-display-in-task-generation.patch Normal file
View File

@ -0,0 +1,149 @@
From 7ef8931b94ce3a8801e00413b35a37691456e800 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Wed, 28 Aug 2024 09:34:49 +0800
Subject: [PATCH 1/1] fix issue with language display in task generation api
---
apollo/database/proxy/task/cve_fix.py | 18 ++++++++++++++++--
apollo/database/proxy/task/cve_rollback.py | 20 +++++++++++++++++---
apollo/function/schema/task.py | 6 +++---
database/aops-apollo.sql | 2 +-
4 files changed, 37 insertions(+), 9 deletions(-)
diff --git a/apollo/database/proxy/task/cve_fix.py b/apollo/database/proxy/task/cve_fix.py
index 1aa5546..fc98633 100644
--- a/apollo/database/proxy/task/cve_fix.py
+++ b/apollo/database/proxy/task/cve_fix.py
@@ -23,7 +23,7 @@ from typing import Dict, Tuple
import sqlalchemy.orm
from elasticsearch import ElasticsearchException
-from flask import g
+from flask import request
from sqlalchemy import func, case
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.sql import or_
@@ -462,6 +462,11 @@ class CveFixTaskProxy(TaskProxy):
}
"""
+ lang_info = request.headers.get("Accept-Language")
+ if lang_info:
+ lang = lang_info.split(',')[0].split(';')[0]
+ else:
+ lang = "en"
task_id = str(uuid.uuid1()).replace('-', '')
task_info = copy.deepcopy(data)
task_info['task_id'] = task_id
@@ -470,8 +475,17 @@ class CveFixTaskProxy(TaskProxy):
task_info["check_items"] = ",".join(task_info["check_items"])
task_info["host_num"] = len(wait_fix_rpms.keys())
task_info["fix_type"] = fix_way
+
+ prefix_map = {
+ "zh": {"hotpatch": "热补丁修复", "coldpatch": "冷补丁修复"},
+ "en": {"hotpatch": "Livepatch Upgrade", "coldpatch": "Normal Upgrade"},
+ }
if subtask:
- task_prefix = "冷补丁修复：" if fix_way == "coldpatch" else "热补丁修复："
+ task_prefix = (
+ f"{prefix_map['en'].get(fix_way,'coldpatch')}："
+ if lang.startswith("en")
+ else f"{prefix_map['zh'].get(fix_way,'coldpatch')}："
+ )
task_info["description"] = task_prefix + task_info["description"]
task_info["task_name"] = task_prefix + task_info["task_name"]
task_info["takeover"] = False if fix_way == "coldpatch" else task_info["takeover"]
diff --git a/apollo/database/proxy/task/cve_rollback.py b/apollo/database/proxy/task/cve_rollback.py
index e93be15..0525602 100644
--- a/apollo/database/proxy/task/cve_rollback.py
+++ b/apollo/database/proxy/task/cve_rollback.py
@@ -14,7 +14,7 @@ from typing import Tuple, Optional
import sqlalchemy.orm
from elasticsearch import ElasticsearchException
-from flask import g
+from flask import request
from sqlalchemy import or_, func, case
from sqlalchemy.exc import SQLAlchemyError
@@ -106,16 +106,30 @@ class CveRollbackTaskProxy(TaskProxy):
@staticmethod
def _gen_task_row(data: dict, cve_fix_task_info: sqlalchemy.orm.Query) -> dict:
+ lang_info = request.headers.get("Accept-Language")
+ if lang_info:
+ lang = lang_info.split(',')[0].split(';')[0]
+ else:
+ lang = "en"
+
fix_task_description = cve_fix_task_info.description
fix_task_name = cve_fix_task_info.task_name
host_num = cve_fix_task_info.host_num
+
+ if lang.startswith("en"):
+ task_name = "ROLLBACK_TASK: %s" % fix_task_name
+ description = "ORIGIN_TASK_DESCRIPTION: %s" % fix_task_description
+ else:
+ task_name = "回滚: %s" % fix_task_name
+ description = "原CVE修复任务描述: %s" % fix_task_description
+
task_data = {
"cluster_id": data["cluster_id"],
"task_id": data["task_id"],
"task_type": data["task_type"],
"create_time": data["create_time"],
- "task_name": "回滚: %s" % fix_task_name,
- "description": "原CVE修复任务描述: %s" % fix_task_description,
+ "task_name": task_name,
+ "description": description,
"host_num": host_num,
"username": data.get("username"),
}
diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py
index a494703..f306cea 100644
--- a/apollo/function/schema/task.py
+++ b/apollo/function/schema/task.py
@@ -91,7 +91,7 @@ class GenerateCveTaskSchema(Schema):
"""
task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20)
- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
+ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100)
accepted = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
check_items = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False)
takeover = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
@@ -178,7 +178,7 @@ class GenerateRepoTaskSchema(Schema):
"""
task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20)
- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
+ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100)
repo_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 36)
host_list = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 36), required=True)
@@ -367,7 +367,7 @@ class HotpatchRemoveInfoSchema(Schema):
class GenerateHotpatchRemoveTaskSchema(Schema):
task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20)
- description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
+ description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100)
info = fields.List(fields.Nested(HotpatchRemoveInfoSchema), required=True, validate=lambda s: len(s) > 0)
diff --git a/database/aops-apollo.sql b/database/aops-apollo.sql
index 4e99d7a..b78bb3a 100644
--- a/database/aops-apollo.sql
+++ b/database/aops-apollo.sql
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS `cve` (
CREATE TABLE IF NOT EXISTS `vul_task` (
`task_id` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
`task_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
- `description` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+ `description` varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
`task_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
`latest_execute_time` int(11) NULL DEFAULT NULL,
`create_time` int(11) NULL DEFAULT NULL,
--
2.33.0

85
0003-update-verification-method-for-host-ip-fieldl.patch
View File

@ -1,85 +0,0 @@
From 47a4c1d6488f07aa55621454fefedb559fc1bbf8 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Wed, 20 Dec 2023 16:26:24 +0800
Subject: [PATCH] update verification method for host ip fieldl;fix repo
field filter error
---
apollo/database/proxy/cve.py | 8 ++++++--
apollo/function/schema/cve.py | 2 +-
apollo/function/schema/task.py | 6 +++---
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index 6210156..5d29544 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -20,7 +20,7 @@ import copy
from collections import defaultdict
from elasticsearch import ElasticsearchException
-from sqlalchemy import func, tuple_, case
+from sqlalchemy import func, tuple_, case, or_
from sqlalchemy.exc import SQLAlchemyError
from vulcanus.database.helper import sort_and_page, judge_return_code
from vulcanus.database.proxy import MysqlProxy, ElasticsearchProxy
@@ -200,7 +200,11 @@ class CveMysqlProxy(MysqlProxy):
if filter_dict.get("host_group"):
filters.add(Host.host_group_name.in_(filter_dict["host_group"]))
if filter_dict.get("repo"):
- filters.add(Host.repo_name.in_(filter_dict["repo"]))
+ if all(filter_dict.get("repo")):
+ filters.add(Host.repo_name.in_(filter_dict["repo"]))
+ else:
+ repo_names = list(filter(None, filter_dict["repo"]))
+ filters.add(or_(Host.repo_name.in_(repo_names), Host.repo_name == None))
return filters
def _query_cve_hosts(self, username: str, cve_id: str, filters: set):
diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py
index 178672e..56d76ed 100644
--- a/apollo/function/schema/cve.py
+++ b/apollo/function/schema/cve.py
@@ -56,7 +56,7 @@ class CveHostFilterSchema(Schema):
host_name = fields.String(required=False, validate=lambda s: len(s) != 0)
host_group = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False)
- repo = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False)
+ repo = fields.List(fields.String(validate=lambda s: len(s) != 0, allow_none=True), required=False)
fixed = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py
index de86194..e136182 100644
--- a/apollo/function/schema/task.py
+++ b/apollo/function/schema/task.py
@@ -18,7 +18,7 @@ Description: For task related restful interfaces schema
from marshmallow import Schema
from marshmallow import fields
from marshmallow import validate
-from vulcanus.restful.serialize.validate import PaginationSchema
+from vulcanus.restful.serialize.validate import PaginationSchema, ValidateRules
from apollo.conf.constant import TaskType, TaskStatus
@@ -248,7 +248,7 @@ class CveFixResultCallbackSchema(Schema):
class CallbackSchma(Schema):
task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32)
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
- host_ip = fields.IP(required=True)
+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check)
host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
status = fields.String(required=True, validate=lambda s: len(s) != 0)
execution_time = fields.Integer(required=True)
@@ -270,7 +270,7 @@ class CheckItemsSchema(Schema):
class RepoSetCallbackSchema(Schema):
task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32)
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
- host_ip = fields.IP(required=True)
+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check)
host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
status = fields.String(required=True, validate=lambda s: len(s) != 0)
execution_time = fields.Integer(required=True)
--
2.33.0

168
0004-fix-TimedCorrectTask.patch
View File

@ -1,168 +0,0 @@
From 8e19d92b6a484ddcf7ca7bf666ce21baa56ab326 Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Wed, 20 Dec 2023 17:21:22 +0800
Subject: [PATCH] fix TimedCorrectTask
---
apollo/cron/timed_correct_manager.py | 12 ++++--
apollo/database/proxy/task/base.py | 50 ++++++++++++++++++-----
apollo/database/proxy/task/timed_proxy.py | 8 ++++
apollo/tests/database/test_task.py | 2 +-
4 files changed, 57 insertions(+), 15 deletions(-)
diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py
index db0b4c1..ae3a1e3 100644
--- a/apollo/cron/timed_correct_manager.py
+++ b/apollo/cron/timed_correct_manager.py
@@ -41,11 +41,15 @@ class TimedCorrectTask(TimedTask):
"""
Start the correct after the specified time of day.
"""
- LOGGER.info("Begin to correct the whole host in %s.", str(datetime.datetime.now()))
+ LOGGER.info(
+ "Begin to correct the status of timeout tasks and scan timeout host in %s.",
+ str(datetime.datetime.now()))
abnormal_task_ids, abnormal_host_ids = self.get_abnormal_task()
- self._update_host_status(abnormal_host_ids)
- with TimedProxy() as proxy:
- proxy.timed_correct_error_task_status(abnormal_task_ids)
+ if len(abnormal_host_ids) != 0:
+ self._update_host_status(abnormal_host_ids)
+ if len(abnormal_task_ids) != 0:
+ with TimedProxy() as proxy:
+ proxy.timed_correct_error_task_status(abnormal_task_ids)
@staticmethod
def _abnormal_task(tasks):
diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py
index 840c140..a5ddede 100644
--- a/apollo/database/proxy/task/base.py
+++ b/apollo/database/proxy/task/base.py
@@ -861,17 +861,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
raise EsOperationError("Delete task from elasticsearch failed due to internal error.")
- def get_running_task_form_task_cve_host(self) -> list:
+ def get_running_task_form_hotpatch_remove_task(self) -> list:
"""
- Get all CVE repair tasks with running status under Username
+ Get all hotpatch remove tasks with running status under Username
Returns:
list: task id list
"""
- task_cve_query = (
- self.session.query(HotpatchRemoveTask).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all()
+ hotpatch_remove_query = (
+ self.session.query(HotpatchRemoveTask.task_id).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all()
)
- task_id_list = [task.task_id for task in task_cve_query]
+ task_id_list = [task.task_id for task in hotpatch_remove_query]
return task_id_list
def get_running_task_form_task_host_repo(self) -> list:
@@ -882,13 +882,39 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
list: task id list
"""
host_repo_query = (
- self.session.query(TaskHostRepoAssociation)
+ self.session.query(TaskHostRepoAssociation.task_id)
.filter(TaskHostRepoAssociation.status == TaskStatus.RUNNING)
.all()
)
task_id_list = [task.task_id for task in host_repo_query]
return task_id_list
+ def get_running_task_form_cve_fix_task(self) -> list:
+ """
+ Get all CVE fix tasks with running status
+
+ Returns:
+ list: task id list
+ """
+ cve_fix_query = (
+ self.session.query(CveFixTask.task_id).filter(CveFixTask.status == TaskStatus.RUNNING).all()
+ )
+ task_id_list = [task.task_id for task in cve_fix_query]
+ return task_id_list
+
+ def get_running_task_form_cve_rollback_task(self) -> list:
+ """
+ Get all CVE rollback tasks with running status
+
+ Returns:
+ list: task id list
+ """
+ cve_rollback_query = (
+ self.session.query(CveRollbackTask.task_id).filter(CveRollbackTask.status == TaskStatus.RUNNING).all()
+ )
+ task_id_list = [task.task_id for task in cve_rollback_query]
+ return task_id_list
+
def get_scanning_status_and_time_from_host(self) -> list:
"""
Get all host id and time with scanning status from the host table
@@ -907,13 +933,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
Returns:
list: Each element is a task information, including the task ID, task type, creation time
"""
- task_cve_id_list = self.get_running_task_form_task_cve_host()
- task_repo_id_list = self.get_running_task_form_task_host_repo()
host_info_list = self.get_scanning_status_and_time_from_host()
- task_id_list = task_cve_id_list + task_repo_id_list
+
+ task_cve_id_list = self.get_running_task_form_hotpatch_remove_task()
+ task_repo_id_list = self.get_running_task_form_task_host_repo()
+ task_cve_fix_list = self.get_running_task_form_cve_fix_task()
+ task_cve_rollback_list = self.get_running_task_form_cve_rollback_task()
+
+ task_id_list = task_cve_id_list + task_repo_id_list + task_cve_fix_list + task_cve_rollback_list
task_query = self.session.query(Task).filter(Task.task_id.in_(task_id_list)).all()
- running_task_list = [(task.task_id, task.create_time) for task in task_query]
+ running_task_list = [(task.task_id, task.latest_execute_time) for task in task_query]
return running_task_list, host_info_list
def validate_cves(self, cve_id: list) -> bool:
diff --git a/apollo/database/proxy/task/timed_proxy.py b/apollo/database/proxy/task/timed_proxy.py
index 436c3bd..fd396d1 100644
--- a/apollo/database/proxy/task/timed_proxy.py
+++ b/apollo/database/proxy/task/timed_proxy.py
@@ -22,6 +22,8 @@ from apollo.conf.constant import TaskStatus
from apollo.database.table import (
HotpatchRemoveTask,
TaskHostRepoAssociation,
+ CveFixTask,
+ CveRollbackTask,
)
@@ -42,6 +44,12 @@ class TimedProxy(MysqlProxy):
self.session.query(TaskHostRepoAssociation).filter(TaskHostRepoAssociation.task_id.in_(task_ids)).update(
{TaskHostRepoAssociation.status: TaskStatus.UNKNOWN}, synchronize_session=False
)
+ self.session.query(CveFixTask).filter(CveFixTask.task_id.in_(task_ids)).update(
+ {CveFixTask.status: TaskStatus.UNKNOWN}, synchronize_session=False
+ )
+ self.session.query(CveRollbackTask).filter(CveRollbackTask.task_id.in_(task_ids)).update(
+ {CveRollbackTask.status: TaskStatus.UNKNOWN}, synchronize_session=False
+ )
self.session.commit()
except SQLAlchemyError as error:
self.session.rollback()
diff --git a/apollo/tests/database/test_task.py b/apollo/tests/database/test_task.py
index 35d923b..ceb84ab 100644
--- a/apollo/tests/database/test_task.py
+++ b/apollo/tests/database/test_task.py
@@ -401,7 +401,7 @@ class TestTaskMysqlFirst(DatabaseTestCase):
def test_get_running_task_form_task_cve_host(self):
self.assertEqual(
- self.task_database.get_running_task_form_task_cve_host(),
+ self.task_database.get_running_task_form_hotpatch_remove_task(),
["1111111111poiuytrewqasdfghjklmnb"],
)
--
2.33.0

115
0004-fix-repo-query-error-and-adjust-schema.patch Normal file
View File

@ -0,0 +1,115 @@
From fde9a0d9ed3b3ad99fdde047d9b8928e97e9af2f Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Sat, 7 Sep 2024 16:52:45 +0800
Subject: [PATCH 1/1] Fixed the error of the repo query interface; Adaptable to Copilot adjustments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/database/proxy/task/base.py | 1 +
apollo/function/schema/cve.py | 2 +-
apollo/function/schema/host.py | 4 +++-
apollo/function/schema/repo.py | 1 -
apollo/function/schema/task.py | 4 +++-
apollo/handler/repo_handler/view.py | 15 ++-------------
6 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py
index 8d86614..3f9a0ba 100644
--- a/apollo/database/proxy/task/base.py
+++ b/apollo/database/proxy/task/base.py
@@ -504,6 +504,7 @@ class TaskMysqlProxy(MysqlProxy):
"takeover": row.takeover,
"cluster_id": row.cluster_id,
"cluster_name": cluster_dict_info.get(row.cluster_id),
+ "task_type": row.task_type,
}
return task_info
diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py
index b5a0270..4a18916 100644
--- a/apollo/function/schema/cve.py
+++ b/apollo/function/schema/cve.py
@@ -89,7 +89,7 @@ class CveTaskHostSchemaOfCveInfo(Schema):
"""
cve_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20)
- rpms = fields.List(fields.Nested(PackageInfoSchema), required=True)
+ rpms = fields.List(fields.Nested(PackageInfoSchema), required=False, missing=[])
class GetCveTaskHostSchema(Schema):
diff --git a/apollo/function/schema/host.py b/apollo/function/schema/host.py
index bccdb65..684a55c 100644
--- a/apollo/function/schema/host.py
+++ b/apollo/function/schema/host.py
@@ -37,7 +37,9 @@ class ScanHostSchema(Schema):
validators for parameter of /vulnerability/host/scan
"""
- host_list = fields.List(fields.String(validate=lambda s: 0 < len(s) <= 36, required=True), required=True)
+ host_list = fields.List(
+ fields.String(validate=lambda s: 0 < len(s) <= 36, required=True), required=False, missing=[]
+ )
filter = fields.Nested(ScanHostFilterSchema, required=False)
diff --git a/apollo/function/schema/repo.py b/apollo/function/schema/repo.py
index bdd4c3d..618470f 100644
--- a/apollo/function/schema/repo.py
+++ b/apollo/function/schema/repo.py
@@ -34,7 +34,6 @@ class GetYumRepoSchema(Schema):
"""
repo_id_list = fields.List(fields.String(validate=lambda s: 0 < len(s) <= 36), required=False)
- search_key = fields.String(required=False, validate=lambda s: 0 < len(s) <= 32)
class UpdateYumRepoSchema(Schema):
diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py
index f306cea..be116ad 100644
--- a/apollo/function/schema/task.py
+++ b/apollo/function/schema/task.py
@@ -93,7 +93,9 @@ class GenerateCveTaskSchema(Schema):
task_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 20)
description = fields.String(required=True, validate=lambda s: 0 < len(s) <= 100)
accepted = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
- check_items = fields.List(fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False)
+ check_items = fields.List(
+ fields.String(required=True, validate=lambda s: 0 < len(s) <= 32), required=False, missing=[]
+ )
takeover = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
info = fields.List(fields.Nested(CveInfoDictSchema), required=True, validate=lambda s: len(s) > 0)
diff --git a/apollo/handler/repo_handler/view.py b/apollo/handler/repo_handler/view.py
index 6a06c1d..6909e24 100644
--- a/apollo/handler/repo_handler/view.py
+++ b/apollo/handler/repo_handler/view.py
@@ -93,21 +93,10 @@ class VulGetYumRepo(BaseResponse):
Query repo info handle
"""
cluster_info_dic = cache.get_user_clusters()
- if cluster_info_dic is None:
- return DATABASE_QUERY_ERROR, []
-
- cluster_list = []
- if params.get("search_key"):
- for cluster_id, info in cluster_info_dic.items():
- if params.get("search_key") in info.get("cluster_name"):
- cluster_list.append(cluster_id)
- else:
- cluster_list = list(cluster_info_dic.keys())
-
- if not cluster_list:
+ if not cluster_info_dic:
return SUCCEED, []
- status_code, result = proxy.get_repo(params.get("repo_id_list", []), cluster_list)
+ status_code, result = proxy.get_repo(params.get("repo_id_list", []), list(cluster_info_dic.keys()))
if status_code != SUCCEED:
return status_code, []
--
2.33.0

45
0005-add-reboot-field-to-query-host-info-api.patch
View File

@ -1,45 +0,0 @@
From 5c803953928ad16d56795dcf7158bb9f3d340e1d Mon Sep 17 00:00:00 2001
From: gongzt <gong_zhengtang@163.com>
Date: Thu, 21 Dec 2023 17:33:23 +0800
Subject: [PATCH] add reboot field to query host info api
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/database/proxy/host.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py
index ca709a8..6015a04 100644
--- a/apollo/database/proxy/host.py
+++ b/apollo/database/proxy/host.py
@@ -341,7 +341,8 @@ class HostMysqlProxy(MysqlProxy):
"repo": "20.03-update",
"affected_cve_num": 12,
"unaffected_cve_num": 1,
- "last_scan": 1111111111
+ "last_scan": 1111111111,
+ "reboot": true/false
}
}
"""
@@ -412,6 +413,7 @@ class HostMysqlProxy(MysqlProxy):
Host.host_group_name,
Host.repo_name,
Host.last_scan,
+ Host.reboot,
func.COUNT(func.IF(subquery.c.fixed == True, 1, None)).label("fixed_cve_num"),
func.COUNT(func.IF(and_(subquery.c.fixed == False, subquery.c.affected == True), 1, None)).label(
"affected_cve_num"
@@ -437,6 +439,7 @@ class HostMysqlProxy(MysqlProxy):
"unaffected_cve_num": row.unaffected_cve_num,
"last_scan": row.last_scan,
"fixed_cve_num": row.fixed_cve_num,
+ "reboot": row.reboot,
}
return host_info
--
Gitee

26
0005-set-uwsgi-buffer-size.patch Normal file
View File

@ -0,0 +1,26 @@
From 8b2b1f8eb4cf6a6b47103acee81d218022e3166a Mon Sep 17 00:00:00 2001
From: rearcher <123781007@qq.com>
Date: Mon, 18 Nov 2024 15:50:54 +0800
Subject: [PATCH] =?UTF-8?q?=E8=AE=BE=E7=BD=AEuwsgi=20buffer-size=E4=B8=BA3?=
=?UTF-8?q?2k?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
conf/aops-apollo.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/conf/aops-apollo.yml b/conf/aops-apollo.yml
index f7a39c4..1f34aa4 100644
--- a/conf/aops-apollo.yml
+++ b/conf/aops-apollo.yml
@@ -4,3 +4,4 @@ uwsgi:
processes: 1
gevent: 100
port: 11116
+ buffer_size: 32768
\ No newline at end of file
--
Gitee

BIN
aops-apollo-v1.4.1.tar.gz
View File

Binary file not shown.

BIN
aops-apollo-v2.0.0.tar.gz Normal file
View File

Binary file not shown.

43
aops-apollo.spec
View File

@ -1,23 +1,19 @@
Name: aops-apollo
Version: v1.4.1
Release: 4
Version: v2.0.0
Release: 6
Summary: Cve management service, monitor machine vulnerabilities and provide fix functions.
License: MulanPSL2
URL: https://gitee.com/openeuler/%{name}
Source0: %{name}-%{version}.tar.gz
Patch0001: 0001-fix-cve-list-sort.patch
Patch0002: 0002-fix-the-query-error-of-cve-associated-host.patch
Patch0003: 0003-update-verification-method-for-host-ip-fieldl.patch
Patch0004: 0004-fix-TimedCorrectTask.patch
Patch0005: 0005-add-reboot-field-to-query-host-info-api.patch
Patch0006: 0006-fix-bug-with-host-count-in-cve-fix-task.patch
Patch0001: 0001-fix-security-advisory-parsing-error.patch
Patch0002: 0002-fix-bug-with-host-count-in-cve-fix-task.patch
Patch0003: 0003-fix-issue-with-language-display-in-task-generation.patch
Patch0004: 0004-fix-repo-query-error-and-adjust-schema.patch
Patch0005: 0005-set-uwsgi-buffer-size.patch
BuildRequires: python3-setuptools
Requires: aops-vulcanus >= v1.3.0
Requires: python3-elasticsearch >= 7 python3-flask-restful python3-marshmallow >= 3.13.0
Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0
Requires: python3-PyYAML python3-flask python3-gevent python3-uWSGI
Requires: python3-retrying python3-lxml
Requires: aops-vulcanus >= v2.0.0
Requires: python3-gevent python3-uWSGI python3-celery aops-zeus >= v2.0.0
Provides: aops-apollo
@ -57,10 +53,8 @@ popd
%files
%doc README.*
%attr(0644,root,root) %{_sysconfdir}/aops/apollo.ini
%attr(0644,root,root) %{_sysconfdir}/aops/apollo_crontab.yml
%attr(0755,root,root) %{_bindir}/aops-apollo
%attr(0755,root,root) /usr/lib/systemd/system/aops-apollo.service
%attr(0644,root,root) %{_sysconfdir}/aops/conf.d/aops-apollo.yml
%attr(0755,root,root) %{_unitdir}/aops-apollo.service
%{python3_sitelib}/aops_apollo*.egg-info/*
%{python3_sitelib}/apollo/*
%attr(0755, root, root) /opt/aops/database/*
@ -73,8 +67,19 @@ popd
%{python3_sitelib}/aops_apollo_tool/*
%changelog
* Wed Jul 24 2024 wenxin<wenxin32@foxmail.com> - v1.4.1-4
- fix bug with host count in cve fix task
* Tue Nov 19 2024 luxuexian<luxuexian@huawei.com> - v2.0.0-6
- set-uwsgi-buffer-size to 32k
* Mon Sep 09 2024 wenxin<wenxin32@foxmail.com> - v2.0.0-5
- Fix issue with querying repo info api
- Adjust some schema validation logic
* Fri Aug 30 2024 wenxin<wenxin32@foxmail.com> - v2.0.0-4
- Fix issue with language display in task generation api
* Fri Aug 16 2024 wenxin<wenxin32@foxmail.com> - v2.0.0-3
- Added support for cluster features.
- Adjusted Task Module logic to use Celery for task management and execution.
* Fri Dec 22 2023 wenxin<wenxin32@foxmail.com> - v1.4.1-3
- fix the query error of cve associated host