Update to 4.1.7

This commit is contained in:
starlet-dx 2023-04-11 10:34:23 +08:00
parent a7e7789438
commit fd50a1f961
5 changed files with 6 additions and 564 deletions

View File

@ -1,41 +0,0 @@
diff -Naur a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py
--- a/django/db/backends/base/operations.py 2022-12-06 17:12:38.000000000 +0800
+++ b/django/db/backends/base/operations.py 2022-12-09 11:49:05.347986919 +0800
@@ -9,6 +9,7 @@
from django.db.backends import utils
from django.utils import timezone
from django.utils.encoding import force_str
+from django.utils.regex_helper import _lazy_re_compile
class BaseDatabaseOperations:
@@ -53,6 +54,8 @@
# Prefix for EXPLAIN queries, or None EXPLAIN isn't supported.
explain_prefix = None
+ extract_trunc_lookup_pattern = _lazy_re_compile(r"[\w\-_()]+")
+
def __init__(self, connection):
self.connection = connection
diff -Naur a/django/db/models/functions/datetime.py b/django/db/models/functions/datetime.py
--- a/django/db/models/functions/datetime.py 2022-12-06 17:12:38.000000000 +0800
+++ b/django/db/models/functions/datetime.py 2022-12-09 11:50:50.011981885 +0800
@@ -51,6 +51,8 @@
super().__init__(expression, **extra)
def as_sql(self, compiler, connection):
+ if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.lookup_name):
+ raise ValueError("Invalid lookup_name: %s" % self.lookup_name)
sql, params = compiler.compile(self.lhs)
lhs_output_field = self.lhs.output_field
if isinstance(lhs_output_field, DateTimeField):
@@ -243,6 +245,8 @@
super().__init__(expression, output_field=output_field, **extra)
def as_sql(self, compiler, connection):
+ if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.kind):
+ raise ValueError("Invalid kind: %s" % self.kind)
sql, params = compiler.compile(self.lhs)
tzname = None
if isinstance(self.lhs.output_field, DateTimeField):

View File

@ -1,100 +0,0 @@
From ffde9b888863f30c62c728c6f9538a09d7396ed9 Mon Sep 17 00:00:00 2001
From: starlet-dx <15929766099@163.com>
Date: Mon, 13 Feb 2023 19:26:27 +0800
Subject: [PATCH 1/1] [4.1.x] Fixed CVE-2023-23969 -- Prevented DoS with pathological values for Accept-Language.
The parsed values of Accept-Language headers are cached in order to avoid repetitive parsing. This leads to a potential denial-of-service vector via excessive memory usage if the raw value of Accept-Language headers is very large.
Accept-Language headers are now limited to a maximum length in order to
avoid this issue.
---
django/utils/translation/trans_real.py | 31 +++++++++++++++++++++++++-
tests/i18n/tests.py | 12 ++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py
index 423f30e..2974cc6 100644
--- a/django/utils/translation/trans_real.py
+++ b/django/utils/translation/trans_real.py
@@ -30,6 +30,11 @@ _default = None
# magic gettext number to separate context from message
CONTEXT_SEPARATOR = "\x04"
+# Maximum number of characters that will be parsed from the Accept-Language
+# header to prevent possible denial of service or memory exhaustion attacks.
+# About 10x longer than the longest value shown on MDNs Accept-Language page.
+ACCEPT_LANGUAGE_HEADER_MAX_LENGTH = 500
+
# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9
# and RFC 3066, section 2.1
accept_language_re = _lazy_re_compile(
@@ -586,7 +591,7 @@ def get_language_from_request(request, check_path=False):
@functools.lru_cache(maxsize=1000)
-def parse_accept_lang_header(lang_string):
+def _parse_accept_lang_header(lang_string):
"""
Parse the lang_string, which is the body of an HTTP Accept-Language
header, and return a tuple of (lang, q-value), ordered by 'q' values.
@@ -608,3 +613,27 @@ def parse_accept_lang_header(lang_string):
result.append((lang, priority))
result.sort(key=lambda k: k[1], reverse=True)
return tuple(result)
+
+
+def parse_accept_lang_header(lang_string):
+ """
+ Parse the value of the Accept-Language header up to a maximum length.
+
+ The value of the header is truncated to a maximum length to avoid potential
+ denial of service and memory exhaustion attacks. Excessive memory could be
+ used if the raw value is very large as it would be cached due to the use of
+ functools.lru_cache() to avoid repetitive parsing of common header values.
+ """
+ # If the header value doesn't exceed the maximum allowed length, parse it.
+ if len(lang_string) <= ACCEPT_LANGUAGE_HEADER_MAX_LENGTH:
+ return _parse_accept_lang_header(lang_string)
+
+ # If there is at least one comma in the value, parse up to the last comma
+ # before the max length, skipping any truncated parts at the end of the
+ # header value.
+ if (index := lang_string.rfind(",", 0, ACCEPT_LANGUAGE_HEADER_MAX_LENGTH)) > 0:
+ return _parse_accept_lang_header(lang_string[:index])
+
+ # Don't attempt to parse if there is only one language-range value which is
+ # longer than the maximum allowed length and so truncated.
+ return ()
diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py
index 40fc306..b361a84 100644
--- a/tests/i18n/tests.py
+++ b/tests/i18n/tests.py
@@ -1730,6 +1730,14 @@ class MiscTests(SimpleTestCase):
("de;q=0.", [("de", 0.0)]),
("en; q=1,", [("en", 1.0)]),
("en; q=1.0, * ; q=0.5", [("en", 1.0), ("*", 0.5)]),
+ (
+ "en" + "-x" * 20,
+ [("en-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x", 1.0)],
+ ),
+ (
+ ", ".join(["en; q=1.0"] * 20),
+ [("en", 1.0)] * 20,
+ ),
# Bad headers
("en-gb;q=1.0000", []),
("en;q=0.1234", []),
@@ -1746,6 +1754,10 @@ class MiscTests(SimpleTestCase):
("", []),
("en;q=1e0", []),
("en-au;q=.", []),
+ # Invalid as language-range value too long.
+ ("xxxxxxxx" + "-xxxxxxxx" * 500, []),
+ # Header value too long, only parse up to limit.
+ (", ".join(["en; q=1.0"] * 500), [("en", 1.0)] * 45),
]
for value, expected in tests:
with self.subTest(value=value):
--
2.30.0

View File

@ -1,415 +0,0 @@
From 628b33a854a9c68ec8a0c51f382f304a0044ec92 Mon Sep 17 00:00:00 2001
From: Markus Holtermann <info@markusholtermann.eu>
Date: Tue, 13 Dec 2022 10:27:39 +0100
Subject: [PATCH] [4.1.x] Fixed CVE-2023-24580 -- Prevented DoS with too many
uploaded files.
Thanks to Jakob Ackermann for the report.
---
django/conf/global_settings.py | 4 ++
django/core/exceptions.py | 9 +++
django/core/handlers/exception.py | 3 +-
django/http/multipartparser.py | 64 ++++++++++++++++-----
django/http/request.py | 8 ++-
docs/ref/exceptions.txt | 5 ++
docs/ref/settings.txt | 23 ++++++++
docs/releases/3.2.18.txt | 10 +++-
docs/releases/4.0.10.txt | 10 +++-
docs/releases/4.1.7.txt | 14 ++++-
tests/handlers/test_exception.py | 31 +++++++++-
tests/requests/test_data_upload_settings.py | 55 +++++++++++++++++-
12 files changed, 213 insertions(+), 23 deletions(-)
diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
index 40b34bb71c90..7ac700228f37 100644
--- a/django/conf/global_settings.py
+++ b/django/conf/global_settings.py
@@ -313,6 +313,10 @@ def gettext_noop(s):
# SuspiciousOperation (TooManyFieldsSent) is raised.
DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000
+# Maximum number of files encoded in a multipart upload that will be read
+# before a SuspiciousOperation (TooManyFilesSent) is raised.
+DATA_UPLOAD_MAX_NUMBER_FILES = 100
+
# Directory in which upload streamed files will be temporarily saved. A value of
# `None` will make Django use the operating system's default temporary directory
# (i.e. "/tmp" on *nix systems).
diff --git a/django/core/exceptions.py b/django/core/exceptions.py
index 7be4e16bc55a..e06b33e7bc2d 100644
--- a/django/core/exceptions.py
+++ b/django/core/exceptions.py
@@ -67,6 +67,15 @@ class TooManyFieldsSent(SuspiciousOperation):
pass
+class TooManyFilesSent(SuspiciousOperation):
+ """
+ The number of fields in a GET or POST request exceeded
+ settings.DATA_UPLOAD_MAX_NUMBER_FILES.
+ """
+
+ pass
+
+
class RequestDataTooBig(SuspiciousOperation):
"""
The size of the request (excluding any file uploads) exceeded
diff --git a/django/core/handlers/exception.py b/django/core/handlers/exception.py
index 79577c2d0a6d..fd64584bbafb 100644
--- a/django/core/handlers/exception.py
+++ b/django/core/handlers/exception.py
@@ -13,6 +13,7 @@
RequestDataTooBig,
SuspiciousOperation,
TooManyFieldsSent,
+ TooManyFilesSent,
)
from django.http import Http404
from django.http.multipartparser import MultiPartParserError
@@ -111,7 +112,7 @@ def response_for_exception(request, exc):
exception=exc,
)
elif isinstance(exc, SuspiciousOperation):
- if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent)):
+ if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent)):
# POST data can't be accessed again, otherwise the original
# exception would be raised.
request._mark_post_parse_error()
diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py
index 26fb2bc41f86..944ca4aa6c2f 100644
--- a/django/http/multipartparser.py
+++ b/django/http/multipartparser.py
@@ -15,6 +15,7 @@
RequestDataTooBig,
SuspiciousMultipartForm,
TooManyFieldsSent,
+ TooManyFilesSent,
)
from django.core.files.uploadhandler import SkipFile, StopFutureHandlers, StopUpload
from django.utils.datastructures import MultiValueDict
@@ -39,6 +40,7 @@ class InputStreamExhausted(Exception):
RAW = "raw"
FILE = "file"
FIELD = "field"
+FIELD_TYPES = frozenset([FIELD, RAW])
class MultiPartParser:
@@ -111,6 +113,22 @@ def __init__(self, META, input_data, upload_handlers, encoding=None):
self._upload_handlers = upload_handlers
def parse(self):
+ # Call the actual parse routine and close all open files in case of
+ # errors. This is needed because if exceptions are thrown the
+ # MultiPartParser will not be garbage collected immediately and
+ # resources would be kept alive. This is only needed for errors because
+ # the Request object closes all uploaded files at the end of the
+ # request.
+ try:
+ return self._parse()
+ except Exception:
+ if hasattr(self, "_files"):
+ for _, files in self._files.lists():
+ for fileobj in files:
+ fileobj.close()
+ raise
+
+ def _parse(self):
"""
Parse the POST data and break it into a FILES MultiValueDict and a POST
MultiValueDict.
@@ -156,6 +174,8 @@ def parse(self):
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
+ # To count the number of files in the request.
+ num_files = 0
# To limit the amount of data read from the request.
read_size = None
# Whether a file upload is finished.
@@ -171,6 +191,20 @@ def parse(self):
old_field_name = None
uploaded_file = True
+ if (
+ item_type in FIELD_TYPES
+ and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
+ ):
+ # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
+ num_post_keys += 1
+ # 2 accounts for empty raw fields before and after the
+ # last boundary.
+ if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys:
+ raise TooManyFieldsSent(
+ "The number of GET/POST parameters exceeded "
+ "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS."
+ )
+
try:
disposition = meta_data["content-disposition"][1]
field_name = disposition["name"].strip()
@@ -183,17 +217,6 @@ def parse(self):
field_name = force_str(field_name, encoding, errors="replace")
if item_type == FIELD:
- # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
- num_post_keys += 1
- if (
- settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
- and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys
- ):
- raise TooManyFieldsSent(
- "The number of GET/POST parameters exceeded "
- "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS."
- )
-
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = (
@@ -228,6 +251,16 @@ def parse(self):
field_name, force_str(data, encoding, errors="replace")
)
elif item_type == FILE:
+ # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES.
+ num_files += 1
+ if (
+ settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None
+ and num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES
+ ):
+ raise TooManyFilesSent(
+ "The number of files exceeded "
+ "settings.DATA_UPLOAD_MAX_NUMBER_FILES."
+ )
# This is a file, use the handler...
file_name = disposition.get("filename")
if file_name:
@@ -305,8 +338,13 @@ def parse(self):
# Handle file upload completions on next iteration.
old_field_name = field_name
else:
- # If this is neither a FIELD or a FILE, just exhaust the stream.
- exhaust(stream)
+ # If this is neither a FIELD nor a FILE, exhaust the field
+ # stream. Note: There could be an error here at some point,
+ # but there will be at least two RAW types (before and
+ # after the other boundaries). This branch is usually not
+ # reached at all, because a missing content-disposition
+ # header will skip the whole boundary.
+ exhaust(field_stream)
except StopUpload as e:
self._close_files()
if not e.connection_reset:
diff --git a/django/http/request.py b/django/http/request.py
index 4b160bc5f4e9..0789b24c154e 100644
--- a/django/http/request.py
+++ b/django/http/request.py
@@ -13,7 +13,11 @@
TooManyFieldsSent,
)
from django.core.files import uploadhandler
-from django.http.multipartparser import MultiPartParser, MultiPartParserError
+from django.http.multipartparser import (
+ MultiPartParser,
+ MultiPartParserError,
+ TooManyFilesSent,
+)
from django.utils.datastructures import (
CaseInsensitiveMapping,
ImmutableList,
@@ -367,7 +371,7 @@ def _load_post_and_files(self):
data = self
try:
self._post, self._files = self.parse_file_upload(self.META, data)
- except MultiPartParserError:
+ except (MultiPartParserError, TooManyFilesSent):
# An error occurred while parsing POST data. Since when
# formatting the error the request handler might access
# self.POST, set self._post and self._file to prevent
diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt
index 2b567414e6d0..a2bf41499b0d 100644
--- a/docs/ref/exceptions.txt
+++ b/docs/ref/exceptions.txt
@@ -84,12 +84,17 @@ Django core exception classes are defined in ``django.core.exceptions``.
* ``SuspiciousMultipartForm``
* ``SuspiciousSession``
* ``TooManyFieldsSent``
+ * ``TooManyFilesSent``
If a ``SuspiciousOperation`` exception reaches the ASGI/WSGI handler level
it is logged at the ``Error`` level and results in
a :class:`~django.http.HttpResponseBadRequest`. See the :doc:`logging
documentation </topics/logging/>` for more information.
+.. versionchanged:: 3.2.18
+
+ ``SuspiciousOperation`` is raised when too many files are submitted.
+
``PermissionDenied``
--------------------
diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt
index a61612f49ba2..218de8f1d76a 100644
--- a/docs/ref/settings.txt
+++ b/docs/ref/settings.txt
@@ -1108,6 +1108,28 @@ could be used as a denial-of-service attack vector if left unchecked. Since web
servers don't typically perform deep request inspection, it's not possible to
perform a similar check at that level.
+.. setting:: DATA_UPLOAD_MAX_NUMBER_FILES
+
+``DATA_UPLOAD_MAX_NUMBER_FILES``
+--------------------------------
+
+.. versionadded:: 3.2.18
+
+Default: ``100``
+
+The maximum number of files that may be received via POST in a
+``multipart/form-data`` encoded request before a
+:exc:`~django.core.exceptions.SuspiciousOperation` (``TooManyFiles``) is
+raised. You can set this to ``None`` to disable the check. Applications that
+are expected to receive an unusually large number of file fields should tune
+this setting.
+
+The number of accepted files is correlated to the amount of time and memory
+needed to process the request. Large requests could be used as a
+denial-of-service attack vector if left unchecked. Since web servers don't
+typically perform deep request inspection, it's not possible to perform a
+similar check at that level.
+
.. setting:: DATABASE_ROUTERS
``DATABASE_ROUTERS``
@@ -3727,6 +3749,7 @@ HTTP
----
* :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE`
* :setting:`DATA_UPLOAD_MAX_NUMBER_FIELDS`
+* :setting:`DATA_UPLOAD_MAX_NUMBER_FILES`
* :setting:`DEFAULT_CHARSET`
* :setting:`DISALLOWED_USER_AGENTS`
* :setting:`FORCE_SCRIPT_NAME`
diff --git a/tests/handlers/test_exception.py b/tests/handlers/test_exception.py
index 3a483be78441..878fff7cc0c8 100644
--- a/tests/handlers/test_exception.py
+++ b/tests/handlers/test_exception.py
@@ -1,6 +1,11 @@
from django.core.handlers.wsgi import WSGIHandler
from django.test import SimpleTestCase, override_settings
-from django.test.client import FakePayload
+from django.test.client import (
+ BOUNDARY,
+ MULTIPART_CONTENT,
+ FakePayload,
+ encode_multipart,
+)
class ExceptionHandlerTests(SimpleTestCase):
@@ -24,3 +29,27 @@ def test_data_upload_max_memory_size_exceeded(self):
def test_data_upload_max_number_fields_exceeded(self):
response = WSGIHandler()(self.get_suspicious_environ(), lambda *a, **k: None)
self.assertEqual(response.status_code, 400)
+
+ @override_settings(DATA_UPLOAD_MAX_NUMBER_FILES=2)
+ def test_data_upload_max_number_files_exceeded(self):
+ payload = FakePayload(
+ encode_multipart(
+ BOUNDARY,
+ {
+ "a.txt": "Hello World!",
+ "b.txt": "Hello Django!",
+ "c.txt": "Hello Python!",
+ },
+ )
+ )
+ environ = {
+ "REQUEST_METHOD": "POST",
+ "CONTENT_TYPE": MULTIPART_CONTENT,
+ "CONTENT_LENGTH": len(payload),
+ "wsgi.input": payload,
+ "SERVER_NAME": "test",
+ "SERVER_PORT": "8000",
+ }
+
+ response = WSGIHandler()(environ, lambda *a, **k: None)
+ self.assertEqual(response.status_code, 400)
diff --git a/tests/requests/test_data_upload_settings.py b/tests/requests/test_data_upload_settings.py
index 0199296293d9..e89af0a39b82 100644
--- a/tests/requests/test_data_upload_settings.py
+++ b/tests/requests/test_data_upload_settings.py
@@ -1,6 +1,10 @@
from io import BytesIO
-from django.core.exceptions import RequestDataTooBig, TooManyFieldsSent
+from django.core.exceptions import (
+ RequestDataTooBig,
+ TooManyFieldsSent,
+ TooManyFilesSent,
+)
from django.core.handlers.wsgi import WSGIRequest
from django.test import SimpleTestCase
from django.test.client import FakePayload
@@ -8,6 +12,9 @@
TOO_MANY_FIELDS_MSG = (
"The number of GET/POST parameters exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS."
)
+TOO_MANY_FILES_MSG = (
+ "The number of files exceeded settings.DATA_UPLOAD_MAX_NUMBER_FILES."
+)
TOO_MUCH_DATA_MSG = "Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
@@ -191,6 +198,52 @@ def test_no_limit(self):
self.request._load_post_and_files()
+class DataUploadMaxNumberOfFilesMultipartPost(SimpleTestCase):
+ def setUp(self):
+ payload = FakePayload(
+ "\r\n".join(
+ [
+ "--boundary",
+ (
+ 'Content-Disposition: form-data; name="name1"; '
+ 'filename="name1.txt"'
+ ),
+ "",
+ "value1",
+ "--boundary",
+ (
+ 'Content-Disposition: form-data; name="name2"; '
+ 'filename="name2.txt"'
+ ),
+ "",
+ "value2",
+ "--boundary--",
+ ]
+ )
+ )
+ self.request = WSGIRequest(
+ {
+ "REQUEST_METHOD": "POST",
+ "CONTENT_TYPE": "multipart/form-data; boundary=boundary",
+ "CONTENT_LENGTH": len(payload),
+ "wsgi.input": payload,
+ }
+ )
+
+ def test_number_exceeded(self):
+ with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=1):
+ with self.assertRaisesMessage(TooManyFilesSent, TOO_MANY_FILES_MSG):
+ self.request._load_post_and_files()
+
+ def test_number_not_exceeded(self):
+ with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=2):
+ self.request._load_post_and_files()
+
+ def test_no_limit(self):
+ with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=None):
+ self.request._load_post_and_files()
+
+
class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase):
def setUp(self):
payload = FakePayload("\r\n".join(["a=1&a=2&a=3", ""]))

Binary file not shown.

View File

@ -1,16 +1,11 @@
%global _empty_manifest_terminate_build 0 %global _empty_manifest_terminate_build 0
Name: python-django Name: python-django
Version: 4.1.4 Version: 4.1.7
Release: 3 Release: 1
Summary: A high-level Python Web framework that encourages rapid development and clean, pragmatic design. Summary: A high-level Python Web framework that encourages rapid development and clean, pragmatic design.
License: Apache-2.0 and Python-2.0 and BSD-3-Clause License: Apache-2.0 and Python-2.0 and BSD-3-Clause
URL: https://www.djangoproject.com/ URL: https://www.djangoproject.com/
Source0: https://github.com/django/django/archive/refs/tags/4.1.4.tar.gz Source0: https://github.com/django/django/archive/%{version}/django-%{version}.tar.gz
#https://github.com/django/django/commit/a9010fe5555e6086a9d9ae50069579400ef0685e
Patch0: CVE-2022-34265.patch
Patch1: CVE-2023-23969.patch
Patch2: CVE-2023-24580.patch
BuildArch: noarch BuildArch: noarch
%description %description
@ -77,6 +72,9 @@ mv %{buildroot}/doclist.lst .
%{_docdir}/* %{_docdir}/*
%changelog %changelog
* Tue Apr 11 2023 yaoxin <yao_xin001@hoperun.com> - 4.1.7-1
- Update to 4.1.7
* Sat Feb 25 2023 yaoxin <yaoxin30@h-partners.com> - 4.1.4-3 * Sat Feb 25 2023 yaoxin <yaoxin30@h-partners.com> - 4.1.4-3
- Fix CVE-2023-24580 - Fix CVE-2023-24580