diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000000..bde8b64da0f0 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# Configuration for the Read The Docs (RTD) builds of the documentation. +# Ref: https://docs.readthedocs.io/en/stable/config-file/v2.html +# The python.install.requirements pins the version of Sphinx used. +version: 2 + +build: + os: ubuntu-20.04 + tools: + python: "3.8" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements.txt + +formats: all diff --git a/AUTHORS b/AUTHORS index 20cc0c75c7ca..b8c688cc0c62 100644 --- a/AUTHORS +++ b/AUTHORS @@ -585,6 +585,7 @@ answer newbie questions, and generally made Django that much better: mattycakes@gmail.com Max Burstein Max Derkachev + Max Smolens Maxime Lorant Maxime Turcotte Maximilian Merz diff --git a/README.rst b/README.rst index 2a33fa2e640a..6776b7c39b99 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ ticket here: https://code.djangoproject.com/newticket To get more help: -* Join the ``#django`` channel on irc.freenode.net. Lots of helpful people hang +* Join the ``#django`` channel on ``irc.libera.chat``. Lots of helpful people out there. See https://en.wikipedia.org/wiki/Wikipedia:IRC/Tutorial if you're new to IRC. diff --git a/django/__init__.py b/django/__init__.py index fd771d5ae421..30a42af5324d 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (2, 2, 13, 'final', 0) +VERSION = (2, 2, 28, 'final', 0) __version__ = get_version(VERSION) diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 0474c38fd4d4..5986717d9517 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -15,6 +15,7 @@ from django.http import Http404 from django.template.engine import Engine from django.urls import get_mod_func, get_resolver, get_urlconf +from django.utils._os import safe_join from django.utils.decorators import method_decorator from django.utils.inspect import ( func_accepts_kwargs, func_accepts_var_args, get_func_full_args, @@ -328,7 +329,7 @@ def get_context_data(self, **kwargs): else: # This doesn't account for template loaders (#24128). for index, directory in enumerate(default_engine.dirs): - template_file = Path(directory) / template + template_file = Path(safe_join(directory, template)) if template_file.exists(): with template_file.open() as f: template_contents = f.read() diff --git a/django/contrib/auth/password_validation.py b/django/contrib/auth/password_validation.py index 948ded6dbc39..a80214ded77c 100644 --- a/django/contrib/auth/password_validation.py +++ b/django/contrib/auth/password_validation.py @@ -115,6 +115,36 @@ def get_help_text(self): ) % {'min_length': self.min_length} +def exceeds_maximum_length_ratio(password, max_similarity, value): + """ + Test that value is within a reasonable range of password. + + The following ratio calculations are based on testing SequenceMatcher like + this: + + for i in range(0,6): + print(10**i, SequenceMatcher(a='A', b='A'*(10**i)).quick_ratio()) + + which yields: + + 1 1.0 + 10 0.18181818181818182 + 100 0.019801980198019802 + 1000 0.001998001998001998 + 10000 0.00019998000199980003 + 100000 1.999980000199998e-05 + + This means a length_ratio of 10 should never yield a similarity higher than + 0.2, for 100 this is down to 0.02 and for 1000 it is 0.002. This can be + calculated via 2 / length_ratio. As a result we avoid the potentially + expensive sequence matching. + """ + pwd_len = len(password) + length_bound_similarity = max_similarity / 2 * pwd_len + value_len = len(value) + return pwd_len >= 10 * value_len and value_len < length_bound_similarity + + class UserAttributeSimilarityValidator: """ Validate whether the password is sufficiently different from the user's @@ -130,19 +160,25 @@ class UserAttributeSimilarityValidator: def __init__(self, user_attributes=DEFAULT_USER_ATTRIBUTES, max_similarity=0.7): self.user_attributes = user_attributes + if max_similarity < 0.1: + raise ValueError('max_similarity must be at least 0.1') self.max_similarity = max_similarity def validate(self, password, user=None): if not user: return + password = password.lower() for attribute_name in self.user_attributes: value = getattr(user, attribute_name, None) if not value or not isinstance(value, str): continue - value_parts = re.split(r'\W+', value) + [value] + value_lower = value.lower() + value_parts = re.split(r'\W+', value_lower) + [value_lower] for value_part in value_parts: - if SequenceMatcher(a=password.lower(), b=value_part.lower()).quick_ratio() >= self.max_similarity: + if exceeds_maximum_length_ratio(password, self.max_similarity, value_part): + continue + if SequenceMatcher(a=password, b=value_part).quick_ratio() >= self.max_similarity: try: verbose_name = str(user._meta.get_field(attribute_name).verbose_name) except FieldDoesNotExist: diff --git a/django/contrib/messages/storage/cookie.py b/django/contrib/messages/storage/cookie.py index 9e0c93e436f3..057d573d3fa8 100644 --- a/django/contrib/messages/storage/cookie.py +++ b/django/contrib/messages/storage/cookie.py @@ -89,7 +89,11 @@ def _update_cookie(self, encoded_data, response): samesite=settings.SESSION_COOKIE_SAMESITE, ) else: - response.delete_cookie(self.cookie_name, domain=settings.SESSION_COOKIE_DOMAIN) + response.delete_cookie( + self.cookie_name, + domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) def _store(self, messages, response, remove_oldest=True, *args, **kwargs): """ diff --git a/django/contrib/sessions/middleware.py b/django/contrib/sessions/middleware.py index 6795354cc5f9..a464b44245f4 100644 --- a/django/contrib/sessions/middleware.py +++ b/django/contrib/sessions/middleware.py @@ -39,6 +39,7 @@ def process_response(self, request, response): settings.SESSION_COOKIE_NAME, path=settings.SESSION_COOKIE_PATH, domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, ) else: if accessed: diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index 70de0a8b709d..245c2fb82891 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -287,6 +287,6 @@ def memcache_key_warnings(key): if ord(char) < 33 or ord(char) == 127: yield ( 'Cache key contains characters that will cause errors if ' - 'used with memcached: %r' % key, CacheKeyWarning + 'used with memcached: %r' % key ) break diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index ca8b0065771d..012b54e8cf78 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -114,10 +114,15 @@ def _cull(self): def _createdir(self): if not os.path.exists(self._dir): + # Set the umask because os.makedirs() doesn't apply the "mode" argument + # to intermediate-level directories. + old_umask = os.umask(0o077) try: os.makedirs(self._dir, 0o700) except FileExistsError: pass + finally: + os.umask(old_umask) def _key_to_file(self, key, version=None): """ diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 9a7d8793fc7b..ea5bbc82d0e2 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -1,4 +1,5 @@ import os +import pathlib from datetime import datetime from urllib.parse import urljoin @@ -6,6 +7,7 @@ from django.core.exceptions import SuspiciousFileOperation from django.core.files import File, locks from django.core.files.move import file_move_safe +from django.core.files.utils import validate_file_name from django.core.signals import setting_changed from django.utils import timezone from django.utils._os import safe_join @@ -49,7 +51,10 @@ def save(self, name, content, max_length=None): content = File(content, name) name = self.get_available_name(name, max_length=max_length) - return self._save(name, content) + name = self._save(name, content) + # Ensure that the name returned from the storage system is still valid. + validate_file_name(name, allow_relative_path=True) + return name # These methods are part of the public API, with default implementations. @@ -65,7 +70,11 @@ def get_available_name(self, name, max_length=None): Return a filename that's free on the target storage system and available for new content to be written to. """ + name = str(name).replace('\\', '/') dir_name, file_name = os.path.split(name) + if '..' in pathlib.PurePath(dir_name).parts: + raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dir_name) + validate_file_name(file_name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a random 7 # character alphanumeric string (before the file extension, if one @@ -96,8 +105,11 @@ def generate_filename(self, filename): Validate the filename by calling get_valid_name() and return a filename to be passed to the save() method. """ + filename = str(filename).replace('\\', '/') # `filename` may include a path as returned by FileField.upload_to. dirname, filename = os.path.split(filename) + if '..' in pathlib.PurePath(dirname).parts: + raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dirname) return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename))) def path(self, name): @@ -231,9 +243,9 @@ def _save(self, name, content): if not os.path.exists(directory): try: if self.directory_permissions_mode is not None: - # os.makedirs applies the global umask, so we reset it, - # for consistency with file_permissions_mode behavior. - old_umask = os.umask(0) + # Set the umask because os.makedirs() doesn't apply the "mode" + # argument to intermediate-level directories. + old_umask = os.umask(0o777 & ~self.directory_permissions_mode) try: os.makedirs(directory, self.directory_permissions_mode) finally: @@ -289,6 +301,8 @@ def _save(self, name, content): if self.file_permissions_mode is not None: os.chmod(full_path, self.file_permissions_mode) + # Ensure the saved path is always relative to the storage root. + name = os.path.relpath(full_path, self.location) # Store filenames with forward slashes, even on Windows. return name.replace('\\', '/') diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py index 48007b86823d..f452bcd9a4a1 100644 --- a/django/core/files/uploadedfile.py +++ b/django/core/files/uploadedfile.py @@ -8,6 +8,7 @@ from django.conf import settings from django.core.files import temp as tempfile from django.core.files.base import File +from django.core.files.utils import validate_file_name __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile') @@ -47,6 +48,8 @@ def _set_name(self, name): ext = ext[:255] name = name[:255 - len(ext)] + ext + name = validate_file_name(name) + self._name = name name = property(_get_name, _set_name) diff --git a/django/core/files/utils.py b/django/core/files/utils.py index de896071759b..f28cea107758 100644 --- a/django/core/files/utils.py +++ b/django/core/files/utils.py @@ -1,3 +1,29 @@ +import os +import pathlib + +from django.core.exceptions import SuspiciousFileOperation + + +def validate_file_name(name, allow_relative_path=False): + # Remove potentially dangerous names + if os.path.basename(name) in {'', '.', '..'}: + raise SuspiciousFileOperation("Could not derive file name from '%s'" % name) + + if allow_relative_path: + # Use PurePosixPath() because this branch is checked only in + # FileField.generate_filename() where all file paths are expected to be + # Unix style (with forward slashes). + path = pathlib.PurePosixPath(name) + if path.is_absolute() or '..' in path.parts: + raise SuspiciousFileOperation( + "Detected path traversal attempt in '%s'" % name + ) + elif name != os.path.basename(name): + raise SuspiciousFileOperation("File name '%s' includes path elements" % name) + + return name + + class FileProxyMixin: """ A mixin class used to forward file methods to an underlaying file diff --git a/django/core/mail/message.py b/django/core/mail/message.py index a0e80acce9c7..7a3ee799f5d3 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -10,7 +10,9 @@ from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formatdate, getaddresses, make_msgid, parseaddr +from email.utils import ( + formataddr, formatdate, getaddresses, make_msgid, parseaddr, +) from io import BytesIO, StringIO from pathlib import Path @@ -103,7 +105,15 @@ def sanitize_address(addr, encoding): addr = parseaddr(addr) nm, addr = addr localpart, domain = None, None - nm = Header(nm, encoding).encode() + if '\n' in nm or '\r' in nm: + raise ValueError('Invalid address; address parts cannot contain newlines.') + + # Avoid UTF-8 encode, if it's possible. + try: + nm.encode('ascii') + nm = Header(nm).encode() + except UnicodeEncodeError: + nm = Header(nm, encoding).encode() try: addr.encode('ascii') except UnicodeEncodeError: # IDN or non-ascii in the local part @@ -112,15 +122,20 @@ def sanitize_address(addr, encoding): # An `email.headerregistry.Address` object is used since # email.utils.formataddr() naively encodes the name as ascii (see #25986). if localpart and domain: - address = Address(nm, username=localpart, domain=domain) - return str(address) + address_parts = localpart + domain + if '\n' in address_parts or '\r' in address_parts: + raise ValueError('Invalid address; address parts cannot contain newlines.') + address = Address(username=localpart, domain=domain) + return formataddr((nm, address.addr_spec)) try: - address = Address(nm, addr_spec=addr) + if '\n' in addr or '\r' in addr: + raise ValueError('Invalid address; address parts cannot contain newlines.') + address = Address(addr_spec=addr) except (InvalidHeaderDefect, NonASCIILocalPartDefect): localpart, domain = split_addr(addr, encoding) - address = Address(nm, username=localpart, domain=domain) - return str(address) + address = Address(username=localpart, domain=domain) + return formataddr((nm, address.addr_spec)) class MIMEMixin: diff --git a/django/core/validators.py b/django/core/validators.py index 38e4b6aa1d7a..2da0688e28b8 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -75,7 +75,7 @@ class URLValidator(RegexValidator): ul = '\u00a1-\uffff' # unicode letters range (must not be a raw string) # IP patterns - ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' + ipv4_re = r'(?:0|25[0-5]|2[0-4]\d|1\d?\d?|[1-9]\d?)(?:\.(?:0|25[0-5]|2[0-4]\d|1\d?\d?|[1-9]\d?)){3}' ipv6_re = r'\[[0-9a-f:\.]+\]' # (simple regex, validated later) # Host patterns @@ -101,6 +101,7 @@ class URLValidator(RegexValidator): r'\Z', re.IGNORECASE) message = _('Enter a valid URL.') schemes = ['http', 'https', 'ftp', 'ftps'] + unsafe_chars = frozenset('\t\r\n') def __init__(self, schemes=None, **kwargs): super().__init__(**kwargs) @@ -108,7 +109,9 @@ def __init__(self, schemes=None, **kwargs): self.schemes = schemes def __call__(self, value): - # Check first if the scheme is valid + if isinstance(value, str) and self.unsafe_chars.intersection(value): + raise ValidationError(self.message, code=self.code) + # Check if the scheme is valid. scheme = value.split('://')[0].lower() if scheme not in self.schemes: raise ValidationError(self.message, code=self.code) @@ -253,6 +256,18 @@ def validate_ipv4_address(value): ipaddress.IPv4Address(value) except ValueError: raise ValidationError(_('Enter a valid IPv4 address.'), code='invalid') + else: + # Leading zeros are forbidden to avoid ambiguity with the octal + # notation. This restriction is included in Python 3.9.5+. + # TODO: Remove when dropping support for PY39. + if any( + octet != '0' and octet[0] == '0' + for octet in value.split('.') + ): + raise ValidationError( + _('Enter a valid IPv4 address.'), + code='invalid', + ) def validate_ipv6_address(value): diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index 5c8701c396d4..9f63ca6b0ce1 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -53,7 +53,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_over_clause = True supports_aggregate_filter_clause = True supported_explain_formats = {'JSON', 'TEXT', 'XML', 'YAML'} - validates_explain_options = False # A query will error on invalid options. @cached_property def is_postgresql_9_5(self): diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index 66e5482be6ba..66ac2d5d108c 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -8,6 +8,18 @@ class DatabaseOperations(BaseDatabaseOperations): cast_char_field_without_max_length = 'varchar' explain_prefix = 'EXPLAIN' + explain_options = frozenset( + [ + "ANALYZE", + "BUFFERS", + "COSTS", + "SETTINGS", + "SUMMARY", + "TIMING", + "VERBOSE", + "WAL", + ] + ) cast_data_types = { 'AutoField': 'integer', 'BigAutoField': 'bigint', @@ -267,15 +279,20 @@ def window_frame_range_start_end(self, start=None, end=None): return start_, end_ def explain_query_prefix(self, format=None, **options): - prefix = super().explain_query_prefix(format) extra = {} - if format: - extra['FORMAT'] = format + # Normalize options. if options: - extra.update({ + options = { name.upper(): 'true' if value else 'false' for name, value in options.items() - }) + } + for valid_option in self.explain_options: + value = options.pop(valid_option, None) + if value is not None: + extra[valid_option.upper()] = value + prefix = super().explain_query_prefix(format, **options) + if format: + extra['FORMAT'] = format if extra: prefix += ' (%s)' % ', '.join('%s %s' % i for i in extra.items()) return prefix diff --git a/django/db/models/base.py b/django/db/models/base.py index fccc6633af2b..92a1135a1abd 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -544,7 +544,10 @@ def __reduce__(self): def __getstate__(self): """Hook to allow choosing the attributes to pickle.""" - return self.__dict__ + state = self.__dict__.copy() + state['_state'] = copy.copy(state['_state']) + state['_state'].fields_cache = state['_state'].fields_cache.copy() + return state def __setstate__(self, state): msg = None diff --git a/django/db/models/fields/files.py b/django/db/models/fields/files.py index bd8da95e4649..0f8c3fe48420 100644 --- a/django/db/models/fields/files.py +++ b/django/db/models/fields/files.py @@ -6,6 +6,7 @@ from django.core.files.base import File from django.core.files.images import ImageFile from django.core.files.storage import default_storage +from django.core.files.utils import validate_file_name from django.db.models import signals from django.db.models.fields import Field from django.utils.translation import gettext_lazy as _ @@ -304,6 +305,7 @@ def generate_filename(self, instance, filename): else: dirname = datetime.datetime.now().strftime(self.upload_to) filename = posixpath.join(dirname, filename) + filename = validate_file_name(filename, allow_relative_path=True) return self.storage.generate_filename(filename) def save_form_data(self, instance, data): diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index e5c726676ad8..f3d66cff33be 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -948,7 +948,8 @@ def get_select_for_update_of_arguments(self): the query. """ def _get_parent_klass_info(klass_info): - for parent_model, parent_link in klass_info['model']._meta.parents.items(): + concrete_model = klass_info['model']._meta.concrete_model + for parent_model, parent_link in concrete_model._meta.parents.items(): parent_list = parent_model._meta.get_parent_list() yield { 'model': parent_model, @@ -973,8 +974,9 @@ def _get_first_selected_col_from_model(klass_info): select_fields is filled recursively, so it also contains fields from the parent models. """ + concrete_model = klass_info['model']._meta.concrete_model for select_index in klass_info['select_fields']: - if self.select[select_index][0].target.model == klass_info['model']: + if self.select[select_index][0].target.model == concrete_model: return self.select[select_index][0] def _get_field_choices(): diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index b99f0e90efad..1e823cfe74b1 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -8,6 +8,7 @@ """ import difflib import functools +import re from collections import Counter, OrderedDict, namedtuple from collections.abc import Iterator, Mapping from itertools import chain, count, product @@ -40,6 +41,14 @@ __all__ = ['Query', 'RawQuery'] +# Quotation marks ('"`[]), whitespace characters, semicolons, or inline +# SQL comments are forbidden in column aliases. +FORBIDDEN_ALIAS_PATTERN = re.compile(r"['`\"\]\[;\s]|--|/\*|\*/") + +# Inspired from +# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS +EXPLAIN_OPTIONS_PATTERN = re.compile(r"[\w\-]+") + def get_field_names_from_opts(opts): return set(chain.from_iterable( @@ -523,6 +532,12 @@ def has_results(self, using): def explain(self, using, format=None, **options): q = self.clone() + for option_name in options: + if ( + not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name) or + "--" in option_name + ): + raise ValueError("Invalid option name: '%s'." % option_name) q.explain_query = True q.explain_format = format q.explain_options = options @@ -994,8 +1009,16 @@ def join_parent_model(self, opts, model, alias, seen): alias = seen[int_model] = join_info.joins[-1] return alias or seen[None] + def check_alias(self, alias): + if FORBIDDEN_ALIAS_PATTERN.search(alias): + raise ValueError( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + def add_annotation(self, annotation, alias, is_summary=False): """Add a single annotation expression to the Query.""" + self.check_alias(alias) annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None, summarize=is_summary) self.append_annotation_mask([alias]) @@ -1873,6 +1896,7 @@ def add_extra(self, select, select_params, where, params, tables, order_by): else: param_iter = iter([]) for name, entry in select.items(): + self.check_alias(name) entry = str(entry) entry_params = [] pos = entry.find("%s") diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index f6f12ca71860..259128acefce 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -7,6 +7,7 @@ import base64 import binascii import cgi +import html from urllib.parse import unquote from django.conf import settings @@ -18,7 +19,6 @@ ) from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_text -from django.utils.text import unescape_entities __all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted') @@ -205,7 +205,7 @@ def parse(self): file_name = disposition.get('filename') if file_name: file_name = force_text(file_name, encoding, errors='replace') - file_name = self.IE_sanitize(unescape_entities(file_name)) + file_name = self.sanitize_file_name(file_name) if not file_name: continue @@ -240,6 +240,8 @@ def parse(self): remaining = len(stripped_chunk) % 4 while remaining != 0: over_chunk = field_stream.read(4 - remaining) + if not over_chunk: + break stripped_chunk += b"".join(over_chunk.split()) remaining = len(stripped_chunk) % 4 @@ -293,9 +295,28 @@ def handle_file_complete(self, old_field_name, counters): self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj) break - def IE_sanitize(self, filename): - """Cleanup filename from Internet Explorer full paths.""" - return filename and filename[filename.rfind("\\") + 1:].strip() + def sanitize_file_name(self, file_name): + """ + Sanitize the filename of an upload. + + Remove all possible path separators, even though that might remove more + than actually required by the target system. Filenames that could + potentially cause problems (current/parent dir) are also discarded. + + It should be noted that this function could still return a "filepath" + like "C:some_file.txt" which is handled later on by the storage layer. + So while this function does sanitize filenames to some extent, the + resulting filename should still be considered as untrusted user input. + """ + file_name = html.unescape(file_name) + file_name = file_name.rsplit('/')[-1] + file_name = file_name.rsplit('\\')[-1] + + if file_name in {'', '.', '..'}: + return None + return file_name + + IE_sanitize = sanitize_file_name def _close_files(self): # Free up all file handles. diff --git a/django/http/response.py b/django/http/response.py index f7d248e93328..5a693b178602 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -210,13 +210,13 @@ def set_signed_cookie(self, key, value, salt='', **kwargs): value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) - def delete_cookie(self, key, path='/', domain=None): + def delete_cookie(self, key, path='/', domain=None, samesite=None): # Most browsers ignore the Set-Cookie header if the cookie name starts # with __Host- or __Secure- and the cookie doesn't use the secure flag. secure = key.startswith(('__Secure-', '__Host-')) self.set_cookie( key, max_age=0, path=path, domain=domain, secure=secure, - expires='Thu, 01 Jan 1970 00:00:00 GMT', + expires='Thu, 01 Jan 1970 00:00:00 GMT', samesite=samesite, ) # Common methods used by subclasses diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index f82c08348a75..a1d77f5e692e 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -22,7 +22,7 @@ from django.utils.timesince import timesince, timeuntil from django.utils.translation import gettext, ngettext -from .base import Variable, VariableDoesNotExist +from .base import VARIABLE_ATTRIBUTE_SEPARATOR from .library import Library register = Library() @@ -465,7 +465,7 @@ def striptags(value): def _property_resolver(arg): """ When arg is convertible to float, behave like operator.itemgetter(arg) - Otherwise, behave like Variable(arg).resolve + Otherwise, chain __getitem__() and getattr(). >>> _property_resolver(1)('abc') 'b' @@ -483,7 +483,19 @@ def _property_resolver(arg): try: float(arg) except ValueError: - return Variable(arg).resolve + if VARIABLE_ATTRIBUTE_SEPARATOR + '_' in arg or arg[0] == '_': + raise AttributeError('Access to private variables is forbidden.') + parts = arg.split(VARIABLE_ATTRIBUTE_SEPARATOR) + + def resolve(value): + for part in parts: + try: + value = value[part] + except (AttributeError, IndexError, KeyError, TypeError, ValueError): + value = getattr(value, part) + return value + + return resolve else: return itemgetter(arg) @@ -496,7 +508,7 @@ def dictsort(value, arg): """ try: return sorted(value, key=_property_resolver(arg)) - except (TypeError, VariableDoesNotExist): + except (AttributeError, TypeError): return '' @@ -508,7 +520,7 @@ def dictsortreversed(value, arg): """ try: return sorted(value, key=_property_resolver(arg), reverse=True) - except (TypeError, VariableDoesNotExist): + except (AttributeError, TypeError): return '' diff --git a/django/template/defaulttags.py b/django/template/defaulttags.py index c4a37c25dde2..31fa279ca045 100644 --- a/django/template/defaulttags.py +++ b/django/template/defaulttags.py @@ -8,7 +8,7 @@ from django.conf import settings from django.utils import timezone -from django.utils.html import conditional_escape, format_html +from django.utils.html import conditional_escape, escape, format_html from django.utils.lorem_ipsum import paragraphs, words from django.utils.safestring import mark_safe @@ -94,10 +94,13 @@ def reset(self, context): class DebugNode(Node): def render(self, context): + if not settings.DEBUG: + return '' + from pprint import pformat - output = [pformat(val) for val in context] + output = [escape(pformat(val)) for val in context] output.append('\n\n') - output.append(pformat(sys.modules)) + output.append(escape(pformat(sys.modules))) return ''.join(output) diff --git a/django/urls/resolvers.py b/django/urls/resolvers.py index 5b722474c9ec..3f8f6c00ea89 100644 --- a/django/urls/resolvers.py +++ b/django/urls/resolvers.py @@ -147,7 +147,11 @@ def __init__(self, regex, name=None, is_endpoint=False): self.converters = {} def match(self, path): - match = self.regex.search(path) + match = ( + self.regex.fullmatch(path) + if self._is_endpoint and self.regex.pattern.endswith('$') + else self.regex.search(path) + ) if match: # If there are any named groups, use those as kwargs, ignoring # non-named groups. Otherwise, pass all non-named arguments as @@ -230,7 +234,7 @@ def _route_to_regex(route, is_endpoint=False): converters[parameter] = converter parts.append('(?P<' + parameter + '>' + converter.regex + ')') if is_endpoint: - parts.append('$') + parts.append(r'\Z') return ''.join(parts), converters diff --git a/django/utils/archive.py b/django/utils/archive.py index 5b9998f89cc3..f2f153a1fc3d 100644 --- a/django/utils/archive.py +++ b/django/utils/archive.py @@ -27,6 +27,8 @@ import tarfile import zipfile +from django.core.exceptions import SuspiciousOperation + class ArchiveException(Exception): """ @@ -133,6 +135,13 @@ def has_leading_dir(self, paths): return False return True + def target_filename(self, to_path, name): + target_path = os.path.abspath(to_path) + filename = os.path.abspath(os.path.join(target_path, name)) + if not filename.startswith(target_path): + raise SuspiciousOperation("Archive contains invalid path: '%s'" % name) + return filename + def extract(self): raise NotImplementedError('subclasses of BaseArchive must provide an extract() method') @@ -155,7 +164,7 @@ def extract(self, to_path): name = member.name if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) + filename = self.target_filename(to_path, name) if member.isdir(): if filename and not os.path.exists(filename): os.makedirs(filename) @@ -198,11 +207,13 @@ def extract(self, to_path): info = self._archive.getinfo(name) if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) + if not name: + continue + filename = self.target_filename(to_path, name) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - if filename.endswith(('/', '\\')): + if name.endswith(('/', '\\')): # A directory if not os.path.exists(filename): os.makedirs(filename) diff --git a/django/utils/http.py b/django/utils/http.py index de1ea713685a..74bc5cb8ab56 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -41,7 +41,7 @@ RFC3986_GENDELIMS = ":/?#[]@" RFC3986_SUBDELIMS = "!$&'()*+,;=" -FIELDS_MATCH = re.compile('[&;]') +FIELDS_MATCH = re.compile('&') @keep_lazy_text diff --git a/django/utils/text.py b/django/utils/text.py index 853436a38f3f..1fae7b252255 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -4,6 +4,7 @@ from gzip import GzipFile from io import BytesIO +from django.core.exceptions import SuspiciousFileOperation from django.utils.functional import SimpleLazyObject, keep_lazy_text, lazy from django.utils.translation import gettext as _, gettext_lazy, pgettext @@ -216,7 +217,7 @@ def _truncate_html(self, length, truncate, text, truncate_len, words): @keep_lazy_text -def get_valid_filename(s): +def get_valid_filename(name): """ Return the given string converted to a string that can be used for a clean filename. Remove leading and trailing spaces; convert other spaces to @@ -225,8 +226,11 @@ def get_valid_filename(s): >>> get_valid_filename("john's portrait in 2004.jpg") 'johns_portrait_in_2004.jpg' """ - s = str(s).strip().replace(' ', '_') - return re.sub(r'(?u)[^-\w.]', '', s) + s = str(name).strip().replace(' ', '_') + s = re.sub(r'(?u)[^-\w.]', '', s) + if s in {'', '.', '..'}: + raise SuspiciousFileOperation("Could not derive file name from '%s'" % name) + return s @keep_lazy_text diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index cc40c40cd8d6..1fa3e4bf5e97 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -8,7 +8,7 @@ from docutils import nodes from docutils.parsers.rst import Directive from docutils.statemachine import ViewList -from sphinx import addnodes +from sphinx import addnodes, version_info as sphinx_version from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.directives.code import CodeBlock from sphinx.domains.std import Cmdoption @@ -114,11 +114,17 @@ class DjangoHTMLTranslator(HTMLTranslator): def visit_table(self, node): self.context.append(self.compact_p) self.compact_p = True - self._table_row_index = 0 # Needed by Sphinx + # Needed by Sphinx. + if sphinx_version >= (4, 3): + self._table_row_indices.append(0) + else: + self._table_row_index = 0 self.body.append(self.starttag(node, 'table', CLASS='docutils')) def depart_table(self, node): self.compact_p = self.context.pop() + if sphinx_version >= (4, 3): + self._table_row_indices.pop() self.body.append('\n') def visit_desc_parameterlist(self, node): diff --git a/docs/conf.py b/docs/conf.py index 11d637cb5c09..52fa18fc16ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -95,8 +95,9 @@ def django_release(): django_next_version = '3.0' extlinks = { + 'bpo': ('https://bugs.python.org/issue%s', 'bpo-'), 'commit': ('https://github.com/django/django/commit/%s', ''), - 'cve': ('https://nvd.nist.gov/view/vuln/detail?vulnId=%s', 'CVE-'), + 'cve': ('https://nvd.nist.gov/vuln/detail/CVE-%s', 'CVE-'), # A file or directory. GitHub redirects from blob to tree if needed. 'source': ('https://github.com/django/django/blob/stable/' + version + '.x/%s', ''), 'ticket': ('https://code.djangoproject.com/ticket/%s', '#'), @@ -117,7 +118,7 @@ def django_release(): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', '_theme'] +exclude_patterns = ['_build', '_theme', 'requirements.txt'] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None @@ -233,7 +234,6 @@ def django_release(): # Appended to every page rst_epilog = """ .. |django-users| replace:: :ref:`django-users ` -.. |django-core-mentorship| replace:: :ref:`django-core-mentorship ` .. |django-developers| replace:: :ref:`django-developers ` .. |django-announce| replace:: :ref:`django-announce ` .. |django-updates| replace:: :ref:`django-updates ` diff --git a/docs/faq/help.txt b/docs/faq/help.txt index e2626894eef9..fe76ba6e1e12 100644 --- a/docs/faq/help.txt +++ b/docs/faq/help.txt @@ -9,10 +9,9 @@ If this FAQ doesn't contain an answer to your question, you might want to try the |django-users| mailing list. Feel free to ask any question related to installing, using, or debugging Django. -If you prefer IRC, the `#django IRC channel`_ on the Freenode IRC network is an -active community of helpful individuals who may be able to solve your problem. - -.. _`#django IRC channel`: irc://irc.freenode.net/django +If you prefer IRC, the `#django IRC channel`_ on the Libera.Chat IRC network is +an active community of helpful individuals who may be able to solve your +problem. .. _message-does-not-appear-on-django-users: @@ -40,7 +39,7 @@ As with most open-source mailing lists, the folks on |django-users| are volunteers. If nobody has answered your question, it may be because nobody knows the answer, it may be because nobody can understand the question, or it may be that everybody that can help is busy. One thing you might try is to ask -the question on IRC -- visit the `#django IRC channel`_ on the Freenode IRC +the question on IRC -- visit the `#django IRC channel`_ on the Libera.Chat IRC network. You might notice we have a second mailing list, called |django-developers| -- @@ -69,3 +68,4 @@ while a defect is outstanding, we would like to minimize any damage that could be inflicted through public knowledge of that defect. .. _`policy for handling security issues`: ../contributing/#reporting-security-issues +.. _`#django IRC channel`: irc://irc.libera.chat/django diff --git a/docs/faq/install.txt b/docs/faq/install.txt index 5f322bc7e639..d3f79c22466b 100644 --- a/docs/faq/install.txt +++ b/docs/faq/install.txt @@ -49,7 +49,7 @@ Django version Python versions 1.11 2.7, 3.4, 3.5, 3.6, 3.7 (added in 1.11.17) 2.0 3.4, 3.5, 3.6, 3.7 2.1 3.5, 3.6, 3.7 -2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8) +2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8), 3.9 (added in 2.2.17) ============== =============== For each version of Python, only the latest micro release (A.B.C) is officially diff --git a/docs/index.txt b/docs/index.txt index 6139c3e9b889..9c00dc438578 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -23,7 +23,7 @@ Having trouble? We'd like to help! .. _archives: https://groups.google.com/group/django-users/ .. _post a question: https://groups.google.com/d/forum/django-users -.. _#django IRC channel: irc://irc.freenode.net/django +.. _#django IRC channel: irc://irc.libera.chat/django .. _ticket tracker: https://code.djangoproject.com/ How the documentation is organized diff --git a/docs/internals/contributing/bugs-and-features.txt b/docs/internals/contributing/bugs-and-features.txt index 858de4ad0860..dcaa78ee93a8 100644 --- a/docs/internals/contributing/bugs-and-features.txt +++ b/docs/internals/contributing/bugs-and-features.txt @@ -166,4 +166,4 @@ Votes on technical matters should be announced and held in public on the .. _searching: https://code.djangoproject.com/search .. _custom queries: https://code.djangoproject.com/query -.. _#django: irc://irc.freenode.net/django +.. _#django: irc://irc.libera.chat/django diff --git a/docs/internals/contributing/index.txt b/docs/internals/contributing/index.txt index 9a1e5d64d7f5..1915c99dddde 100644 --- a/docs/internals/contributing/index.txt +++ b/docs/internals/contributing/index.txt @@ -16,7 +16,7 @@ contribute in many ways: friendly and helpful atmosphere. If you're new to the Django community, you should read the `posting guidelines`_. -* Join the `#django IRC channel`_ on Freenode and answer questions. By +* Join the `#django IRC channel`_ on Libera.Chat and answer questions. By explaining Django to other users, you're going to learn a lot about the framework yourself. @@ -68,8 +68,8 @@ Browse the following sections to find out how: committing-code .. _posting guidelines: https://code.djangoproject.com/wiki/UsingTheMailingList -.. _#django IRC channel: irc://irc.freenode.net/django -.. _#django-dev IRC channel: irc://irc.freenode.net/django-dev +.. _#django IRC channel: irc://irc.libera.chat/django +.. _#django-dev IRC channel: irc://irc.libera.chat/django-dev .. _community page: https://www.djangoproject.com/community/ .. _Django forum: https://forum.djangoproject.com/ .. _register it here: https://www.djangoproject.com/community/add/blogs/ diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 25a9a15ab854..4fe32950cba5 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -235,7 +235,7 @@ dependencies: * geoip2_ * jinja2_ 2.7+ * numpy_ -* Pillow_ +* Pillow_ 4.2.0+ * PyYAML_ * pytz_ (required) * pywatchman_ @@ -243,7 +243,7 @@ dependencies: * memcached_, plus a :ref:`supported Python binding ` * gettext_ (:ref:`gettext_on_windows`) * selenium_ -* sqlparse_ (required) +* sqlparse_ 0.2.2+ (required) * tblib_ 1.5.0+ You can find these dependencies in `pip requirements files`_ inside the diff --git a/docs/internals/mailing-lists.txt b/docs/internals/mailing-lists.txt index d5b9ab5f9ced..cdf5b6d26451 100644 --- a/docs/internals/mailing-lists.txt +++ b/docs/internals/mailing-lists.txt @@ -35,23 +35,6 @@ installation, usage, or debugging of Django. .. _django-users subscription email address: mailto:django-users+subscribe@googlegroups.com .. _django-users posting email: mailto:django-users@googlegroups.com -.. _django-core-mentorship-mailing-list: - -``django-core-mentorship`` -========================== - -The Django Core Mentorship list is intended to provide a welcoming -introductory environment for community members interested in contributing to -the Django Project. - -* `django-core-mentorship mailing archive`_ -* `django-core-mentorship subscription email address`_ -* `django-core-mentorship posting email`_ - -.. _django-core-mentorship mailing archive: https://groups.google.com/d/forum/django-core-mentorship -.. _django-core-mentorship subscription email address: mailto:django-core-mentorship+subscribe@googlegroups.com -.. _django-core-mentorship posting email: mailto:django-core-mentorship@googlegroups.com - .. _django-developers-mailing-list: ``django-developers`` diff --git a/docs/internals/organization.txt b/docs/internals/organization.txt index b2d399255f66..b124b4b5fe93 100644 --- a/docs/internals/organization.txt +++ b/docs/internals/organization.txt @@ -21,170 +21,280 @@ and its community. .. _Django Code of Conduct: https://www.djangoproject.com/conduct/ .. _Django Software Foundation: https://www.djangoproject.com/foundation/ -The Django core team makes the decisions, nominates its new members, and -elects its technical board. While it holds decision power in theory, it aims -at using it as rarely as possible in practice. Rough consensus should be the -norm and formal voting an exception. +.. _mergers-team: -.. _core-team: - -Core team -========= +Mergers +======= Role ---- -The core team is the group of trusted volunteers who manage the Django -Project. They assume many roles required to achieve the project's goals, -especially those that require a high level of trust. They make the decisions -that shape the future of the project. - -Core team members are expected to act as role models for the community and -custodians of the project, on behalf of the community and all those who rely -on Django. - -They will intervene, where necessary, in online discussions or at official -Django events on the rare occasions that a situation arises that requires -intervention. - -They have authority over the Django Project infrastructure, including the -Django Project website itself, the Django GitHub organization and -repositories, the Trac bug tracker, the mailing lists, IRC channels, etc. +Mergers_ are a small set of people who merge pull requests to the `Django Git +repository`_. Prerogatives ------------ -Core team members may participate in formal votes, typically to nominate new -team members and to elect the technical board. +Mergers hold the following prerogatives: -Some contributions don't require commit access. Depending on the reasons why a -contributor joins the team, they may or may not have commit permissions to the -Django code repository. +- Merging any pull request which constitutes a `minor change`_ (small enough + not to require the use of the `DEP process`_). A Merger must not merge a + change primarily authored by that Merger, unless the pull request has been + approved by: -However, should the need arise, any team member may ask for commit access by -writing to the core team's mailing list. Access will be granted unless the -person withdraws their request or the technical board vetoes the proposal. + - another Merger, + - a technical board member, + - a member of the `triage & review team`_, or + - a member of the `security team`_. -Core team members who have commit access are referred to as "committers" or -"core developers". +- Initiating discussion of a minor change in the appropriate venue, and request + that other Mergers refrain from merging it while discussion proceeds. +- Requesting a vote of the technical board regarding any minor change if, in + the Merger's opinion, discussion has failed to reach a consensus. +- Requesting a vote of the technical board when a `major change`_ (significant + enough to require the use of the `DEP process`_) reaches one of its + implementation milestones and is intended to merge. -Other permissions, such as access to the servers, are granted to those who -need them through the same process. +.. _`minor change`: https://github.com/django/deps/blob/main/accepted/0010-new-governance.rst#terminology +.. _`major change`: https://github.com/django/deps/blob/main/accepted/0010-new-governance.rst#terminology Membership ---------- -`Django team members `_ -demonstrate: - -- a good grasp of the philosophy of the Django Project -- a solid track record of being constructive and helpful -- significant contributions to the project's goals, in any form -- willingness to dedicate some time to improving Django - -As the project matures, contributions go way beyond code. Here's an incomplete -list of areas where contributions may be considered for joining the core team, -in no particular order: - -- Working on community management and outreach -- Providing support on the mailing-lists and on IRC -- Triaging tickets -- Writing patches (code, docs, or tests) -- Reviewing patches (code, docs, or tests) -- Participating in design decisions -- Providing expertise in a particular domain (security, i18n, etc.) -- Managing the continuous integration infrastructure -- Managing the servers (website, tracker, documentation, etc.) -- Maintaining related projects (djangoproject.com site, ex-contrib apps, etc.) -- Creating visual designs - -Very few areas are reserved to core team members: - -- Reviewing security reports -- Merging patches (code, docs, or tests) -- Packaging releases - -Core team membership acknowledges sustained and valuable efforts that align -well with the philosophy and the goals of the Django Project. - -It is granted by a four fifths majority of votes cast in a core team vote and -no veto by the technical board. - -Core team members are always looking for promising contributors, teaching them -how the project is managed, and submitting their names to the core team's vote -when they're ready. If you would like to join the core team, you can contact a -core team member privately or ask for guidance on the :ref:`Django Core -Mentorship mailing-list `. - -There's no time limit on core team membership. However, in order to provide -the general public with a reasonable idea of how many people maintain Django, -core team members who have stopped contributing are encouraged to declare -themselves as "past team members". Those who haven't made any non-trivial -contribution in two years may be asked to move themselves to this category, -and moved there if they don't respond. Past team members lose their privileges -such as voting rights and commit access. +`The technical board`_ selects Mergers_ as necessary to maintain their number +at a minimum of three, in order to spread the workload and avoid over-burdening +or burning out any individual Merger. There is no upper limit to the number of +Mergers. -.. _technical-board: +It's not a requirement that a Merger is also a Django Fellow, but the Django +Software Foundation has the power to use funding of Fellow positions as a way +to make the role of Merger sustainable. -Technical board -=============== +The following restrictions apply to the role of Merger: + +- A person must not simultaneously serve as a member of the technical board. If + a Merger is elected to the technical board, they shall cease to be a Merger + immediately upon taking up membership in the technical board. +- A person may serve in the roles of Releaser and Merger simultaneously. + +The selection process, when a vacancy occurs or when the technical board deems +it necessary to select additional persons for such a role, occur as follows: + +- Any member in good standing of an appropriate discussion venue, or the Django + Software Foundation board acting with the input of the DSF's Fellowship + committee, may suggest a person for consideration. +- The technical board considers the suggestions put forth, and then any member + of the technical board formally nominates a candidate for the role. +- The technical board votes on nominees. + +Mergers may resign their role at any time, but should endeavor to provide some +advance notice in order to allow the selection of a replacement. Termination of +the contract of a Django Fellow by the Django Software Foundation temporarily +suspends that person's Merger role until such time as the technical board can +vote on their nomination. + +Otherwise, a Merger may be removed by: + +- Becoming disqualified due to election to the technical board. +- Becoming disqualified due to actions taken by the Code of Conduct committee + of the Django Software Foundation. +- A vote of the technical board. + +.. _releasers-team: + +Releasers +========= Role ---- -The technical board is a group of experienced and active committers who steer -technical choices. Their main concern is to maintain the quality and stability -of the Django Web Framework. +Releasers_ are a small set of people who have the authority to upload packaged +releases of Django to the `Python Package Index`_, and to the +`djangoproject.com`_ website. Prerogatives ------------ -The technical board holds two prerogatives: +Releasers_ :doc:`build Django releases ` and +upload them to the `Python Package Index`_, and to the `djangoproject.com`_ +website. + +Membership +---------- + +`The technical board`_ selects Releasers_ as necessary to maintain their number +at a minimum of three, in order to spread the workload and avoid over-burdening +or burning out any individual Releaser. There is no upper limit to the number +of Releasers. -- Making major technical decisions when no consensus is found otherwise. This - happens on the |django-developers| mailing-list. -- Veto a grant of commit access or remove commit access. This happens on the - ``django-core`` mailing-list. +It's not a requirement that a Releaser is also a Django Fellow, but the Django +Software Foundation has the power to use funding of Fellow positions as a way +to make the role of Releaser sustainable. -In both cases, the technical board is a last resort. In these matters, it -fulfills a similar function to the former Benevolent Dictators For Life. +A person may serve in the roles of Releaser and Merger simultaneously. -When the board wants to exercise one of these prerogatives, it must hold a -private, simple majority vote on the resolution. The quorum is the full -committee — each member must cast a vote or abstain explicitly. Then the board -communicates the result, and if possible the reasons, on the appropriate -mailing-list. There's no appeal for such decisions. +The selection process, when a vacancy occurs or when the technical board deems +it necessary to select additional persons for such a role, occur as follows: -In addition, at its discretion, the technical board may act in an advisory -capacity on non-technical decisions. +- Any member in good standing of an appropriate discussion venue, or the Django + Software Foundation board acting with the input of the DSF's Fellowship + committee, may suggest a person for consideration. +- The technical board considers the suggestions put forth, and then any member + of the technical board formally nominates a candidate for the role. +- The technical board votes on nominees. -Membership ----------- +Releasers may resign their role at any time, but should endeavor to provide +some advance notice in order to allow the selection of a replacement. +Termination of the contract of a Django Fellow by the Django Software +Foundation temporarily suspends that person's Releaser role until such time as +the technical board can vote on their nomination. + +Otherwise, a Releaser may be removed by: -`The technical board`_ is an elected group of five committers. They're expected -to be experienced but there's no formal seniority requirement. +- Becoming disqualified due to actions taken by the Code of Conduct committee + of the Django Software Foundation. +- A vote of the technical board. -A new board is elected after each feature release of Django. The election -process is managed by a returns officer nominated by the outgoing technical -board. The election process works as follows: +.. _`Python Package Index`: https://pypi.org/project/Django/ +.. _djangoproject.com: https://www.djangoproject.com/download/ -#. Candidates advertise their application for the technical board to the team. +.. _technical-board: - They must be committers already. There's no term limit for technical board - members. +Technical board +=============== -#. Each team member can vote for zero to five people among the candidates. - Candidates are ranked by the total number of votes they received. +Role +---- - In case of a tie, the person who joined the core team earlier wins. +The technical board is a group of experienced contributors who: -Both the application and the voting period last between one and two weeks, at -the outgoing board's discretion. +- provide oversight of Django's development and release process, +- assist in setting the direction of feature development and releases, +- take part in filling certain roles, and +- have a tie-breaking vote when other decision-making processes fail. -.. _the technical board: https://www.djangoproject.com/foundation/teams/#technical-board-team +Their main concern is to maintain the quality and stability of the Django Web +Framework. + +Prerogatives +------------ + +The technical board holds the following prerogatives: + +- Making a binding decision regarding any question of a technical change to + Django. +- Vetoing the merging of any particular piece of code into Django or ordering + the reversion of any particular merge or commit. +- Announcing calls for proposals and ideas for the future technical direction + of Django. +- Setting and adjusting the schedule of releases of Django. +- Selecting and removing mergers and releasers. +- Participating in the removal of members of the technical board, when deemed + appropriate. +- Calling elections of the technical board outside of those which are + automatically triggered, at times when the technical board deems an election + is appropriate. +- Participating in modifying Django's governance (see + :ref:`organization-change`). +- Declining to vote on a matter the technical board feels is unripe for a + binding decision, or which the technical board feels is outside the scope of + its powers. +- Taking charge of the governance of other technical teams within the Django + open-source project, and governing those teams accordingly. + +Membership +---------- + +`The technical board`_ is an elected group of five experienced contributors +who demonstrate: + +- A history of technical contributions to Django or the Django ecosystem. This + history must begin at least 18 months prior to the individual's candidacy for + the technical board. +- A history of participation in Django's development outside of contributions + merged to the `Django Git repository`_. This may include, but is not + restricted to: + + - Participation in discussions on the |django-developers| mailing list or + the `Django forum`_. + - Reviewing and offering feedback on pull requests in the Django source-code + repository. + - Assisting in triage and management of the Django bug tracker. + +- A history of recent engagement with the direction and development of Django. + Such engagement must have occurred within a period of no more than two years + prior to the individual's candidacy for the technical board. + +A new board is elected after each release cycle of Django. The election process +works as follows: + +#. The technical board direct one of its members to notify the Secretary of the + Django Software Foundation, in writing, of the triggering of the election, + and the condition which triggered it. The Secretary post to the appropriate + venue -- the |django-developers| mailing list and the `Django forum`_ to + announce the election and its timeline. +#. As soon as the election is announced, the `DSF Board`_ begin a period of + voter registration. All `individual members of the DSF`_ are automatically + registered and need not explicitly register. All other persons who believe + themselves eligible to vote, but who have not yet registered to vote, may + make an application to the DSF Board for voting privileges. The voter + registration form and roll of voters is maintained by the DSF Board. The DSF + Board may challenge and reject the registration of voters it believes are + registering in bad faith or who it believes have falsified their + qualifications or are otherwise unqualified. +#. Registration of voters close one week after the announcement of the + election. At that point, registration of candidates begin. Any qualified + person may register as a candidate. The candidate registration form and + roster of candidates are maintained by the DSF Board, and candidates must + provide evidence of their qualifications as part of registration. The DSF + Board may challenge and reject the registration of candidates it believes do + not meet the qualifications of members of the Technical Board, or who it + believes are registering in bad faith. +#. Registration of candidates close one week after it has opened. One week + after registration of candidates closes, the Secretary of the DSF publish + the roster of candidates to the |django-developers| mailing list and the + `Django forum`_, and the election begin. The DSF Board provide a voting form + accessible to registered voters, and is the custodian of the votes. +#. Voting is by secret ballot containing the roster of candidates, and any + relevant materials regarding the candidates, in a randomized order. Each + voter may vote for up to five candidates on the ballot. +#. The election conclude one week after it begins. The DSF Board then tally the + votes and produce a summary, including the total number of votes cast and + the number received by each candidate. This summary is ratified by a + majority vote of the DSF Board, then posted by the Secretary of the DSF to + the |django-developers| mailing list and the Django Forum. The five + candidates with the highest vote totals are immediately become the new + technical board. + +A member of the technical board may be removed by: + +- Becoming disqualified due to actions taken by the Code of Conduct committee + of the Django Software Foundation. +- Determining that they did not possess the qualifications of a member of the + technical board. This determination must be made jointly by the other members + of the technical board, and the `DSF Board`_. A valid determination of + ineligibility requires that all other members of the technical board and all + members of the DSF Board vote who can vote on the issue (the affected person, + if a DSF Board member, must not vote) vote "yes" on a motion that the person + in question is ineligible. + +.. _`Django forum`: https://forum.djangoproject.com/ +.. _`Django Git repository`: https://github.com/django/django/ +.. _`DSF Board`: https://www.djangoproject.com/foundation/#board +.. _`individual members of the DSF`: https://www.djangoproject.com/foundation/individual-members/ +.. _mergers: https://www.djangoproject.com/foundation/teams/#mergers-team +.. _releasers: https://www.djangoproject.com/foundation/teams/#releasers-team +.. _`security team`: https://www.djangoproject.com/foundation/teams/#security-team +.. _`the technical board`: https://www.djangoproject.com/foundation/teams/#technical-board-team +.. _`triage & review team`: https://www.djangoproject.com/foundation/teams/#triage-review-team + +.. _organization-change: Changing the organization ========================= -Changes to this document require a four fifths majority of votes cast in a -core team vote and no veto by the technical board. +Changes to this document require the use of the `DEP process`_, with +modifications described in `DEP 0010`_. + +.. _`DEP process`: https://github.com/django/deps/blob/main/final/0001-dep-process.rst +.. _`DEP 0010`: https://github.com/django/deps/blob/main/accepted/0010-new-governance.rst#changing-this-governance-process diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt index eb00190ad764..ee889db0993f 100644 --- a/docs/intro/contributing.txt +++ b/docs/intro/contributing.txt @@ -40,11 +40,11 @@ so that it can be of use to the widest audience. .. admonition:: Where to get help: If you're having trouble going through this tutorial, please post a message - to |django-developers| or drop by `#django-dev on irc.freenode.net`__ to + to |django-developers| or drop by `#django-dev on irc.libera.chat`__ to chat with other Django users who might be able to help. __ https://diveinto.org/python3/table-of-contents.html -__ irc://irc.freenode.net/django-dev +__ irc://irc.libera.chat/django-dev What does this tutorial cover? ------------------------------ diff --git a/docs/intro/tutorial01.txt b/docs/intro/tutorial01.txt index 56de527fac3e..e795e2cf44f9 100644 --- a/docs/intro/tutorial01.txt +++ b/docs/intro/tutorial01.txt @@ -36,8 +36,8 @@ older versions of Django and install a newer one. .. admonition:: Where to get help: If you're having trouble going through this tutorial, please post a message - to |django-users| or drop by `#django on irc.freenode.net - `_ to chat with other Django users who might + to |django-users| or drop by `#django on irc.libera.chat + `_ to chat with other Django users who might be able to help. Creating a project diff --git a/docs/intro/whatsnext.txt b/docs/intro/whatsnext.txt index 7d3346a12a70..5b05af4a9f9b 100644 --- a/docs/intro/whatsnext.txt +++ b/docs/intro/whatsnext.txt @@ -127,7 +127,7 @@ particular Django setup, try the |django-users| mailing list or the `#django IRC channel`_ instead. .. _ticket system: https://code.djangoproject.com/ -.. _#django IRC channel: irc://irc.freenode.net/django +.. _#django IRC channel: irc://irc.libera.chat/django In plain text ------------- diff --git a/docs/ref/contrib/gis/install/index.txt b/docs/ref/contrib/gis/install/index.txt index 941afe82d83f..fa1125abbaac 100644 --- a/docs/ref/contrib/gis/install/index.txt +++ b/docs/ref/contrib/gis/install/index.txt @@ -109,9 +109,9 @@ Troubleshooting If you can't find the solution to your problem here then participate in the community! You can: -* Join the ``#geodjango`` IRC channel on Freenode. Please be patient and polite - -- while you may not get an immediate response, someone will attempt to answer - your question as soon as they see it. +* Join the ``#django-geo`` IRC channel on Libera.Chat. Please be patient and + polite -- while you may not get an immediate response, someone will attempt + to answer your question as soon as they see it. * Ask your question on the `GeoDjango`__ mailing list. * File a ticket on the `Django trac`__ if you think there's a bug. Make sure to provide a complete description of the problem, versions used, diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index a511b5828e81..dc8f6165dacb 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -102,10 +102,10 @@ below for information on how to set up your database correctly. PostgreSQL notes ================ -Django supports PostgreSQL 9.4 and higher. `psycopg2`_ 2.5.4 or higher is -required, though the latest release is recommended. +Django supports PostgreSQL 9.4 and higher. `psycopg2`_ 2.5.4 through 2.8.6 is +required, though 2.8.6 is recommended. -.. _psycopg2: http://initd.org/psycopg/ +.. _psycopg2: https://www.psycopg.org/ PostgreSQL connection settings ------------------------------- @@ -768,7 +768,7 @@ Oracle notes ============ Django supports `Oracle Database Server`_ versions 12.1 and higher. Version -6.0 or higher of the `cx_Oracle`_ Python driver is required. +6.0 through 7.3 of the `cx_Oracle`_ Python driver is required. .. _`Oracle Database Server`: https://www.oracle.com/ .. _`cx_Oracle`: https://oracle.github.io/python-cx_Oracle/ diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 08eaf62ca4ca..c0fb0133f3f7 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -841,7 +841,7 @@ Methods you will need to remember to pass it to the corresponding :meth:`HttpRequest.get_signed_cookie` call. -.. method:: HttpResponse.delete_cookie(key, path='/', domain=None) +.. method:: HttpResponse.delete_cookie(key, path='/', domain=None, samesite=None) Deletes the cookie with the given key. Fails silently if the key doesn't exist. @@ -850,6 +850,10 @@ Methods values you used in ``set_cookie()`` -- otherwise the cookie may not be deleted. + .. versionchanged:: 2.2.15 + + The ``samesite`` argument was added. + .. method:: HttpResponse.close() This method is called at the end of the request directly by the WSGI diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 65a162e3b06a..c4b0fa398770 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -194,7 +194,13 @@ from its first value when it's next encountered. --------- Outputs a whole load of debugging information, including the current context -and imported modules. +and imported modules. ``{% debug %}`` outputs nothing when the :setting:`DEBUG` +setting is ``False``. + +.. versionchanged:: 2.2.27 + + In older versions, debugging information was displayed when the + :setting:`DEBUG` setting was ``False``. .. templatetag:: extends @@ -1575,6 +1581,13 @@ produce empty output:: {{ values|dictsort:"0" }} +Ordering by elements at specified index is not supported on dictionaries. + +.. versionchanged:: 2.2.26 + + In older versions, ordering elements at specified index was supported on + dictionaries. + .. templatefilter:: dictsortreversed ``dictsortreversed`` diff --git a/docs/ref/urls.txt b/docs/ref/urls.txt index 1527a347202e..36bc1a7de0d7 100644 --- a/docs/ref/urls.txt +++ b/docs/ref/urls.txt @@ -72,9 +72,18 @@ groups from the regular expression are passed to the view -- as named arguments if the groups are named, and as positional arguments otherwise. The values are passed as strings, without any type conversion. +When a ``route`` ends with ``$`` the whole requested URL, matching against +:attr:`~django.http.HttpRequest.path_info`, must match the regular expression +pattern (:py:func:`re.fullmatch` is used). + The ``view``, ``kwargs`` and ``name`` arguments are the same as for :func:`~django.urls.path()`. +.. versionchanged:: 2.2.25 + + In older versions, a full-match wasn't required for a ``route`` which ends + with ``$``. + ``include()`` ============= diff --git a/docs/releases/0.95.txt b/docs/releases/0.95.txt index 21fdd15320aa..4b9b91570856 100644 --- a/docs/releases/0.95.txt +++ b/docs/releases/0.95.txt @@ -109,9 +109,9 @@ many common questions appear with some regularity, and any particular problem may already have been answered. Finally, for those who prefer the more immediate feedback offered by IRC, -there's a `#django` channel on irc.freenode.net that is regularly populated -by Django users and developers from around the world. Friendly people are -usually available at any hour of the day -- to help, or just to chat. +there's a ``#django`` channel on ``irc.libera.chat`` that is regularly +populated by Django users and developers from around the world. Friendly people +are usually available at any hour of the day -- to help, or just to chat. .. _Django website: https://www.djangoproject.com/ .. _django-users: https://groups.google.com/group/django-users diff --git a/docs/releases/1.1.txt b/docs/releases/1.1.txt index 49c375b5ce17..e55ef9c903ef 100644 --- a/docs/releases/1.1.txt +++ b/docs/releases/1.1.txt @@ -441,7 +441,7 @@ What's next? We'll take a short break, and then work on Django 1.2 will begin -- no rest for the weary! If you'd like to help, discussion of Django development, including progress toward the 1.2 release, takes place daily on the |django-developers| -mailing list and in the ``#django-dev`` IRC channel on ``irc.freenode.net``. +mailing list and in the ``#django-dev`` IRC channel on ``irc.libera.chat``. Feel free to join the discussions! Django's online documentation also includes pointers on how to contribute to diff --git a/docs/releases/1.5.1.txt b/docs/releases/1.5.1.txt index cc961ac304bf..66d78997626a 100644 --- a/docs/releases/1.5.1.txt +++ b/docs/releases/1.5.1.txt @@ -10,10 +10,8 @@ compatible with Django 1.5, but includes a handful of fixes. The biggest fix is for a memory leak introduced in Django 1.5. Under certain circumstances, repeated iteration over querysets could leak memory - sometimes quite a bit of it. If you'd like more information, the details are in -:ticket:`our ticket tracker <19895>` (and in `a related issue`__ in Python -itself). - -__ https://bugs.python.org/issue17468 +:ticket:`our ticket tracker <19895>` (and in :bpo:`a related issue <17468>` in +Python itself). If you've noticed memory problems under Django 1.5, upgrading to 1.5.1 should fix those issues. diff --git a/docs/releases/1.6.11.txt b/docs/releases/1.6.11.txt index 8cf81f89bfdb..1bf2bf89110b 100644 --- a/docs/releases/1.6.11.txt +++ b/docs/releases/1.6.11.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that diff --git a/docs/releases/1.7.7.txt b/docs/releases/1.7.7.txt index f20ee127bcc5..bfd54563a1ee 100644 --- a/docs/releases/1.7.7.txt +++ b/docs/releases/1.7.7.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that diff --git a/docs/releases/2.2.14.txt b/docs/releases/2.2.14.txt new file mode 100644 index 000000000000..38683cf30198 --- /dev/null +++ b/docs/releases/2.2.14.txt @@ -0,0 +1,13 @@ +=========================== +Django 2.2.14 release notes +=========================== + +*July 1, 2020* + +Django 2.2.14 fixes a bug in 2.2.13. + +Bugfixes +======== + +* Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` + warnings raised by cache key validation (:ticket:`31654`). diff --git a/docs/releases/2.2.15.txt b/docs/releases/2.2.15.txt new file mode 100644 index 000000000000..c36d746d5db9 --- /dev/null +++ b/docs/releases/2.2.15.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.15 release notes +=========================== + +*August 3, 2020* + +Django 2.2.15 fixes two bugs in 2.2.14. + +Bugfixes +======== + +* Allowed setting the ``SameSite`` cookie flag in + :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). + +* Fixed crash when sending emails to addresses with display names longer than + 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+ (:ticket:`31784`). diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt new file mode 100644 index 000000000000..31231fb0655d --- /dev/null +++ b/docs/releases/2.2.16.txt @@ -0,0 +1,36 @@ +=========================== +Django 2.2.16 release notes +=========================== + +*September 1, 2020* + +Django 2.2.16 fixes two security issues and two data loss bugs in 2.2.15. + +CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ +====================================================================================== + +On Python 3.7+, :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` mode was not +applied to intermediate-level directories created in the process of uploading +files and to intermediate-level collected static directories when using the +:djadmin:`collectstatic` management command. + +You should review and manually fix permissions on existing intermediate-level +directories. + +CVE-2020-24584: Permission escalation in intermediate-level directories of the file system cache on Python 3.7+ +=============================================================================================================== + +On Python 3.7+, the intermediate-level directories of the file system cache had +the system's standard umask rather than ``0o077`` (no group or others +permissions). + +Bugfixes +======== + +* Fixed a data loss possibility in the + :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using + related fields pointing to a proxy model in the ``of`` argument, the + corresponding model was not locked (:ticket:`31866`). + +* Fixed a data loss possibility, following a regression in Django 2.0, when + copying model instances with a cached fields value (:ticket:`31863`). diff --git a/docs/releases/2.2.17.txt b/docs/releases/2.2.17.txt new file mode 100644 index 000000000000..4bea2eaed43a --- /dev/null +++ b/docs/releases/2.2.17.txt @@ -0,0 +1,7 @@ +=========================== +Django 2.2.17 release notes +=========================== + +*November 2, 2020* + +Django 2.2.17 adds compatibility with Python 3.9. diff --git a/docs/releases/2.2.18.txt b/docs/releases/2.2.18.txt new file mode 100644 index 000000000000..45df4fb83c9f --- /dev/null +++ b/docs/releases/2.2.18.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.18 release notes +=========================== + +*February 1, 2021* + +Django 2.2.18 fixes a security issue with severity "low" in 2.2.17. + +CVE-2021-3281: Potential directory-traversal via ``archive.extract()`` +====================================================================== + +The ``django.utils.archive.extract()`` function, used by +:option:`startapp --template` and :option:`startproject --template`, allowed +directory-traversal via an archive with absolute paths or relative paths with +dot segments. diff --git a/docs/releases/2.2.19.txt b/docs/releases/2.2.19.txt new file mode 100644 index 000000000000..feaffd996cac --- /dev/null +++ b/docs/releases/2.2.19.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.19 release notes +=========================== + +*February 19, 2021* + +Django 2.2.19 fixes a security issue in 2.2.18. + +CVE-2021-23336: Web cache poisoning via ``django.utils.http.limited_parse_qsl()`` +================================================================================= + +Django contains a copy of :func:`urllib.parse.parse_qsl` which was added to +backport some security fixes. A further security fix has been issued recently +such that ``parse_qsl()`` no longer allows using ``;`` as a query parameter +separator by default. Django now includes this fix. See :bpo:`42967` for +further details. diff --git a/docs/releases/2.2.20.txt b/docs/releases/2.2.20.txt new file mode 100644 index 000000000000..a67c51502181 --- /dev/null +++ b/docs/releases/2.2.20.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.20 release notes +=========================== + +*April 6, 2021* + +Django 2.2.20 fixes a security issue with severity "low" in 2.2.19. + +CVE-2021-28658: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser`` allowed directory-traversal via uploaded files with +suitably crafted file names. + +Built-in upload handlers were not affected by this vulnerability. diff --git a/docs/releases/2.2.21.txt b/docs/releases/2.2.21.txt new file mode 100644 index 000000000000..2302df428520 --- /dev/null +++ b/docs/releases/2.2.21.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.21 release notes +=========================== + +*May 4, 2021* + +Django 2.2.21 fixes a security issue in 2.2.20. + +CVE-2021-31542: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser``, ``UploadedFile``, and ``FieldFile`` allowed +directory-traversal via uploaded files with suitably crafted file names. + +In order to mitigate this risk, stricter basename and path sanitation is now +applied. diff --git a/docs/releases/2.2.22.txt b/docs/releases/2.2.22.txt new file mode 100644 index 000000000000..6808a267afeb --- /dev/null +++ b/docs/releases/2.2.22.txt @@ -0,0 +1,22 @@ +=========================== +Django 2.2.22 release notes +=========================== + +*May 6, 2021* + +Django 2.2.22 fixes a security issue in 2.2.21. + +CVE-2021-32052: Header injection possibility since ``URLValidator`` accepted newlines in input on Python 3.9.5+ +=============================================================================================================== + +On Python 3.9.5+, :class:`~django.core.validators.URLValidator` didn't prohibit +newlines and tabs. If you used values with newlines in HTTP response, you could +suffer from header injection attacks. Django itself wasn't vulnerable because +:class:`~django.http.HttpResponse` prohibits newlines in HTTP headers. + +Moreover, the ``URLField`` form field which uses ``URLValidator`` silently +removes newlines and tabs on Python 3.9.5+, so the possibility of newlines +entering your data only existed if you are using this validator outside of the +form fields. + +This issue was introduced by the :bpo:`43882` fix. diff --git a/docs/releases/2.2.23.txt b/docs/releases/2.2.23.txt new file mode 100644 index 000000000000..6c39361e5fc7 --- /dev/null +++ b/docs/releases/2.2.23.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.23 release notes +=========================== + +*May 13, 2021* + +Django 2.2.23 fixes a regression in 2.2.21. + +Bugfixes +======== + +* Fixed a regression in Django 2.2.21 where saving ``FileField`` would raise a + ``SuspiciousFileOperation`` even when a custom + :attr:`~django.db.models.FileField.upload_to` returns a valid file path + (:ticket:`32718`). diff --git a/docs/releases/2.2.24.txt b/docs/releases/2.2.24.txt new file mode 100644 index 000000000000..1064fc53a004 --- /dev/null +++ b/docs/releases/2.2.24.txt @@ -0,0 +1,32 @@ +=========================== +Django 2.2.24 release notes +=========================== + +*June 2, 2021* + +Django 2.2.24 fixes two security issues in 2.2.23. + +CVE-2021-33203: Potential directory traversal via ``admindocs`` +=============================================================== + +Staff members could use the :mod:`~django.contrib.admindocs` +``TemplateDetailView`` view to check the existence of arbitrary files. +Additionally, if (and only if) the default admindocs templates have been +customized by the developers to also expose the file contents, then not only +the existence but also the file contents would have been exposed. + +As a mitigation, path sanitation is now applied and only files within the +template root directories can be loaded. + +CVE-2021-33571: Possible indeterminate SSRF, RFI, and LFI attacks since validators accepted leading zeros in IPv4 addresses +=========================================================================================================================== + +:class:`~django.core.validators.URLValidator`, +:func:`~django.core.validators.validate_ipv4_address`, and +:func:`~django.core.validators.validate_ipv46_address` didn't prohibit leading +zeros in octal literals. If you used such values you could suffer from +indeterminate SSRF, RFI, and LFI attacks. + +:func:`~django.core.validators.validate_ipv4_address` and +:func:`~django.core.validators.validate_ipv46_address` validators were not +affected on Python 3.9.5+. diff --git a/docs/releases/2.2.25.txt b/docs/releases/2.2.25.txt new file mode 100644 index 000000000000..1662451a3064 --- /dev/null +++ b/docs/releases/2.2.25.txt @@ -0,0 +1,13 @@ +=========================== +Django 2.2.25 release notes +=========================== + +*December 7, 2021* + +Django 2.2.25 fixes a security issue with severity "low" in 2.2.24. + +CVE-2021-44420: Potential bypass of an upstream access control based on URL paths +================================================================================= + +HTTP requests for URLs with trailing newlines could bypass an upstream access +control based on URL paths. diff --git a/docs/releases/2.2.26.txt b/docs/releases/2.2.26.txt new file mode 100644 index 000000000000..7fbdc02089de --- /dev/null +++ b/docs/releases/2.2.26.txt @@ -0,0 +1,47 @@ +=========================== +Django 2.2.26 release notes +=========================== + +*January 4, 2022* + +Django 2.2.26 fixes one security issue with severity "medium" and two security +issues with severity "low" in 2.2.25. + +CVE-2021-45115: Denial-of-service possibility in ``UserAttributeSimilarityValidator`` +===================================================================================== + +:class:`.UserAttributeSimilarityValidator` incurred significant overhead +evaluating submitted password that were artificially large in relative to the +comparison values. On the assumption that access to user registration was +unrestricted this provided a potential vector for a denial-of-service attack. + +In order to mitigate this issue, relatively long values are now ignored by +``UserAttributeSimilarityValidator``. + +This issue has severity "medium" according to the :ref:`Django security policy +`. + +CVE-2021-45116: Potential information disclosure in ``dictsort`` template filter +================================================================================ + +Due to leveraging the Django Template Language's variable resolution logic, the +:tfilter:`dictsort` template filter was potentially vulnerable to information +disclosure or unintended method calls, if passed a suitably crafted key. + +In order to avoid this possibility, ``dictsort`` now works with a restricted +resolution logic, that will not call methods, nor allow indexing on +dictionaries. + +As a reminder, all untrusted user input should be validated before use. + +This issue has severity "low" according to the :ref:`Django security policy +`. + +CVE-2021-45452: Potential directory-traversal via ``Storage.save()`` +==================================================================== + +``Storage.save()`` allowed directory-traversal if directly passed suitably +crafted file names. + +This issue has severity "low" according to the :ref:`Django security policy +`. diff --git a/docs/releases/2.2.27.txt b/docs/releases/2.2.27.txt new file mode 100644 index 000000000000..2a9b24be421e --- /dev/null +++ b/docs/releases/2.2.27.txt @@ -0,0 +1,23 @@ +=========================== +Django 2.2.27 release notes +=========================== + +*February 1, 2022* + +Django 2.2.27 fixes two security issues with severity "medium" in 2.2.26. + +CVE-2022-22818: Possible XSS via ``{% debug %}`` template tag +============================================================= + +The ``{% debug %}`` template tag didn't properly encode the current context, +posing an XSS attack vector. + +In order to avoid this vulnerability, ``{% debug %}`` no longer outputs +information when the ``DEBUG`` setting is ``False``, and it ensures all context +variables are correctly escaped when the ``DEBUG`` setting is ``True``. + +CVE-2022-23833: Denial-of-service possibility in file uploads +============================================================= + +Passing certain inputs to multipart forms could result in an infinite loop when +parsing files. diff --git a/docs/releases/2.2.28.txt b/docs/releases/2.2.28.txt new file mode 100644 index 000000000000..43270fc5c080 --- /dev/null +++ b/docs/releases/2.2.28.txt @@ -0,0 +1,22 @@ +=========================== +Django 2.2.28 release notes +=========================== + +*April 11, 2022* + +Django 2.2.28 fixes two security issues with severity "high" in 2.2.27. + +CVE-2022-28346: Potential SQL injection in ``QuerySet.annotate()``, ``aggregate()``, and ``extra()`` +==================================================================================================== + +:meth:`.QuerySet.annotate`, :meth:`~.QuerySet.aggregate`, and +:meth:`~.QuerySet.extra` methods were subject to SQL injection in column +aliases, using a suitably crafted dictionary, with dictionary expansion, as the +``**kwargs`` passed to these methods. + +CVE-2022-28347: Potential SQL injection via ``QuerySet.explain(**options)`` on PostgreSQL +========================================================================================= + +:meth:`.QuerySet.explain` method was subject to SQL injection in option names, +using a suitably crafted dictionary, with dictionary expansion, as the +``**options`` argument. diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt index e83c162b0b53..f602fd0b46ec 100644 --- a/docs/releases/2.2.txt +++ b/docs/releases/2.2.txt @@ -23,9 +23,9 @@ end in April 2020. Python compatibility ==================== -Django 2.2 supports Python 3.5, 3.6, 3.7, and 3.8 (as of 2.2.8). We -**highly recommend** and only officially support the latest release of each -series. +Django 2.2 supports Python 3.5, 3.6, 3.7, 3.8 (as of 2.2.8), and 3.9 (as of +2.2.17). We **highly recommend** and only officially support the latest release +of each series. .. _whats-new-2.2: @@ -331,7 +331,7 @@ change shouldn't have an impact on your tests unless you've customized ``sqlparse`` is required dependency ----------------------------------- -To simplify a few parts of Django's database handling, `sqlparse +To simplify a few parts of Django's database handling, `sqlparse 0.2.2+ `_ is now a required dependency. It's automatically installed along with Django. @@ -475,6 +475,14 @@ Miscellaneous * Providing an integer in the ``key`` argument of the :meth:`.cache.delete` or :meth:`.cache.get` now raises :exc:`ValueError`. +* Plural equations for some languages are changed, because the latest versions + from Transifex are incorporated. + + .. note:: + + The ability to handle ``.po`` files containing different plural equations + for the same language was added in Django 2.2.12. + .. _deprecated-features-2.2: Features deprecated in 2.2 diff --git a/docs/releases/index.txt b/docs/releases/index.txt index d9fccd441e3a..589aa670407a 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,21 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.28 + 2.2.27 + 2.2.26 + 2.2.25 + 2.2.24 + 2.2.23 + 2.2.22 + 2.2.21 + 2.2.20 + 2.2.19 + 2.2.18 + 2.2.17 + 2.2.16 + 2.2.15 + 2.2.14 2.2.13 2.2.12 2.2.11 diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 340aba041b02..8b85b4a981b4 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1082,3 +1082,232 @@ Versions affected * Django 3.0 :commit:`(patch) <26a5cf834526e291db00385dd33d319b8271fc4c>` * Django 2.2 :commit:`(patch) ` * Django 1.11 :commit:`(patch) <02d97f3c9a88adc890047996e5606180bd1c6166>` + +June 3, 2020 - :cve:`2020-13254` +-------------------------------- + +Potential data leakage via malformed memcached keys. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <84b2da5552e100ae3294f564f6c862fef8d0e693>` +* Django 2.2 :commit:`(patch) <07e59caa02831c4569bbebb9eb773bdd9cb4b206>` + +June 3, 2020 - :cve:`2020-13596` +-------------------------------- + +Possible XSS via admin ``ForeignKeyRawIdWidget``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <1f2dd37f6fcefdd10ed44cb233b2e62b520afb38>` +* Django 2.2 :commit:`(patch) <6d61860b22875f358fac83d903dc629897934815>` + +September 1, 2020 - :cve:`2020-24583` +------------------------------------- + +Incorrect permissions on intermediate-level directories on Python 3.7+. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <934430d22aa5d90c2ba33495ff69a6a1d997d584>` +* Django 3.0 :commit:`(patch) <08892bffd275c79ee1f8f67639eb170aaaf1181e>` +* Django 2.2 :commit:`(patch) <375657a71c889c588f723469bd868bd1d40c369f>` + +September 1, 2020 - :cve:`2020-24584` +------------------------------------- + +Permission escalation in intermediate-level directories of the file system +cache on Python 3.7+. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <2b099caa5923afa8cfb5f1e8c0d56b6e0e81915b>` +* Django 3.0 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` + +February 1, 2021 - :cve:`2021-3281` +----------------------------------- + +Potential directory-traversal via ``archive.extract()``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <02e6592835b4559909aa3aaaf67988fef435f624>` +* Django 3.0 :commit:`(patch) <52e409ed17287e9aabda847b6afe58be2fa9f86a>` +* Django 2.2 :commit:`(patch) <21e7622dec1f8612c85c2fc37fe8efbfd3311e37>` + +February 19, 2021 - :cve:`2021-23336` +------------------------------------- + +Web cache poisoning via ``django.utils.http.limited_parse_qsl()``. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <8f6d431b08cbb418d9144b976e7b972546607851>` +* Django 3.0 :commit:`(patch) <326a926beef869d3341bc9ef737887f0449b6b71>` +* Django 2.2 :commit:`(patch) ` + +April 6, 2021 - :cve:`2021-28658` +--------------------------------- + +Potential directory-traversal via uploaded files. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <2820fd1be5dfccbf1216c3845fad8580502473e1>` +* Django 3.1 :commit:`(patch) ` +* Django 3.0 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) <4036d62bda0e9e9f6172943794b744a454ca49c2>` + +May 4, 2021 - :cve:`2021-31542` +------------------------------- + +Potential directory-traversal via uploaded files. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <25d84d64122c15050a0ee739e859f22ddab5ac48>` +* Django 2.2 :commit:`(patch) <04ac1624bdc2fa737188401757cf95ced122d26d>` + +May 6, 2021 - :cve:`2021-32052` +------------------------------- + +Header injection possibility since ``URLValidator`` accepted newlines in input +on Python 3.9.5+. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <2d2c1d0c97832860fbd6597977e2aae17dd7e5b2>` +* Django 3.1 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` + +June 2, 2021 - :cve:`2021-33203` +-------------------------------- + +Potential directory traversal via ``admindocs``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <20c67a0693c4ede2b09af02574823485e82e4c8f>` +* Django 2.2 :commit:`(patch) <053cc9534d174dc89daba36724ed2dcb36755b90>` + +June 2, 2021 - :cve:`2021-33571` +-------------------------------- + +Possible indeterminate SSRF, RFI, and LFI attacks since validators accepted +leading zeros in IPv4 addresses. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <9f75e2e562fa0c0482f3dde6fc7399a9070b4a3d>` +* Django 3.1 :commit:`(patch) <203d4ab9ebcd72fc4d6eb7398e66ed9e474e118e>` +* Django 2.2 :commit:`(patch) ` + +December 7, 2021 - :cve:`2021-44420` +------------------------------------ + +Potential bypass of an upstream access control based on URL paths. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) <333c65603032c377e682cdbd7388657a5463a05a>` +* Django 3.1 :commit:`(patch) <22bd17488159601bf0741b70ae7932bffea8eced>` +* Django 2.2 :commit:`(patch) <7cf7d74e8a754446eeb85cacf2fef1247e0cb6d7>` + +January 4, 2022 - :cve:`2021-45115` +------------------------------------ + +Denial-of-service possibility in ``UserAttributeSimilarityValidator``. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 4.0 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) <2135637fdd5ce994de110affef9e67dffdf77277>` + +January 4, 2022 - :cve:`2021-45116` +------------------------------------ + +Potential information disclosure in ``dictsort`` template filter. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 4.0 :commit:`(patch) <2a8ec7f546d6d5806e221ec948c5146b55bd7489>` +* Django 3.2 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` + +January 4, 2022 - :cve:`2021-45452` +------------------------------------ + +Potential directory-traversal via ``Storage.save()``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 4.0 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) <8d2f7cff76200cbd2337b2cf1707e383eb1fb54b>` +* Django 2.2 :commit:`(patch) <4cb35b384ceef52123fc66411a73c36a706825e1>` + +February 1, 2022 - :cve:`2022-22818` +------------------------------------ + +Possible XSS via ``{% debug %}`` template tag. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 4.0 :commit:`(patch) <01422046065d2b51f8f613409cad2c81b39487e5>` +* Django 3.2 :commit:`(patch) <1a1e8278c46418bde24c86a65443b0674bae65e2>` +* Django 2.2 :commit:`(patch) ` + +February 1, 2022 - :cve:`2022-23833` +------------------------------------ + +Denial-of-service possibility in file uploads. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 4.0 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000000..6ea13726807a --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +pyenchant +Sphinx>=3.1.0 +sphinxcontrib-spelling diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index f8a718a3bcf6..d3ab8e1c1fbc 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -217,12 +217,12 @@ flatpages Flatpages followup fooapp +formatter formatters formfield formset formsets formtools -freenode Frysian functionalities gdal @@ -320,6 +320,7 @@ Kyngesburye latin lawrence lexer +Libera lifecycle lifecycles linearize diff --git a/docs/topics/auth/passwords.txt b/docs/topics/auth/passwords.txt index bcf20a976d8b..c509a3a52220 100644 --- a/docs/topics/auth/passwords.txt +++ b/docs/topics/auth/passwords.txt @@ -522,10 +522,16 @@ Django includes four validators: is used: ``'username', 'first_name', 'last_name', 'email'``. Attributes that don't exist are ignored. - The minimum similarity of a rejected password can be set on a scale of 0 to - 1 with the ``max_similarity`` parameter. A setting of 0 rejects all - passwords, whereas a setting of 1 rejects only passwords that are identical - to an attribute's value. + The maximum allowed similarity of passwords can be set on a scale of 0.1 + to 1.0 with the ``max_similarity`` parameter. This is compared to the + result of :meth:`difflib.SequenceMatcher.quick_ratio`. A value of 0.1 + rejects passwords unless they are substantially different from the + ``user_attributes``, whereas a value of 1.0 rejects only passwords that are + identical to an attribute's value. + + .. versionchanged:: 2.2.26 + + The ``max_similarity`` parameter was limited to a minimum value of 0.1. .. class:: CommonPasswordValidator(password_list_path=DEFAULT_PASSWORD_LIST_PATH) diff --git a/docs/topics/db/sql.txt b/docs/topics/db/sql.txt index 84420c7e4b30..13d5a83c2c53 100644 --- a/docs/topics/db/sql.txt +++ b/docs/topics/db/sql.txt @@ -23,8 +23,8 @@ __ `executing custom SQL directly`_ :doc:`custom query expressions `. Before using raw SQL, explore :doc:`the ORM `. Ask on - |django-users| or the `#django IRC channel - `_ to see if the ORM supports your use case. + one of :doc:`the support channels ` to see if the ORM supports + your use case. .. warning:: diff --git a/setup.py b/setup.py index 0dcfa4724333..676aba5f1b7b 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def read(fname): entry_points={'console_scripts': [ 'django-admin = django.core.management:execute_from_command_line', ]}, - install_requires=['pytz', 'sqlparse'], + install_requires=['pytz', 'sqlparse >= 0.2.2'], extras_require={ "bcrypt": ["bcrypt"], "argon2": ["argon2-cffi >= 16.1.0"], @@ -102,6 +102,7 @@ def read(fname): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', diff --git a/tests/admin_docs/test_views.py b/tests/admin_docs/test_views.py index bcadff7d8a62..dc6d3c127b18 100644 --- a/tests/admin_docs/test_views.py +++ b/tests/admin_docs/test_views.py @@ -134,6 +134,22 @@ def test_no_sites_framework(self): self.assertContains(response, 'View documentation') +@unittest.skipUnless(utils.docutils_is_available, 'no docutils installed.') +class AdminDocViewDefaultEngineOnly(TestDataMixin, AdminDocsTestCase): + + def setUp(self): + self.client.force_login(self.superuser) + + def test_template_detail_path_traversal(self): + cases = ['/etc/passwd', '../passwd'] + for fpath in cases: + with self.subTest(path=fpath): + response = self.client.get( + reverse('django-admindocs-templates', args=[fpath]), + ) + self.assertEqual(response.status_code, 400) + + @override_settings(TEMPLATES=[{ 'NAME': 'ONE', 'BACKEND': 'django.template.backends.django.DjangoTemplates', diff --git a/tests/admin_widgets/tests.py b/tests/admin_widgets/tests.py index b6be85f39c3d..4c14a47280e2 100644 --- a/tests/admin_widgets/tests.py +++ b/tests/admin_widgets/tests.py @@ -590,11 +590,14 @@ def test_proper_manager_for_label_lookup(self): def test_render_unsafe_limit_choices_to(self): rel = UnsafeLimitChoicesTo._meta.get_field('band').remote_field w = widgets.ForeignKeyRawIdWidget(rel, widget_admin_site) + parameters = w.url_parameters() + parameters['name'] = '%22%26%3E%3Cescapeme' self.assertHTMLEqual( w.render('test', None), - '\n' - '' + '' + '' + % '&'.join('%s=%s' % (k, v) for k, v in parameters.items()) ) diff --git a/tests/aggregation/tests.py b/tests/aggregation/tests.py index 3820496c9fd5..501a18700baf 100644 --- a/tests/aggregation/tests.py +++ b/tests/aggregation/tests.py @@ -1114,3 +1114,12 @@ def test_arguments_must_be_expressions(self): Book.objects.aggregate(is_book=True) with self.assertRaisesMessage(TypeError, msg % ', '.join([str(FloatField()), 'True'])): Book.objects.aggregate(FloatField(), Avg('price'), is_book=True) + + def test_alias_sql_injection(self): + crafted_alias = """injected_name" from "aggregation_author"; --""" + msg = ( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + with self.assertRaisesMessage(ValueError, msg): + Author.objects.aggregate(**{crafted_alias: Avg("age")}) diff --git a/tests/annotations/tests.py b/tests/annotations/tests.py index 021f59d2d71d..27cd7ebfb826 100644 --- a/tests/annotations/tests.py +++ b/tests/annotations/tests.py @@ -598,3 +598,37 @@ def test_annotation_filter_with_subquery(self): total_books=Subquery(long_books_qs, output_field=IntegerField()), ).values('name') self.assertCountEqual(publisher_books_qs, [{'name': 'Sams'}, {'name': 'Morgan Kaufmann'}]) + + def test_alias_sql_injection(self): + crafted_alias = """injected_name" from "annotations_book"; --""" + msg = ( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + with self.assertRaisesMessage(ValueError, msg): + Book.objects.annotate(**{crafted_alias: Value(1)}) + + def test_alias_forbidden_chars(self): + tests = [ + 'al"ias', + "a'lias", + "ali`as", + "alia s", + "alias\t", + "ali\nas", + "alias--", + "ali/*as", + "alias*/", + "alias;", + # [] are used by MSSQL. + "alias[", + "alias]", + ] + msg = ( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + for crafted_alias in tests: + with self.subTest(crafted_alias): + with self.assertRaisesMessage(ValueError, msg): + Book.objects.annotate(**{crafted_alias: Value(1)}) diff --git a/tests/auth_tests/test_hashers.py b/tests/auth_tests/test_hashers.py index 7feff16da98d..ee6441b237f6 100644 --- a/tests/auth_tests/test_hashers.py +++ b/tests/auth_tests/test_hashers.py @@ -16,7 +16,7 @@ crypt = None else: # On some platforms (e.g. OpenBSD), crypt.crypt() always return None. - if crypt.crypt('', '') is None: + if crypt.crypt('') is None: crypt = None try: diff --git a/tests/auth_tests/test_validators.py b/tests/auth_tests/test_validators.py index 1c2c6b4afff1..777e51ebde4a 100644 --- a/tests/auth_tests/test_validators.py +++ b/tests/auth_tests/test_validators.py @@ -150,13 +150,10 @@ def test_validate(self): max_similarity=1, ).validate(user.first_name, user=user) self.assertEqual(cm.exception.messages, [expected_error % "first name"]) - # max_similarity=0 rejects all passwords. - with self.assertRaises(ValidationError) as cm: - UserAttributeSimilarityValidator( - user_attributes=['first_name'], - max_similarity=0, - ).validate('XXX', user=user) - self.assertEqual(cm.exception.messages, [expected_error % "first name"]) + # Very low max_similarity is rejected. + msg = 'max_similarity must be at least 0.1' + with self.assertRaisesMessage(ValueError, msg): + UserAttributeSimilarityValidator(max_similarity=0.09) # Passes validation. self.assertIsNone( UserAttributeSimilarityValidator(user_attributes=['first_name']).validate('testclient', user=user) diff --git a/tests/cache/tests.py b/tests/cache/tests.py index 4bc428a98b96..539247d6af18 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -6,11 +6,13 @@ import pickle import re import shutil +import sys import tempfile import threading import time import unittest -from unittest import mock +from pathlib import Path +from unittest import mock, skipIf from django.conf import settings from django.core import management, signals @@ -619,8 +621,9 @@ def func(key, *args): cache.key_func = func try: - with self.assertWarnsMessage(CacheKeyWarning, expected_warning): + with self.assertWarns(CacheKeyWarning) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.warning), expected_warning) finally: cache.key_func = old_func @@ -1256,9 +1259,10 @@ def _perform_invalid_key_test(self, key, expected_warning): Whilst other backends merely warn, memcached should raise for an invalid key. """ - msg = expected_warning.replace(key, ':1:%s' % key) - with self.assertRaisesMessage(InvalidCacheKey, msg): + msg = expected_warning.replace(key, cache.make_key(key)) + with self.assertRaises(InvalidCacheKey) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.exception), msg) def test_default_never_expiring_timeout(self): # Regression test for #22845 @@ -1428,6 +1432,28 @@ def test_get_ignores_enoent(self): # Returns the default instead of erroring. self.assertEqual(cache.get('foo', 'baz'), 'baz') + @skipIf( + sys.platform == 'win32', + 'Windows only partially supports umasks and chmod.', + ) + def test_cache_dir_permissions(self): + os.rmdir(self.dirname) + dir_path = Path(self.dirname) / 'nested' / 'filebasedcache' + for cache_params in settings.CACHES.values(): + cache_params['LOCATION'] = str(dir_path) + setting_changed.send(self.__class__, setting='CACHES', enter=False) + cache.set('foo', 'bar') + self.assertIs(dir_path.exists(), True) + tests = [ + dir_path, + dir_path.parent, + dir_path.parent.parent, + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o700) + def test_get_does_not_ignore_non_filenotfound_exceptions(self): with mock.patch('builtins.open', side_effect=IOError): with self.assertRaises(IOError): diff --git a/tests/expressions/test_queryset_values.py b/tests/expressions/test_queryset_values.py index e26459796807..0804531869d9 100644 --- a/tests/expressions/test_queryset_values.py +++ b/tests/expressions/test_queryset_values.py @@ -27,6 +27,15 @@ def test_values_expression(self): [{'salary': 10}, {'salary': 20}, {'salary': 30}], ) + def test_values_expression_alias_sql_injection(self): + crafted_alias = """injected_name" from "expressions_company"; --""" + msg = ( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + with self.assertRaisesMessage(ValueError, msg): + Company.objects.values(**{crafted_alias: F("ceo__salary")}) + def test_values_expression_group_by(self): # values() applies annotate() first, so values selected are grouped by # id, not firstname. diff --git a/tests/file_storage/test_generate_filename.py b/tests/file_storage/test_generate_filename.py index b4222f412162..fd8da6debf40 100644 --- a/tests/file_storage/test_generate_filename.py +++ b/tests/file_storage/test_generate_filename.py @@ -1,7 +1,8 @@ import os +from django.core.exceptions import SuspiciousFileOperation from django.core.files.base import ContentFile -from django.core.files.storage import Storage +from django.core.files.storage import FileSystemStorage, Storage from django.db.models import FileField from django.test import SimpleTestCase @@ -36,6 +37,69 @@ def generate_filename(self, filename): class GenerateFilenameStorageTests(SimpleTestCase): + def test_storage_dangerous_paths(self): + candidates = [ + ('/tmp/..', '..'), + ('/tmp/.', '.'), + ('', ''), + ] + s = FileSystemStorage() + msg = "Could not derive file name from '%s'" + for file_name, base_name in candidates: + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg % base_name): + s.get_available_name(file_name) + with self.assertRaisesMessage(SuspiciousFileOperation, msg % base_name): + s.generate_filename(file_name) + + def test_storage_dangerous_paths_dir_name(self): + candidates = [ + ('tmp/../path', 'tmp/..'), + ('tmp\\..\\path', 'tmp/..'), + ('/tmp/../path', '/tmp/..'), + ('\\tmp\\..\\path', '/tmp/..'), + ] + s = FileSystemStorage() + for file_name, path in candidates: + msg = "Detected path traversal attempt in '%s'" % path + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + s.get_available_name(file_name) + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + s.generate_filename(file_name) + + def test_filefield_dangerous_filename(self): + candidates = [ + ('..', 'some/folder/..'), + ('.', 'some/folder/.'), + ('', 'some/folder/'), + ('???', '???'), + ('$.$.$', '$.$.$'), + ] + f = FileField(upload_to='some/folder/') + for file_name, msg_file_name in candidates: + msg = "Could not derive file name from '%s'" % msg_file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + + def test_filefield_dangerous_filename_dot_segments(self): + f = FileField(upload_to='some/folder/') + msg = "Detected path traversal attempt in 'some/folder/../path'" + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, '../path') + + def test_filefield_generate_filename_absolute_path(self): + f = FileField(upload_to='some/folder/') + candidates = [ + '/tmp/path', + '/tmp/../path', + ] + for file_name in candidates: + msg = "Detected path traversal attempt in '%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) def test_filefield_generate_filename(self): f = FileField(upload_to='some/folder/') @@ -54,6 +118,57 @@ def upload_to(instance, filename): os.path.normpath('some/folder/test_with_space.txt') ) + def test_filefield_generate_filename_upload_to_overrides_dangerous_filename(self): + def upload_to(instance, filename): + return 'test.txt' + + f = FileField(upload_to=upload_to) + candidates = [ + '/tmp/.', + '/tmp/..', + '/tmp/../path', + '/tmp/path', + 'some/folder/', + 'some/folder/.', + 'some/folder/..', + 'some/folder/???', + 'some/folder/$.$.$', + 'some/../test.txt', + '', + ] + for file_name in candidates: + with self.subTest(file_name=file_name): + self.assertEqual(f.generate_filename(None, file_name), 'test.txt') + + def test_filefield_generate_filename_upload_to_absolute_path(self): + def upload_to(instance, filename): + return '/tmp/' + filename + + f = FileField(upload_to=upload_to) + candidates = [ + 'path', + '../path', + '???', + '$.$.$', + ] + for file_name in candidates: + msg = "Detected path traversal attempt in '/tmp/%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + + def test_filefield_generate_filename_upload_to_dangerous_filename(self): + def upload_to(instance, filename): + return '/tmp/' + filename + + f = FileField(upload_to=upload_to) + candidates = ['..', '.', ''] + for file_name in candidates: + msg = "Could not derive file name from '/tmp/%s'" % file_name + with self.subTest(file_name=file_name): + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + f.generate_filename(None, file_name) + def test_filefield_awss3_storage(self): """ Simulate a FileField with an S3 storage which uses keys rather than diff --git a/tests/file_storage/tests.py b/tests/file_storage/tests.py index 8c50abc9b0b6..4c6f6920ed2d 100644 --- a/tests/file_storage/tests.py +++ b/tests/file_storage/tests.py @@ -7,6 +7,7 @@ import unittest from datetime import datetime, timedelta from io import StringIO +from pathlib import Path from urllib.request import urlopen from django.core.cache import cache @@ -290,6 +291,12 @@ def test_file_save_with_path(self): self.storage.delete('path/to/test.file') + def test_file_save_abs_path(self): + test_name = 'path/to/test.file' + f = ContentFile('file saved with path') + f_name = self.storage.save(os.path.join(self.temp_dir, test_name), f) + self.assertEqual(f_name, test_name) + def test_save_doesnt_close(self): with TemporaryUploadedFile('test', 'text/plain', 1, 'utf8') as file: file.write(b'1') @@ -901,16 +908,19 @@ def test_file_upload_default_permissions(self): @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765) def test_file_upload_directory_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o765) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + self.assertEqual(file_path.parent.stat().st_mode & 0o777, 0o765) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, 0o765) @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=None) def test_file_upload_directory_default_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o777 & ~self.umask) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + expected_mode = 0o777 & ~self.umask + self.assertEqual(file_path.parent.stat().st_mode & 0o777, expected_mode) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, expected_mode) class FileStoragePathParsing(SimpleTestCase): diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index ea4976dc0a83..6be88679b8f4 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -8,8 +8,9 @@ from io import BytesIO, StringIO from urllib.parse import quote +from django.core.exceptions import SuspiciousFileOperation from django.core.files import temp as tempfile -from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.files.uploadedfile import SimpleUploadedFile, UploadedFile from django.http.multipartparser import ( MultiPartParser, MultiPartParserError, parse_header, ) @@ -22,6 +23,31 @@ MEDIA_ROOT = sys_tempfile.mkdtemp() UPLOAD_TO = os.path.join(MEDIA_ROOT, 'test_upload') +CANDIDATE_TRAVERSAL_FILE_NAMES = [ + '/tmp/hax0rd.txt', # Absolute path, *nix-style. + 'C:\\Windows\\hax0rd.txt', # Absolute path, win-style. + 'C:/Windows/hax0rd.txt', # Absolute path, broken-style. + '\\tmp\\hax0rd.txt', # Absolute path, broken in a different way. + '/tmp\\hax0rd.txt', # Absolute path, broken by mixing. + 'subdir/hax0rd.txt', # Descendant path, *nix-style. + 'subdir\\hax0rd.txt', # Descendant path, win-style. + 'sub/dir\\hax0rd.txt', # Descendant path, mixed. + '../../hax0rd.txt', # Relative path, *nix-style. + '..\\..\\hax0rd.txt', # Relative path, win-style. + '../..\\hax0rd.txt', # Relative path, mixed. + '../hax0rd.txt', # HTML entities. +] + +CANDIDATE_INVALID_FILE_NAMES = [ + '/tmp/', # Directory, *nix-style. + 'c:\\tmp\\', # Directory, win-style. + '/tmp/.', # Directory dot, *nix-style. + 'c:\\tmp\\.', # Directory dot, *nix-style. + '/tmp/..', # Parent directory, *nix-style. + 'c:\\tmp\\..', # Parent directory, win-style. + '', # Empty filename. +] + @override_settings(MEDIA_ROOT=MEDIA_ROOT, ROOT_URLCONF='file_uploads.urls', MIDDLEWARE=[]) class FileUploadTests(TestCase): @@ -37,6 +63,22 @@ def tearDownClass(cls): shutil.rmtree(MEDIA_ROOT) super().tearDownClass() + def test_upload_name_is_validated(self): + candidates = [ + '/tmp/', + '/tmp/..', + '/tmp/.', + ] + if sys.platform == 'win32': + candidates.extend([ + 'c:\\tmp\\', + 'c:\\tmp\\..', + 'c:\\tmp\\.', + ]) + for file_name in candidates: + with self.subTest(file_name=file_name): + self.assertRaises(SuspiciousFileOperation, UploadedFile, name=file_name) + def test_simple_upload(self): with open(__file__, 'rb') as fp: post_data = { @@ -100,6 +142,26 @@ def test_big_base64_upload(self): def test_big_base64_newlines_upload(self): self._test_base64_upload("Big data" * 68000, encode=base64.encodebytes) + def test_base64_invalid_upload(self): + payload = client.FakePayload('\r\n'.join([ + '--' + client.BOUNDARY, + 'Content-Disposition: form-data; name="file"; filename="test.txt"', + 'Content-Type: application/octet-stream', + 'Content-Transfer-Encoding: base64', + '' + ])) + payload.write(b'\r\n!\r\n') + payload.write('--' + client.BOUNDARY + '--\r\n') + r = { + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': client.MULTIPART_CONTENT, + 'PATH_INFO': '/echo_content/', + 'REQUEST_METHOD': 'POST', + 'wsgi.input': payload, + } + response = self.client.request(**r) + self.assertEqual(response.json()['file'], '') + def test_unicode_file_name(self): with sys_tempfile.TemporaryDirectory() as temp_dir: # This file contains Chinese symbols and an accented char in the name. @@ -205,22 +267,8 @@ def test_dangerous_file_names(self): # a malicious payload with an invalid file name (containing os.sep or # os.pardir). This similar to what an attacker would need to do when # trying such an attack. - scary_file_names = [ - "/tmp/hax0rd.txt", # Absolute path, *nix-style. - "C:\\Windows\\hax0rd.txt", # Absolute path, win-style. - "C:/Windows/hax0rd.txt", # Absolute path, broken-style. - "\\tmp\\hax0rd.txt", # Absolute path, broken in a different way. - "/tmp\\hax0rd.txt", # Absolute path, broken by mixing. - "subdir/hax0rd.txt", # Descendant path, *nix-style. - "subdir\\hax0rd.txt", # Descendant path, win-style. - "sub/dir\\hax0rd.txt", # Descendant path, mixed. - "../../hax0rd.txt", # Relative path, *nix-style. - "..\\..\\hax0rd.txt", # Relative path, win-style. - "../..\\hax0rd.txt" # Relative path, mixed. - ] - payload = client.FakePayload() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): payload.write('\r\n'.join([ '--' + client.BOUNDARY, 'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name), @@ -240,7 +288,7 @@ def test_dangerous_file_names(self): response = self.client.request(**r) # The filenames should have been sanitized by the time it got to the view. received = response.json() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): got = received["file%s" % i] self.assertEqual(got, "hax0rd.txt") @@ -518,6 +566,36 @@ def test_filename_case_preservation(self): # shouldn't differ. self.assertEqual(os.path.basename(obj.testfile.path), 'MiXeD_cAsE.txt') + def test_filename_traversal_upload(self): + os.makedirs(UPLOAD_TO, exist_ok=True) + self.addCleanup(shutil.rmtree, MEDIA_ROOT) + file_name = '../test.txt', + payload = client.FakePayload() + payload.write( + '\r\n'.join([ + '--' + client.BOUNDARY, + 'Content-Disposition: form-data; name="my_file"; ' + 'filename="%s";' % file_name, + 'Content-Type: text/plain', + '', + 'file contents.\r\n', + '\r\n--' + client.BOUNDARY + '--\r\n', + ]), + ) + r = { + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': client.MULTIPART_CONTENT, + 'PATH_INFO': '/upload_traversal/', + 'REQUEST_METHOD': 'POST', + 'wsgi.input': payload, + } + response = self.client.request(**r) + result = response.json() + self.assertEqual(response.status_code, 200) + self.assertEqual(result['file_name'], 'test.txt') + self.assertIs(os.path.exists(os.path.join(MEDIA_ROOT, 'test.txt')), False) + self.assertIs(os.path.exists(os.path.join(UPLOAD_TO, 'test.txt')), True) + @override_settings(MEDIA_ROOT=MEDIA_ROOT) class DirectoryCreationTests(SimpleTestCase): @@ -591,6 +669,24 @@ def test_bad_type_content_length(self): }, StringIO('x'), [], 'utf-8') self.assertEqual(multipart_parser._content_length, 0) + def test_sanitize_file_name(self): + parser = MultiPartParser({ + 'CONTENT_TYPE': 'multipart/form-data; boundary=_foo', + 'CONTENT_LENGTH': '1' + }, StringIO('x'), [], 'utf-8') + for file_name in CANDIDATE_TRAVERSAL_FILE_NAMES: + with self.subTest(file_name=file_name): + self.assertEqual(parser.sanitize_file_name(file_name), 'hax0rd.txt') + + def test_sanitize_invalid_file_name(self): + parser = MultiPartParser({ + 'CONTENT_TYPE': 'multipart/form-data; boundary=_foo', + 'CONTENT_LENGTH': '1', + }, StringIO('x'), [], 'utf-8') + for file_name in CANDIDATE_INVALID_FILE_NAMES: + with self.subTest(file_name=file_name): + self.assertIsNone(parser.sanitize_file_name(file_name)) + def test_rfc2231_parsing(self): test_data = ( (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", diff --git a/tests/file_uploads/uploadhandler.py b/tests/file_uploads/uploadhandler.py index 7c6199fd16d3..65d70c648c44 100644 --- a/tests/file_uploads/uploadhandler.py +++ b/tests/file_uploads/uploadhandler.py @@ -1,6 +1,8 @@ """ Upload handlers to test the upload API. """ +import os +from tempfile import NamedTemporaryFile from django.core.files.uploadhandler import FileUploadHandler, StopUpload @@ -35,3 +37,32 @@ class ErroringUploadHandler(FileUploadHandler): """A handler that raises an exception.""" def receive_data_chunk(self, raw_data, start): raise CustomUploadError("Oops!") + + +class TraversalUploadHandler(FileUploadHandler): + """A handler with potential directory-traversal vulnerability.""" + def __init__(self, request=None): + from .views import UPLOAD_TO + + super().__init__(request) + self.upload_dir = UPLOAD_TO + + def file_complete(self, file_size): + self.file.seek(0) + self.file.size = file_size + with open(os.path.join(self.upload_dir, self.file_name), 'wb') as fp: + fp.write(self.file.read()) + return self.file + + def new_file( + self, field_name, file_name, content_type, content_length, charset=None, + content_type_extra=None, + ): + super().new_file( + file_name, file_name, content_length, content_length, charset, + content_type_extra, + ) + self.file = NamedTemporaryFile(suffix='.upload', dir=self.upload_dir) + + def receive_data_chunk(self, raw_data, start): + self.file.write(raw_data) diff --git a/tests/file_uploads/urls.py b/tests/file_uploads/urls.py index 3e7985d2f9db..eaac1dae3d4b 100644 --- a/tests/file_uploads/urls.py +++ b/tests/file_uploads/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path('upload/', views.file_upload_view), + path('upload_traversal/', views.file_upload_traversal_view), path('verify/', views.file_upload_view_verify), path('unicode_name/', views.file_upload_unicode_name), path('echo/', views.file_upload_echo), diff --git a/tests/file_uploads/views.py b/tests/file_uploads/views.py index d4947e413404..137c6f3a4b46 100644 --- a/tests/file_uploads/views.py +++ b/tests/file_uploads/views.py @@ -6,7 +6,9 @@ from .models import FileModel from .tests import UNICODE_FILENAME, UPLOAD_TO -from .uploadhandler import ErroringUploadHandler, QuotaUploadHandler +from .uploadhandler import ( + ErroringUploadHandler, QuotaUploadHandler, TraversalUploadHandler, +) def file_upload_view(request): @@ -158,3 +160,11 @@ def file_upload_fd_closing(request, access): if access == 't': request.FILES # Trigger file parsing. return HttpResponse('') + + +def file_upload_traversal_view(request): + request.upload_handlers.insert(0, TraversalUploadHandler()) + request.FILES # Trigger file parsing. + return JsonResponse( + {'file_name': request.upload_handlers[0].file_name}, + ) diff --git a/tests/files/tests.py b/tests/files/tests.py index b50061649ae4..c60d69bf6a6a 100644 --- a/tests/files/tests.py +++ b/tests/files/tests.py @@ -17,9 +17,11 @@ ) try: - from PIL import Image + from PIL import Image, features + HAS_WEBP = features.check('webp') except ImportError: Image = None + HAS_WEBP = False else: from django.core.files import images @@ -343,6 +345,7 @@ def test_valid_image(self): size = images.get_image_dimensions(fh) self.assertEqual(size, (None, None)) + @unittest.skipUnless(HAS_WEBP, 'WEBP not installed') def test_webp(self): img_path = os.path.join(os.path.dirname(__file__), 'test.webp') with open(img_path, 'rb') as fh: diff --git a/tests/forms_tests/field_tests/test_filefield.py b/tests/forms_tests/field_tests/test_filefield.py index fc5c4b5c1e1d..33574446f4cb 100644 --- a/tests/forms_tests/field_tests/test_filefield.py +++ b/tests/forms_tests/field_tests/test_filefield.py @@ -20,10 +20,12 @@ def test_filefield_1(self): f.clean(None, '') self.assertEqual('files/test2.pdf', f.clean(None, 'files/test2.pdf')) no_file_msg = "'No file was submitted. Check the encoding type on the form.'" + file = SimpleUploadedFile(None, b'') + file._name = '' with self.assertRaisesMessage(ValidationError, no_file_msg): - f.clean(SimpleUploadedFile('', b'')) + f.clean(file) with self.assertRaisesMessage(ValidationError, no_file_msg): - f.clean(SimpleUploadedFile('', b''), '') + f.clean(file, '') self.assertEqual('files/test3.pdf', f.clean(None, 'files/test3.pdf')) with self.assertRaisesMessage(ValidationError, no_file_msg): f.clean('some content that is not a file') diff --git a/tests/gis_tests/test_geoip2.py b/tests/gis_tests/test_geoip2.py index 54f8aa00b008..930a9250071a 100644 --- a/tests/gis_tests/test_geoip2.py +++ b/tests/gis_tests/test_geoip2.py @@ -19,8 +19,8 @@ "GeoIP is required along with the GEOIP_PATH setting." ) class GeoIPTest(SimpleTestCase): - addr = '75.41.39.1' - fqdn = 'tmc.edu' + addr = '129.237.192.1' + fqdn = 'ku.edu' def test01_init(self): "GeoIP initialization." @@ -99,7 +99,7 @@ def test03_country(self, gethostbyname): @mock.patch('socket.gethostbyname') def test04_city(self, gethostbyname): "GeoIP city querying methods." - gethostbyname.return_value = '75.41.39.1' + gethostbyname.return_value = '129.237.192.1' g = GeoIP2(country='') for query in (self.fqdn, self.addr): @@ -124,8 +124,8 @@ def test04_city(self, gethostbyname): self.assertEqual('NA', d['continent_code']) self.assertEqual('North America', d['continent_name']) self.assertEqual('US', d['country_code']) - self.assertEqual('Dallas', d['city']) - self.assertEqual('TX', d['region']) + self.assertEqual('Lawrence', d['city']) + self.assertEqual('KS', d['region']) self.assertEqual('America/Chicago', d['time_zone']) geom = g.geos(query) self.assertIsInstance(geom, GEOSGeometry) diff --git a/tests/handlers/test_exception.py b/tests/handlers/test_exception.py index 7afd4acc6b08..0c1e76399045 100644 --- a/tests/handlers/test_exception.py +++ b/tests/handlers/test_exception.py @@ -6,7 +6,7 @@ class ExceptionHandlerTests(SimpleTestCase): def get_suspicious_environ(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') return { 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/i18n/commands/code.sample b/tests/i18n/commands/code.sample index a5f1520ecba5..2c305a3a1dcf 100644 --- a/tests/i18n/commands/code.sample +++ b/tests/i18n/commands/code.sample @@ -1,4 +1,4 @@ from django.utils.translation import gettext -# This will generate an xgettext warning -my_string = gettext("This string contain two placeholders: %s and %s" % ('a', 'b')) +# This will generate an xgettext "Empty msgid" warning. +my_string = gettext('') diff --git a/tests/mail/tests.py b/tests/mail/tests.py index f06e0530745e..e62141c04036 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -720,7 +720,7 @@ def test_sanitize_address(self): ) self.assertEqual( sanitize_address(('A name', 'to@example.com'), 'utf-8'), - '=?utf-8?q?A_name?= ' + 'A name ' ) # Unicode characters are are supported in RFC-6532. @@ -732,6 +732,37 @@ def test_sanitize_address(self): sanitize_address(('Tó Example', 'tó@example.com'), 'utf-8'), '=?utf-8?q?T=C3=B3_Example?= <=?utf-8?b?dMOz?=@example.com>' ) + # Addresses with long unicode display names. + self.assertEqual( + sanitize_address('Tó Example very long' * 4 + ' ', 'utf-8'), + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT=C3=' + 'B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + ) + self.assertEqual( + sanitize_address(('Tó Example very long' * 4, 'to@example.com'), 'utf-8'), + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT=C3=' + 'B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + ) + # Address with long display name and unicode domain. + self.assertEqual( + sanitize_address(('To Example very long' * 4, 'to@exampl€.com'), 'utf-8'), + 'To Example very longTo Example very longTo Example very longTo Ex' + 'ample very\n' + ' long ' + ) + + def test_sanitize_address_header_injection(self): + msg = 'Invalid address; address parts cannot contain newlines.' + tests = [ + ('Name\nInjection', 'to@xample.com'), + ('Name', 'to\ninjection@example.com'), + ] + for email_address in tests: + with self.subTest(email_address=email_address): + with self.assertRaisesMessage(ValueError, msg): + sanitize_address(email_address, encoding='utf-8') @requires_tz_support diff --git a/tests/messages_tests/test_cookie.py b/tests/messages_tests/test_cookie.py index 211d33f04c54..7456e03a70fc 100644 --- a/tests/messages_tests/test_cookie.py +++ b/tests/messages_tests/test_cookie.py @@ -1,5 +1,6 @@ import json +from django.conf import settings from django.contrib.messages import constants from django.contrib.messages.storage.base import Message from django.contrib.messages.storage.cookie import ( @@ -85,6 +86,10 @@ def test_cookie_setings(self): self.assertEqual(response.cookies['messages'].value, '') self.assertEqual(response.cookies['messages']['domain'], '.example.com') self.assertEqual(response.cookies['messages']['expires'], 'Thu, 01 Jan 1970 00:00:00 GMT') + self.assertEqual( + response.cookies['messages']['samesite'], + settings.SESSION_COOKIE_SAMESITE, + ) def test_get_bad_cookie(self): request = self.get_request() diff --git a/tests/model_fields/test_filefield.py b/tests/model_fields/test_filefield.py index 9330a2eba25c..0afef7284ee5 100644 --- a/tests/model_fields/test_filefield.py +++ b/tests/model_fields/test_filefield.py @@ -1,8 +1,10 @@ import os import sys +import tempfile import unittest -from django.core.files import temp +from django.core.exceptions import SuspiciousFileOperation +from django.core.files import File, temp from django.core.files.base import ContentFile from django.core.files.uploadedfile import TemporaryUploadedFile from django.db.utils import IntegrityError @@ -59,6 +61,15 @@ def test_refresh_from_db(self): d.refresh_from_db() self.assertIs(d.myfile.instance, d) + @unittest.skipIf(sys.platform == 'win32', "Crashes with OSError on Windows.") + def test_save_without_name(self): + with tempfile.NamedTemporaryFile(suffix='.txt') as tmp: + document = Document.objects.create(myfile='something.txt') + document.myfile = File(tmp) + msg = "Detected path traversal attempt in '%s'" % tmp.name + with self.assertRaisesMessage(SuspiciousFileOperation, msg): + document.save() + def test_defer(self): Document.objects.create(myfile='something.txt') self.assertEqual(Document.objects.defer('myfile')[0].myfile, 'something.txt') diff --git a/tests/model_regress/tests.py b/tests/model_regress/tests.py index 28eed87008a4..87df240d8153 100644 --- a/tests/model_regress/tests.py +++ b/tests/model_regress/tests.py @@ -1,3 +1,4 @@ +import copy import datetime from operator import attrgetter @@ -256,3 +257,17 @@ def test_model_with_evaluate_method(self): dept = Department.objects.create(pk=1, name='abc') dept.evaluate = 'abc' Worker.objects.filter(department=dept) + + +class ModelFieldsCacheTest(TestCase): + def test_fields_cache_reset_on_copy(self): + department1 = Department.objects.create(id=1, name='department1') + department2 = Department.objects.create(id=2, name='department2') + worker1 = Worker.objects.create(name='worker', department=department1) + worker2 = copy.copy(worker1) + + self.assertEqual(worker2.department, department1) + # Changing related fields doesn't mutate the base object. + worker2.department = department2 + self.assertEqual(worker2.department, department2) + self.assertEqual(worker1.department, department1) diff --git a/tests/queries/test_explain.py b/tests/queries/test_explain.py index 9428bd88e9c3..209c1923071e 100644 --- a/tests/queries/test_explain.py +++ b/tests/queries/test_explain.py @@ -41,8 +41,8 @@ def test_basic(self): @skipUnlessDBFeature('validates_explain_options') def test_unknown_options(self): - with self.assertRaisesMessage(ValueError, 'Unknown options: test, test2'): - Tag.objects.all().explain(test=1, test2=1) + with self.assertRaisesMessage(ValueError, "Unknown options: TEST, TEST2"): + Tag.objects.all().explain(**{"TEST": 1, "TEST2": 1}) def test_unknown_format(self): msg = 'DOES NOT EXIST is not a recognized format.' @@ -71,6 +71,35 @@ def test_postgres_options(self): option = '{} {}'.format(name.upper(), 'true' if value else 'false') self.assertIn(option, captured_queries[0]['sql']) + def test_option_sql_injection(self): + qs = Tag.objects.filter(name="test") + options = {"SUMMARY true) SELECT 1; --": True} + msg = "Invalid option name: 'SUMMARY true) SELECT 1; --'" + with self.assertRaisesMessage(ValueError, msg): + qs.explain(**options) + + def test_invalid_option_names(self): + qs = Tag.objects.filter(name="test") + tests = [ + 'opt"ion', + "o'ption", + "op`tion", + "opti on", + "option--", + "optio\tn", + "o\nption", + "option;", + "你 好", + # [] are used by MSSQL. + "option[", + "option]", + ] + for invalid_option in tests: + with self.subTest(invalid_option): + msg = "Invalid option name: '%s'" % invalid_option + with self.assertRaisesMessage(ValueError, msg): + qs.explain(**{invalid_option: True}) + @unittest.skipUnless(connection.vendor == 'mysql', 'MySQL specific') def test_mysql_text_to_traditional(self): # Initialize the cached property, if needed, to prevent a query for diff --git a/tests/queries/tests.py b/tests/queries/tests.py index e72ecaa654c8..99ab57f4fc2e 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -1737,6 +1737,15 @@ def test_extra_select_literal_percent_s(self): 'bar %s' ) + def test_extra_select_alias_sql_injection(self): + crafted_alias = """injected_name" from "queries_note"; --""" + msg = ( + "Column aliases cannot contain whitespace characters, quotation marks, " + "semicolons, or SQL comments." + ) + with self.assertRaisesMessage(ValueError, msg): + Note.objects.extra(select={crafted_alias: "1"}) + class SelectRelatedTests(TestCase): def test_tickets_3045_3288(self): diff --git a/tests/requests/test_data_upload_settings.py b/tests/requests/test_data_upload_settings.py index f60f1850ea25..44897cc9fa97 100644 --- a/tests/requests/test_data_upload_settings.py +++ b/tests/requests/test_data_upload_settings.py @@ -11,7 +11,7 @@ class DataUploadMaxMemorySizeFormPostTests(SimpleTestCase): def setUp(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', @@ -117,7 +117,7 @@ def test_get_max_fields_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -126,7 +126,7 @@ def test_get_max_fields_not_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -168,7 +168,7 @@ def test_no_limit(self): class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase): def setUp(self): - payload = FakePayload("\r\n".join(['a=1&a=2;a=3', ''])) + payload = FakePayload("\r\n".join(['a=1&a=2&a=3', ''])) self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/requirements/oracle.txt b/tests/requirements/oracle.txt index 763baa8579ca..65d8fee5dd57 100644 --- a/tests/requirements/oracle.txt +++ b/tests/requirements/oracle.txt @@ -1 +1 @@ -cx_oracle >= 6.0 +cx_oracle >= 6.0, < 8 diff --git a/tests/requirements/postgres.txt b/tests/requirements/postgres.txt index 820d85bb44df..844349958063 100644 --- a/tests/requirements/postgres.txt +++ b/tests/requirements/postgres.txt @@ -1 +1 @@ -psycopg2-binary>=2.5.4 +psycopg2-binary>=2.5.4, < 2.9 diff --git a/tests/requirements/py3.txt b/tests/requirements/py3.txt index bb335791f445..84a6bf03446d 100644 --- a/tests/requirements/py3.txt +++ b/tests/requirements/py3.txt @@ -1,10 +1,10 @@ argon2-cffi >= 16.1.0 bcrypt docutils -geoip2 +geoip2 < 4.0.0 jinja2 >= 2.9.2 numpy -Pillow != 5.4.0 +Pillow >=4.2.0, != 5.4.0 # pylibmc/libmemcached can't be built on Windows. pylibmc; sys.platform != 'win32' python-memcached >= 1.59 @@ -12,5 +12,5 @@ pytz pywatchman; sys.platform != 'win32' PyYAML selenium -sqlparse +sqlparse >= 0.2.2 tblib >= 1.5.0 diff --git a/tests/responses/test_cookie.py b/tests/responses/test_cookie.py index a5092c3bbf64..68927a4ee2bb 100644 --- a/tests/responses/test_cookie.py +++ b/tests/responses/test_cookie.py @@ -102,6 +102,7 @@ def test_default(self): self.assertEqual(cookie['path'], '/') self.assertEqual(cookie['secure'], '') self.assertEqual(cookie['domain'], '') + self.assertEqual(cookie['samesite'], '') def test_delete_cookie_secure_prefix(self): """ @@ -115,3 +116,8 @@ def test_delete_cookie_secure_prefix(self): cookie_name = '__%s-c' % prefix response.delete_cookie(cookie_name) self.assertEqual(response.cookies[cookie_name]['secure'], True) + + def test_delete_cookie_samesite(self): + response = HttpResponse() + response.delete_cookie('c', samesite='lax') + self.assertEqual(response.cookies['c']['samesite'], 'lax') diff --git a/tests/schema/tests.py b/tests/schema/tests.py index dfc684c4b0d7..a1d364bcf4c1 100644 --- a/tests/schema/tests.py +++ b/tests/schema/tests.py @@ -92,8 +92,12 @@ def delete_tables(self): with connection.schema_editor() as editor: connection.disable_constraint_checking() table_names = connection.introspection.table_names() + if connection.features.ignores_table_name_case: + table_names = [table_name.lower() for table_name in table_names] for model in itertools.chain(SchemaTests.models, self.local_models): tbl = converter(model._meta.db_table) + if connection.features.ignores_table_name_case: + tbl = tbl.lower() if tbl in table_names: editor.delete_model(model) table_names.remove(tbl) diff --git a/tests/select_for_update/models.py b/tests/select_for_update/models.py index 305e8cac490b..0a6396bc7066 100644 --- a/tests/select_for_update/models.py +++ b/tests/select_for_update/models.py @@ -23,6 +23,20 @@ class EUCity(models.Model): country = models.ForeignKey(EUCountry, models.CASCADE) +class CountryProxy(Country): + class Meta: + proxy = True + + +class CountryProxyProxy(CountryProxy): + class Meta: + proxy = True + + +class CityCountryProxy(models.Model): + country = models.ForeignKey(CountryProxyProxy, models.CASCADE) + + class Person(models.Model): name = models.CharField(max_length=30) born = models.ForeignKey(City, models.CASCADE, related_name='+') diff --git a/tests/select_for_update/tests.py b/tests/select_for_update/tests.py index 7ef0319477d4..70511b09a123 100644 --- a/tests/select_for_update/tests.py +++ b/tests/select_for_update/tests.py @@ -15,7 +15,9 @@ ) from django.test.utils import CaptureQueriesContext -from .models import City, Country, EUCity, EUCountry, Person, PersonProfile +from .models import ( + City, CityCountryProxy, Country, EUCity, EUCountry, Person, PersonProfile, +) class SelectForUpdateTests(TransactionTestCase): @@ -195,6 +197,21 @@ def test_for_update_sql_multilevel_model_inheritance_ptr_generated_of(self): expected = [connection.ops.quote_name(value) for value in expected] self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') + def test_for_update_sql_model_proxy_generated_of(self): + with transaction.atomic(), CaptureQueriesContext(connection) as ctx: + list(CityCountryProxy.objects.select_related( + 'country', + ).select_for_update( + of=('country',), + )) + if connection.features.select_for_update_of_column: + expected = ['select_for_update_country"."entity_ptr_id'] + else: + expected = ['select_for_update_country'] + expected = [connection.ops.quote_name(value) for value in expected] + self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') def test_for_update_of_followed_by_values(self): with transaction.atomic(): @@ -353,6 +370,19 @@ def test_model_inheritance_of_argument_raises_error_ptr_in_choices(self): with transaction.atomic(): EUCountry.objects.select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') + def test_model_proxy_of_argument_raises_error_proxy_field_in_choices(self): + msg = ( + 'Invalid field name(s) given in select_for_update(of=(...)): ' + 'name. Only relational fields followed in the query are allowed. ' + 'Choices are: self, country, country__entity_ptr.' + ) + with self.assertRaisesMessage(FieldError, msg): + with transaction.atomic(): + CityCountryProxy.objects.select_related( + 'country', + ).select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') def test_reverse_one_to_one_of_arguments(self): """ @@ -471,7 +501,7 @@ def test_block(self): # Check the thread has finished. Assuming it has, we should # find that it has updated the person's name. - self.assertFalse(thread.isAlive()) + self.assertFalse(thread.is_alive()) # We must commit the transaction to ensure that MySQL gets a fresh read, # since by default it runs in REPEATABLE READ mode diff --git a/tests/sessions_tests/tests.py b/tests/sessions_tests/tests.py index 733f5adb1dc5..901995fa88fb 100644 --- a/tests/sessions_tests/tests.py +++ b/tests/sessions_tests/tests.py @@ -743,8 +743,9 @@ def test_session_delete_on_end(self): # Set-Cookie: sessionid=; expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/ self.assertEqual( 'Set-Cookie: {}=""; expires=Thu, 01 Jan 1970 00:00:00 GMT; ' - 'Max-Age=0; Path=/'.format( + 'Max-Age=0; Path=/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) @@ -772,8 +773,9 @@ def test_session_delete_on_end_with_custom_domain_and_path(self): # Path=/example/ self.assertEqual( 'Set-Cookie: {}=""; Domain=.example.local; expires=Thu, ' - '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/'.format( + '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) diff --git a/tests/staticfiles_tests/project/documents/nested/css/base.css b/tests/staticfiles_tests/project/documents/nested/css/base.css new file mode 100644 index 000000000000..06041ca25f1e --- /dev/null +++ b/tests/staticfiles_tests/project/documents/nested/css/base.css @@ -0,0 +1 @@ +html {height: 100%;} diff --git a/tests/staticfiles_tests/test_storage.py b/tests/staticfiles_tests/test_storage.py index 97e3b9113d64..fe44348dff9d 100644 --- a/tests/staticfiles_tests/test_storage.py +++ b/tests/staticfiles_tests/test_storage.py @@ -4,6 +4,7 @@ import tempfile import unittest from io import StringIO +from pathlib import Path from django.conf import settings from django.contrib.staticfiles import finders, storage @@ -508,12 +509,19 @@ def run_collectstatic(self, **kwargs): ) def test_collect_static_files_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o655) - self.assertEqual(dir_mode, 0o765) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o765) @override_settings( FILE_UPLOAD_PERMISSIONS=None, @@ -521,12 +529,19 @@ def test_collect_static_files_permissions(self): ) def test_collect_static_files_default_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o666 & ~self.umask) - self.assertEqual(dir_mode, 0o777 & ~self.umask) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o777 & ~self.umask) @override_settings( FILE_UPLOAD_PERMISSIONS=0o655, @@ -535,12 +550,19 @@ def test_collect_static_files_default_permissions(self): ) def test_collect_static_files_subclass_of_static_storage(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o640) - self.assertEqual(dir_mode, 0o740) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o740) @override_settings( diff --git a/tests/template_tests/filter_tests/test_dictsort.py b/tests/template_tests/filter_tests/test_dictsort.py index 00c2bd42cbd8..3de247fd86fa 100644 --- a/tests/template_tests/filter_tests/test_dictsort.py +++ b/tests/template_tests/filter_tests/test_dictsort.py @@ -1,9 +1,58 @@ -from django.template.defaultfilters import dictsort +from django.template.defaultfilters import _property_resolver, dictsort from django.test import SimpleTestCase +class User: + password = 'abc' + + _private = 'private' + + @property + def test_property(self): + return 'cde' + + def test_method(self): + """This is just a test method.""" + + class FunctionTests(SimpleTestCase): + def test_property_resolver(self): + user = User() + dict_data = {'a': { + 'b1': {'c': 'result1'}, + 'b2': user, + 'b3': {'0': 'result2'}, + 'b4': [0, 1, 2], + }} + list_data = ['a', 'b', 'c'] + tests = [ + ('a.b1.c', dict_data, 'result1'), + ('a.b2.password', dict_data, 'abc'), + ('a.b2.test_property', dict_data, 'cde'), + # The method should not get called. + ('a.b2.test_method', dict_data, user.test_method), + ('a.b3.0', dict_data, 'result2'), + (0, list_data, 'a'), + ] + for arg, data, expected_value in tests: + with self.subTest(arg=arg): + self.assertEqual(_property_resolver(arg)(data), expected_value) + # Invalid lookups. + fail_tests = [ + ('a.b1.d', dict_data, AttributeError), + ('a.b2.password.0', dict_data, AttributeError), + ('a.b2._private', dict_data, AttributeError), + ('a.b4.0', dict_data, AttributeError), + ('a', list_data, AttributeError), + ('0', list_data, TypeError), + (4, list_data, IndexError), + ] + for arg, data, expected_exception in fail_tests: + with self.subTest(arg=arg): + with self.assertRaises(expected_exception): + _property_resolver(arg)(data) + def test_sort(self): sorted_dicts = dictsort( [{'age': 23, 'name': 'Barbara-Ann'}, @@ -21,7 +70,7 @@ def test_sort(self): def test_dictsort_complex_sorting_key(self): """ - Since dictsort uses template.Variable under the hood, it can sort + Since dictsort uses dict.get()/getattr() under the hood, it can sort on keys like 'foo.bar'. """ data = [ @@ -60,3 +109,9 @@ def test_invalid_values(self): self.assertEqual(dictsort('Hello!', 'age'), '') self.assertEqual(dictsort({'a': 1}, 'age'), '') self.assertEqual(dictsort(1, 'age'), '') + + def test_invalid_args(self): + """Fail silently if invalid lookups are passed.""" + self.assertEqual(dictsort([{}], '._private'), '') + self.assertEqual(dictsort([{'_private': 'test'}], '_private'), '') + self.assertEqual(dictsort([{'nested': {'_private': 'test'}}], 'nested._private'), '') diff --git a/tests/template_tests/filter_tests/test_dictsortreversed.py b/tests/template_tests/filter_tests/test_dictsortreversed.py index ada199e127d2..e2e24e312849 100644 --- a/tests/template_tests/filter_tests/test_dictsortreversed.py +++ b/tests/template_tests/filter_tests/test_dictsortreversed.py @@ -46,3 +46,9 @@ def test_invalid_values(self): self.assertEqual(dictsortreversed('Hello!', 'age'), '') self.assertEqual(dictsortreversed({'a': 1}, 'age'), '') self.assertEqual(dictsortreversed(1, 'age'), '') + + def test_invalid_args(self): + """Fail silently if invalid lookups are passed.""" + self.assertEqual(dictsortreversed([{}], '._private'), '') + self.assertEqual(dictsortreversed([{'_private': 'test'}], '_private'), '') + self.assertEqual(dictsortreversed([{'nested': {'_private': 'test'}}], 'nested._private'), '') diff --git a/tests/template_tests/syntax_tests/test_debug.py b/tests/template_tests/syntax_tests/test_debug.py new file mode 100644 index 000000000000..17fea44b6832 --- /dev/null +++ b/tests/template_tests/syntax_tests/test_debug.py @@ -0,0 +1,46 @@ +from django.contrib.auth.models import Group +from django.test import SimpleTestCase, override_settings + +from ..utils import setup + + +@override_settings(DEBUG=True) +class DebugTests(SimpleTestCase): + + @override_settings(DEBUG=False) + @setup({'non_debug': '{% debug %}'}) + def test_non_debug(self): + output = self.engine.render_to_string('non_debug', {}) + self.assertEqual(output, '') + + @setup({'modules': '{% debug %}'}) + def test_modules(self): + output = self.engine.render_to_string('modules', {}) + self.assertIn( + ''django': <module 'django' ', + output, + ) + + @setup({'plain': '{% debug %}'}) + def test_plain(self): + output = self.engine.render_to_string('plain', {'a': 1}) + self.assertTrue(output.startswith( + '{'a': 1}' + '{'False': False, 'None': None, ' + ''True': True}\n\n{' + )) + + @setup({'non_ascii': '{% debug %}'}) + def test_non_ascii(self): + group = Group(name="清風") + output = self.engine.render_to_string('non_ascii', {'group': group}) + self.assertTrue(output.startswith( + '{'group': <Group: 清風>}' + )) + + @setup({'script': '{% debug %}'}) + def test_script(self): + output = self.engine.render_to_string('script', {'frag': '