From b26eabe52dfe74d3d0a3e8e71a0c7eae18751793 Mon Sep 17 00:00:00 2001 From: David Lord Date: Tue, 8 Oct 2019 13:42:48 -0700 Subject: [PATCH 001/372] explain what autoescaping does --- docs/api.rst | 86 +++++++++++++------------------------- docs/extensions.rst | 2 +- docs/faq.rst | 2 +- docs/templates.rst | 12 +++--- jinja2/__init__.py | 3 +- jinja2/asyncsupport.py | 4 +- jinja2/compiler.py | 9 ++-- jinja2/environment.py | 18 ++++---- jinja2/ext.py | 4 +- jinja2/filters.py | 8 ++-- jinja2/nativetypes.py | 5 ++- jinja2/nodes.py | 3 +- jinja2/runtime.py | 4 +- jinja2/sandbox.py | 8 ++-- jinja2/utils.py | 68 +++++++++++++++++------------- tests/test_asyncfilters.py | 4 +- tests/test_filters.py | 5 ++- tests/test_security.py | 4 +- tests/test_tests.py | 4 +- 19 files changed, 130 insertions(+), 123 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 3f56e5ffd..7faf08a8b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -36,8 +36,8 @@ This will create a template environment with the default settings and a loader that looks up the templates in the `templates` folder inside the `yourapplication` python package. Different loaders are available and you can also write your own if you want to load templates from a -database or other resources. This also enables autoescaping for HTML and -XML files. +database or other resources. This also enables :ref:`autoescaping` for +HTML and XML files. To load a template from this environment you just have to call the :meth:`get_template` method which then returns the loaded :class:`Template`:: @@ -52,12 +52,6 @@ Using a template loader rather than passing strings to :class:`Template` or :meth:`Environment.from_string` has multiple advantages. Besides being a lot easier to use it also enables template inheritance. -.. admonition:: Notes on Autoescaping - - In future versions of Jinja2 we might enable autoescaping by default - for security reasons. As such you are encouraged to explicitly - configure autoescaping now instead of relying on the default. - Unicode ------- @@ -254,44 +248,37 @@ useful if you want to dig deeper into Jinja2 or :ref:`develop extensions :members: disable_buffering, enable_buffering, dump +.. _autoescaping: + Autoescaping ------------ -.. versionchanged:: 2.4 - -Jinja2 now comes with autoescaping support. As of Jinja 2.9 the -autoescape extension is removed and built-in. However autoescaping is -not yet enabled by default though this will most likely change in the -future. It's recommended to configure a sensible default for -autoescaping. This makes it possible to enable and disable autoescaping -on a per-template basis (HTML versus text for instance). +Autoescaping is a feature to mitigate injection attacks when rendering +HTML and XML documents. When autoescaping is enabled, Jinja will use +`MarkupSafe`_ to escape unsafe characters in the output of expressions, +unless the output is marked safe. For example, if a comment contained +````, the tags would be rendered with +escapes like ``<script>``. In a user's browser, the comment would +display as text, rather than being interpreted as a script. + +Because Jinja can be used to render any type of document for many types +of applications, not just HTML with untrusted input, autoescaping is not +enabled by default. You should configure a sensible default based on +your use case when creating the environment. The +:func:`~jinja2.select_autoescape` function can be used to enable +autoescaping for HTML templates while disabling it in text templates. .. autofunction:: jinja2.select_autoescape -Here a recommended setup that enables autoescaping for templates ending -in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default -for all other extensions. You can use the :func:`~jinja2.select_autoescape` -function for this:: +You can also pass ``autoescape=True`` to enable it unconditionally, or +pass your own function that takes the name of the template and returns +whether it should be enabled. When writing a function, make sure to +accept ``None`` as a name, as that will be passed for string templates. - from jinja2 import Environment, select_autoescape - env = Environment(autoescape=select_autoescape(['html', 'htm', 'xml']), - loader=PackageLoader('mypackage')) +Inside a template the behaviour can be temporarily changed by using +the ``autoescape`` block, see :ref:`autoescape-overrides`. -The :func:`~jinja.select_autoescape` function returns a function that -works rougly like this:: - - def autoescape(template_name): - if template_name is None: - return False - if template_name.endswith(('.html', '.htm', '.xml')) - -When implementing a guessing autoescape function, make sure you also -accept `None` as valid template name. This will be passed when generating -templates from strings. You should always configure autoescaping as -defaults in the future might change. - -Inside the templates the behaviour can be temporarily changed by using -the `autoescape` block (see :ref:`autoescape-overrides`). +.. _MarkupSafe: https://markupsafe.palletsprojects.com/ .. _identifier-naming: @@ -637,28 +624,14 @@ functions to a Jinja2 environment. .. autofunction:: jinja2.evalcontextfunction -.. function:: escape(s) - - Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s` - to HTML-safe sequences. Use this if you need to display text that might - contain such characters in HTML. This function will not escaped objects - that do have an HTML representation such as already escaped data. - - The return value is a :class:`Markup` string. - .. autofunction:: jinja2.clear_caches .. autofunction:: jinja2.is_undefined -.. autoclass:: jinja2.Markup([string]) - :members: escape, unescape, striptags - -.. admonition:: Note - - The Jinja2 :class:`Markup` class is compatible with at least Pylons and - Genshi. It's expected that more template engines and framework will pick - up the `__html__` concept soon. +.. autofunction:: markupsafe.escape +.. autoclass:: markupsafe.Markup + :members: escape, unescape, striptags Exceptions ---------- @@ -738,7 +711,8 @@ paragraphs and marks the return value as safe HTML string if autoescaping is enabled:: import re - from jinja2 import evalcontextfilter, Markup, escape + from jinja2 import evalcontextfilter + from markupsafe import Markup, escape _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}') diff --git a/docs/extensions.rst b/docs/extensions.rst index 12c589076..821185816 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -132,7 +132,7 @@ extraction tools in case you are not using Babel's. What's the big difference between standard and newstyle gettext calls? In general they are less to type and less error prone. Also if they are used -in an autoescaping environment they better support automatic escaping. +in an :ref:`autescaping` environment they better support automatic escaping. Here are some common differences between old and new calls: standard gettext: diff --git a/docs/faq.rst b/docs/faq.rst index 743a6cbca..73d46598e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -95,7 +95,7 @@ means that you will less likely have an XSS problem it also causes a huge amount of extra processing in the template engine which can cause serious performance problems. As Python doesn't provide a way to mark strings as unsafe Jinja has to hack around that limitation by providing a custom -string class (the :class:`Markup` string) that safely interacts with safe +string class (the :class:`~markupsafe.Markup` string) that safely interacts with safe and unsafe strings. With explicit escaping however the template engine doesn't have to perform diff --git a/docs/templates.rst b/docs/templates.rst index 4f6146980..fd3e39f7f 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -391,7 +391,7 @@ this template, it first locates the parent. The extends tag should be the first tag in the template. Everything before it is printed out normally and may cause confusion. For details about this behavior and how to take advantage of it, see :ref:`null-master-fallback`. Also a block will always be -filled in regardless of whether the surrounding condition is evaluated to be true +filled in regardless of whether the surrounding condition is evaluated to be true or false. The filename of the template depends on the template loader. For example, the @@ -527,7 +527,7 @@ When automatic escaping is enabled, everything is escaped by default except for values explicitly marked as safe. Variables and expressions can be marked as safe either in: -a. the context dictionary by the application with `MarkupSafe.Markup`, or +a. the context dictionary by the application with :class:`markupsafe.Markup`, or b. the template, with the `|safe` filter The main problem with this approach is that Python itself doesn't have the @@ -543,7 +543,7 @@ data that is marked as safe. String literals in templates with automatic escaping are considered unsafe because native Python strings (``str``, ``unicode``, ``basestring``) are not -`MarkupSafe.Markup` strings with an ``__html__`` attribute. +:class:`markupsafe.Markup` strings with an ``__html__`` attribute. .. _list-of-control-structures: @@ -1672,8 +1672,8 @@ Autoescape Overrides .. versionadded:: 2.4 -If you want you can activate and deactivate the autoescaping from within -the templates. +If you want you can activate and deactivate :ref:`autoescaping` from within +a template. Example:: @@ -1685,7 +1685,7 @@ Example:: Autoescaping is inactive within this block {% endautoescape %} -After an `endautoescape` the behavior is reverted to what it was before. +After an ``endautoescape`` the behavior is reverted to what it was before. .. admonition:: Extension diff --git a/jinja2/__init__.py b/jinja2/__init__.py index 0eaf72149..3ce857a6f 100644 --- a/jinja2/__init__.py +++ b/jinja2/__init__.py @@ -53,9 +53,10 @@ # decorators and public utilities from jinja2.filters import environmentfilter, contextfilter, \ evalcontextfilter -from jinja2.utils import Markup, escape, clear_caches, \ +from jinja2.utils import clear_caches, \ environmentfunction, evalcontextfunction, contextfunction, \ is_undefined, select_autoescape +from markupsafe import Markup, escape __all__ = [ 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', diff --git a/jinja2/asyncsupport.py b/jinja2/asyncsupport.py index b1e7b5ce9..7ceddf417 100644 --- a/jinja2/asyncsupport.py +++ b/jinja2/asyncsupport.py @@ -14,7 +14,9 @@ import inspect from functools import update_wrapper -from jinja2.utils import concat, internalcode, Markup +from markupsafe import Markup + +from jinja2.utils import concat, internalcode from jinja2.environment import TemplateModule from jinja2.runtime import LoopContextBase, _last_iteration diff --git a/jinja2/compiler.py b/jinja2/compiler.py index d534a8273..10aed134b 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -9,21 +9,22 @@ :license: BSD, see LICENSE for more details. """ from itertools import chain -from copy import deepcopy -from keyword import iskeyword as is_python_keyword from functools import update_wrapper +from keyword import iskeyword as is_python_keyword + +from markupsafe import Markup, escape + from jinja2 import nodes from jinja2.nodes import EvalContext from jinja2.visitor import NodeVisitor from jinja2.optimizer import Optimizer from jinja2.exceptions import TemplateAssertionError -from jinja2.utils import Markup, concat, escape +from jinja2.utils import concat from jinja2._compat import range_type, text_type, string_types, \ iteritems, NativeStringIO, imap, izip from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \ VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED - operators = { 'eq': '==', 'ne': '!=', diff --git a/jinja2/environment.py b/jinja2/environment.py index 549d9afab..b7f8b2faa 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -12,6 +12,9 @@ import sys import weakref from functools import reduce, partial + +from markupsafe import Markup + from jinja2 import nodes from jinja2.defaults import BLOCK_START_STRING, \ BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ @@ -26,7 +29,7 @@ from jinja2.runtime import Undefined, new_context, Context from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ TemplatesNotFound, TemplateRuntimeError -from jinja2.utils import import_string, LRUCache, Markup, missing, \ +from jinja2.utils import import_string, LRUCache, missing, \ concat, consume, internalcode, have_async_gen from jinja2._compat import imap, ifilter, string_types, iteritems, \ text_type, reraise, implements_iterator, implements_to_string, \ @@ -189,15 +192,14 @@ class Environment(object): ``None`` implicitly into an empty string here. `autoescape` - If set to ``True`` the XML/HTML autoescaping feature is enabled by - default. For more details about autoescaping see - :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also - be a callable that is passed the template name and has to - return ``True`` or ``False`` depending on autoescape should be - enabled by default. + If set to ``True`` the HTML/XML autoescaping feature is + enabled by default. For more details about autoescaping see + :ref:`autoescaping`. This can also be a callable that is + passed the template name and returns whether autoescaping + should be enabled. .. versionchanged:: 2.4 - `autoescape` can now be a function + Can be a function. `loader` The template loader for this environment. diff --git a/jinja2/ext.py b/jinja2/ext.py index 0734a84f7..b70b50b4a 100644 --- a/jinja2/ext.py +++ b/jinja2/ext.py @@ -12,6 +12,8 @@ """ import re +from markupsafe import Markup + from jinja2 import nodes from jinja2.defaults import BLOCK_START_STRING, \ BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ @@ -21,7 +23,7 @@ from jinja2.environment import Environment from jinja2.runtime import concat from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.utils import contextfunction, import_string, Markup +from jinja2.utils import contextfunction, import_string from jinja2._compat import with_metaclass, string_types, iteritems diff --git a/jinja2/filters.py b/jinja2/filters.py index 267ddddaa..bcb651f5b 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -12,14 +12,16 @@ import math import random import warnings - from itertools import groupby, chain from collections import namedtuple -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ + +from markupsafe import Markup, escape, soft_unicode + +from jinja2.utils import pformat, urlize, \ unicode_urlencode, htmlsafe_json_dumps from jinja2.runtime import Undefined from jinja2.exceptions import FilterArgumentError -from jinja2._compat import imap, string_types, text_type, iteritems, PY2 +from jinja2._compat import imap, string_types, text_type, iteritems _word_re = re.compile(r'\w+', re.UNICODE) diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py index fe17e4138..eab95d614 100644 --- a/jinja2/nativetypes.py +++ b/jinja2/nativetypes.py @@ -1,11 +1,14 @@ import sys from ast import literal_eval + +from markupsafe import escape + from itertools import islice, chain from jinja2 import nodes from jinja2._compat import text_type from jinja2.compiler import CodeGenerator, has_safe_repr from jinja2.environment import Environment, Template -from jinja2.utils import concat, escape +from jinja2.utils import concat def native_concat(nodes): diff --git a/jinja2/nodes.py b/jinja2/nodes.py index 4d9a01ad8..801764afa 100644 --- a/jinja2/nodes.py +++ b/jinja2/nodes.py @@ -15,8 +15,9 @@ import types import operator +from markupsafe import Markup + from collections import deque -from jinja2.utils import Markup from jinja2._compat import izip, with_metaclass, text_type, PY2 diff --git a/jinja2/runtime.py b/jinja2/runtime.py index 5e313369e..c925b71e2 100644 --- a/jinja2/runtime.py +++ b/jinja2/runtime.py @@ -13,8 +13,10 @@ from itertools import chain from types import MethodType +from markupsafe import Markup, escape, soft_unicode + from jinja2.nodes import EvalContext, _context_function_types -from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \ +from jinja2.utils import missing, concat, \ internalcode, object_type_repr, evalcontextfunction, Namespace from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \ TemplateNotFound diff --git a/jinja2/sandbox.py b/jinja2/sandbox.py index 08c22f4f1..88f83ae26 100644 --- a/jinja2/sandbox.py +++ b/jinja2/sandbox.py @@ -14,13 +14,13 @@ """ import types import operator +from string import Formatter + +from markupsafe import Markup, EscapeFormatter + from jinja2.environment import Environment from jinja2.exceptions import SecurityError from jinja2._compat import string_types, PY2, abc, range_type -from jinja2.utils import Markup - -from markupsafe import EscapeFormatter -from string import Formatter #: maximum number of items a range may produce diff --git a/jinja2/utils.py b/jinja2/utils.py index db9c5d062..8c5978d34 100644 --- a/jinja2/utils.py +++ b/jinja2/utils.py @@ -13,6 +13,9 @@ import errno from collections import deque from threading import Lock + +from markupsafe import Markup, escape + from jinja2._compat import text_type, string_types, implements_iterator, \ url_quote, abc @@ -483,25 +486,28 @@ def __reversed__(self): abc.MutableMapping.register(LRUCache) -def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), - disabled_extensions=(), - default_for_string=True, - default=False): - """Intelligently sets the initial value of autoescaping based on the - filename of the template. This is the recommended way to configure +def select_autoescape( + enabled_extensions=("html", "htm", "xml"), + disabled_extensions=(), + default_for_string=True, + default=False, +): + """Set the initial value of autoescaping based on the name of the + template, case insensitive. This is the recommended way to configure autoescaping if you do not want to write a custom function yourself. - If you want to enable it for all templates created from strings or - for all templates with `.html` and `.xml` extensions:: + The defaults will enable autoescaping only for template names ending + with ".html", ".htm", and ".xml", as well as templates from strings. + + .. code-block:: python from jinja2 import Environment, select_autoescape - env = Environment(autoescape=select_autoescape( - enabled_extensions=('html', 'xml'), - default_for_string=True, - )) + env = Environment(autoescape=select_autoescape()) - Example configuration to turn it on at all times except if the template - ends with `.txt`:: + The following configuration enables it for all templates except if + the name ends with ".txt". + + .. code-block:: python from jinja2 import Environment, select_autoescape env = Environment(autoescape=select_autoescape( @@ -510,30 +516,36 @@ def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), default=True, )) - The `enabled_extensions` is an iterable of all the extensions that - autoescaping should be enabled for. Likewise `disabled_extensions` is - a list of all templates it should be disabled for. If a template is - loaded from a string then the default from `default_for_string` is used. - If nothing matches then the initial value of autoescaping is set to the - value of `default`. - - For security reasons this function operates case insensitive. + :param enabled_extensions: Template names ending in these extensions + will have autoescaping enabled. A "." is prepended to each value + if it's missing. + :param disabled_extensions: Template names ending in these + extensions will have autoescaping disabled. A "." is prepended + to each value if it's missing. + :param default_for_string: What to do if the template is loaded from + a string and doesn't have a name. + :param default: What to do if the name does not match any of the + other rules. .. versionadded:: 2.9 """ - enabled_patterns = tuple('.' + x.lstrip('.').lower() - for x in enabled_extensions) - disabled_patterns = tuple('.' + x.lstrip('.').lower() - for x in disabled_extensions) + enabled_patterns = tuple("." + x.lstrip(".").lower() for x in enabled_extensions) + disabled_patterns = tuple("." + x.lstrip(".").lower() for x in disabled_extensions) + def autoescape(template_name): if template_name is None: return default_for_string + template_name = template_name.lower() + if template_name.endswith(enabled_patterns): return True + if template_name.endswith(disabled_patterns): return False + return default + return autoescape @@ -636,7 +648,3 @@ def __repr__(self): have_async_gen = True except SyntaxError: have_async_gen = False - - -# Imported here because that's where it was in the past -from markupsafe import Markup, escape, soft_unicode diff --git a/tests/test_asyncfilters.py b/tests/test_asyncfilters.py index 162cc6d4d..9796caa7f 100644 --- a/tests/test_asyncfilters.py +++ b/tests/test_asyncfilters.py @@ -1,6 +1,8 @@ import pytest + +from markupsafe import Markup + from jinja2 import Environment -from jinja2.utils import Markup async def make_aiter(iter): diff --git a/tests/test_filters.py b/tests/test_filters.py index 8962ced30..2769e9c1a 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -10,7 +10,10 @@ """ import random import pytest -from jinja2 import Markup, Environment + +from markupsafe import Markup + +from jinja2 import Environment from jinja2._compat import text_type, implements_to_string diff --git a/tests/test_security.py b/tests/test_security.py index 5c8639c4e..6e6b2f2be 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -10,10 +10,12 @@ """ import pytest +from markupsafe import Markup + from jinja2 import Environment from jinja2.sandbox import SandboxedEnvironment, \ ImmutableSandboxedEnvironment, unsafe -from jinja2 import Markup, escape +from jinja2 import escape from jinja2.exceptions import SecurityError, TemplateSyntaxError, \ TemplateRuntimeError from jinja2.nodes import EvalContext diff --git a/tests/test_tests.py b/tests/test_tests.py index 84df5ea75..119851647 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -10,7 +10,9 @@ """ import pytest -from jinja2 import Markup, Environment +from markupsafe import Markup + +from jinja2 import Environment @pytest.mark.test_tests From 4b8d839e1ec7ac4c7e42241e9e4279826bcc3580 Mon Sep 17 00:00:00 2001 From: David Lord Date: Mon, 27 Jan 2020 13:51:27 -0800 Subject: [PATCH 002/372] start version 3.0.0 --- CHANGES.rst | 10 ++++++++++ setup.py | 17 +++-------------- src/jinja2/__init__.py | 3 +-- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 21eb77651..9e26dff6f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,15 @@ .. currentmodule:: jinja2 +Version 3.0.0 +------------- + +Unreleased + +- Drop support for Python 2.7 and 3.5. +- Bump MarkupSafe dependency to >=1.1. +- Bump Babel optional dependency to >=2.1. + + Version 2.11.0 -------------- diff --git a/setup.py b/setup.py index 7d94cd3ae..050d3e227 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,6 @@ "Issue tracker": "https://github.com/pallets/jinja/issues", }, license="BSD-3-Clause", - author="Armin Ronacher", - author_email="armin.ronacher@active-4.com", maintainer="Pallets", maintainer_email="contact@palletsprojects.com", description="A very fast and expressive template engine.", @@ -33,15 +31,6 @@ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP :: Dynamic Content", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing :: Markup :: HTML", @@ -49,8 +38,8 @@ packages=find_packages("src"), package_dir={"": "src"}, include_package_data=True, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", - install_requires=["MarkupSafe>=0.23"], - extras_require={"i18n": ["Babel>=0.8"]}, + python_requires=">=3.6", + install_requires=["MarkupSafe>=1.1"], + extras_require={"i18n": ["Babel>=2.1"]}, entry_points={"babel.extractors": ["jinja2 = jinja2.ext:babel_extract[i18n]"]}, ) diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py index 7233387d4..8fa05183e 100644 --- a/src/jinja2/__init__.py +++ b/src/jinja2/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Jinja is a template engine written in pure Python. It provides a non-XML syntax that supports inline expressions and an optional sandboxed environment. @@ -41,4 +40,4 @@ from .utils import is_undefined from .utils import select_autoescape -__version__ = "2.11.0" +__version__ = "3.0.0a1" From 4ca745e6c80903316a5e660ef428bfed213d19a4 Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 30 Jan 2020 09:28:40 -0800 Subject: [PATCH 003/372] remove unsupported test envs --- .azure-pipelines.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 680a15cbf..565961196 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -6,7 +6,6 @@ variables: vmImage: ubuntu-latest python.version: '3.8' TOXENV: py - hasTestResults: 'true' strategy: matrix: @@ -16,22 +15,16 @@ strategy: vmImage: windows-latest Python 3.8 Mac: vmImage: macos-latest - PyPy 3 Linux: - python.version: pypy3 Python 3.7 Linux: python.version: '3.7' Python 3.6 Linux: python.version: '3.6' - Python 3.5 Linux: - python.version: '3.5' - Python 2.7 Linux: - python.version: '2.7' + PyPy 3 Linux: + python.version: pypy3 Docs: TOXENV: docs - hasTestResults: 'false' Style: TOXENV: style - hasTestResults: 'false' pool: vmImage: $[ variables.vmImage ] From f28b25ea623fdc08dcdf573af837048314360bb7 Mon Sep 17 00:00:00 2001 From: David Lord Date: Mon, 27 Jan 2020 14:12:52 -0800 Subject: [PATCH 004/372] remove Python 2 from docs --- docs/api.rst | 120 +++++++++----------------------------- docs/extensions.rst | 10 ++-- docs/faq.rst | 40 ++++--------- docs/index.rst | 29 +-------- docs/intro.rst | 99 +++++++++++++------------------ docs/switching.rst | 12 ++-- docs/templates.rst | 5 +- src/jinja2/bccache.py | 2 +- src/jinja2/environment.py | 12 ++-- src/jinja2/ext.py | 4 +- src/jinja2/lexer.py | 4 +- src/jinja2/loaders.py | 12 ++-- src/jinja2/nodes.py | 4 +- src/jinja2/runtime.py | 4 +- src/jinja2/sandbox.py | 3 +- 15 files changed, 115 insertions(+), 245 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 871b326f5..501a2c6d1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -60,63 +60,6 @@ a lot easier to use it also enables template inheritance. configure autoescaping now instead of relying on the default. -Unicode -------- - -Jinja is using Unicode internally which means that you have to pass Unicode -objects to the render function or bytestrings that only consist of ASCII -characters. Additionally newlines are normalized to one end of line -sequence which is per default UNIX style (``\n``). - -Python 2.x supports two ways of representing string objects. One is the -`str` type and the other is the `unicode` type, both of which extend a type -called `basestring`. Unfortunately the default is `str` which should not -be used to store text based information unless only ASCII characters are -used. With Python 2.6 it is possible to make `unicode` the default on a per -module level and with Python 3 it will be the default. - -To explicitly use a Unicode string you have to prefix the string literal -with a `u`: ``u'Hänsel und Gretel sagen Hallo'``. That way Python will -store the string as Unicode by decoding the string with the character -encoding from the current Python module. If no encoding is specified this -defaults to 'ASCII' which means that you can't use any non ASCII identifier. - -To set a better module encoding add the following comment to the first or -second line of the Python module using the Unicode literal:: - - # -*- coding: utf-8 -*- - -We recommend utf-8 as Encoding for Python modules and templates as it's -possible to represent every Unicode character in utf-8 and because it's -backwards compatible to ASCII. For Jinja the default encoding of templates -is assumed to be utf-8. - -It is not possible to use Jinja to process non-Unicode data. The reason -for this is that Jinja uses Unicode already on the language level. For -example Jinja treats the non-breaking space as valid whitespace inside -expressions which requires knowledge of the encoding or operating on an -Unicode string. - -For more details about Unicode in Python have a look at the excellent -`Unicode documentation`_. - -Another important thing is how Jinja is handling string literals in -templates. A naive implementation would be using Unicode strings for -all string literals but it turned out in the past that this is problematic -as some libraries are typechecking against `str` explicitly. For example -`datetime.strftime` does not accept Unicode arguments. To not break it -completely Jinja is returning `str` for strings that fit into ASCII and -for everything else `unicode`: - ->>> m = Template(u"{% set a, b = 'foo', 'föö' %}").module ->>> m.a -'foo' ->>> m.b -u'f\xf6\xf6' - - -.. _Unicode documentation: https://docs.python.org/3/howto/unicode.html - High Level API -------------- @@ -301,12 +244,12 @@ Notes on Identifiers -------------------- Jinja uses Python naming rules. Valid identifiers can be any combination -of Unicode characters accepted by Python. +of characters accepted by Python. Filters and tests are looked up in separate namespaces and have slightly modified identifier syntax. Filters and tests may contain dots to group filters and tests by topic. For example it's perfectly valid to add a -function into the filter dict and call it `to.unicode`. The regular +function into the filter dict and call it `to.str`. The regular expression for filter and test identifiers is ``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```. @@ -328,8 +271,8 @@ disallows all operations beside testing if it's an undefined object. .. attribute:: _undefined_hint - Either `None` or an unicode string with the error message for - the undefined object. + Either `None` or a string with the error message for the + undefined object. .. attribute:: _undefined_obj @@ -367,27 +310,32 @@ Undefined objects are created by calling :attr:`undefined`. .. admonition:: Implementation - :class:`Undefined` objects are implemented by overriding the special - `__underscore__` methods. For example the default :class:`Undefined` - class implements `__unicode__` in a way that it returns an empty - string, however `__int__` and others still fail with an exception. To - allow conversion to int by returning ``0`` you can implement your own:: + :class:`Undefined` is implemented by overriding the special + ``__underscore__`` methods. For example the default + :class:`Undefined` class implements ``__str__`` to returns an empty + string, while ``__int__`` and others fail with an exception. To + allow conversion to int by returning ``0`` you can implement your + own subclass. + + .. code-block:: python class NullUndefined(Undefined): def __int__(self): return 0 + def __float__(self): return 0.0 - To disallow a method, just override it and raise - :attr:`~Undefined._undefined_exception`. Because this is a very common - idiom in undefined objects there is the helper method - :meth:`~Undefined._fail_with_undefined_error` that does the error raising - automatically. Here a class that works like the regular :class:`Undefined` - but chokes on iteration:: + To disallow a method, override it and raise + :attr:`~Undefined._undefined_exception`. Because this is very + common there is the helper method + :meth:`~Undefined._fail_with_undefined_error` that raises the error + with the correct information. Here's a class that works like the + regular :class:`Undefined` but fails on iteration:: class NonIterableUndefined(Undefined): - __iter__ = Undefined._fail_with_undefined_error + def __iter__(self): + self._fail_with_undefined_error() The Context @@ -575,16 +523,6 @@ Example:: env.policies['urlize.rel'] = 'nofollow noopener' -``compiler.ascii_str``: - This boolean controls on Python 2 if Jinja should store ASCII only - literals as bytestring instead of unicode strings. This used to be - always enabled for Jinja versions below 2.9 and now can be changed. - Traditionally it was done this way since some APIs in Python 2 failed - badly for unicode strings (for instance the datetime strftime API). - Now however sometimes the inverse is true (for instance str.format). - If this is set to False then all strings are stored as unicode - internally. - ``truncate.leeway``: Configures the leeway default for the `truncate` filter. Leeway as introduced in 2.9 but to restore compatibility with older templates @@ -676,24 +614,20 @@ Exceptions .. attribute:: message - The error message as utf-8 bytestring. + The error message. .. attribute:: lineno - The line number where the error occurred + The line number where the error occurred. .. attribute:: name - The load name for the template as unicode string. + The load name for the template. .. attribute:: filename - The filename that loaded the template as bytestring in the encoding - of the file system (most likely utf-8 or mbcs on Windows systems). - - The reason why the filename and error message are bytestrings and not - unicode strings is that Python 2.x is not using unicode for exceptions - and tracebacks as well as the compiler. This will change with Python 3. + The filename that loaded the template in the encoding of the + file system (most likely utf-8, or mbcs on Windows systems). .. autoexception:: jinja2.TemplateRuntimeError @@ -894,7 +828,7 @@ don't recommend using any of those. that has to be created by :meth:`new_context` of the same template or a compatible template. This render function is generated by the compiler from the template code and returns a generator that yields - unicode strings. + strings. If an exception in the template code happens the template engine will not rewrite the exception but pass through the original one. As a diff --git a/docs/extensions.rst b/docs/extensions.rst index 7abed658b..bb81f217b 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -44,8 +44,7 @@ additional methods: .. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False) Installs a translation globally for the environment. The - ``translations`` object must implement ``gettext`` and ``ngettext`` - (or ``ugettext`` and ``ungettext`` for Python 2). + ``translations`` object must implement ``gettext`` and ``ngettext``. :class:`gettext.NullTranslations`, :class:`gettext.GNUTranslations`, and `Babel`_\s ``Translations`` are supported. @@ -63,8 +62,7 @@ additional methods: Install the given ``gettext`` and ``ngettext`` callables into the environment. They should behave exactly like - :func:`gettext.gettext` and :func:`gettext.ngettext` (or - ``ugettext`` and ``ungettext`` for Python 2). + :func:`gettext.gettext` and :func:`gettext.ngettext`. If ``newstyle`` is activated, the callables are wrapped to work like newstyle callables. See :ref:`newstyle-gettext` for more information. @@ -86,8 +84,8 @@ additional methods: found. - ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code). - - ``message`` is the string itself (``unicode`` on Python 2), or a - tuple of strings for functions with multiple arguments. + - ``message`` is the string itself, or a tuple of strings for + functions with multiple arguments. If `Babel`_ is installed, see :ref:`babel-integration` to extract the strings. diff --git a/docs/faq.rst b/docs/faq.rst index 294fef1d6..1e29e12f9 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -125,19 +125,18 @@ instead that one can assign to a variable by using set:: {% set comments = get_latest_comments() %} -My tracebacks look weird. What's happening? --------------------------------------------- +My tracebacks look weird. What's happening? +------------------------------------------- -If the debugsupport module is not compiled and you are using a Python -installation without ctypes (Python 2.4 without ctypes, Jython or Google's -AppEngine) Jinja is unable to provide correct debugging information and -the traceback may be incomplete. There is currently no good workaround -for Jython or the AppEngine as ctypes is unavailable there and it's not -possible to use the debugsupport extension. +Jinja can rewrite tracebacks so they show the template lines numbers and +source rather than the underlying compiled code, but this requires +special Python support. CPython <3.7 requires ``ctypes``, and PyPy +requires transparent proxy support. -If you are working in the Google AppEngine development server you can -whitelist the ctypes module to restore the tracebacks. This however won't -work in production environments:: +If you are using Google App Engine, ``ctypes`` is not available. You can +make it available in development, but not in production. + +.. code-block:: python import os if os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'): @@ -147,25 +146,6 @@ work in production environments:: Credit for this snippet goes to `Thomas Johansson `_ -Why is there no Python 2.3/2.4/2.5/2.6/3.1/3.2/3.3 support? ------------------------------------------------------------ - -Python 2.3 is missing a lot of features that are used heavily in Jinja. This -decision was made as with the upcoming Python 2.6 and 3.0 versions it becomes -harder to maintain the code for older Python versions. If you really need -Python 2.3 support you either have to use Jinja 1 or other templating -engines that still support 2.3. - -Python 2.4/2.5/3.1/3.2 support was removed when we switched to supporting -Python 2 and 3 by the same sourcecode (without using 2to3). It was required to -drop support because only Python 2.6/2.7 and >=3.3 support byte and unicode -literals in a way compatible to each other version. If you really need support -for older Python 2 (or 3) versions, you can just use Jinja 2.6. - -Python 2.6/3.3 support was dropped because it got dropped in various upstream -projects (such as wheel or pytest), which would make it difficult to continue -supporting it. Jinja 2.10 was the last version supporting Python 2.6/3.3. - My Macros are overridden by something ------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index 65d5d3d4a..dcaa9ffd3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,29 +7,9 @@ Jinja :align: center :target: https://palletsprojects.com/p/jinja/ -Jinja is a modern and designer-friendly templating language for Python, -modelled after Django's templates. It is fast, widely used and secure -with the optional sandboxed template execution environment: - -.. sourcecode:: html+jinja - - {% block title %}{% endblock %} - - -Features: - -- sandboxed execution -- powerful automatic HTML escaping system for XSS prevention -- template inheritance -- compiles down to the optimal python code just in time -- optional ahead-of-time template compilation -- easy to debug. Line numbers of exceptions directly point to - the correct line in the template. -- configurable syntax +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. .. toctree:: :maxdepth: 2 @@ -46,6 +26,3 @@ Features: tricks faq changelog - -* :ref:`genindex` -* :ref:`search` diff --git a/docs/intro.rst b/docs/intro.rst index c20c5e910..25c2b580d 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -1,82 +1,63 @@ Introduction ============ -This is the documentation for the Jinja general purpose templating language. -Jinja is a library for Python that is designed to be flexible, fast and secure. +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. -If you have any exposure to other text-based template languages, such as Smarty or -Django, you should feel right at home with Jinja. It's both designer and -developer friendly by sticking to Python's principles and adding functionality -useful for templating environments. - -Prerequisites -------------- - -Jinja works with Python 2.7.x and >= 3.5. If you are using Python -3.2 you can use an older release of Jinja (2.6) as support for Python 3.2 -was dropped in Jinja version 2.7. The last release which supported Python 2.6 -and 3.3 was Jinja 2.10. - -If you wish to use the :class:`~jinja2.PackageLoader` class, you will also -need `setuptools`_ or `distribute`_ installed at runtime. Installation ------------ -You can install the most recent Jinja version using `pip`_:: +We recommend using the latest version of Python. Jinja supports Python +3.6 and newer. We also recommend using a `virtual environment`_ in order +to isolate your project dependencies from other projects and the system. - pip install Jinja2 +.. _virtual environment: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments -This will install Jinja in your Python installation's site-packages directory. +Install the most recent Jinja version using pip: -Installing the development version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: text -1. Install `git`_ -2. ``git clone git://github.com/pallets/jinja.git`` -3. ``cd jinja2`` -4. ``ln -s jinja2 /usr/lib/python2.X/site-packages`` + $ pip install Jinja2 -As an alternative to steps 4 you can also do ``python setup.py develop`` -which will install the package via `distribute` in development mode. This also -has the advantage that the C extensions are compiled. -.. _distribute: https://pypi.org/project/distribute/ -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ -.. _git: https://git-scm.com/ +Dependencies +~~~~~~~~~~~~ +These will be installed automatically when installing Jinja. -MarkupSafe Dependency -~~~~~~~~~~~~~~~~~~~~~ - -As of version 2.7 Jinja depends on the `MarkupSafe`_ module. If you install -Jinja via ``pip`` it will be installed automatically for you. +- `MarkupSafe`_ escapes untrusted input when rendering templates to + avoid injection attacks. .. _MarkupSafe: https://markupsafe.palletsprojects.com/ -Basic API Usage ---------------- - -This section gives you a brief introduction to the Python API for Jinja -templates. -The most basic way to create a template and render it is through -:class:`~jinja2.Template`. This however is not the recommended way to -work with it if your templates are not loaded from strings but the file -system or another data source: +Optional Dependencies +~~~~~~~~~~~~~~~~~~~~~ ->>> from jinja2 import Template ->>> template = Template('Hello {{ name }}!') ->>> template.render(name='John Doe') -u'Hello John Doe!' +These distributions will not be installed automatically. -By creating an instance of :class:`~jinja2.Template` you get back a new template -object that provides a method called :meth:`~jinja2.Template.render` which when -called with a dict or keyword arguments expands the template. The dict -or keywords arguments passed to the template are the so-called "context" -of the template. +- `Babel`_ provides translation support in templates. -What you can see here is that Jinja is using unicode internally and the -return value is an unicode string. So make sure that your application is -indeed using unicode internally. +.. _Babel: http://babel.pocoo.org/ diff --git a/docs/switching.rst b/docs/switching.rst index 8225b2ee0..b9ff954cc 100644 --- a/docs/switching.rst +++ b/docs/switching.rst @@ -32,12 +32,12 @@ Loading templates from strings with optional additional configuration. Automatic unicode conversion - Jinja 1 performed automatic conversion of bytestrings in a given encoding - into unicode objects. This conversion is no longer implemented as it - was inconsistent as most libraries are using the regular Python ASCII - bytestring to Unicode conversion. An application powered by Jinja 2 - *has to* use unicode internally everywhere or make sure that Jinja 2 only - gets unicode strings passed. + Jinja 1 performed automatic conversion of bytes in a given encoding + into unicode objects. This conversion is no longer implemented as it + was inconsistent as most libraries are using the regular Python + ASCII bytes to Unicode conversion. An application powered by Jinja 2 + *has to* use unicode internally everywhere or make sure that Jinja 2 + only gets unicode strings passed. i18n Jinja 1 used custom translators for internationalization. i18n is now diff --git a/docs/templates.rst b/docs/templates.rst index 89c2a5060..c08ce4b44 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -602,9 +602,8 @@ you have data that is already safe but not marked, be sure to wrap it in Jinja functions (macros, `super`, `self.BLOCKNAME`) always return template data that is marked as safe. -String literals in templates with automatic escaping are considered unsafe -because native Python strings (``str``, ``unicode``, ``basestring``) are not -`MarkupSafe.Markup` strings with an ``__html__`` attribute. +String literals in templates with automatic escaping are considered +unsafe because native Python strings are not safe. .. _list-of-control-structures: diff --git a/src/jinja2/bccache.py b/src/jinja2/bccache.py index 9c0661030..ff4d606fd 100644 --- a/src/jinja2/bccache.py +++ b/src/jinja2/bccache.py @@ -284,7 +284,7 @@ class MemcachedBytecodeCache(BytecodeCache): - `python-memcached `_ (Unfortunately the django cache interface is not compatible because it - does not support storing binary data, only unicode. You can however pass + does not support storing binary data, only text. You can however pass the underlying cache client to the bytecode cache which is available as `django.core.cache.cache._client`.) diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 8430390ee..3f75816c8 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -1081,7 +1081,7 @@ def render(self, *args, **kwargs): template.render(knights='that say nih') template.render({'knights': 'that say nih'}) - This will return the rendered template as unicode string. + This will return the rendered template as a string. """ vars = dict(*args, **kwargs) try: @@ -1113,7 +1113,7 @@ def generate(self, *args, **kwargs): """For very large templates it can be useful to not render the whole template at once but evaluate each statement after another and yield piece for piece. This method basically does exactly that and returns - a generator that yields one item after another as unicode strings. + a generator that yields one item after another as strings. It accepts the same arguments as :meth:`render`. """ @@ -1223,7 +1223,7 @@ def __repr__(self): class TemplateModule(object): """Represents an imported template. All the exported names of the template are available as attributes on this object. Additionally - converting it into an unicode- or bytestrings renders the contents. + converting it into a string renders the contents. """ def __init__(self, template, context, body_stream=None): @@ -1278,10 +1278,10 @@ class TemplateStream(object): """A template stream works pretty much like an ordinary python generator but it can buffer multiple items to reduce the number of total iterations. Per default the output is unbuffered which means that for every unbuffered - instruction in the template one unicode string is yielded. + instruction in the template one string is yielded. If buffering is enabled with a buffer size of 5, five items are combined - into a new unicode string. This is mainly useful if you are streaming + into a new string. This is mainly useful if you are streaming big templates to a client via WSGI which flushes after each iteration. """ @@ -1291,7 +1291,7 @@ def __init__(self, gen): def dump(self, fp, encoding=None, errors="strict"): """Dump the complete stream into a file or file-like object. - Per default unicode strings are written, if you want to encode + Per default strings are written, if you want to encode before writing specify an `encoding`. Example usage:: diff --git a/src/jinja2/ext.py b/src/jinja2/ext.py index 9141be4da..99ecb34f1 100644 --- a/src/jinja2/ext.py +++ b/src/jinja2/ext.py @@ -538,8 +538,8 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and - * ``message`` is the string itself (a ``unicode`` object, or a tuple - of ``unicode`` objects for functions with multiple string arguments). + * ``message`` is the string, or a tuple of strings for functions + with multiple string arguments. This extraction function operates on the AST and is because of that unable to extract any comments. For comment support you have to use the babel diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index a2b44e926..8e73be820 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -607,7 +607,9 @@ def c(x): } def _normalize_newlines(self, value): - """Called for strings and template data to normalize it to unicode.""" + """Replace all newlines with the configured sequence in strings + and template data. + """ return newline_re.sub(self.newline_sequence, value) def tokenize(self, source, name=None, filename=None, state=None): diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index ce5537a03..0a5538ad9 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -77,9 +77,9 @@ def get_source(self, environment, template): `TemplateNotFound` error if it can't locate the template. The source part of the returned tuple must be the source of the - template as unicode string or a ASCII bytestring. The filename should - be the name of the file on the filesystem if it was loaded from there, - otherwise `None`. The filename is used by python for the tracebacks + template as a string. The filename should be the name of the + file on the filesystem if it was loaded from there, otherwise + ``None``. The filename is used by Python for the tracebacks if no loader extension is used. The last item in the tuple is the `uptodate` function. If auto @@ -357,8 +357,8 @@ def list_templates(self): class DictLoader(BaseLoader): - """Loads a template from a python dict. It's passed a dict of unicode - strings bound to template names. This loader is useful for unittesting: + """Loads a template from a Python dict mapping template names to + template source. This loader is useful for unittesting: >>> loader = DictLoader({'index.html': 'source here'}) @@ -381,7 +381,7 @@ def list_templates(self): class FunctionLoader(BaseLoader): """A loader that is passed a function which does the loading. The function receives the name of the template and has to return either - an unicode string with the template source, a tuple in the form ``(source, + a string with the template source, a tuple in the form ``(source, filename, uptodatefunc)`` or `None` if the template does not exist. >>> def load_template(name): diff --git a/src/jinja2/nodes.py b/src/jinja2/nodes.py index 95bd614a1..c0b6d77e4 100644 --- a/src/jinja2/nodes.py +++ b/src/jinja2/nodes.py @@ -788,8 +788,8 @@ def const(obj): class Concat(Expr): - """Concatenates the list of expressions provided after converting them to - unicode. + """Concatenates the list of expressions provided after converting + them to strings. """ fields = ("nodes",) diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index 3ad796862..8df42c590 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -60,7 +60,7 @@ def identity(x): def markup_join(seq): - """Concatenation that escapes if necessary and converts to unicode.""" + """Concatenation that escapes if necessary and converts to string.""" buf = [] iterator = imap(soft_unicode, seq) for arg in iterator: @@ -71,7 +71,7 @@ def markup_join(seq): def unicode_join(seq): - """Simple args to unicode conversion and concatenation.""" + """Simple args to string conversion and concatenation.""" return concat(imap(text_type, seq)) diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index cfd7993ae..3bb014555 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -244,8 +244,7 @@ def modifies_known_mutable(obj, attr): >>> modifies_known_mutable([], "index") False - If called with an unsupported object (such as unicode) `False` is - returned. + If called with an unsupported object, ``False`` is returned. >>> modifies_known_mutable("foo", "upper") False From e52b42d3efb9ace10ea5543e3e856eff09057a93 Mon Sep 17 00:00:00 2001 From: David Lord Date: Mon, 27 Jan 2020 14:19:30 -0800 Subject: [PATCH 005/372] remove unsupported test envs --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 679ebebe9..736b6e757 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38,37,36,35,27,py3,py} + py{38,37,36,py3} style docs skip_missing_interpreters = true From b0015c72d5acbf93b9d99a1ce6167889338db85b Mon Sep 17 00:00:00 2001 From: David Lord Date: Mon, 27 Jan 2020 14:28:14 -0800 Subject: [PATCH 006/372] remove deprecated code --- CHANGES.rst | 2 ++ setup.cfg | 1 - src/jinja2/asyncsupport.py | 12 --------- src/jinja2/environment.py | 38 ++------------------------ src/jinja2/exceptions.py | 4 +-- src/jinja2/filters.py | 12 +-------- src/jinja2/sandbox.py | 6 ----- src/jinja2/utils.py | 55 -------------------------------------- tests/test_loader.py | 6 ++--- tests/test_utils.py | 7 ----- 10 files changed, 9 insertions(+), 134 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2253fe9c8..afd43e5f1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,8 @@ Unreleased - Drop support for Python 2.7 and 3.5. - Bump MarkupSafe dependency to >=1.1. - Bump Babel optional dependency to >=2.1. +- Remove code that was marked deprecated. +- 2.11.2 diff --git a/setup.cfg b/setup.cfg index 2769e96ad..9dff382cf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,6 @@ universal = true testpaths = tests filterwarnings = error - ignore:the sets module:DeprecationWarning:jinja2.sandbox [coverage:run] branch = True diff --git a/src/jinja2/asyncsupport.py b/src/jinja2/asyncsupport.py index 78ba3739d..ae4958f6d 100644 --- a/src/jinja2/asyncsupport.py +++ b/src/jinja2/asyncsupport.py @@ -249,16 +249,4 @@ async def __anext__(self): return rv, self -async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): - import warnings - - warnings.warn( - "This template must be recompiled with at least Jinja 2.11, or" - " it will fail in 3.0.", - DeprecationWarning, - stacklevel=2, - ) - return AsyncLoopContext(iterable, undefined, recurse, depth0) - - patch_all() diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 3f75816c8..63ac0c8be 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -15,8 +15,6 @@ from ._compat import implements_iterator from ._compat import implements_to_string from ._compat import iteritems -from ._compat import PY2 -from ._compat import PYPY from ._compat import reraise from ._compat import string_types from ._compat import text_type @@ -689,7 +687,6 @@ def compile_templates( zip="deflated", log_function=None, ignore_errors=True, - py_compile=False, ): """Finds all the templates the loader can find, compiles them and stores them in `target`. If `zip` is `None`, instead of in a @@ -706,11 +703,6 @@ def compile_templates( syntax errors to abort the compilation you can set `ignore_errors` to `False` and you will get an exception on syntax errors. - If `py_compile` is set to `True` .pyc files will be written to the - target instead of standard .py files. This flag does not do anything - on pypy and Python 3 where pyc files are not picked up by itself and - don't give much benefit. - .. versionadded:: 2.4 """ from .loaders import ModuleLoader @@ -720,27 +712,6 @@ def compile_templates( def log_function(x): pass - if py_compile: - if not PY2 or PYPY: - import warnings - - warnings.warn( - "'py_compile=True' has no effect on PyPy or Python" - " 3 and will be removed in version 3.0", - DeprecationWarning, - stacklevel=2, - ) - py_compile = False - else: - import imp - import marshal - - py_header = imp.get_magic() + u"\xff\xff\xff\xff".encode("iso-8859-15") - - # Python 3.3 added a source filesize to the header - if sys.version_info >= (3, 3): - py_header += u"\x00\x00\x00\x00".encode("iso-8859-15") - def write_file(filename, data): if zip: info = ZipInfo(filename) @@ -778,13 +749,8 @@ def write_file(filename, data): filename = ModuleLoader.get_module_filename(name) - if py_compile: - c = self._compile(code, encode_filename(filename)) - write_file(filename + "c", py_header + marshal.dumps(c)) - log_function('Byte-compiled "%s" as %s' % (name, filename + "c")) - else: - write_file(filename, code) - log_function('Compiled "%s" as %s' % (name, filename)) + write_file(filename, code) + log_function('Compiled "%s" as %s' % (name, filename)) finally: if zip: zip_file.close() diff --git a/src/jinja2/exceptions.py b/src/jinja2/exceptions.py index 0bf2003e3..ac62c3b48 100644 --- a/src/jinja2/exceptions.py +++ b/src/jinja2/exceptions.py @@ -47,8 +47,8 @@ class TemplateNotFound(IOError, LookupError, TemplateError): provided, an :exc:`UndefinedError` is raised. """ - # looks weird, but removes the warning descriptor that just - # bogusly warns us about message being deprecated + # Silence the Python warning about message being deprecated since + # it's not valid here. message = None def __init__(self, name, message=None): diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index 1af7ac88a..963c03ab5 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -3,7 +3,6 @@ import math import random import re -import warnings from collections import namedtuple from itertools import chain from itertools import groupby @@ -617,7 +616,7 @@ def do_urlize( return rv -def do_indent(s, width=4, first=False, blank=False, indentfirst=None): +def do_indent(s, width=4, first=False, blank=False): """Return a copy of the string with each line indented by 4 spaces. The first line and blank lines are not indented by default. @@ -630,15 +629,6 @@ def do_indent(s, width=4, first=False, blank=False, indentfirst=None): Rename the ``indentfirst`` argument to ``first``. """ - if indentfirst is not None: - warnings.warn( - "The 'indentfirst' argument is renamed to 'first' and will" - " be removed in version 3.0.", - DeprecationWarning, - stacklevel=2, - ) - first = indentfirst - indention = u" " * width newline = u"\n" diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index 3bb014555..4629143ba 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -4,7 +4,6 @@ """ import operator import types -import warnings from collections import deque from string import Formatter @@ -47,11 +46,6 @@ #: unsafe attributes on async generators UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = {"ag_code", "ag_frame"} -# make sure we don't warn in python 2.6 about stuff we don't care about -warnings.filterwarnings( - "ignore", "the sets module", DeprecationWarning, module=__name__ -) - _mutable_set_types = (set,) _mutable_mapping_types = (dict,) _mutable_sequence_types = (list,) diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index e3285e8ed..7e1ad9117 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -2,7 +2,6 @@ import json import os import re -import warnings from collections import deque from random import choice from random import randrange @@ -465,56 +464,14 @@ def items(self): result.reverse() return result - def iteritems(self): - """Iterate over all items.""" - warnings.warn( - "'iteritems()' will be removed in version 3.0. Use" - " 'iter(cache.items())' instead.", - DeprecationWarning, - stacklevel=2, - ) - return iter(self.items()) - def values(self): """Return a list of all values.""" return [x[1] for x in self.items()] - def itervalue(self): - """Iterate over all values.""" - warnings.warn( - "'itervalue()' will be removed in version 3.0. Use" - " 'iter(cache.values())' instead.", - DeprecationWarning, - stacklevel=2, - ) - return iter(self.values()) - - def itervalues(self): - """Iterate over all values.""" - warnings.warn( - "'itervalues()' will be removed in version 3.0. Use" - " 'iter(cache.values())' instead.", - DeprecationWarning, - stacklevel=2, - ) - return iter(self.values()) - def keys(self): """Return a list of all keys ordered by most recent usage.""" return list(self) - def iterkeys(self): - """Iterate over all keys in the cache dict, ordered by - the most recent usage. - """ - warnings.warn( - "'iterkeys()' will be removed in version 3.0. Use" - " 'iter(cache.keys())' instead.", - DeprecationWarning, - stacklevel=2, - ) - return iter(self) - def __iter__(self): return reversed(tuple(self._queue)) @@ -713,15 +670,3 @@ def __repr__(self): have_async_gen = True except SyntaxError: have_async_gen = False - - -def soft_unicode(s): - from markupsafe import soft_unicode - - warnings.warn( - "'jinja2.utils.soft_unicode' will be removed in version 3.0." - " Use 'markupsafe.soft_unicode' instead.", - DeprecationWarning, - stacklevel=2, - ) - return soft_unicode(s) diff --git a/tests/test_loader.py b/tests/test_loader.py index 4aa6511ef..ef51bb568 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -191,7 +191,7 @@ def test_uses_specified_encoding(self, encoding, expect): class TestModuleLoader(object): archive = None - def compile_down(self, prefix_loader, zip="deflated", py_compile=False): + def compile_down(self, prefix_loader, zip="deflated"): log = [] self.reg_env = Environment(loader=prefix_loader) if zip is not None: @@ -199,9 +199,7 @@ def compile_down(self, prefix_loader, zip="deflated", py_compile=False): os.close(fd) else: self.archive = tempfile.mkdtemp() - self.reg_env.compile_templates( - self.archive, zip=zip, log_function=log.append, py_compile=py_compile - ) + self.reg_env.compile_templates(self.archive, zip=zip, log_function=log.append) self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive)) return "".join(log) diff --git a/tests/test_utils.py b/tests/test_utils.py index 58165efbf..6ddd1a412 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -31,13 +31,6 @@ def test_simple(self): assert len(d) == 3 assert "a" in d and "c" in d and "d" in d and "b" not in d - def test_itervalue_deprecated(self): - cache = LRUCache(3) - cache["a"] = 1 - cache["b"] = 2 - with pytest.deprecated_call(): - cache.itervalue() - def test_itervalues(self): cache = LRUCache(3) cache["b"] = 1 From 148a19138c197233cbd3a2cb1f543d468479f1bc Mon Sep 17 00:00:00 2001 From: David Lord Date: Mon, 27 Jan 2020 22:05:00 -0800 Subject: [PATCH 007/372] remove _compat module --- src/jinja2/_compat.py | 132 ------------------------------- src/jinja2/bccache.py | 24 +++--- src/jinja2/compiler.py | 110 ++++++++------------------ src/jinja2/debug.py | 12 ++- src/jinja2/defaults.py | 3 +- src/jinja2/environment.py | 37 +++------ src/jinja2/exceptions.py | 44 ++--------- src/jinja2/ext.py | 11 +-- src/jinja2/filters.py | 74 ++++++++--------- src/jinja2/idtracking.py | 5 +- src/jinja2/lexer.py | 16 ++-- src/jinja2/loaders.py | 35 +++----- src/jinja2/meta.py | 12 ++- src/jinja2/nativetypes.py | 5 +- src/jinja2/nodes.py | 27 ++----- src/jinja2/parser.py | 3 +- src/jinja2/runtime.py | 162 +++++++++++--------------------------- src/jinja2/sandbox.py | 63 +++------------ src/jinja2/tests.py | 17 ++-- src/jinja2/utils.py | 40 +++++----- tests/test_ext.py | 8 +- tests/test_features.py | 29 ------- tests/test_filters.py | 10 +-- tests/test_lexnparse.py | 58 ++++++-------- tests/test_loader.py | 24 +----- tests/test_nativetypes.py | 5 +- tests/test_regression.py | 12 +-- tests/test_security.py | 42 +--------- tests/test_utils.py | 14 ++-- 29 files changed, 266 insertions(+), 768 deletions(-) delete mode 100644 src/jinja2/_compat.py diff --git a/src/jinja2/_compat.py b/src/jinja2/_compat.py deleted file mode 100644 index 1f044954a..000000000 --- a/src/jinja2/_compat.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import marshal -import sys - -PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, "pypy_translation_info") -_identity = lambda x: x - -if not PY2: - unichr = chr - range_type = range - text_type = str - string_types = (str,) - integer_types = (int,) - - iterkeys = lambda d: iter(d.keys()) - itervalues = lambda d: iter(d.values()) - iteritems = lambda d: iter(d.items()) - - import pickle - from io import BytesIO, StringIO - - NativeStringIO = StringIO - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - - ifilter = filter - imap = map - izip = zip - intern = sys.intern - - implements_iterator = _identity - implements_to_string = _identity - encode_filename = _identity - - marshal_dump = marshal.dump - marshal_load = marshal.load - -else: - unichr = unichr - text_type = unicode - range_type = xrange - string_types = (str, unicode) - integer_types = (int, long) - - iterkeys = lambda d: d.iterkeys() - itervalues = lambda d: d.itervalues() - iteritems = lambda d: d.iteritems() - - import cPickle as pickle - from cStringIO import StringIO as BytesIO, StringIO - - NativeStringIO = BytesIO - - exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") - - from itertools import imap, izip, ifilter - - intern = intern - - def implements_iterator(cls): - cls.next = cls.__next__ - del cls.__next__ - return cls - - def implements_to_string(cls): - cls.__unicode__ = cls.__str__ - cls.__str__ = lambda x: x.__unicode__().encode("utf-8") - return cls - - def encode_filename(filename): - if isinstance(filename, unicode): - return filename.encode("utf-8") - return filename - - def marshal_dump(code, f): - if isinstance(f, file): - marshal.dump(code, f) - else: - f.write(marshal.dumps(code)) - - def marshal_load(f): - if isinstance(f, file): - return marshal.load(f) - return marshal.loads(f.read()) - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a - # dummy metaclass for one level of class instantiation that replaces - # itself with the actual metaclass. - class metaclass(type): - def __new__(cls, name, this_bases, d): - return meta(name, bases, d) - - return type.__new__(metaclass, "temporary_class", (), {}) - - -try: - from urllib.parse import quote_from_bytes as url_quote -except ImportError: - from urllib import quote as url_quote - - -try: - from collections import abc -except ImportError: - import collections as abc - - -try: - from os import fspath -except ImportError: - try: - from pathlib import PurePath - except ImportError: - PurePath = None - - def fspath(path): - if hasattr(path, "__fspath__"): - return path.__fspath__() - - # Python 3.5 doesn't have __fspath__ yet, use str. - if PurePath is not None and isinstance(path, PurePath): - return str(path) - - return path diff --git a/src/jinja2/bccache.py b/src/jinja2/bccache.py index ff4d606fd..b328b3b08 100644 --- a/src/jinja2/bccache.py +++ b/src/jinja2/bccache.py @@ -8,19 +8,15 @@ """ import errno import fnmatch +import marshal import os +import pickle import stat import sys import tempfile from hashlib import sha1 -from os import listdir -from os import path - -from ._compat import BytesIO -from ._compat import marshal_dump -from ._compat import marshal_load -from ._compat import pickle -from ._compat import text_type +from io import BytesIO + from .utils import open_if_exists bc_version = 4 @@ -67,7 +63,7 @@ def load_bytecode(self, f): return # if marshal_load fails then we need to reload try: - self.code = marshal_load(f) + self.code = marshal.load(f) except (EOFError, ValueError, TypeError): self.reset() return @@ -78,7 +74,7 @@ def write_bytecode(self, f): raise TypeError("can't write empty bucket") f.write(bc_magic) pickle.dump(self.checksum, f, 2) - marshal_dump(self.code, f) + marshal.dump(self.code, f) def bytecode_from_string(self, string): """Load bytecode from a string.""" @@ -145,7 +141,7 @@ def get_cache_key(self, name, filename=None): hash = sha1(name.encode("utf-8")) if filename is not None: filename = "|" + filename - if isinstance(filename, text_type): + if isinstance(filename, str): filename = filename.encode("utf-8") hash.update(filename) return hash.hexdigest() @@ -241,7 +237,7 @@ def _unsafe_dir(): return actual_dir def _get_cache_filename(self, bucket): - return path.join(self.directory, self.pattern % bucket.key) + return os.path.join(self.directory, self.pattern % bucket.key) def load_bytecode(self, bucket): f = open_if_exists(self._get_cache_filename(bucket), "rb") @@ -264,10 +260,10 @@ def clear(self): # normally. from os import remove - files = fnmatch.filter(listdir(self.directory), self.pattern % "*") + files = fnmatch.filter(os.listdir(self.directory), self.pattern % "*") for filename in files: try: - remove(path.join(self.directory, filename)) + remove(os.path.join(self.directory, filename)) except OSError: pass diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 63297b42c..8965b320d 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -2,6 +2,7 @@ """Compiles nodes from the parser into Python code.""" from collections import namedtuple from functools import update_wrapper +from io import StringIO from itertools import chain from keyword import iskeyword as is_python_keyword @@ -9,13 +10,6 @@ from markupsafe import Markup from . import nodes -from ._compat import imap -from ._compat import iteritems -from ._compat import izip -from ._compat import NativeStringIO -from ._compat import range_type -from ._compat import string_types -from ._compat import text_type from .exceptions import TemplateAssertionError from .idtracking import Symbols from .idtracking import VAR_LOAD_ALIAS @@ -38,30 +32,6 @@ "notin": "not in", } -# what method to iterate over items do we want to use for dict iteration -# in generated code? on 2.x let's go with iteritems, on 3.x with items -if hasattr(dict, "iteritems"): - dict_item_iter = "iteritems" -else: - dict_item_iter = "items" - -code_features = ["division"] - -# does this python version support generator stops? (PEP 0479) -try: - exec("from __future__ import generator_stop") - code_features.append("generator_stop") -except SyntaxError: - pass - -# does this python version support yield from? -try: - exec("def f(): yield from x()") -except SyntaxError: - supports_yield_from = False -else: - supports_yield_from = True - def optimizeconst(f): def new_func(self, node, frame, **kwargs): @@ -93,20 +63,16 @@ def has_safe_repr(value): """Does the node have a safe representation?""" if value is None or value is NotImplemented or value is Ellipsis: return True - if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: - return True - if type(value) in (tuple, list, set, frozenset): - for item in value: - if not has_safe_repr(item): - return False - return True - elif type(value) is dict: - for key, value in iteritems(value): - if not has_safe_repr(key): - return False - if not has_safe_repr(value): - return False + + if type(value) in {bool, int, float, complex, range, str, Markup}: return True + + if type(value) in {tuple, list, set, frozenset}: + return all(has_safe_repr(v) for v in value) + + if type(value) is dict: + return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) + return False @@ -249,7 +215,7 @@ def __init__( self, environment, name, filename, stream=None, defer_init=False, optimized=True ): if stream is None: - stream = NativeStringIO() + stream = StringIO() self.environment = environment self.name = name self.filename = filename @@ -432,7 +398,7 @@ def signature(self, node, frame, extra_kwargs=None): self.write(", ") self.visit(kwarg, frame) if extra_kwargs is not None: - for key, value in iteritems(extra_kwargs): + for key, value in extra_kwargs.items(): self.write(", %s=%s" % (key, value)) if node.dyn_args: self.write(", *") @@ -448,7 +414,7 @@ def signature(self, node, frame, extra_kwargs=None): self.visit(kwarg.value, frame) self.write(", ") if extra_kwargs is not None: - for key, value in iteritems(extra_kwargs): + for key, value in extra_kwargs.items(): self.write("%r: %s, " % (key, value)) if node.dyn_kwargs is not None: self.write("}, **") @@ -477,7 +443,7 @@ def pull_dependencies(self, nodes): def enter_frame(self, frame): undefs = [] - for target, (action, param) in iteritems(frame.symbols.loads): + for target, (action, param) in frame.symbols.loads.items(): if action == VAR_LOAD_PARAMETER: pass elif action == VAR_LOAD_RESOLVE: @@ -494,7 +460,7 @@ def enter_frame(self, frame): def leave_frame(self, frame, with_python_scope=False): if not with_python_scope: undefs = [] - for target, _ in iteritems(frame.symbols.loads): + for target in frame.symbols.loads: undefs.append(target) if undefs: self.writeline("%s = missing" % " = ".join(undefs)) @@ -612,7 +578,7 @@ def position(self, node): def dump_local_context(self, frame): return "{%s}" % ", ".join( "%r: %s" % (name, target) - for name, target in iteritems(frame.symbols.dump_stores()) + for name, target in frame.symbols.dump_stores().items() ) def write_commons(self): @@ -704,7 +670,7 @@ def pop_assign_tracking(self, frame): else: self.writeline( "context.exported_vars.update((%s))" - % ", ".join(imap(repr, public_names)) + % ", ".join(map(repr, public_names)) ) # -- Statement Visitors @@ -715,7 +681,6 @@ def visit_Template(self, node, frame=None): from .runtime import exported - self.writeline("from __future__ import %s" % ", ".join(code_features)) self.writeline("from jinja2.runtime import " + ", ".join(exported)) if self.environment.is_async: @@ -781,7 +746,7 @@ def visit_Template(self, node, frame=None): self.indent() self.writeline("if parent_template is not None:") self.indent() - if supports_yield_from and not self.environment.is_async: + if not self.environment.is_async: self.writeline("yield from parent_template.root_render_func(context)") else: self.writeline( @@ -795,7 +760,7 @@ def visit_Template(self, node, frame=None): self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. - for name, block in iteritems(self.blocks): + for name, block in self.blocks.items(): self.writeline( "%s(context, missing=missing%s):" % (self.func("block_" + name), envenv), @@ -851,11 +816,7 @@ def visit_Block(self, node, frame): else: context = self.get_context_ref() - if ( - supports_yield_from - and not self.environment.is_async - and frame.buffer is None - ): + if not self.environment.is_async and frame.buffer is None: self.writeline( "yield from context.blocks[%r][0](%s)" % (node.name, context), node ) @@ -900,9 +861,7 @@ def visit_Extends(self, node, frame): self.writeline("parent_template = environment.get_template(", node) self.visit(node.template, frame) self.write(", %r)" % self.name) - self.writeline( - "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter - ) + self.writeline("for name, parent_block in parent_template.blocks.items():") self.indent() self.writeline("context.blocks.setdefault(name, []).append(parent_block)") self.outdent() @@ -924,7 +883,7 @@ def visit_Include(self, node, frame): func_name = "get_or_select_template" if isinstance(node.template, nodes.Const): - if isinstance(node.template.value, string_types): + if isinstance(node.template.value, str): func_name = "get_template" elif isinstance(node.template.value, (tuple, list)): func_name = "select_template" @@ -958,13 +917,8 @@ def visit_Include(self, node, frame): "._body_stream:" ) else: - if supports_yield_from: - self.writeline("yield from template._get_default_module()._body_stream") - skip_event_yield = True - else: - self.writeline( - "for event in template._get_default_module()._body_stream:" - ) + self.writeline("yield from template._get_default_module()._body_stream") + skip_event_yield = True if not skip_event_yield: self.indent() @@ -1071,7 +1025,7 @@ def visit_FromImport(self, node, frame): else: self.writeline( "context.exported_vars.difference_" - "update((%s))" % ", ".join(imap(repr, discarded_names)) + "update((%s))" % ", ".join(map(repr, discarded_names)) ) def visit_For(self, node, frame): @@ -1262,7 +1216,7 @@ def visit_With(self, node, frame): with_frame = frame.inner() with_frame.symbols.analyze_node(node) self.enter_frame(with_frame) - for target, expr in izip(node.targets, node.values): + for target, expr in zip(node.targets, node.values): self.newline() self.visit(target, with_frame) self.write(" = ") @@ -1278,7 +1232,7 @@ def visit_ExprStmt(self, node, frame): #: The default finalize function if the environment isn't configured #: with one. Or if the environment has one, this is called on that #: function's output for constants. - _default_finalize = text_type + _default_finalize = str _finalize = None def _make_finalize(self): @@ -1344,7 +1298,7 @@ def _output_child_to_const(self, node, frame, finalize): # Template data doesn't go through finalize. if isinstance(node, nodes.TemplateData): - return text_type(const) + return str(const) return finalize.const(const) @@ -1353,11 +1307,11 @@ def _output_child_pre(self, node, frame, finalize): ``Output`` node. """ if frame.eval_ctx.volatile: - self.write("(escape if context.eval_ctx.autoescape else to_string)(") + self.write("(escape if context.eval_ctx.autoescape else str)(") elif frame.eval_ctx.autoescape: self.write("escape(") else: - self.write("to_string(") + self.write("str(") if finalize.src is not None: self.write(finalize.src) @@ -1615,11 +1569,11 @@ def visitor(self, node, frame): @optimizeconst def visit_Concat(self, node, frame): if frame.eval_ctx.volatile: - func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" + func_name = "(context.eval_ctx.volatile and markup_join or str_join)" elif frame.eval_ctx.autoescape: func_name = "markup_join" else: - func_name = "unicode_join" + func_name = "str_join" self.write("%s((" % func_name) for arg in node.nodes: self.visit(arg, frame) diff --git a/src/jinja2/debug.py b/src/jinja2/debug.py index d2c5a06bf..2854a9f77 100644 --- a/src/jinja2/debug.py +++ b/src/jinja2/debug.py @@ -1,8 +1,8 @@ +import platform import sys from types import CodeType from . import TemplateSyntaxError -from ._compat import PYPY from .utils import internal_code from .utils import missing @@ -14,13 +14,11 @@ def rewrite_traceback_stack(source=None): This must be called within an ``except`` block. - :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, - the current ``exc_info`` is used. :param source: For ``TemplateSyntaxError``, the original source if known. - :return: A :meth:`sys.exc_info` tuple that can be re-raised. + :return: The original exception with the rewritten traceback. """ - exc_type, exc_value, tb = sys.exc_info() + _, exc_value, tb = sys.exc_info() if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: exc_value.translated = True @@ -70,7 +68,7 @@ def rewrite_traceback_stack(source=None): for tb in reversed(stack): tb_next = tb_set_next(tb, tb_next) - return exc_type, exc_value, tb_next + return exc_value.with_traceback(tb_next) def fake_traceback(exc_value, tb, filename, lineno): @@ -215,7 +213,7 @@ def tb_set_next(tb, tb_next): return tb -elif PYPY: +elif platform.python_implementation() == "PyPy": # PyPy might have special support, and won't work with ctypes. try: import tputil diff --git a/src/jinja2/defaults.py b/src/jinja2/defaults.py index 8e0e7d771..72a935789 100644 --- a/src/jinja2/defaults.py +++ b/src/jinja2/defaults.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from ._compat import range_type from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 from .tests import TESTS as DEFAULT_TESTS # noqa: F401 from .utils import Cycler @@ -24,7 +23,7 @@ # default filters, tests and namespace DEFAULT_NAMESPACE = { - "range": range_type, + "range": range, "dict": dict, "lipsum": generate_lorem_ipsum, "cycler": Cycler, diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 63ac0c8be..da7a09f3e 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -11,13 +11,6 @@ from markupsafe import Markup from . import nodes -from ._compat import encode_filename -from ._compat import implements_iterator -from ._compat import implements_to_string -from ._compat import iteritems -from ._compat import reraise -from ._compat import string_types -from ._compat import text_type from .compiler import CodeGenerator from .compiler import generate from .defaults import BLOCK_END_STRING @@ -102,7 +95,7 @@ def load_extensions(environment, extensions): """ result = {} for extension in extensions: - if isinstance(extension, string_types): + if isinstance(extension, str): extension = import_string(extension) result[extension.identifier] = extension(environment) return result @@ -376,7 +369,7 @@ def extend(self, **attributes): yet. This is used by :ref:`extensions ` to register callbacks and configuration values without breaking inheritance. """ - for key, value in iteritems(attributes): + for key, value in attributes.items(): if not hasattr(self, key): setattr(self, key, value) @@ -421,7 +414,7 @@ def overlay( rv.overlayed = True rv.linked_to = self - for key, value in iteritems(args): + for key, value in args.items(): if value is not missing: setattr(rv, key, value) @@ -431,7 +424,7 @@ def overlay( rv.cache = copy_cache(self.cache) rv.extensions = {} - for key, value in iteritems(self.extensions): + for key, value in self.extensions.items(): rv.extensions[key] = value.bind(rv) if extensions is not missing: rv.extensions.update(load_extensions(rv, extensions)) @@ -449,7 +442,7 @@ def getitem(self, obj, argument): try: return obj[argument] except (AttributeError, TypeError, LookupError): - if isinstance(argument, string_types): + if isinstance(argument, str): try: attr = str(argument) except Exception: @@ -534,7 +527,7 @@ def parse(self, source, name=None, filename=None): def _parse(self, source, name, filename): """Internal parsing function used by `parse` and `compile`.""" - return Parser(self, source, name, encode_filename(filename)).parse() + return Parser(self, source, name, filename).parse() def lex(self, source, name=None, filename=None): """Lex the given sourcecode and return a generator that yields @@ -546,7 +539,7 @@ def lex(self, source, name=None, filename=None): of the extensions to be applied you have to filter source through the :meth:`preprocess` method. """ - source = text_type(source) + source = str(source) try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: @@ -560,7 +553,7 @@ def preprocess(self, source, name=None, filename=None): return reduce( lambda s, e: e.preprocess(s, name, filename), self.iter_extensions(), - text_type(source), + str(source), ) def _tokenize(self, source, name, filename=None, state=None): @@ -621,7 +614,7 @@ def compile(self, source, name=None, filename=None, raw=False, defer_init=False) """ source_hint = None try: - if isinstance(source, string_types): + if isinstance(source, str): source_hint = source source = self._parse(source, name, filename) source = self._generate(source, name, filename, defer_init=defer_init) @@ -629,8 +622,6 @@ def compile(self, source, name=None, filename=None, raw=False, defer_init=False) return source if filename is None: filename = "