diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml deleted file mode 100644 index 680a15cbf..000000000 --- a/.azure-pipelines.yml +++ /dev/null @@ -1,49 +0,0 @@ -trigger: - - master - - '*.x' - -variables: - vmImage: ubuntu-latest - python.version: '3.8' - TOXENV: py - hasTestResults: 'true' - -strategy: - matrix: - Python 3.8 Linux: - vmImage: ubuntu-latest - Python 3.8 Windows: - vmImage: windows-latest - Python 3.8 Mac: - vmImage: macos-latest - PyPy 3 Linux: - python.version: pypy3 - Python 3.7 Linux: - python.version: '3.7' - Python 3.6 Linux: - python.version: '3.6' - Python 3.5 Linux: - python.version: '3.5' - Python 2.7 Linux: - python.version: '2.7' - Docs: - TOXENV: docs - hasTestResults: 'false' - Style: - TOXENV: style - hasTestResults: 'false' - -pool: - vmImage: $[ variables.vmImage ] - -steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: $(python.version) - displayName: Use Python $(python.version) - - - script: pip --disable-pip-version-check install -U tox - displayName: Install tox - - - script: tox - displayName: Run tox diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..45281036b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "pallets/jinja", + "image": "mcr.microsoft.com/devcontainers/python:3", + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "${workspaceFolder}/.venv", + "python.terminal.activateEnvInCurrentTerminal": true, + "python.terminal.launchArgs": [ + "-X", + "dev" + ] + } + } + }, + "onCreateCommand": ".devcontainer/on-create-command.sh" +} diff --git a/.devcontainer/on-create-command.sh b/.devcontainer/on-create-command.sh new file mode 100755 index 000000000..eaebea618 --- /dev/null +++ b/.devcontainer/on-create-command.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +python3 -m venv --upgrade-deps .venv +. .venv/bin/activate +pip install -r requirements/dev.txt +pip install -e . +pre-commit install --install-hooks diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..2ff985a67 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 +max_line_length = 88 + +[*.{css,html,js,json,jsx,scss,ts,tsx,yaml,yml}] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 4273496d3..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,33 +0,0 @@ -The issue tracker is a tool to address bugs in Jinja itself. -Please use the #pocoo IRC channel on freenode or Stack Overflow for general -questions about using Jinja or issues not related to Jinja. - -If you'd like to report a bug in Jinja, fill out the template below and provide -any extra information that may be useful / related to your problem. -Ideally, you create an [MCVE](http://stackoverflow.com/help/mcve) reproducing -the problem before opening an issue to ensure it's not caused by something in -your code. - ---- - -## Expected Behavior -Tell us what should happen - -## Actual Behavior -Tell us what happens instead - -## Template Code -```jinja -Paste the template code (ideally a minimal example) that causes the issue - -``` - -## Full Traceback -```pytb -Paste the full traceback in case there is an exception - -``` - -## Your Environment -* Python version: -* Jinja version: diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 000000000..75575a4eb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Report a bug in Jinja (not other projects which depend on Jinja) +--- + + + + + + + +Environment: + +- Python version: +- Jinja version: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..b5350a42c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Questions on Discussions + url: https://github.com/pallets/jinja/discussions/ + about: Ask questions about your own code on the Discussions tab. + - name: Questions on Chat + url: https://discord.gg/pallets + about: Ask questions about your own code on our Discord chat. diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 000000000..3791e5f98 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,15 @@ +--- +name: Feature request +about: Suggest a new feature for Jinja +--- + + + + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..eb124d251 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ + + + + + diff --git a/.github/workflows/lock.yaml b/.github/workflows/lock.yaml new file mode 100644 index 000000000..22228a1cd --- /dev/null +++ b/.github/workflows/lock.yaml @@ -0,0 +1,23 @@ +name: Lock inactive closed issues +# Lock closed issues that have not received any further activity for two weeks. +# This does not close open issues, only humans may do that. It is easier to +# respond to new issues with fresh examples rather than continuing discussions +# on old issues. + +on: + schedule: + - cron: '0 0 * * *' +permissions: + issues: write + pull-requests: write +concurrency: + group: lock +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1 + with: + issue-inactive-days: 14 + pr-inactive-days: 14 + discussion-inactive-days: 14 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000..d609abdb6 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,69 @@ +name: Publish +on: + push: + tags: + - '*' +jobs: + build: + runs-on: ubuntu-latest + outputs: + hash: ${{ steps.hash.outputs.hash }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.x' + cache: pip + cache-dependency-path: requirements*/*.txt + - run: pip install -r requirements/build.txt + # Use the commit date instead of the current date during the build. + - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - run: python -m build + # Generate hashes used for provenance. + - name: generate hash + id: hash + run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + path: ./dist + provenance: + needs: [build] + permissions: + actions: read + id-token: write + contents: write + # Can't pin with hash due to how this workflow works. + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 + with: + base64-subjects: ${{ needs.build.outputs.hash }} + create-release: + # Upload the sdist, wheels, and provenance to a GitHub release. They remain + # available as build artifacts for a while as well. + needs: [provenance] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - name: create release + run: > + gh release create --draft --repo ${{ github.repository }} + ${{ github.ref_name }} + *.intoto.jsonl/* artifact/* + env: + GH_TOKEN: ${{ github.token }} + publish-pypi: + needs: [provenance] + # Wait for approval before attempting to upload to PyPI. This allows reviewing the + # files in the draft release. + environment: + name: publish + url: https://pypi.org/project/Jinja2/${{ github.ref_name }} + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 + with: + packages-dir: artifact/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 000000000..1062ebe44 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,51 @@ +name: Tests +on: + push: + branches: [main, stable] + paths-ignore: ['docs/**', '*.md', '*.rst'] + pull_request: + paths-ignore: [ 'docs/**', '*.md', '*.rst' ] +jobs: + tests: + name: ${{ matrix.name || matrix.python }} + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + include: + - {python: '3.13'} + - {python: '3.12'} + - {name: Windows, python: '3.12', os: windows-latest} + - {name: Mac, python: '3.12', os: macos-latest} + - {python: '3.11'} + - {python: '3.10'} + - {python: '3.9'} + - {python: '3.8'} + - {python: '3.7'} + - {name: PyPy, python: 'pypy-3.10', tox: pypy310} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python }} + allow-prereleases: true + cache: pip + cache-dependency-path: requirements*/*.txt + - run: pip install tox + - run: tox run -e ${{ matrix.tox || format('py{0}', matrix.python) }} + typing: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.x' + cache: pip + cache-dependency-path: requirements*/*.txt + - name: cache mypy + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ./.mypy_cache + key: mypy|${{ hashFiles('pyproject.toml') }} + - run: pip install tox + - run: tox run -e typing diff --git a/.gitignore b/.gitignore index 81752e0e9..62c1b887d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,10 @@ -*.so -docs/_build/ -*.pyc -*.pyo -*.egg-info/ -*.egg -build/ +.idea/ +.vscode/ +.venv*/ +venv*/ +__pycache__/ dist/ -.DS_Store +.coverage* +htmlcov/ .tox/ -.cache/ -.idea/ -env/ -venv/ -venv-*/ -.coverage -.coverage.* -htmlcov -.pytest_cache/ -/.vscode/ +docs/_build/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a341a82f..a9f102b5e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,16 @@ +ci: + autoupdate_schedule: monthly repos: - - repo: https://github.com/asottile/reorder_python_imports - rev: v1.9.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 hooks: - - id: reorder-python-imports - args: ["--application-directories", "src"] - - repo: https://github.com/ambv/black - rev: 19.10b0 - hooks: - - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 - hooks: - - id: flake8 - additional_dependencies: [flake8-bugbear] + - id: ruff + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v5.0.0 hooks: - - id: check-byte-order-marker + - id: check-merge-conflict + - id: debug-statements + - id: fix-byte-order-marker - id: trailing-whitespace - id: end-of-file-fixer diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..865c68597 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 +build: + os: ubuntu-22.04 + tools: + python: '3.12' +python: + install: + - requirements: requirements/docs.txt + - method: pip + path: . +sphinx: + builder: dirhtml + fail_on_warning: true diff --git a/CHANGES.rst b/CHANGES.rst index 511b22be6..e1b339198 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,269 @@ .. currentmodule:: jinja2 +Version 3.1.5 +------------- + +Released 2024-12-21 + +- The sandboxed environment handles indirect calls to ``str.format``, such as + by passing a stored reference to a filter that calls its argument. + :ghsa:`q2x7-8rv6-6q7h` +- Escape template name before formatting it into error messages, to avoid + issues with names that contain f-string syntax. + :issue:`1792`, :ghsa:`gmj6-6f8f-6699` +- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence + types. :issue:`2032` +- Calling sync ``render`` for an async template uses ``asyncio.run``. + :pr:`1952` +- Avoid unclosed ``auto_aiter`` warnings. :pr:`1960` +- Return an ``aclose``-able ``AsyncGenerator`` from + ``Template.generate_async``. :pr:`1960` +- Avoid leaving ``root_render_func()`` unclosed in + ``Template.generate_async``. :pr:`1960` +- Avoid leaving async generators unclosed in blocks, includes and extends. + :pr:`1960` +- The runtime uses the correct ``concat`` function for the current environment + when calling block references. :issue:`1701` +- Make ``|unique`` async-aware, allowing it to be used after another + async-aware filter. :issue:`1781` +- ``|int`` filter handles ``OverflowError`` from scientific notation. + :issue:`1921` +- Make compiling deterministic for tuple unpacking in a ``{% set ... %}`` + call. :issue:`2021` +- Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined`` + objects. :issue:`2025` +- Fix `copy`/`pickle` support for the internal ``missing`` object. + :issue:`2027` +- ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` +- The error message from ``FileSystemLoader`` includes the paths that were + searched. :issue:`1661` +- ``PackageLoader`` shows a clearer error message when the package does not + contain the templates directory. :issue:`1705` +- Improve annotations for methods returning copies. :pr:`1880` +- ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` +- Tests decorated with `@pass_context`` can be used with the ``|select`` + filter. :issue:`1624` +- Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the + target is a namespace attribute. :issue:`1413` +- Using ``set`` in all branches of ``{% if %}{% elif %}{% else %}`` blocks + does not cause the variable to be considered initially undefined. + :issue:`1253` + + +Version 3.1.4 +------------- + +Released 2024-05-05 + +- The ``xmlattr`` filter does not allow keys with ``/`` solidus, ``>`` + greater-than sign, or ``=`` equals sign, in addition to disallowing spaces. + Regardless of any validation done by Jinja, user input should never be used + as keys to this filter, or must be separately validated first. + :ghsa:`h75v-3vvj-5mfj` + + +Version 3.1.3 +------------- + +Released 2024-01-10 + +- Fix compiler error when checking if required blocks in parent templates are + empty. :pr:`1858` +- ``xmlattr`` filter does not allow keys with spaces. :ghsa:`h5c8-rqwp-cp95` +- Make error messages stemming from invalid nesting of ``{% trans %}`` blocks + more helpful. :pr:`1918` + + +Version 3.1.2 +------------- + +Released 2022-04-28 + +- Add parameters to ``Environment.overlay`` to match ``__init__``. + :issue:`1645` +- Handle race condition in ``FileSystemBytecodeCache``. :issue:`1654` + + +Version 3.1.1 +------------- + +Released 2022-03-25 + +- The template filename on Windows uses the primary path separator. + :issue:`1637` + + +Version 3.1.0 +------------- + +Released 2022-03-24 + +- Drop support for Python 3.6. :pr:`1534` +- Remove previously deprecated code. :pr:`1544` + + - ``WithExtension`` and ``AutoEscapeExtension`` are built-in now. + - ``contextfilter`` and ``contextfunction`` are replaced by + ``pass_context``. ``evalcontextfilter`` and + ``evalcontextfunction`` are replaced by ``pass_eval_context``. + ``environmentfilter`` and ``environmentfunction`` are replaced + by ``pass_environment``. + - ``Markup`` and ``escape`` should be imported from MarkupSafe. + - Compiled templates from very old Jinja versions may need to be + recompiled. + - Legacy resolve mode for ``Context`` subclasses is no longer + supported. Override ``resolve_or_missing`` instead of + ``resolve``. + - ``unicode_urlencode`` is renamed to ``url_quote``. + +- Add support for native types in macros. :issue:`1510` +- The ``{% trans %}`` tag can use ``pgettext`` and ``npgettext`` by + passing a context string as the first token in the tag, like + ``{% trans "title" %}``. :issue:`1430` +- Update valid identifier characters from Python 3.6 to 3.7. + :pr:`1571` +- Filters and tests decorated with ``@async_variant`` are pickleable. + :pr:`1612` +- Add ``items`` filter. :issue:`1561` +- Subscriptions (``[0]``, etc.) can be used after filters, tests, and + calls when the environment is in async mode. :issue:`1573` +- The ``groupby`` filter is case-insensitive by default, matching + other comparison filters. Added the ``case_sensitive`` parameter to + control this. :issue:`1463` +- Windows drive-relative path segments in template names will not + result in ``FileSystemLoader`` and ``PackageLoader`` loading from + drive-relative paths. :pr:`1621` + + +Version 3.0.3 +------------- + +Released 2021-11-09 + +- Fix traceback rewriting internals for Python 3.10 and 3.11. + :issue:`1535` +- Fix how the native environment treats leading and trailing spaces + when parsing values on Python 3.10. :pr:`1537` +- Improve async performance by avoiding checks for common types. + :issue:`1514` +- Revert change to ``hash(Node)`` behavior. Nodes are hashed by id + again :issue:`1521` +- ``PackageLoader`` works when the package is a single module file. + :issue:`1512` + + +Version 3.0.2 +------------- + +Released 2021-10-04 + +- Fix a loop scoping bug that caused assignments in nested loops + to still be referenced outside of it. :issue:`1427` +- Make ``compile_templates`` deterministic for filter and import + names. :issue:`1452, 1453` +- Revert an unintended change that caused ``Undefined`` to act like + ``StrictUndefined`` for the ``in`` operator. :issue:`1448` +- Imported macros have access to the current template globals in async + environments. :issue:`1494` +- ``PackageLoader`` will not include a current directory (.) path + segment. This allows loading templates from the root of a zip + import. :issue:`1467` + + +Version 3.0.1 +------------- + +Released 2021-05-18 + +- Update MarkupSafe dependency to >= 2.0. :pr:`1418` +- Mark top-level names as exported so type checking understands + imports in user projects. :issue:`1426` +- Fix some types that weren't available in Python 3.6.0. :issue:`1433` +- The deprecation warning for unneeded ``autoescape`` and ``with_`` + extensions shows more relevant context. :issue:`1429` +- Fixed calling deprecated ``jinja2.Markup`` without an argument. + Use ``markupsafe.Markup`` instead. :issue:`1438` +- Calling sync ``render`` for an async template uses ``asyncio.new_event_loop`` + This fixes a deprecation that Python 3.10 introduces. :issue:`1443` + + +Version 3.0.0 +------------- + +Released 2021-05-11 + +- Drop support for Python 2.7 and 3.5. +- Bump MarkupSafe dependency to >=1.1. +- Bump Babel optional dependency to >=2.1. +- Remove code that was marked deprecated. +- Add type hinting. :pr:`1412` +- Use :pep:`451` API to load templates with + :class:`~loaders.PackageLoader`. :issue:`1168` +- Fix a bug that caused imported macros to not have access to the + current template's globals. :issue:`688` +- Add ability to ignore ``trim_blocks`` using ``+%}``. :issue:`1036` +- Fix a bug that caused custom async-only filters to fail with + constant input. :issue:`1279` +- Fix UndefinedError incorrectly being thrown on an undefined variable + instead of ``Undefined`` being returned on + ``NativeEnvironment`` on Python 3.10. :issue:`1335` +- Blocks can be marked as ``required``. They must be overridden at + some point, but not necessarily by the direct child. :issue:`1147` +- Deprecate the ``autoescape`` and ``with`` extensions, they are + built-in to the compiler. :issue:`1203` +- The ``urlize`` filter recognizes ``mailto:`` links and takes + ``extra_schemes`` (or ``env.policies["urlize.extra_schemes"]``) to + recognize other schemes. It tries to balance parentheses within a + URL instead of ignoring trailing characters. The parsing in general + has been updated to be more efficient and match more cases. URLs + without a scheme are linked as ``https://`` instead of ``http://``. + :issue:`522, 827, 1172`, :pr:`1195` +- Filters that get attributes, such as ``map`` and ``groupby``, can + use a false or empty value as a default. :issue:`1331` +- Fix a bug that prevented variables set in blocks or loops from + being accessed in custom context functions. :issue:`768` +- Fix a bug that caused scoped blocks from accessing special loop + variables. :issue:`1088` +- Update the template globals when calling + ``Environment.get_template(globals=...)`` even if the template was + already loaded. :issue:`295` +- Do not raise an error for undefined filters in unexecuted + if-statements and conditional expressions. :issue:`842` +- Add ``is filter`` and ``is test`` tests to test if a name is a + registered filter or test. This allows checking if a filter is + available in a template before using it. Test functions can be + decorated with ``@pass_environment``, ``@pass_eval_context``, + or ``@pass_context``. :issue:`842`, :pr:`1248` +- Support ``pgettext`` and ``npgettext`` (message contexts) in i18n + extension. :issue:`441` +- The ``|indent`` filter's ``width`` argument can be a string to + indent by. :pr:`1167` +- The parser understands hex, octal, and binary integer literals. + :issue:`1170` +- ``Undefined.__contains__`` (``in``) raises an ``UndefinedError`` + instead of a ``TypeError``. :issue:`1198` +- ``Undefined`` is iterable in an async environment. :issue:`1294` +- ``NativeEnvironment`` supports async mode. :issue:`1362` +- Template rendering only treats ``\n``, ``\r\n`` and ``\r`` as line + breaks. Other characters are left unchanged. :issue:`769, 952, 1313` +- ``|groupby`` filter takes an optional ``default`` argument. + :issue:`1359` +- The function and filter decorators have been renamed and unified. + The old names are deprecated. :issue:`1381` + + - ``pass_context`` replaces ``contextfunction`` and + ``contextfilter``. + - ``pass_eval_context`` replaces ``evalcontextfunction`` and + ``evalcontextfilter`` + - ``pass_environment`` replaces ``environmentfunction`` and + ``environmentfilter``. + +- Async support no longer requires Jinja to patch itself. It must + still be enabled with ``Environment(enable_async=True)``. + :issue:`1390` +- Overriding ``Context.resolve`` is deprecated, override + ``resolve_or_missing`` instead. :issue:`1380` + + Version 2.11.3 -------------- @@ -19,7 +283,7 @@ Released 2020-04-13 :class:`~unittest.mock.Mock` to be treated as a :func:`contextfunction`. :issue:`1145` - Update ``wordcount`` filter to trigger :class:`Undefined` methods - by wrapping the input in :func:`soft_unicode`. :pr:`1160` + by wrapping the input in :func:`soft_str`. :pr:`1160` - Fix a hang when displaying tracebacks on Python 32-bit. :issue:`1162` - Showing an undefined error for an object that raises @@ -294,7 +558,7 @@ Released 2017-01-08 possible. For more information and a discussion see :issue:`641` - Resolved an issue where ``block scoped`` would not take advantage of the new scoping rules. In some more exotic cases a variable - overriden in a local scope would not make it into a block. + overridden in a local scope would not make it into a block. - Change the code generation of the ``with`` statement to be in line with the new scoping rules. This resolves some unlikely bugs in edge cases. This also introduces a new internal ``With`` node that can be @@ -771,7 +1035,7 @@ Released 2008-07-17, codename Jinjavitus evaluates to ``false``. - Improved error reporting for undefined values by providing a position. -- ``filesizeformat`` filter uses decimal prefixes now per default and +- ``filesizeformat`` filter uses decimal prefixes now by default and can be set to binary mode with the second parameter. - Fixed bug in finalizer diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index f4ba197de..000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at report@palletsprojects.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 000000000..5f835032f --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,222 @@ +How to contribute to Jinja +========================== + +Thank you for considering contributing to Jinja! + + +Support questions +----------------- + +Please don't use the issue tracker for this. The issue tracker is a +tool to address bugs and feature requests in Jinja itself. Use one of +the following resources for questions about using Jinja or issues with +your own code: + +- The ``#get-help`` channel on our Discord chat: + https://discord.gg/pallets +- The mailing list flask@python.org for long term discussion or larger + issues. +- Ask on `Stack Overflow`_. Search with Google first using: + ``site:stackoverflow.com jinja {search term, exception message, etc.}`` + +.. _Stack Overflow: https://stackoverflow.com/questions/tagged/jinja?tab=Frequent + + +Reporting issues +---------------- + +Include the following information in your post: + +- Describe what you expected to happen. +- If possible, include a `minimal reproducible example`_ to help us + identify the issue. This also helps check that the issue is not with + your own code. +- Describe what actually happened. Include the full traceback if there + was an exception. +- List your Python and Jinja versions. If possible, check if this + issue is already fixed in the latest releases or the latest code in + the repository. + +.. _minimal reproducible example: https://stackoverflow.com/help/minimal-reproducible-example + + +Submitting patches +------------------ + +If there is not an open issue for what you want to submit, prefer +opening one for discussion before working on a PR. You can work on any +issue that doesn't have an open PR linked to it or a maintainer assigned +to it. These show up in the sidebar. No need to ask if you can work on +an issue that interests you. + +Include the following in your patch: + +- Use `Black`_ to format your code. This and other tools will run + automatically if you install `pre-commit`_ using the instructions + below. +- Include tests if your patch adds or changes code. Make sure the test + fails without your patch. +- Update any relevant docs pages and docstrings. Docs pages and + docstrings should be wrapped at 72 characters. +- Add an entry in ``CHANGES.rst``. Use the same style as other + entries. Also include ``.. versionchanged::`` inline changelogs in + relevant docstrings. + +.. _Black: https://black.readthedocs.io +.. _pre-commit: https://pre-commit.com + + +First time setup +~~~~~~~~~~~~~~~~ + +- Download and install the `latest version of git`_. +- Configure git with your `username`_ and `email`_. + + .. code-block:: text + + $ git config --global user.name 'your name' + $ git config --global user.email 'your email' + +- Make sure you have a `GitHub account`_. +- Fork Jinja to your GitHub account by clicking the `Fork`_ button. +- `Clone`_ the main repository locally. + + .. code-block:: text + + $ git clone https://github.com/pallets/jinja + $ cd jinja + +- Add your fork as a remote to push your work to. Replace + ``{username}`` with your username. This names the remote "fork", the + default Pallets remote is "origin". + + .. code-block:: text + + $ git remote add fork https://github.com/{username}/jinja + +- Create a virtualenv. + + .. code-block:: text + + $ python3 -m venv env + $ . env/bin/activate + + On Windows, activating is different. + + .. code-block:: text + + > env\Scripts\activate + +- Upgrade pip and setuptools. + + .. code-block:: text + + $ python -m pip install --upgrade pip setuptools + +- Install the development dependencies, then install Jinja in editable + mode. + + .. code-block:: text + + $ pip install -r requirements/dev.txt && pip install -e . + +- Install the pre-commit hooks. + + .. code-block:: text + + $ pre-commit install + +.. _latest version of git: https://git-scm.com/downloads +.. _username: https://docs.github.com/en/github/using-git/setting-your-username-in-git +.. _email: https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address +.. _GitHub account: https://github.com/join +.. _Fork: https://github.com/pallets/jinja/fork +.. _Clone: https://docs.github.com/en/github/getting-started-with-github/fork-a-repo#step-2-create-a-local-clone-of-your-fork + + +Start coding +~~~~~~~~~~~~ + +- Create a branch to identify the issue you would like to work on. If + you're submitting a bug or documentation fix, branch off of the + latest ".x" branch. + + .. code-block:: text + + $ git fetch origin + $ git checkout -b your-branch-name origin/3.0.x + + If you're submitting a feature addition or change, branch off of the + "main" branch. + + .. code-block:: text + + $ git fetch origin + $ git checkout -b your-branch-name origin/main + +- Using your favorite editor, make your changes, + `committing as you go`_. +- Include tests that cover any code changes you make. Make sure the + test fails without your patch. Run the tests as described below. +- Push your commits to your fork on GitHub and + `create a pull request`_. Link to the issue being addressed with + ``fixes #123`` in the pull request. + + .. code-block:: text + + $ git push --set-upstream fork your-branch-name + +.. _committing as you go: https://dont-be-afraid-to-commit.readthedocs.io/en/latest/git/commandlinegit.html#commit-your-changes +.. _create a pull request: https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request + + +Running the tests +~~~~~~~~~~~~~~~~~ + +Run the basic test suite with pytest. + +.. code-block:: text + + $ pytest + +This runs the tests for the current environment, which is usually +sufficient. CI will run the full suite when you submit your pull +request. You can run the full test suite with tox if you don't want to +wait. + +.. code-block:: text + + $ tox + + +Running test coverage +~~~~~~~~~~~~~~~~~~~~~ + +Generating a report of lines that do not have test coverage can indicate +where to start contributing. Run ``pytest`` using ``coverage`` and +generate a report. + +.. code-block:: text + + $ pip install coverage + $ coverage run -m pytest + $ coverage html + +Open ``htmlcov/index.html`` in your browser to explore the report. + +Read more about `coverage `__. + + +Building the docs +~~~~~~~~~~~~~~~~~ + +Build the docs in the ``docs`` directory using Sphinx. + +.. code-block:: text + + $ cd docs + $ make html + +Open ``_build/html/index.html`` in your browser to view the docs. + +Read more about `Sphinx `__. diff --git a/LICENSE.rst b/LICENSE.txt similarity index 100% rename from LICENSE.rst rename to LICENSE.txt diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 909102a77..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include CHANGES.rst -include tox.ini -graft artwork -graft docs -prune docs/_build -graft examples -graft ext -graft tests -global-exclude *.pyc diff --git a/README.rst b/README.md similarity index 54% rename from README.rst rename to README.md index 060b19efe..f4aa7cbea 100644 --- a/README.rst +++ b/README.md @@ -1,5 +1,4 @@ -Jinja -===== +# Jinja Jinja is a fast, expressive, extensible templating engine. Special placeholders in the template allow writing code similar to Python @@ -26,41 +25,25 @@ possible, it shouldn't make the template designer's job difficult by restricting functionality too much. -Installing ----------- +## In A Nutshell -Install and update using `pip`_: +```jinja +{% extends "base.html" %} +{% block title %}Members{% endblock %} +{% block content %} + +{% endblock %} +``` -.. code-block:: text +## Donate - $ pip install -U Jinja2 +The Pallets organization develops and supports Jinja and other popular +packages. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, [please +donate today][]. -.. _pip: https://pip.pypa.io/en/stable/quickstart/ - - -In A Nutshell -------------- - -.. code-block:: jinja - - {% extends "base.html" %} - {% block title %}Members{% endblock %} - {% block content %} - - {% endblock %} - - -Links ------ - -- Website: https://palletsprojects.com/p/jinja/ -- Documentation: https://jinja.palletsprojects.com/ -- Releases: https://pypi.org/project/Jinja2/ -- Code: https://github.com/pallets/jinja -- Issue tracker: https://github.com/pallets/jinja/issues -- Test status: https://dev.azure.com/pallets/jinja/_build -- Official chat: https://discord.gg/t6rrQZH +[please donate today]: https://palletsprojects.com/donate diff --git a/docs/api.rst b/docs/api.rst index 9d901d8a7..c0fa163a0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -25,30 +25,40 @@ initialization and use that to load templates. In some cases however, it's useful to have multiple environments side by side, if different configurations are in use. -The simplest way to configure Jinja to load templates for your application -looks roughly like this:: +The simplest way to configure Jinja to load templates for your +application is to use :class:`~loaders.PackageLoader`. + +.. code-block:: python from jinja2 import Environment, PackageLoader, select_autoescape env = Environment( - loader=PackageLoader('yourapplication', 'templates'), - autoescape=select_autoescape(['html', 'xml']) + loader=PackageLoader("yourapp"), + autoescape=select_autoescape() ) -This will create a template environment with the default settings and a -loader that looks up the templates in the `templates` folder inside the -`yourapplication` python package. Different loaders are available -and you can also write your own if you want to load templates from a -database or other resources. This also enables autoescaping for HTML and -XML files. +This will create a template environment with a loader that looks up +templates in the ``templates`` folder inside the ``yourapp`` Python +package (or next to the ``yourapp.py`` Python module). It also enables +autoescaping for HTML files. This loader only requires that ``yourapp`` +is importable, it figures out the absolute path to the folder for you. + +Different loaders are available to load templates in other ways or from +other locations. They're listed in the `Loaders`_ section below. You can +also write your own if you want to load templates from a source that's +more specialized to your project. + +To load a template from this environment, call the :meth:`get_template` +method, which returns the loaded :class:`Template`. + +.. code-block:: python -To load a template from this environment you just have to call the -:meth:`get_template` method which then returns the loaded :class:`Template`:: + template = env.get_template("mytemplate.html") - template = env.get_template('mytemplate.html') +To render it with some variables, call the :meth:`render` method. -To render it with some variables, just call the :meth:`render` method:: +.. code-block:: python - print(template.render(the='variables', go='here')) + print(template.render(the="variables", go="here")) Using a template loader rather than passing strings to :class:`Template` or :meth:`Environment.from_string` has multiple advantages. Besides being @@ -61,63 +71,6 @@ a lot easier to use it also enables template inheritance. configure autoescaping now instead of relying on the default. -Unicode -------- - -Jinja is using Unicode internally which means that you have to pass Unicode -objects to the render function or bytestrings that only consist of ASCII -characters. Additionally newlines are normalized to one end of line -sequence which is per default UNIX style (``\n``). - -Python 2.x supports two ways of representing string objects. One is the -`str` type and the other is the `unicode` type, both of which extend a type -called `basestring`. Unfortunately the default is `str` which should not -be used to store text based information unless only ASCII characters are -used. With Python 2.6 it is possible to make `unicode` the default on a per -module level and with Python 3 it will be the default. - -To explicitly use a Unicode string you have to prefix the string literal -with a `u`: ``u'Hänsel und Gretel sagen Hallo'``. That way Python will -store the string as Unicode by decoding the string with the character -encoding from the current Python module. If no encoding is specified this -defaults to 'ASCII' which means that you can't use any non ASCII identifier. - -To set a better module encoding add the following comment to the first or -second line of the Python module using the Unicode literal:: - - # -*- coding: utf-8 -*- - -We recommend utf-8 as Encoding for Python modules and templates as it's -possible to represent every Unicode character in utf-8 and because it's -backwards compatible to ASCII. For Jinja the default encoding of templates -is assumed to be utf-8. - -It is not possible to use Jinja to process non-Unicode data. The reason -for this is that Jinja uses Unicode already on the language level. For -example Jinja treats the non-breaking space as valid whitespace inside -expressions which requires knowledge of the encoding or operating on an -Unicode string. - -For more details about Unicode in Python have a look at the excellent -`Unicode documentation`_. - -Another important thing is how Jinja is handling string literals in -templates. A naive implementation would be using Unicode strings for -all string literals but it turned out in the past that this is problematic -as some libraries are typechecking against `str` explicitly. For example -`datetime.strftime` does not accept Unicode arguments. To not break it -completely Jinja is returning `str` for strings that fit into ASCII and -for everything else `unicode`: - ->>> m = Template(u"{% set a, b = 'foo', 'föö' %}").module ->>> m.a -'foo' ->>> m.b -u'f\xf6\xf6' - - -.. _Unicode documentation: https://docs.python.org/3/howto/unicode.html - High Level API -------------- @@ -161,10 +114,10 @@ useful if you want to dig deeper into Jinja or :ref:`develop extensions .. attribute:: globals - A dict of global variables. These variables are always available - in a template. As long as no template was loaded it's safe - to modify this dict. For more details see :ref:`global-namespace`. - For valid object names have a look at :ref:`identifier-naming`. + A dict of variables that are available in every template loaded + by the environment. As long as no template was loaded it's safe + to modify this. For more details see :ref:`global-namespace`. + For valid object names see :ref:`identifier-naming`. .. attribute:: policies @@ -227,9 +180,20 @@ useful if you want to dig deeper into Jinja or :ref:`develop extensions .. attribute:: globals - The dict with the globals of that template. It's unsafe to modify - this dict as it may be shared with other templates or the environment - that loaded the template. + A dict of variables that are available every time the template + is rendered, without needing to pass them during render. This + should not be modified, as depending on how the template was + loaded it may be shared with the environment and other + templates. + + Defaults to :attr:`Environment.globals` unless extra values are + passed to :meth:`Environment.get_template`. + + Globals are only intended for data that is common to every + render of the template. Specific data should be passed to + :meth:`render`. + + See :ref:`global-namespace`. .. attribute:: name @@ -302,14 +266,14 @@ Notes on Identifiers -------------------- Jinja uses Python naming rules. Valid identifiers can be any combination -of Unicode characters accepted by Python. +of characters accepted by Python. Filters and tests are looked up in separate namespaces and have slightly modified identifier syntax. Filters and tests may contain dots to group filters and tests by topic. For example it's perfectly valid to add a -function into the filter dict and call it `to.unicode`. The regular +function into the filter dict and call it `to.str`. The regular expression for filter and test identifiers is -``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```. +``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*``. Undefined Types @@ -329,8 +293,8 @@ disallows all operations beside testing if it's an undefined object. .. attribute:: _undefined_hint - Either `None` or an unicode string with the error message for - the undefined object. + Either `None` or a string with the error message for the + undefined object. .. attribute:: _undefined_obj @@ -368,34 +332,39 @@ Undefined objects are created by calling :attr:`undefined`. .. admonition:: Implementation - :class:`Undefined` objects are implemented by overriding the special - `__underscore__` methods. For example the default :class:`Undefined` - class implements `__unicode__` in a way that it returns an empty - string, however `__int__` and others still fail with an exception. To - allow conversion to int by returning ``0`` you can implement your own:: + :class:`Undefined` is implemented by overriding the special + ``__underscore__`` methods. For example the default + :class:`Undefined` class implements ``__str__`` to returns an empty + string, while ``__int__`` and others fail with an exception. To + allow conversion to int by returning ``0`` you can implement your + own subclass. + + .. code-block:: python class NullUndefined(Undefined): def __int__(self): return 0 + def __float__(self): return 0.0 - To disallow a method, just override it and raise - :attr:`~Undefined._undefined_exception`. Because this is a very common - idiom in undefined objects there is the helper method - :meth:`~Undefined._fail_with_undefined_error` that does the error raising - automatically. Here a class that works like the regular :class:`Undefined` - but chokes on iteration:: + To disallow a method, override it and raise + :attr:`~Undefined._undefined_exception`. Because this is very + common there is the helper method + :meth:`~Undefined._fail_with_undefined_error` that raises the error + with the correct information. Here's a class that works like the + regular :class:`Undefined` but fails on iteration:: class NonIterableUndefined(Undefined): - __iter__ = Undefined._fail_with_undefined_error + def __iter__(self): + self._fail_with_undefined_error() The Context ----------- .. autoclass:: jinja2.runtime.Context() - :members: resolve, get_exported, get_all + :members: get, resolve, resolve_or_missing, get_exported, get_all .. attribute:: parent @@ -441,16 +410,19 @@ The Context .. automethod:: jinja2.runtime.Context.call(callable, \*args, \**kwargs) -.. admonition:: Implementation +The context is immutable, it prevents modifications, and if it is +modified somehow despite that those changes may not show up. For +performance, Jinja does not use the context as data storage for, only as +a primary data source. Variables that the template does not define are +looked up in the context, but variables the template does define are +stored locally. - Context is immutable for the same reason Python's frame locals are - immutable inside functions. Both Jinja and Python are not using the - context / frame locals as data storage for variables but only as primary - data source. +Instead of modifying the context directly, a function should return +a value that can be assigned to a variable within the template itself. - When a template accesses a variable the template does not define, Jinja - looks up the variable in the context, after that the variable is treated - as if it was defined in the template. +.. code-block:: jinja + + {% set comments = get_latest_comments() %} .. _loaders: @@ -543,13 +515,10 @@ environment to compile different code behind the scenes in order to handle async and sync code in an asyncio event loop. This has the following implications: -- Template rendering requires an event loop to be available to the - current thread. :func:`asyncio.get_event_loop` must return an event - loop. - The compiled code uses ``await`` for functions and attributes, and uses ``async for`` loops. In order to support using both async and sync functions in this context, a small wrapper is placed around - all calls and access, which add overhead compared to purely async + all calls and access, which adds overhead compared to purely async code. - Sync methods and filters become wrappers around their corresponding async implementations where needed. For example, ``render`` invokes @@ -577,16 +546,6 @@ Example:: env.policies['urlize.rel'] = 'nofollow noopener' -``compiler.ascii_str``: - This boolean controls on Python 2 if Jinja should store ASCII only - literals as bytestring instead of unicode strings. This used to be - always enabled for Jinja versions below 2.9 and now can be changed. - Traditionally it was done this way since some APIs in Python 2 failed - badly for unicode strings (for instance the datetime strftime API). - Now however sometimes the inverse is true (for instance str.format). - If this is set to False then all strings are stored as unicode - internally. - ``truncate.leeway``: Configures the leeway default for the `truncate` filter. Leeway as introduced in 2.9 but to restore compatibility with older templates @@ -602,6 +561,10 @@ Example:: The default target that is issued for links from the `urlize` filter if no other target is defined by the call explicitly. +``urlize.extra_schemes``: + Recognize URLs that start with these schemes in addition to the + default ``http://``, ``https://``, and ``mailto:``. + ``json.dumps_function``: If this is set to a value other than `None` then the `tojson` filter will dump with this function instead of the default one. Note that @@ -628,40 +591,16 @@ Utilities These helper functions and classes are useful if you add custom filters or functions to a Jinja environment. -.. autofunction:: jinja2.environmentfilter - -.. autofunction:: jinja2.contextfilter - -.. autofunction:: jinja2.evalcontextfilter - -.. autofunction:: jinja2.environmentfunction - -.. autofunction:: jinja2.contextfunction - -.. autofunction:: jinja2.evalcontextfunction +.. autofunction:: jinja2.pass_context -.. function:: escape(s) +.. autofunction:: jinja2.pass_eval_context - Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s` - to HTML-safe sequences. Use this if you need to display text that might - contain such characters in HTML. This function will not escaped objects - that do have an HTML representation such as already escaped data. - - The return value is a :class:`Markup` string. +.. autofunction:: jinja2.pass_environment .. autofunction:: jinja2.clear_caches .. autofunction:: jinja2.is_undefined -.. autoclass:: jinja2.Markup([string]) - :members: escape, unescape, striptags - -.. admonition:: Note - - The Jinja :class:`Markup` class is compatible with at least Pylons and - Genshi. It's expected that more template engines and framework will pick - up the `__html__` concept soon. - Exceptions ---------- @@ -678,24 +617,20 @@ Exceptions .. attribute:: message - The error message as utf-8 bytestring. + The error message. .. attribute:: lineno - The line number where the error occurred + The line number where the error occurred. .. attribute:: name - The load name for the template as unicode string. + The load name for the template. .. attribute:: filename - The filename that loaded the template as bytestring in the encoding - of the file system (most likely utf-8 or mbcs on Windows systems). - - The reason why the filename and error message are bytestrings and not - unicode strings is that Python 2.x is not using unicode for exceptions - and tracebacks as well as the compiler. This will change with Python 3. + The filename that loaded the template in the encoding of the + file system (most likely utf-8, or mbcs on Windows systems). .. autoexception:: jinja2.TemplateRuntimeError @@ -707,54 +642,119 @@ Exceptions Custom Filters -------------- -Custom filters are just regular Python functions that take the left side of -the filter as first argument and the arguments passed to the filter as -extra arguments or keyword arguments. +Filters are Python functions that take the value to the left of the +filter as the first argument and produce a new value. Arguments passed +to the filter are passed after the value. -For example in the filter ``{{ 42|myfilter(23) }}`` the function would be -called with ``myfilter(42, 23)``. Here for example a simple filter that can -be applied to datetime objects to format them:: +For example, the filter ``{{ 42|myfilter(23) }}`` is called behind the +scenes as ``myfilter(42, 23)``. - def datetimeformat(value, format='%H:%M / %d-%m-%Y'): - return value.strftime(format) +Jinja comes with some :ref:`built-in filters `. To use +a custom filter, write a function that takes at least a ``value`` +argument, then register it in :attr:`Environment.filters`. + +Here's a filter that formats datetime objects: -You can register it on the template environment by updating the -:attr:`~Environment.filters` dict on the environment:: +.. code-block:: python - environment.filters['datetimeformat'] = datetimeformat + def datetime_format(value, format="%H:%M %d-%m-%y"): + return value.strftime(format) + + environment.filters["datetime_format"] = datetime_format -Inside the template it can then be used as follows: +Now it can be used in templates: .. sourcecode:: jinja - written on: {{ article.pub_date|datetimeformat }} - publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }} + {{ article.pub_date|datetime_format }} + {{ article.pub_date|datetime_format("%B %Y") }} -Filters can also be passed the current template context or environment. This -is useful if a filter wants to return an undefined value or check the current -:attr:`~Environment.autoescape` setting. For this purpose three decorators -exist: :func:`environmentfilter`, :func:`contextfilter` and -:func:`evalcontextfilter`. +Some decorators are available to tell Jinja to pass extra information to +the filter. The object is passed as the first argument, making the value +being filtered the second argument. -Here a small example filter that breaks a text into HTML line breaks and -paragraphs and marks the return value as safe HTML string if autoescaping is -enabled:: +- :func:`pass_environment` passes the :class:`Environment`. +- :func:`pass_eval_context` passes the :ref:`eval-context`. +- :func:`pass_context` passes the current + :class:`~jinja2.runtime.Context`. - import re - from jinja2 import evalcontextfilter, Markup, escape +Here's a filter that converts line breaks into HTML ``
`` and ``

`` +tags. It uses the eval context to check if autoescape is currently +enabled before escaping the input and marking the output safe. + +.. code-block:: python - _paragraph_re = re.compile(r'(?:\r\n|\r(?!\n)|\n){2,}') + import re + from jinja2 import pass_eval_context + from markupsafe import Markup, escape - @evalcontextfilter + @pass_eval_context def nl2br(eval_ctx, value): - result = u'\n\n'.join(u'

%s

' % p.replace('\n', Markup('
\n')) - for p in _paragraph_re.split(escape(value))) + br = "
\n" + if eval_ctx.autoescape: - result = Markup(result) - return result + value = escape(value) + br = Markup(br) + + result = "\n\n".join( + f"

{br.join(p.splitlines())}<\p>" + for p in re.split(r"(?:\r\n|\r(?!\n)|\n){2,}", value) + ) + return Markup(result) if autoescape else result + + +.. _writing-tests: + +Custom Tests +------------ + +Test are Python functions that take the value to the left of the test as +the first argument, and return ``True`` or ``False``. Arguments passed +to the test are passed after the value. + +For example, the test ``{{ 42 is even }}`` is called behind the scenes +as ``is_even(42)``. + +Jinja comes with some :ref:`built-in tests `. To use a +custom tests, write a function that takes at least a ``value`` argument, +then register it in :attr:`Environment.tests`. + +Here's a test that checks if a value is a prime number: + +.. code-block:: python + + import math + + def is_prime(n): + if n == 2: + return True + + for i in range(2, int(math.ceil(math.sqrt(n))) + 1): + if n % i == 0: + return False + + return True + + environment.tests["prime"] = is_prime + +Now it can be used in templates: + +.. sourcecode:: jinja + + {% if value is prime %} + {{ value }} is a prime number + {% else %} + {{ value }} is not a prime number + {% endif %} + +Some decorators are available to tell Jinja to pass extra information to +the test. The object is passed as the first argument, making the value +being tested the second argument. -Context filters work the same just that the first argument is the current -active :class:`Context` rather than the environment. +- :func:`pass_environment` passes the :class:`Environment`. +- :func:`pass_eval_context` passes the :ref:`eval-context`. +- :func:`pass_context` passes the current + :class:`~jinja2.runtime.Context`. .. _eval-context: @@ -762,44 +762,53 @@ active :class:`Context` rather than the environment. Evaluation Context ------------------ -The evaluation context (short eval context or eval ctx) is a new object -introduced in Jinja 2.4 that makes it possible to activate and deactivate -compiled features at runtime. +The evaluation context (short eval context or eval ctx) makes it +possible to activate and deactivate compiled features at runtime. -Currently it is only used to enable and disable the automatic escaping but -can be used for extensions as well. +Currently it is only used to enable and disable automatic escaping, but +it can be used by extensions as well. -In previous Jinja versions filters and functions were marked as -environment callables in order to check for the autoescape status from the -environment. In new versions it's encouraged to check the setting from the -evaluation context instead. +The ``autoescape`` setting should be checked on the evaluation context, +not the environment. The evaluation context will have the computed value +for the current template. -Previous versions:: +Instead of ``pass_environment``: - @environmentfilter +.. code-block:: python + + @pass_environment def filter(env, value): result = do_something(value) + if env.autoescape: result = Markup(result) + return result -In new versions you can either use a :func:`contextfilter` and access the -evaluation context from the actual context, or use a -:func:`evalcontextfilter` which directly passes the evaluation context to -the function:: +Use ``pass_eval_context`` if you only need the setting: - @contextfilter - def filter(context, value): +.. code-block:: python + + @pass_eval_context + def filter(eval_ctx, value): result = do_something(value) - if context.eval_ctx.autoescape: + + if eval_ctx.autoescape: result = Markup(result) + return result - @evalcontextfilter - def filter(eval_ctx, value): +Or use ``pass_context`` if you need other context behavior as well: + +.. code-block:: python + + @pass_context + def filter(context, value): result = do_something(value) - if eval_ctx.autoescape: + + if context.eval_ctx.autoescape: result = Markup(result) + return result The evaluation context must not be modified at runtime. Modifications @@ -819,57 +828,32 @@ eval context object itself. time. At runtime this should always be `False`. -.. _writing-tests: - -Custom Tests ------------- - -Tests work like filters just that there is no way for a test to get access -to the environment or context and that they can't be chained. The return -value of a test should be `True` or `False`. The purpose of a test is to -give the template designers the possibility to perform type and conformability -checks. - -Here a simple test that checks if a variable is a prime number:: - - import math - - def is_prime(n): - if n == 2: - return True - for i in range(2, int(math.ceil(math.sqrt(n))) + 1): - if n % i == 0: - return False - return True - - -You can register it on the template environment by updating the -:attr:`~Environment.tests` dict on the environment:: - - environment.tests['prime'] = is_prime - -A template designer can then use the test like this: - -.. sourcecode:: jinja - - {% if 42 is prime %} - 42 is a prime number - {% else %} - 42 is not a prime number - {% endif %} - - .. _global-namespace: The Global Namespace -------------------- -Variables stored in the :attr:`Environment.globals` dict are special as they -are available for imported templates too, even if they are imported without -context. This is the place where you can put variables and functions -that should be available all the time. Additionally :attr:`Template.globals` -exist that are variables available to a specific template that are available -to all :meth:`~Template.render` calls. +The global namespace stores variables and functions that should be +available without needing to pass them to :meth:`Template.render`. They +are also available to templates that are imported or included without +context. Most applications should only use :attr:`Environment.globals`. + +:attr:`Environment.globals` are intended for data that is common to all +templates loaded by that environment. :attr:`Template.globals` are +intended for data that is common to all renders of that template, and +default to :attr:`Environment.globals` unless they're given in +:meth:`Environment.get_template`, etc. Data that is specific to a +render should be passed as context to :meth:`Template.render`. + +Only one set of globals is used during any specific rendering. If +templates A and B both have template globals, and B extends A, then +only B's globals are used for both when using ``b.render()``. + +Environment globals should not be changed after loading any templates, +and template globals should not be changed at any time after loading the +template. Changing globals after loading a template will result in +unexpected behavior as they may be shared between the environment and +other templates. .. _low-level-api: @@ -896,7 +880,7 @@ don't recommend using any of those. that has to be created by :meth:`new_context` of the same template or a compatible template. This render function is generated by the compiler from the template code and returns a generator that yields - unicode strings. + strings. If an exception in the template code happens the template engine will not rewrite the exception but pass through the original one. As a diff --git a/docs/changelog.rst b/docs/changes.rst similarity index 59% rename from docs/changelog.rst rename to docs/changes.rst index 218fe3339..955deaf27 100644 --- a/docs/changelog.rst +++ b/docs/changes.rst @@ -1,4 +1,4 @@ -Changelog -========= +Changes +======= .. include:: ../CHANGES.rst diff --git a/docs/conf.py b/docs/conf.py index 01e530dc8..02c74a86b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,16 +10,25 @@ # General -------------------------------------------------------------- -master_doc = "index" +default_role = "code" extensions = [ "sphinx.ext.autodoc", + "sphinx.ext.extlinks", "sphinx.ext.intersphinx", - "pallets_sphinx_themes", "sphinxcontrib.log_cabinet", - "sphinx_issues", + "pallets_sphinx_themes", ] -intersphinx_mapping = {"python": ("https://docs.python.org/3/", None)} -issues_github_path = "pallets/jinja" +autodoc_member_order = "bysource" +autodoc_typehints = "description" +autodoc_preserve_defaults = True +extlinks = { + "issue": ("https://github.com/pallets/jinja/issues/%s", "#%s"), + "pr": ("https://github.com/pallets/jinja/pull/%s", "#%s"), + "ghsa": ("https://github.com/advisories/GHSA-%s", "GHSA-%s"), +} +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), +} # HTML ----------------------------------------------------------------- @@ -27,26 +36,20 @@ html_theme_options = {"index_sidebar_logo": False} html_context = { "project_links": [ - ProjectLink("Donate to Pallets", "https://palletsprojects.com/donate"), - ProjectLink("Jinja Website", "https://palletsprojects.com/p/jinja/"), - ProjectLink("PyPI releases", "https://pypi.org/project/Jinja2/"), + ProjectLink("Donate", "https://palletsprojects.com/donate"), + ProjectLink("PyPI Releases", "https://pypi.org/project/Jinja2/"), ProjectLink("Source Code", "https://github.com/pallets/jinja/"), ProjectLink("Issue Tracker", "https://github.com/pallets/jinja/issues/"), + ProjectLink("Chat", "https://discord.gg/pallets"), ] } html_sidebars = { - "index": ["project.html", "localtoc.html", "searchbox.html"], - "**": ["localtoc.html", "relations.html", "searchbox.html"], + "index": ["project.html", "localtoc.html", "searchbox.html", "ethicalads.html"], + "**": ["localtoc.html", "relations.html", "searchbox.html", "ethicalads.html"], } -singlehtml_sidebars = {"index": ["project.html", "localtoc.html"]} +singlehtml_sidebars = {"index": ["project.html", "localtoc.html", "ethicalads.html"]} html_static_path = ["_static"] html_favicon = "_static/jinja-logo-sidebar.png" html_logo = "_static/jinja-logo-sidebar.png" -html_title = "Jinja Documentation ({})".format(version) +html_title = f"Jinja Documentation ({version})" html_show_sourcelink = False - -# LaTeX ---------------------------------------------------------------- - -latex_documents = [ - (master_doc, "Jinja-{}.tex".format(version), html_title, author, "manual") -] diff --git a/docs/examples/cache_extension.py b/docs/examples/cache_extension.py index 387cd4657..46af67ce0 100644 --- a/docs/examples/cache_extension.py +++ b/docs/examples/cache_extension.py @@ -7,7 +7,7 @@ class FragmentCacheExtension(Extension): tags = {"cache"} def __init__(self, environment): - super(FragmentCacheExtension, self).__init__(environment) + super().__init__(environment) # add the defaults to the environment environment.extend(fragment_cache_prefix="", fragment_cache=None) diff --git a/docs/examples/inline_gettext_extension.py b/docs/examples/inline_gettext_extension.py index 47bc9cc1c..80a10d1fd 100644 --- a/docs/examples/inline_gettext_extension.py +++ b/docs/examples/inline_gettext_extension.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re from jinja2.exceptions import TemplateSyntaxError @@ -6,7 +5,6 @@ from jinja2.lexer import count_newlines from jinja2.lexer import Token - _outside_re = re.compile(r"\\?(gettext|_)\(") _inside_re = re.compile(r"\\?[()]") @@ -31,7 +29,7 @@ def filter_stream(self, stream): pos = 0 lineno = token.lineno - while 1: + while True: if not paren_stack: match = _outside_re.search(token.value, pos) else: @@ -54,7 +52,7 @@ def filter_stream(self, stream): else: if gtok == "(" or paren_stack > 1: yield Token(lineno, "data", gtok) - paren_stack += gtok == ")" and -1 or 1 + paren_stack += -1 if gtok == ")" else 1 if not paren_stack: yield Token(lineno, "block_begin", None) yield Token(lineno, "name", "endtrans") diff --git a/docs/extensions.rst b/docs/extensions.rst index 7abed658b..9b15e813e 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -11,14 +11,17 @@ code into a reusable class like adding support for internationalization. Adding Extensions ----------------- -Extensions are added to the Jinja environment at creation time. Once the -environment is created additional extensions cannot be added. To add an +Extensions are added to the Jinja environment at creation time. To add an extension pass a list of extension classes or import paths to the ``extensions`` parameter of the :class:`~jinja2.Environment` constructor. The following example creates a Jinja environment with the i18n extension loaded:: jinja_env = Environment(extensions=['jinja2.ext.i18n']) +To add extensions after creation time, use the :meth:`~jinja2.Environment.add_extension` method:: + + jinja_env.add_extension('jinja2.ext.debug') + .. _i18n-extension: @@ -31,9 +34,15 @@ The i18n extension can be used in combination with `gettext`_ or `Babel`_. When it's enabled, Jinja provides a ``trans`` statement that marks a block as translatable and calls ``gettext``. -After enabling, an application has to provide ``gettext`` and -``ngettext`` functions, either globally or when rendering. A ``_()`` -function is added as an alias to the ``gettext`` function. +After enabling, an application has to provide functions for ``gettext``, +``ngettext``, and optionally ``pgettext`` and ``npgettext``, either +globally or when rendering. A ``_()`` function is added as an alias to +the ``gettext`` function. + +A convenient way to provide these functions is to call one of the below +methods depending on the translation system in use. If you do not require +actual translation, use ``Environment.install_null_translations`` to +install no-op functions. Environment Methods ~~~~~~~~~~~~~~~~~~~ @@ -44,12 +53,16 @@ additional methods: .. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False) Installs a translation globally for the environment. The - ``translations`` object must implement ``gettext`` and ``ngettext`` - (or ``ugettext`` and ``ungettext`` for Python 2). + ``translations`` object must implement ``gettext``, ``ngettext``, + and optionally ``pgettext`` and ``npgettext``. :class:`gettext.NullTranslations`, :class:`gettext.GNUTranslations`, and `Babel`_\s ``Translations`` are supported. - .. versionchanged:: 2.5 Added new-style gettext support. + .. versionchanged:: 3.0 + Added ``pgettext`` and ``npgettext``. + + .. versionchanged:: 2.5 + Added new-style gettext support. .. method:: jinja2.Environment.install_null_translations(newstyle=False) @@ -59,17 +72,21 @@ additional methods: .. versionchanged:: 2.5 Added new-style gettext support. -.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False) +.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False, pgettext=None, npgettext=None) - Install the given ``gettext`` and ``ngettext`` callables into the - environment. They should behave exactly like - :func:`gettext.gettext` and :func:`gettext.ngettext` (or - ``ugettext`` and ``ungettext`` for Python 2). + Install the given ``gettext``, ``ngettext``, ``pgettext``, and + ``npgettext`` callables into the environment. They should behave + exactly like :func:`gettext.gettext`, :func:`gettext.ngettext`, + :func:`gettext.pgettext` and :func:`gettext.npgettext`. If ``newstyle`` is activated, the callables are wrapped to work like newstyle callables. See :ref:`newstyle-gettext` for more information. - .. versionadded:: 2.5 Added new-style gettext support. + .. versionchanged:: 3.0 + Added ``pgettext`` and ``npgettext``. + + .. versionadded:: 2.5 + Added new-style gettext support. .. method:: jinja2.Environment.uninstall_gettext_translations() @@ -86,8 +103,8 @@ additional methods: found. - ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code). - - ``message`` is the string itself (``unicode`` on Python 2), or a - tuple of strings for functions with multiple arguments. + - ``message`` is the string itself, or a tuple of strings for + functions with multiple arguments. If `Babel`_ is installed, see :ref:`babel-integration` to extract the strings. @@ -110,7 +127,7 @@ The usage of the ``i18n`` extension for template designers is covered in :ref:`the template documentation `. .. _gettext: https://docs.python.org/3/library/gettext.html -.. _Babel: http://babel.pocoo.org/ +.. _Babel: https://babel.pocoo.org/ Whitespace Trimming @@ -153,6 +170,10 @@ done with the ``|format`` filter. This requires duplicating work for {{ ngettext( "%(num)d apple", "%(num)d apples", apples|count )|format(num=apples|count) }} + {{ pgettext("greeting", "Hello, World!") }} + {{ npgettext( + "fruit", "%(num)d apple", "%(num)d apples", apples|count + )|format(num=apples|count) }} New style ``gettext`` make formatting part of the call, and behind the scenes enforce more consistency. @@ -162,6 +183,8 @@ scenes enforce more consistency. {{ gettext("Hello, World!") }} {{ gettext("Hello, %(name)s!", name=name) }} {{ ngettext("%(num)d apple", "%(num)d apples", apples|count) }} + {{ pgettext("greeting", "Hello, World!") }} + {{ npgettext("fruit", "%(num)d apple", "%(num)d apples", apples|count) }} The advantages of newstyle gettext are: diff --git a/docs/faq.rst b/docs/faq.rst index 294fef1d6..a53ae12ff 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1,195 +1,77 @@ Frequently Asked Questions ========================== -This page answers some of the often asked questions about Jinja. - -.. highlight:: html+jinja Why is it called Jinja? ----------------------- -The name Jinja was chosen because it's the name of a Japanese temple and -temple and template share a similar pronunciation. It is not named after -the city in Uganda. - -How fast is it? ---------------- - -We really hate benchmarks especially since they don't reflect much. The -performance of a template depends on many factors and you would have to -benchmark different engines in different situations. The benchmarks from the -testsuite show that Jinja has a similar performance to `Mako`_ and is between -10 and 20 times faster than Django's template engine or Genshi. These numbers -should be taken with tons of salt as the benchmarks that took these numbers -only test a few performance related situations such as looping. Generally -speaking the performance of a template engine doesn't matter much as the -usual bottleneck in a web application is either the database or the application -code. - -.. _Mako: https://www.makotemplates.org/ - -How Compatible is Jinja with Django? ------------------------------------- - -The default syntax of Jinja matches Django syntax in many ways. However -this similarity doesn't mean that you can use a Django template unmodified -in Jinja. For example filter arguments use a function call syntax rather -than a colon to separate filter name and arguments. Additionally the -extension interface in Jinja is fundamentally different from the Django one -which means that your custom tags won't work any longer. +"Jinja" is a Japanese `Shinto shrine`_, or temple, and temple and +template share a similar English pronunciation. It is not named after +the `city in Uganda`_. -Generally speaking you will use much less custom extensions as the Jinja -template system allows you to use a certain subset of Python expressions -which can replace most Django extensions. For example instead of using -something like this:: +.. _Shinto shrine: https://en.wikipedia.org/wiki/Shinto_shrine +.. _city in Uganda: https://en.wikipedia.org/wiki/Jinja%2C_Uganda - {% load comments %} - {% get_latest_comments 10 as latest_comments %} - {% for comment in latest_comments %} - ... - {% endfor %} -You will most likely provide an object with attributes to retrieve -comments from the database:: +How fast is Jinja? +------------------ - {% for comment in models.comments.latest(10) %} - ... - {% endfor %} +Jinja is relatively fast among template engines because it compiles and +caches template code to Python code, so that the template does not need +to be parsed and interpreted each time. Rendering a template becomes as +close to executing a Python function as possible. -Or directly provide the model for quick testing:: +Jinja also makes extensive use of caching. Templates are cached by name +after loading, so future uses of the template avoid loading. The +template loading itself uses a bytecode cache to avoid repeated +compiling. The caches can be external to persist across restarts. +Templates can also be precompiled and loaded as fast Python imports. - {% for comment in Comment.objects.order_by('-pub_date')[:10] %} - ... - {% endfor %} +We dislike benchmarks because they don't reflect real use. Performance +depends on many factors. Different engines have different default +configurations and tradeoffs that make it unclear how to set up a useful +comparison. Often, database access, API calls, and data processing have +a much larger effect on performance than the template engine. -Please keep in mind that even though you may put such things into templates -it still isn't a good idea. Queries should go into the view code and not -the template! -Isn't it a terrible idea to put Logic into Templates? ------------------------------------------------------ +Isn't it a bad idea to put logic in templates? +---------------------------------------------- Without a doubt you should try to remove as much logic from templates as -possible. But templates without any logic mean that you have to do all -the processing in the code which is boring and stupid. A template engine -that does that is shipped with Python and called `string.Template`. Comes -without loops and if conditions and is by far the fastest template engine -you can get for Python. - -So some amount of logic is required in templates to keep everyone happy. -And Jinja leaves it pretty much to you how much logic you want to put into -templates. There are some restrictions in what you can do and what not. - -Jinja neither allows you to put arbitrary Python code into templates nor -does it allow all Python expressions. The operators are limited to the -most common ones and more advanced expressions such as list comprehensions -and generator expressions are not supported. This keeps the template engine -easier to maintain and templates more readable. - -Why is Autoescaping not the Default? ------------------------------------- - -There are multiple reasons why automatic escaping is not the default mode -and also not the recommended one. While automatic escaping of variables -means that you will less likely have an XSS problem it also causes a huge -amount of extra processing in the template engine which can cause serious -performance problems. As Python doesn't provide a way to mark strings as -unsafe Jinja has to hack around that limitation by providing a custom -string class (the :class:`Markup` string) that safely interacts with safe -and unsafe strings. - -With explicit escaping however the template engine doesn't have to perform -any safety checks on variables. Also a human knows not to escape integers -or strings that may never contain characters one has to escape or already -HTML markup. For example when iterating over a list over a table of -integers and floats for a table of statistics the template designer can -omit the escaping because he knows that integers or floats don't contain -any unsafe parameters. - -Additionally Jinja is a general purpose template engine and not only used -for HTML/XML generation. For example you may generate LaTeX, emails, -CSS, JavaScript, or configuration files. - -Why is the Context immutable? ------------------------------ - -When writing a :func:`contextfunction` or something similar you may have -noticed that the context tries to stop you from modifying it. If you have -managed to modify the context by using an internal context API you may -have noticed that changes in the context don't seem to be visible in the -template. The reason for this is that Jinja uses the context only as -primary data source for template variables for performance reasons. - -If you want to modify the context write a function that returns a variable -instead that one can assign to a variable by using set:: - - {% set comments = get_latest_comments() %} - -My tracebacks look weird. What's happening? --------------------------------------------- - -If the debugsupport module is not compiled and you are using a Python -installation without ctypes (Python 2.4 without ctypes, Jython or Google's -AppEngine) Jinja is unable to provide correct debugging information and -the traceback may be incomplete. There is currently no good workaround -for Jython or the AppEngine as ctypes is unavailable there and it's not -possible to use the debugsupport extension. - -If you are working in the Google AppEngine development server you can -whitelist the ctypes module to restore the tracebacks. This however won't -work in production environments:: - - import os - if os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'): - from google.appengine.tools.devappserver2.python import sandbox - sandbox._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt'] - -Credit for this snippet goes to `Thomas Johansson -`_ - -Why is there no Python 2.3/2.4/2.5/2.6/3.1/3.2/3.3 support? ------------------------------------------------------------ - -Python 2.3 is missing a lot of features that are used heavily in Jinja. This -decision was made as with the upcoming Python 2.6 and 3.0 versions it becomes -harder to maintain the code for older Python versions. If you really need -Python 2.3 support you either have to use Jinja 1 or other templating -engines that still support 2.3. - -Python 2.4/2.5/3.1/3.2 support was removed when we switched to supporting -Python 2 and 3 by the same sourcecode (without using 2to3). It was required to -drop support because only Python 2.6/2.7 and >=3.3 support byte and unicode -literals in a way compatible to each other version. If you really need support -for older Python 2 (or 3) versions, you can just use Jinja 2.6. - -Python 2.6/3.3 support was dropped because it got dropped in various upstream -projects (such as wheel or pytest), which would make it difficult to continue -supporting it. Jinja 2.10 was the last version supporting Python 2.6/3.3. - -My Macros are overridden by something +possible. With less logic, the template is easier to understand, has +fewer potential side effects, and is faster to compile and render. But a +template without any logic means processing must be done in code before +rendering. A template engine that does that is shipped with Python, +called :class:`string.Template`, and while it's definitely fast it's not +convenient. + +Jinja's features such as blocks, statements, filters, and function calls +make it much easier to write expressive templates, with very few +restrictions. Jinja doesn't allow arbitrary Python code in templates, or +every feature available in the Python language. This keeps the engine +easier to maintain, and keeps templates more readable. + +Some amount of logic is required in templates to keep everyone happy. +Too much logic in the template can make it complex to reason about and +maintain. It's up to you to decide how your application will work and +balance how much logic you want to put in the template. + + +Why is HTML escaping not the default? ------------------------------------- -In some situations the Jinja scoping appears arbitrary: - -layout.tmpl: - -.. sourcecode:: jinja - - {% macro foo() %}LAYOUT{% endmacro %} - {% block body %}{% endblock %} - -child.tmpl: - -.. sourcecode:: jinja +Jinja provides a feature that can be enabled to escape HTML syntax in +rendered templates. However, it is disabled by default. - {% extends 'layout.tmpl' %} - {% macro foo() %}CHILD{% endmacro %} - {% block body %}{{ foo() }}{% endblock %} +Jinja is a general purpose template engine, it is not only used for HTML +documents. You can generate plain text, LaTeX, emails, CSS, JavaScript, +configuration files, etc. HTML escaping wouldn't make sense for any of +these document types. -This will print ``LAYOUT`` in Jinja. This is a side effect of having -the parent template evaluated after the child one. This allows child -templates passing information to the parent template. To avoid this -issue rename the macro or variable in the parent template to have an -uncommon prefix. +While automatic escaping means that you are less likely have an XSS +problem, it also requires significant extra processing during compiling +and rendering, which can reduce performance. Jinja uses `MarkupSafe`_ for +escaping, which provides optimized C code for speed, but it still +introduces overhead to track escaping across methods and formatting. -.. _Jinja 1: https://pypi.org/project/Jinja/ +.. _MarkupSafe: https://markupsafe.palletsprojects.com/ diff --git a/docs/index.rst b/docs/index.rst index 65d5d3d4a..4ce207191 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,29 +7,9 @@ Jinja :align: center :target: https://palletsprojects.com/p/jinja/ -Jinja is a modern and designer-friendly templating language for Python, -modelled after Django's templates. It is fast, widely used and secure -with the optional sandboxed template execution environment: - -.. sourcecode:: html+jinja - - {% block title %}{% endblock %} -

- -Features: - -- sandboxed execution -- powerful automatic HTML escaping system for XSS prevention -- template inheritance -- compiles down to the optimal python code just in time -- optional ahead-of-time template compilation -- easy to debug. Line numbers of exceptions directly point to - the correct line in the template. -- configurable syntax +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. .. toctree:: :maxdepth: 2 @@ -45,7 +25,5 @@ Features: switching tricks faq - changelog - -* :ref:`genindex` -* :ref:`search` + license + changes diff --git a/docs/integration.rst b/docs/integration.rst index 2cfad55f9..d53fb52ba 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -1,35 +1,50 @@ Integration =========== -Jinja provides some code for integration into other tools such as frameworks, -the `Babel`_ library or your favourite editor for fancy code highlighting. -This is a brief description of whats included. -Files to help integration are available -`here. `_ +Flask +----- + +The `Flask`_ web application framework, also maintained by Pallets, uses +Jinja templates by default. Flask sets up a Jinja environment and +template loader for you, and provides functions to easily render +templates from view functions. + +.. _Flask: https://flask.palletsprojects.com + + +Django +------ + +Django supports using Jinja as its template engine, see +https://docs.djangoproject.com/en/stable/topics/templates/#support-for-template-engines. + .. _babel-integration: -Babel Integration ------------------ +Babel +----- -Jinja provides support for extracting gettext messages from templates via a -`Babel`_ extractor entry point called `jinja2.ext.babel_extract`. The Babel -support is implemented as part of the :ref:`i18n-extension` extension. +Jinja provides support for extracting gettext messages from templates +via a `Babel`_ extractor entry point called +``jinja2.ext.babel_extract``. The support is implemented as part of the +:ref:`i18n-extension` extension. -Gettext messages extracted from both `trans` tags and code expressions. +Gettext messages are extracted from both ``trans`` tags and code +expressions. -To extract gettext messages from templates, the project needs a Jinja section -in its Babel extraction method `mapping file`_: +To extract gettext messages from templates, the project needs a Jinja +section in its Babel extraction method `mapping file`_: .. sourcecode:: ini [jinja2: **/templates/**.html] encoding = utf-8 -The syntax related options of the :class:`Environment` are also available as -configuration values in the mapping file. For example to tell the extraction -that templates use ``%`` as `line_statement_prefix` you can use this code: +The syntax related options of the :class:`Environment` are also +available as configuration values in the mapping file. For example, to +tell the extractor that templates use ``%`` as +``line_statement_prefix`` you can use this code: .. sourcecode:: ini @@ -37,70 +52,43 @@ that templates use ``%`` as `line_statement_prefix` you can use this code: encoding = utf-8 line_statement_prefix = % -:ref:`jinja-extensions` may also be defined by passing a comma separated list -of import paths as `extensions` value. The i18n extension is added -automatically. +:ref:`jinja-extensions` may also be defined by passing a comma separated +list of import paths as the ``extensions`` value. The i18n extension is +added automatically. -.. versionchanged:: 2.7 +Template syntax errors are ignored by default. The assumption is that +tests will catch syntax errors in templates. If you don't want to ignore +errors, add ``silent = false`` to the settings. - Until 2.7 template syntax errors were always ignored. This was done - since many people are dropping non template html files into the - templates folder and it would randomly fail. The assumption was that - testsuites will catch syntax errors in templates anyways. If you don't - want that behavior you can add ``silent=false`` to the settings and - exceptions are propagated. +.. _Babel: https://babel.readthedocs.io/ +.. _mapping file: https://babel.readthedocs.io/en/latest/messages.html#extraction-method-mapping-and-configuration -.. _mapping file: http://babel.pocoo.org/en/latest/messages.html#extraction-method-mapping-and-configuration Pylons ------ -With `Pylons`_ 0.9.7 onwards it's incredible easy to integrate Jinja into a -Pylons powered application. +It's easy to integrate Jinja into a `Pylons`_ application. + +The template engine is configured in ``config/environment.py``. The +configuration for Jinja looks something like this: -The template engine is configured in `config/environment.py`. The configuration -for Jinja looks something like that:: +.. code-block:: python from jinja2 import Environment, PackageLoader config['pylons.app_globals'].jinja_env = Environment( loader=PackageLoader('yourapplication', 'templates') ) -After that you can render Jinja templates by using the `render_jinja` function -from the `pylons.templating` module. +After that you can render Jinja templates by using the ``render_jinja`` +function from the ``pylons.templating`` module. -Additionally it's a good idea to set the Pylons' `c` object into strict mode. -Per default any attribute to not existing attributes on the `c` object return -an empty string and not an undefined object. To change this just use this -snippet and add it into your `config/environment.py`:: +Additionally it's a good idea to set the Pylons ``c`` object to strict +mode. By default attribute access on missing attributes on the ``c`` +object returns an empty string and not an undefined object. To change +this add this to ``config/environment.py``: - config['pylons.strict_c'] = True - -.. _Pylons: https://pylonshq.com/ - -TextMate --------- - -There is a `bundle for TextMate`_ that supports syntax highlighting for Jinja 1 -and Jinja 2 for text based templates as well as HTML. It also contains a few -often used snippets. +.. code-block:: python -.. _bundle for TextMate: https://github.com/mitsuhiko/jinja2-tmbundle - -Vim ---- - -A syntax plugin for `Vim`_ is available `from the jinja repository -`_. The script -supports Jinja 1 and Jinja 2. Once installed, two file types are available -(``jinja`` and ``htmljinja``). The first one is for text-based templates and the -second is for HTML templates. For HTML documents, the plugin attempts to -automatically detect Jinja syntax inside of existing HTML documents. - -If you are using a plugin manager like `Pathogen`_, see the `vim-jinja -`_ repository for installing in the -``bundle/`` directory. + config['pylons.strict_c'] = True -.. _Babel: http://babel.pocoo.org/ -.. _Vim: https://www.vim.org/ -.. _Pathogen: https://github.com/tpope/vim-pathogen +.. _Pylons: https://pylonsproject.org/ diff --git a/docs/intro.rst b/docs/intro.rst index c20c5e910..fd6f84ff5 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -1,82 +1,63 @@ Introduction ============ -This is the documentation for the Jinja general purpose templating language. -Jinja is a library for Python that is designed to be flexible, fast and secure. +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- Async support for generating templates that automatically handle + sync and async functions without extra syntax. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. -If you have any exposure to other text-based template languages, such as Smarty or -Django, you should feel right at home with Jinja. It's both designer and -developer friendly by sticking to Python's principles and adding functionality -useful for templating environments. - -Prerequisites -------------- - -Jinja works with Python 2.7.x and >= 3.5. If you are using Python -3.2 you can use an older release of Jinja (2.6) as support for Python 3.2 -was dropped in Jinja version 2.7. The last release which supported Python 2.6 -and 3.3 was Jinja 2.10. - -If you wish to use the :class:`~jinja2.PackageLoader` class, you will also -need `setuptools`_ or `distribute`_ installed at runtime. Installation ------------ -You can install the most recent Jinja version using `pip`_:: +We recommend using the latest version of Python. Jinja supports Python +3.7 and newer. We also recommend using a `virtual environment`_ in order +to isolate your project dependencies from other projects and the system. - pip install Jinja2 +.. _virtual environment: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments -This will install Jinja in your Python installation's site-packages directory. +Install the most recent Jinja version using pip: -Installing the development version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: text -1. Install `git`_ -2. ``git clone git://github.com/pallets/jinja.git`` -3. ``cd jinja2`` -4. ``ln -s jinja2 /usr/lib/python2.X/site-packages`` + $ pip install Jinja2 -As an alternative to steps 4 you can also do ``python setup.py develop`` -which will install the package via `distribute` in development mode. This also -has the advantage that the C extensions are compiled. -.. _distribute: https://pypi.org/project/distribute/ -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ -.. _git: https://git-scm.com/ +Dependencies +~~~~~~~~~~~~ +These will be installed automatically when installing Jinja. -MarkupSafe Dependency -~~~~~~~~~~~~~~~~~~~~~ - -As of version 2.7 Jinja depends on the `MarkupSafe`_ module. If you install -Jinja via ``pip`` it will be installed automatically for you. +- `MarkupSafe`_ escapes untrusted input when rendering templates to + avoid injection attacks. .. _MarkupSafe: https://markupsafe.palletsprojects.com/ -Basic API Usage ---------------- - -This section gives you a brief introduction to the Python API for Jinja -templates. -The most basic way to create a template and render it is through -:class:`~jinja2.Template`. This however is not the recommended way to -work with it if your templates are not loaded from strings but the file -system or another data source: +Optional Dependencies +~~~~~~~~~~~~~~~~~~~~~ ->>> from jinja2 import Template ->>> template = Template('Hello {{ name }}!') ->>> template.render(name='John Doe') -u'Hello John Doe!' +These distributions will not be installed automatically. -By creating an instance of :class:`~jinja2.Template` you get back a new template -object that provides a method called :meth:`~jinja2.Template.render` which when -called with a dict or keyword arguments expands the template. The dict -or keywords arguments passed to the template are the so-called "context" -of the template. +- `Babel`_ provides translation support in templates. -What you can see here is that Jinja is using unicode internally and the -return value is an unicode string. So make sure that your application is -indeed using unicode internally. +.. _Babel: https://babel.pocoo.org/ diff --git a/docs/license.rst b/docs/license.rst new file mode 100644 index 000000000..2a445f9c6 --- /dev/null +++ b/docs/license.rst @@ -0,0 +1,5 @@ +BSD-3-Clause License +==================== + +.. literalinclude:: ../LICENSE.txt + :language: text diff --git a/docs/make.bat b/docs/make.bat index 7893348a1..b16225546 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -21,7 +21,7 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ + echo.https://www.sphinx-doc.org/ exit /b 1 ) diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst index 1a08700b0..fb2a76718 100644 --- a/docs/nativetypes.rst +++ b/docs/nativetypes.rst @@ -55,6 +55,17 @@ Foo >>> print(result.value) 15 +Sandboxed Native Environment +---------------------------- + +You can combine :class:`.SandboxedEnvironment` and :class:`NativeEnvironment` to +get both behaviors. + +.. code-block:: python + + class SandboxedNativeEnvironment(SandboxedEnvironment, NativeEnvironment): + pass + API --- diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index cfe1fd75e..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -Sphinx~=2.1.2 -Pallets-Sphinx-Themes~=1.2.0 -sphinxcontrib-log-cabinet~=1.0.1 -sphinx-issues~=1.2.0 diff --git a/docs/sandbox.rst b/docs/sandbox.rst index 1222d0250..fc9c31fbd 100644 --- a/docs/sandbox.rst +++ b/docs/sandbox.rst @@ -1,18 +1,56 @@ Sandbox ======= -The Jinja sandbox can be used to evaluate untrusted code. Access to unsafe -attributes and methods is prohibited. +The Jinja sandbox can be used to render untrusted templates. Access to +attributes, method calls, operators, mutating data structures, and +string formatting can be intercepted and prohibited. -Assuming `env` is a :class:`SandboxedEnvironment` in the default configuration -the following piece of code shows how it works: +.. code-block:: pycon + + >>> from jinja2.sandbox import SandboxedEnvironment + >>> env = SandboxedEnvironment() + >>> func = lambda: "Hello, Sandbox!" + >>> env.from_string("{{ func() }}").render(func=func) + 'Hello, Sandbox!' + >>> env.from_string("{{ func.__code__.co_code }}").render(func=func) + Traceback (most recent call last): + ... + SecurityError: access to attribute '__code__' of 'function' object is unsafe. + +A sandboxed environment can be useful, for example, to allow users of an +internal reporting system to create custom emails. You would document +what data is available in the templates, then the user would write a +template using that information. Your code would generate the report +data and pass it to the user's sandboxed template to render. + + +Security Considerations +----------------------- + +The sandbox alone is not a solution for perfect security. Keep these +things in mind when using the sandbox. + +Templates can still raise errors when compiled or rendered. Your code +should attempt to catch errors instead of crashing. + +It is possible to construct a relatively small template that renders to +a very large amount of output, which could correspond to a high use of +CPU or memory. You should run your application with limits on resources +such as CPU and memory to mitigate this. + +Jinja only renders text, it does not understand, for example, JavaScript +code. Depending on how the rendered template will be used, you may need +to do other postprocessing to restrict the output. + +Pass only the data that is relevant to the template. Avoid passing +global data, or objects with methods that have side effects. By default +the sandbox prevents private and internal attribute access. You can +override :meth:`~SandboxedEnvironment.is_safe_attribute` to further +restrict attributes access. Decorate methods with :func:`unsafe` to +prevent calling them from templates when passing objects as data. Use +:class:`ImmutableSandboxedEnvironment` to prevent modifying lists and +dictionaries. ->>> env.from_string("{{ func.func_code }}").render(func=lambda:None) -u'' ->>> env.from_string("{{ func.func_code.do_something }}").render(func=lambda:None) -Traceback (most recent call last): - ... -SecurityError: access to attribute 'func_code' of 'function' object is unsafe. API --- @@ -34,61 +72,40 @@ API .. autofunction:: modifies_known_mutable -.. admonition:: Note - - The Jinja sandbox alone is no solution for perfect security. Especially - for web applications you have to keep in mind that users may create - templates with arbitrary HTML in so it's crucial to ensure that (if you - are running multiple users on the same server) they can't harm each other - via JavaScript insertions and much more. - - Also the sandbox is only as good as the configuration. We strongly - recommend only passing non-shared resources to the template and use - some sort of whitelisting for attributes. - - Also keep in mind that templates may raise runtime or compile time errors, - so make sure to catch them. Operator Intercepting --------------------- -.. versionadded:: 2.6 - -For maximum performance Jinja will let operators call directly the type -specific callback methods. This means that it's not possible to have this -intercepted by overriding :meth:`Environment.call`. Furthermore a -conversion from operator to special method is not always directly possible -due to how operators work. For instance for divisions more than one -special method exist. - -With Jinja 2.6 there is now support for explicit operator intercepting. -This can be used to customize specific operators as necessary. In order -to intercept an operator one has to override the -:attr:`SandboxedEnvironment.intercepted_binops` attribute. Once the -operator that needs to be intercepted is added to that set Jinja will -generate bytecode that calls the :meth:`SandboxedEnvironment.call_binop` -function. For unary operators the `unary` attributes and methods have to -be used instead. - -The default implementation of :attr:`SandboxedEnvironment.call_binop` -will use the :attr:`SandboxedEnvironment.binop_table` to translate -operator symbols into callbacks performing the default operator behavior. - -This example shows how the power (``**``) operator can be disabled in -Jinja:: +For performance, Jinja outputs operators directly when compiling. This +means it's not possible to intercept operator behavior by overriding +:meth:`SandboxEnvironment.call ` by default, because +operator special methods are handled by the Python interpreter, and +might not correspond with exactly one method depending on the operator's +use. + +The sandbox can instruct the compiler to output a function to intercept +certain operators instead. Override +:attr:`SandboxedEnvironment.intercepted_binops` and +:attr:`SandboxedEnvironment.intercepted_unops` with the operator symbols +you want to intercept. The compiler will replace the symbols with calls +to :meth:`SandboxedEnvironment.call_binop` and +:meth:`SandboxedEnvironment.call_unop` instead. The default +implementation of those methods will use +:attr:`SandboxedEnvironment.binop_table` and +:attr:`SandboxedEnvironment.unop_table` to translate operator symbols +into :mod:`operator` functions. + +For example, the power (``**``) operator can be disabled: + +.. code-block:: python from jinja2.sandbox import SandboxedEnvironment - class MyEnvironment(SandboxedEnvironment): - intercepted_binops = frozenset(['**']) + intercepted_binops = frozenset(["**"]) def call_binop(self, context, operator, left, right): - if operator == '**': - return self.undefined('the power operator is unavailable') - return SandboxedEnvironment.call_binop(self, context, - operator, left, right) - -Make sure to always call into the super method, even if you are not -intercepting the call. Jinja might internally call the method to -evaluate expressions. + if operator == "**": + return self.undefined("The power (**) operator is unavailable.") + + return super().call_binop(self, context, operator, left, right) diff --git a/docs/switching.rst b/docs/switching.rst index 8225b2ee0..a0ee530e7 100644 --- a/docs/switching.rst +++ b/docs/switching.rst @@ -1,141 +1,73 @@ -Switching from other Template Engines +Switching From Other Template Engines ===================================== -.. highlight:: html+jinja - -If you have used a different template engine in the past and want to switch -to Jinja here is a small guide that shows the basic syntactic and semantic -changes between some common, similar text template engines for Python. - -Jinja 1 -------- - -Jinja 2 is mostly compatible with Jinja 1 in terms of API usage and template -syntax. The differences between Jinja 1 and 2 are explained in the following -list. - -API -~~~ - -Loaders - Jinja 2 uses a different loader API. Because the internal representation - of templates changed there is no longer support for external caching - systems such as memcached. The memory consumed by templates is comparable - with regular Python modules now and external caching doesn't give any - advantage. If you have used a custom loader in the past have a look at - the new :ref:`loader API `. - -Loading templates from strings - In the past it was possible to generate templates from a string with the - default environment configuration by using `jinja.from_string`. Jinja 2 - provides a :class:`Template` class that can be used to do the same, but - with optional additional configuration. - -Automatic unicode conversion - Jinja 1 performed automatic conversion of bytestrings in a given encoding - into unicode objects. This conversion is no longer implemented as it - was inconsistent as most libraries are using the regular Python ASCII - bytestring to Unicode conversion. An application powered by Jinja 2 - *has to* use unicode internally everywhere or make sure that Jinja 2 only - gets unicode strings passed. - -i18n - Jinja 1 used custom translators for internationalization. i18n is now - available as Jinja 2 extension and uses a simpler, more gettext friendly - interface and has support for babel. For more details see - :ref:`i18n-extension`. - -Internal methods - Jinja 1 exposed a few internal methods on the environment object such - as `call_function`, `get_attribute` and others. While they were marked - as being an internal method it was possible to override them. Jinja 2 - doesn't have equivalent methods. - -Sandbox - Jinja 1 was running sandbox mode by default. Few applications actually - used that feature so it became optional in Jinja 2. For more details - about the sandboxed execution see :class:`SandboxedEnvironment`. - -Context - Jinja 1 had a stacked context as storage for variables passed to the - environment. In Jinja 2 a similar object exists but it doesn't allow - modifications nor is it a singleton. As inheritance is dynamic now - multiple context objects may exist during template evaluation. - -Filters and Tests - Filters and tests are regular functions now. It's no longer necessary - and allowed to use factory functions. - - -Templates -~~~~~~~~~ - -Jinja 2 has mostly the same syntax as Jinja 1. What's different is that -macros require parentheses around the argument list now. - -Additionally Jinja 2 allows dynamic inheritance now and dynamic includes. -The old helper function `rendertemplate` is gone now, `include` can be used -instead. Includes no longer import macros and variable assignments, for -that the new `import` tag is used. This concept is explained in the -:ref:`import` documentation. - -Another small change happened in the `for`-tag. The special loop variable -doesn't have a `parent` attribute, instead you have to alias the loop -yourself. See :ref:`accessing-the-parent-loop` for more details. +This is a brief guide on some of the differences between Jinja syntax +and other template languages. See :doc:`/templates` for a comprehensive +guide to Jinja syntax and features. Django ------ If you have previously worked with Django templates, you should find -Jinja very familiar. In fact, most of the syntax elements look and -work the same. +Jinja very familiar. Many of the syntax elements look and work the same. +However, Jinja provides some more syntax elements, and some work a bit +differently. -However, Jinja provides some more syntax elements covered in the -documentation and some work a bit different. +This section covers the template changes. The API, including extension +support, is fundamentally different so it won't be covered here. + +Django supports using Jinja as its template engine, see +https://docs.djangoproject.com/en/stable/topics/templates/#support-for-template-engines. -This section covers the template changes. As the API is fundamentally -different we won't cover it here. Method Calls ~~~~~~~~~~~~ -In Django method calls work implicitly, while Jinja requires the explicit -Python syntax. Thus this Django code:: +In Django, methods are called implicitly, without parentheses. + +.. code-block:: django {% for page in user.get_created_pages %} ... {% endfor %} -...looks like this in Jinja:: +In Jinja, using parentheses is required for calls, like in Python. This +allows you to pass variables to the method, which is not possible +in Django. This syntax is also used for calling macros. + +.. code-block:: jinja {% for page in user.get_created_pages() %} ... {% endfor %} -This allows you to pass variables to the method, which is not possible in -Django. This syntax is also used for macros. Filter Arguments ~~~~~~~~~~~~~~~~ -Jinja provides more than one argument for filters. Also the syntax for -argument passing is different. A template that looks like this in Django:: +In Django, one literal value can be passed to a filter after a colon. + +.. code-block:: django {{ items|join:", " }} -looks like this in Jinja:: +In Jinja, filters can take any number of positional and keyword +arguments in parentheses, like function calls. Arguments can also be +variables instead of literal values. - {{ items|join(', ') }} +.. code-block:: jinja + + {{ items|join(", ") }} -It is a bit more verbose, but it allows different types of arguments - -including variables - and more than one of them. Tests ~~~~~ -In addition to filters there also are tests you can perform using the is -operator. Here are some examples:: +In addition to filters, Jinja also has "tests" used with the ``is`` +operator. This operator is not the same as the Python operator. + +.. code-block:: jinja {% if user.user_id is odd %} {{ user.username|e }} is odd @@ -146,64 +78,85 @@ operator. Here are some examples:: Loops ~~~~~ -For loops work very similarly to Django, but notably the Jinja special -variable for the loop context is called `loop`, not `forloop` as in Django. +In Django, the special variable for the loop context is called +``forloop``, and the ``empty`` is used for no loop items. -In addition, the Django `empty` argument is called `else` in Jinja. For -example, the Django template:: +.. code-block:: django {% for item in items %} - {{ item }} + {{ forloop.counter }}. {{ item }} {% empty %} No items! {% endfor %} -...looks like this in Jinja:: +In Jinja, the special variable for the loop context is called ``loop``, +and the ``else`` block is used for no loop items. + +.. code-block:: jinja {% for item in items %} - {{ item }} + {{ loop.index }}. {{ item }} {% else %} No items! {% endfor %} + Cycle ~~~~~ -The ``{% cycle %}`` tag does not exist in Jinja; however, you can achieve the -same output by using the `cycle` method on the loop context special variable. +In Django, the ``{% cycle %}`` can be used in a for loop to alternate +between values per loop. -The following Django template:: +.. code-block:: django {% for user in users %}
  • {{ user }}
    • {% endfor %} -...looks like this in Jinja:: +In Jinja, the ``loop`` context has a ``cycle`` method. + +.. code-block:: jinja {% for user in users %}
  • {{ user }}
    • {% endfor %} -There is no equivalent of ``{% cycle ... as variable %}``. +A cycler can also be assigned to a variable and used outside or across +loops with the ``cycle()`` global function. Mako ---- -.. highlight:: html+mako +You can configure Jinja to look more like Mako: + +.. code-block:: python -If you have used Mako so far and want to switch to Jinja you can configure -Jinja to look more like Mako: + env = Environment( + block_start_string="<%", + block_end_string="%>", + variable_start_string="${", + variable_end_string="}", + comment_start_string="<%doc>", + commend_end_string="", + line_statement_prefix="%", + line_comment_prefix="##", + ) -.. sourcecode:: python +With an environment configured like that, Jinja should be able to +interpret a small subset of Mako templates without any changes. - env = Environment('<%', '%>', '${', '}', '<%doc>', '', '%', '##') +Jinja does not support embedded Python code, so you would have to move +that out of the template. You could either process the data with the +same code before rendering, or add a global function or filter to the +Jinja environment. -With an environment configured like that, Jinja should be able to interpret -a small subset of Mako templates. Jinja does not support embedded Python -code, so you would have to move that out of the template. The syntax for defs -(which are called macros in Jinja) and template inheritance is different too. -The following Mako template:: +The syntax for defs (which are called macros in Jinja) and template +inheritance is different too. + +The following Mako template: + +.. code-block:: mako <%inherit file="layout.html" /> <%def name="title()">Page Title @@ -213,7 +166,9 @@ The following Mako template:: % endfor -Looks like this in Jinja with the above configuration:: +Looks like this in Jinja with the above configuration: + +.. code-block:: jinja <% extends "layout.html" %> <% block title %>Page Title<% endblock %> diff --git a/docs/templates.rst b/docs/templates.rst index 89c2a5060..9f376a13c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1,8 +1,9 @@ +.. py:currentmodule:: jinja2 +.. highlight:: html+jinja + Template Designer Documentation =============================== -.. highlight:: html+jinja - This document describes the syntax and semantics of the template engine and will be most useful as reference to those creating Jinja templates. As the template engine is very flexible, the configuration from the application can @@ -54,7 +55,11 @@ configured as follows: * ``{% ... %}`` for :ref:`Statements ` * ``{{ ... }}`` for :ref:`Expressions` to print to the template output * ``{# ... #}`` for :ref:`Comments` not included in the template output -* ``# ... ##`` for :ref:`Line Statements ` + +:ref:`Line Statements and Comments ` are also possible, +though they don't have default prefix characters. To use them, set +``line_statement_prefix`` and ``line_comment_prefix`` when creating the +:class:`~jinja2.Environment`. Template File Extension @@ -197,10 +202,11 @@ option can also be set to strip tabs and spaces from the beginning of a line to the start of a block. (Nothing will be stripped if there are other characters before the start of the block.) -With both `trim_blocks` and `lstrip_blocks` enabled, you can put block tags -on their own lines, and the entire block line will be removed when -rendered, preserving the whitespace of the contents. For example, -without the `trim_blocks` and `lstrip_blocks` options, this template:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled (the default), block +tags on their own lines will be removed, but a blank line will remain and the +spaces in the content will be preserved. For example, this template: + +.. code-block:: jinja
    {% if True %} @@ -208,7 +214,10 @@ without the `trim_blocks` and `lstrip_blocks` options, this template:: {% endif %}
    -gets rendered with blank lines inside the div:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled, the template is +rendered with blank lines inside the div: + +.. code-block:: text
    @@ -216,8 +225,10 @@ gets rendered with blank lines inside the div::
    -But with both `trim_blocks` and `lstrip_blocks` enabled, the template block -lines are removed and other whitespace is preserved:: +With both ``trim_blocks`` and ``lstrip_blocks`` enabled, the template block +lines are completely removed: + +.. code-block:: text
    yay @@ -230,6 +241,15 @@ plus sign (``+``) at the start of a block:: {%+ if something %}yay{% endif %}
    +Similarly, you can manually disable the ``trim_blocks`` behavior by +putting a plus sign (``+``) at the end of a block:: + +
    + {% if something +%} + yay + {% endif %} +
    + You can also strip whitespace in templates by hand. If you add a minus sign (``-``) to the start or end of a block (e.g. a :ref:`for-loop` tag), a comment, or a variable expression, the whitespaces before or after @@ -413,7 +433,7 @@ this template "extends" another template. When the template system evaluates this template, it first locates the parent. The extends tag should be the first tag in the template. Everything before it is printed out normally and may cause confusion. For details about this behavior and how to take -advantage of it, see :ref:`null-master-fallback`. Also a block will always be +advantage of it, see :ref:`null-default-fallback`. Also a block will always be filled in regardless of whether the surrounding condition is evaluated to be true or false. @@ -508,8 +528,8 @@ However, the name after the `endblock` word must match the block name. Block Nesting and Scope ~~~~~~~~~~~~~~~~~~~~~~~ -Blocks can be nested for more complex layouts. However, per default blocks -may not access variables from outer scopes:: +Blocks can be nested for more complex layouts. By default, a block may not +access variables from outside the block (outer scopes):: {% for item in seq %}
  • {% block loop_item %}{{ item }}{% endblock %}
    • @@ -531,20 +551,69 @@ modifier to a block declaration:: When overriding a block, the `scoped` modifier does not have to be provided. +Required Blocks +~~~~~~~~~~~~~~~ + +Blocks can be marked as ``required``. They must be overridden at some +point, but not necessarily by the direct child template. Required blocks +may only contain space and comments, and they cannot be rendered +directly. + +.. code-block:: jinja + :caption: ``page.txt`` + + {% block body required %}{% endblock %} + +.. code-block:: jinja + :caption: ``issue.txt`` + + {% extends "page.txt" %} + +.. code-block:: jinja + :caption: ``bug_report.txt`` + + {% extends "issue.txt" %} + {% block body %}Provide steps to demonstrate the bug.{% endblock %} + +Rendering ``page.txt`` or ``issue.txt`` will raise +``TemplateRuntimeError`` because they don't override the ``body`` block. +Rendering ``bug_report.txt`` will succeed because it does override the +block. + +When combined with ``scoped``, the ``required`` modifier must be placed +*after* the scoped modifier. Here are some valid examples: + +.. code-block:: jinja + + {% block body scoped %}{% endblock %} + {% block body required %}{% endblock %} + {% block body scoped required %}{% endblock %} + + Template Objects ~~~~~~~~~~~~~~~~ -.. versionchanged:: 2.4 +``extends``, ``include``, and ``import`` can take a template object +instead of the name of a template to load. This could be useful in some +advanced situations, since you can use Python code to load a template +first and pass it in to ``render``. -If a template object was passed in the template context, you can -extend from that object as well. Assuming the calling code passes -a layout template as `layout_template` to the environment, this -code works:: +.. code-block:: python - {% extends layout_template %} + if debug_mode: + layout = env.get_template("debug_layout.html") + else: + layout = env.get_template("layout.html") + + user_detail = env.get_template("user/detail.html") + return user_detail.render(layout=layout) + +.. code-block:: jinja -Previously, the `layout_template` variable had to be a string with -the layout template's filename for this to work. + {% extends layout %} + +Note how ``extends`` is passed the variable with the template object +that was passed to ``render``, instead of a string. HTML Escaping @@ -602,9 +671,8 @@ you have data that is already safe but not marked, be sure to wrap it in Jinja functions (macros, `super`, `self.BLOCKNAME`) always return template data that is marked as safe. -String literals in templates with automatic escaping are considered unsafe -because native Python strings (``str``, ``unicode``, ``basestring``) are not -`MarkupSafe.Markup` strings with an ``__html__`` attribute. +String literals in templates with automatic escaping are considered +unsafe because native Python strings are not safe. .. _list-of-control-structures: @@ -641,9 +709,17 @@ iterate over containers like `dict`:: {% endfor %} -Note, however, that **Python dicts are not ordered**; so you might want to -either pass a sorted ``list`` of ``tuple`` s -- or a -``collections.OrderedDict`` -- to the template, or use the `dictsort` filter. +Python dicts may not be in the order you want to display them in. If +order matters, use the ``|dictsort`` filter. + +.. code-block:: jinja + +
    + {% for key, value in my_dict | dictsort %} +
    {{ key|e }}
    +
    {{ value|e }}
    + {% endfor %} +
    Inside of a for-loop block, you can access some special variables: @@ -854,9 +930,6 @@ are available on a macro object: `arguments` A tuple of the names of arguments the macro accepts. -`defaults` - A tuple of default values. - `catch_kwargs` This is `true` if the macro accepts extra keyword arguments (i.e.: accesses the special `kwargs` variable). @@ -872,6 +945,23 @@ are available on a macro object: If a macro name starts with an underscore, it's not exported and can't be imported. +Due to how scopes work in Jinja, a macro in a child template does not +override a macro in a parent template. The following will output +"LAYOUT", not "CHILD". + +.. code-block:: jinja + :caption: ``layout.txt`` + + {% macro foo() %}LAYOUT{% endmacro %} + {% block body %}{% endblock %} + +.. code-block:: jinja + :caption: ``child.txt`` + + {% extends 'layout.txt' %} + {% macro foo() %}CHILD{% endmacro %} + {% block body %}{{ foo() }}{% endblock %} + .. _call: @@ -913,9 +1003,9 @@ Here's an example of how a call block can be used with arguments:: {% call(user) dump_users(list_of_user) %}
    -
    Realname
    +
    Realname
    {{ user.realname|e }}
    -
    Description
    +
    Description
    {{ user.description }}
    {% endcall %} @@ -931,6 +1021,9 @@ template data. Just wrap the code in the special `filter` section:: This text becomes uppercase {% endfilter %} +Filters that accept arguments can be called like this:: + + {% filter center(100) %}Center this{% endfilter %} .. _assignments: @@ -993,34 +1086,34 @@ Assignments use the `set` tag and can have multiple targets:: Block Assignments ~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.8 +It's possible to use `set` as a block to assign the content of the block to a +variable. This can be used to create multi-line strings, since Jinja doesn't +support Python's triple quotes (``"""``, ``'''``). -Starting with Jinja 2.8, it's possible to also use block assignments to -capture the contents of a block into a variable name. This can be useful -in some situations as an alternative for macros. In that case, instead of -using an equals sign and a value, you just write the variable name and then -everything until ``{% endset %}`` is captured. +Instead of using an equals sign and a value, you only write the variable name, +and everything until ``{% endset %}`` is captured. -Example:: +.. code-block:: jinja {% set navigation %}
  • Index
  • Downloads {% endset %} -The `navigation` variable then contains the navigation HTML source. - -.. versionchanged:: 2.10 - -Starting with Jinja 2.10, the block assignment supports filters. +Filters applied to the variable name will be applied to the block's content. -Example:: +.. code-block:: jinja {% set reply | wordwrap %} You wrote: {{ message }} {% endset %} +.. versionadded:: 2.8 + +.. versionchanged:: 2.10 + + Block assignment supports filters. .. _extends: @@ -1047,42 +1140,45 @@ at the same time. They are documented in detail in the Include ~~~~~~~ -The `include` tag is useful to include a template and return the -rendered contents of that file into the current namespace:: +The ``include`` tag renders another template and outputs the result into +the current template. + +.. code-block:: jinja {% include 'header.html' %} - Body + Body goes here. {% include 'footer.html' %} -Included templates have access to the variables of the active context by -default. For more details about context behavior of imports and includes, -see :ref:`import-visibility`. +The included template has access to context of the current template by +default. Use ``without context`` to use a separate context instead. +``with context`` is also valid, but is the default behavior. See +:ref:`import-visibility`. + +The included template can ``extend`` another template and override +blocks in that template. However, the current template cannot override +any blocks that the included template outputs. -From Jinja 2.2 onwards, you can mark an include with ``ignore missing``; in -which case Jinja will ignore the statement if the template to be included -does not exist. When combined with ``with`` or ``without context``, it must -be placed *before* the context visibility statement. Here are some valid -examples:: +Use ``ignore missing`` to ignore the statement if the template does not +exist. It must be placed *before* a context visibility statement. +.. code-block:: jinja + + {% include "sidebar.html" without context %} {% include "sidebar.html" ignore missing %} {% include "sidebar.html" ignore missing with context %} {% include "sidebar.html" ignore missing without context %} -.. versionadded:: 2.2 - -You can also provide a list of templates that are checked for existence -before inclusion. The first template that exists will be included. If -`ignore missing` is given, it will fall back to rendering nothing if -none of the templates exist, otherwise it will raise an exception. +If a list of templates is given, each will be tried in order until one +is not missing. This can be used with ``ignore missing`` to ignore if +none of the templates exist. -Example:: +.. code-block:: jinja {% include ['page_detailed.html', 'page.html'] %} {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %} -.. versionchanged:: 2.4 - If a template object was passed to the template context, you can - include that object using `include`. +A variable, with either a template name or template object, can also be +passed to the statement. .. _import: @@ -1278,8 +1374,19 @@ but exists for completeness' sake. The following operators are supported: ``{{ '=' * 80 }}`` would print a bar of 80 equal signs. ``**`` - Raise the left operand to the power of the right operand. ``{{ 2**3 }}`` - would return ``8``. + Raise the left operand to the power of the right operand. + ``{{ 2**3 }}`` would return ``8``. + + Unlike Python, chained pow is evaluated left to right. + ``{{ 3**3**3 }}`` is evaluated as ``(3**3)**3`` in Jinja, but would + be evaluated as ``3**(3**3)`` in Python. Use parentheses in Jinja + to be explicit about what order you want. It is usually preferable + to do extended math in Python and pass the results to ``render`` + rather than doing it in the template. + + This behavior may be changed in the future to match Python, if it's + possible to introduce an upgrade path. + Comparisons ~~~~~~~~~~~ @@ -1305,28 +1412,32 @@ Comparisons Logic ~~~~~ -For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be useful to -combine multiple expressions: +For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be +useful to combine multiple expressions. ``and`` - Return true if the left and the right operand are true. + For ``x and y``, if ``x`` is false, then the value is ``x``, else ``y``. In + a boolean context, this will be treated as ``True`` if both operands are + truthy. ``or`` - Return true if the left or the right operand are true. + For ``x or y``, if ``x`` is true, then the value is ``x``, else ``y``. In a + boolean context, this will be treated as ``True`` if at least one operand is + truthy. ``not`` - negate a statement (see below). + For ``not x``, if ``x`` is false, then the value is ``True``, else + ``False``. -``(expr)`` - Parentheses group an expression. - -.. admonition:: Note - - The ``is`` and ``in`` operators support negation using an infix notation, - too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar`` - and ``not foo in bar``. All other expressions require a prefix notation: + Prefer negating ``is`` and ``in`` using their infix notation: + ``foo is not bar`` instead of ``not foo is bar``; ``foo not in bar`` instead + of ``not foo in bar``. All other expressions require prefix notation: ``not (foo and bar).`` +``(expr)`` + Parentheses group an expression. This is used to change evaluation order, or + to make a long expression easier to read or less ambiguous. + Other Operators ~~~~~~~~~~~~~~~ @@ -1342,10 +1453,10 @@ two categories: ``is`` Performs a :ref:`test `. -``|`` +``|`` (pipe, vertical bar) Applies a :ref:`filter `. -``~`` +``~`` (tilde) Converts all operands into strings and concatenates them. ``{{ "Hello " ~ name ~ "!" }}`` would return (assuming `name` is set @@ -1370,7 +1481,7 @@ It is also possible to use inline `if` expressions. These are useful in some situations. For example, you can use this to extend from one template if a variable is defined, otherwise from the default layout template:: - {% extends layout_template if layout_template is defined else 'master.html' %} + {% extends layout_template if layout_template is defined else 'default.html' %} The general syntax is `` if else ``. @@ -1379,7 +1490,7 @@ The `else` part is optional. If not provided, the else block implicitly evaluates into an :class:`Undefined` object (regardless of what ``undefined`` in the environment is set to): -.. sourcecode:: jinja +.. code-block:: jinja {{ "[{}]".format(page.title) if page.title }} @@ -1389,7 +1500,7 @@ in the environment is set to): Python Methods ~~~~~~~~~~~~~~ -You can also use any of the methods of defined on a variable's type. +You can also use any of the methods defined on a variable's type. The value returned from the method invocation is used as the value of the expression. Here is an example that uses methods defined on strings (where ``page.title`` is a string): @@ -1425,6 +1536,8 @@ is a bit contrived in the context of rendering a template): List of Builtin Filters ----------------------- +.. py:currentmodule:: jinja-filters + .. jinja:filters:: jinja2.defaults.DEFAULT_FILTERS @@ -1433,6 +1546,8 @@ List of Builtin Filters List of Builtin Tests --------------------- +.. py:currentmodule:: jinja-tests + .. jinja:tests:: jinja2.defaults.DEFAULT_TESTS @@ -1443,6 +1558,8 @@ List of Global Functions The following functions are available in the global scope by default: +.. py:currentmodule:: jinja-globals + .. function:: range([start,] stop[, step]) Return a list containing an arithmetic progression of integers. @@ -1504,8 +1621,7 @@ The following functions are available in the global scope by default: .. versionadded:: 2.1 - .. method:: current - :property: + .. property:: current Return the current item. Equivalent to the item that will be returned next time :meth:`next` is called. @@ -1562,10 +1678,15 @@ The following functions are available in the global scope by default: .. versionadded:: 2.10 + .. versionchanged:: 3.2 + Namespace attributes can be assigned to in multiple assignment. + Extensions ---------- +.. py:currentmodule:: jinja2 + The following sections cover the built-in Jinja extensions that may be enabled by an application. An application could also provide further extensions not covered by this documentation; in which case there should @@ -1647,11 +1768,35 @@ to disable it for a block. .. versionadded:: 2.10 The ``trimmed`` and ``notrimmed`` modifiers have been added. +If the translation depends on the context that the message appears in, +the ``pgettext`` and ``npgettext`` functions take a ``context`` string +as the first argument, which is used to select the appropriate +translation. To specify a context with the ``{% trans %}`` tag, provide +a string as the first token after ``trans``. + +.. code-block:: jinja + + {% trans "fruit" %}apple{% endtrans %} + {% trans "fruit" trimmed count -%} + 1 apple + {%- pluralize -%} + {{ count }} apples + {%- endtrans %} + +.. versionadded:: 3.1 + A context can be passed to the ``trans`` tag to use ``pgettext`` and + ``npgettext``. + It's possible to translate strings in expressions with these functions: -- ``gettext``: translate a single string -- ``ngettext``: translate a pluralizable string -- ``_``: alias for ``gettext`` +- ``_(message)``: Alias for ``gettext``. +- ``gettext(message)``: Translate a message. +- ``ngettext(singular, plural, n)``: Translate a singular or plural + message based on a count variable. +- ``pgettext(context, message)``: Like ``gettext()``, but picks the + translation based on the context string. +- ``npgettext(context, singular, plural, n)``: Like ``npgettext()``, + but picks the translation based on the context string. You can print a translated string like this: diff --git a/docs/tricks.rst b/docs/tricks.rst index 78ac40862..3a7084a6d 100644 --- a/docs/tricks.rst +++ b/docs/tricks.rst @@ -7,10 +7,10 @@ This part of the documentation shows some tips and tricks for Jinja templates. -.. _null-master-fallback: +.. _null-default-fallback: -Null-Master Fallback --------------------- +Null-Default Fallback +--------------------- Jinja supports dynamic inheritance and does not distinguish between parent and child template as long as no `extends` tag is visited. While this leads @@ -21,12 +21,12 @@ for a neat trick. Usually child templates extend from one template that adds a basic HTML skeleton. However it's possible to put the `extends` tag into an `if` tag to only extend from the layout template if the `standalone` variable evaluates -to false which it does per default if it's not defined. Additionally a very +to false, which it does by default if it's not defined. Additionally a very basic skeleton is added to the file so that if it's indeed rendered with `standalone` set to `True` a very basic HTML skeleton is added:: - {% if not standalone %}{% extends 'master.html' %}{% endif -%} - + {% if not standalone %}{% extends 'default.html' %}{% endif -%} + {% block title %}The Page Title{% endblock %} {% block body %} @@ -46,7 +46,7 @@ list you can use the `cycle` method on the `loop` object:: {% endfor %} -`cycle` can take an unlimited amount of strings. Each time this +`cycle` can take an unlimited number of strings. Each time this tag is encountered the next item from the list is rendered. @@ -74,8 +74,8 @@ sense to define a default for that variable:: ... ... diff --git a/examples/basic/cycle.py b/examples/basic/cycle.py index 25dcb0b09..1f97e3794 100644 --- a/examples/basic/cycle.py +++ b/examples/basic/cycle.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment env = Environment( diff --git a/examples/basic/debugger.py b/examples/basic/debugger.py index d3c1a60a7..f6a962708 100644 --- a/examples/basic/debugger.py +++ b/examples/basic/debugger.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment from jinja2.loaders import FileSystemLoader diff --git a/examples/basic/inheritance.py b/examples/basic/inheritance.py index 4a881bf8a..6d928df9e 100644 --- a/examples/basic/inheritance.py +++ b/examples/basic/inheritance.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment from jinja2.loaders import DictLoader diff --git a/examples/basic/test.py b/examples/basic/test.py index 80b9d1f05..30f5dd6b3 100644 --- a/examples/basic/test.py +++ b/examples/basic/test.py @@ -1,27 +1,25 @@ -from __future__ import print_function - from jinja2 import Environment from jinja2.loaders import DictLoader env = Environment( loader=DictLoader( { - "child.html": u"""\ -{% extends master_layout or 'master.html' %} -{% include helpers = 'helpers.html' %} + "child.html": """\ +{% extends default_layout or 'default.html' %} +{% import 'helpers.html' as helpers %} {% macro get_the_answer() %}42{% endmacro %} -{% title = 'Hello World' %} +{% set title = 'Hello World' %} {% block body %} {{ get_the_answer() }} {{ helpers.conspirate() }} {% endblock %} """, - "master.html": u"""\ + "default.html": """\ {{ title }} {% block body %}{% endblock %} """, - "helpers.html": u"""\ + "helpers.html": """\ {% macro conspirate() %}23{% endmacro %} """, } diff --git a/examples/basic/test_filter_and_linestatements.py b/examples/basic/test_filter_and_linestatements.py index 673b67ed7..9bbcbcaff 100644 --- a/examples/basic/test_filter_and_linestatements.py +++ b/examples/basic/test_filter_and_linestatements.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment env = Environment( diff --git a/examples/basic/test_loop_filter.py b/examples/basic/test_loop_filter.py index 39be08d61..6bd89fde0 100644 --- a/examples/basic/test_loop_filter.py +++ b/examples/basic/test_loop_filter.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment tmpl = Environment().from_string( diff --git a/examples/basic/translate.py b/examples/basic/translate.py index 71547f464..e6596817c 100644 --- a/examples/basic/translate.py +++ b/examples/basic/translate.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from jinja2 import Environment env = Environment(extensions=["jinja2.ext.i18n"]) @@ -7,7 +5,7 @@ env.globals["ngettext"] = lambda s, p, n: { "%(count)s user": "%(count)d Benutzer", "%(count)s users": "%(count)d Benutzer", -}[n == 1 and s or p] +}[s if n == 1 else p] print( env.from_string( """\ diff --git a/ext/Vim/jinja.vim b/ext/Vim/jinja.vim deleted file mode 100644 index e2a5bbf6c..000000000 --- a/ext/Vim/jinja.vim +++ /dev/null @@ -1,138 +0,0 @@ -" Vim syntax file -" Language: Jinja template -" Maintainer: Armin Ronacher -" Last Change: 2008 May 9 -" Version: 1.1 -" -" Known Bugs: -" because of odd limitations dicts and the modulo operator -" appear wrong in the template. -" -" Changes: -" -" 2008 May 9: Added support for Jinja 2 changes (new keyword rules) - -" .vimrc variable to disable html highlighting -if !exists('g:jinja_syntax_html') - let g:jinja_syntax_html=1 -endif - -" For version 5.x: Clear all syntax items -" For version 6.x: Quit when a syntax file was already loaded -if !exists("main_syntax") - if v:version < 600 - syntax clear - elseif exists("b:current_syntax") - finish - endif - let main_syntax = 'jinja' -endif - -" Pull in the HTML syntax. -if g:jinja_syntax_html - if v:version < 600 - so :p:h/html.vim - else - runtime! syntax/html.vim - unlet b:current_syntax - endif -endif - -syntax case match - -" Jinja template built-in tags and parameters (without filter, macro, is and raw, they -" have special threatment) -syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained and if else in not or recursive as import - -syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained is filter skipwhite nextgroup=jinjaFilter -syn keyword jinjaStatement containedin=jinjaTagBlock contained macro skipwhite nextgroup=jinjaFunction -syn keyword jinjaStatement containedin=jinjaTagBlock contained block skipwhite nextgroup=jinjaBlockName - -" Variable Names -syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[a-zA-Z_][a-zA-Z0-9_]*/ -syn keyword jinjaSpecial containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained false true none False True None loop super caller varargs kwargs - -" Filters -syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite nextgroup=jinjaFilter -syn match jinjaFilter contained /[a-zA-Z_][a-zA-Z0-9_]*/ -syn match jinjaFunction contained /[a-zA-Z_][a-zA-Z0-9_]*/ -syn match jinjaBlockName contained /[a-zA-Z_][a-zA-Z0-9_]*/ - -" Jinja template constants -syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/"/ skip=/\(\\\)\@\)*\\"/ end=/"/ -syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/'/ skip=/\(\\\)\@\)*\\'/ end=/'/ -syn match jinjaNumber containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[0-9]\+\(\.[0-9]\+\)\?/ - -" Operators -syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[+\-*\/<>=!,:]/ -syn match jinjaPunctuation containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[()\[\]]/ -syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /\./ nextgroup=jinjaAttribute -syn match jinjaAttribute contained /[a-zA-Z_][a-zA-Z0-9_]*/ - -" Jinja template tag and variable blocks -syn region jinjaNested matchgroup=jinjaOperator start="(" end=")" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained -syn region jinjaNested matchgroup=jinjaOperator start="\[" end="\]" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained -syn region jinjaNested matchgroup=jinjaOperator start="{" end="}" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained -syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment - -syn region jinjaVarBlock matchgroup=jinjaVarDelim start=/{{-\?/ end=/-\?}}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment - -" Jinja template 'raw' tag -syn region jinjaRaw matchgroup=jinjaRawDelim start="{%\s*raw\s*%}" end="{%\s*endraw\s*%}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment - -" Jinja comments -syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment -" help support folding for some setups -setlocal commentstring={#%s#} -setlocal comments=s:{#,e:#} - -" Block start keywords. A bit tricker. We only highlight at the start of a -" tag block and only if the name is not followed by a comma or equals sign -" which usually means that we have to deal with an assignment. -syn match jinjaStatement containedin=jinjaTagBlock contained /\({%-\?\s*\)\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>\(\s*[,=]\)\@!/ - -" and context modifiers -syn match jinjaStatement containedin=jinjaTagBlock contained /\/ - - -" Define the default highlighting. -" For version 5.7 and earlier: only when not done already -" For version 5.8 and later: only when an item doesn't have highlighting yet -if v:version >= 508 || !exists("did_jinja_syn_inits") - if v:version < 508 - let did_jinja_syn_inits = 1 - command -nargs=+ HiLink hi link - else - command -nargs=+ HiLink hi def link - endif - - HiLink jinjaPunctuation jinjaOperator - HiLink jinjaAttribute jinjaVariable - HiLink jinjaFunction jinjaFilter - - HiLink jinjaTagDelim jinjaTagBlock - HiLink jinjaVarDelim jinjaVarBlock - HiLink jinjaCommentDelim jinjaComment - HiLink jinjaRawDelim jinja - - HiLink jinjaSpecial Special - HiLink jinjaOperator Normal - HiLink jinjaRaw Normal - HiLink jinjaTagBlock PreProc - HiLink jinjaVarBlock PreProc - HiLink jinjaStatement Statement - HiLink jinjaFilter Function - HiLink jinjaBlockName Function - HiLink jinjaVariable Identifier - HiLink jinjaString Constant - HiLink jinjaNumber Constant - HiLink jinjaComment Comment - - delcommand HiLink -endif - -let b:current_syntax = "jinja" - -if main_syntax ==# 'jinja' - unlet main_syntax -endif diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..a7cfa7f93 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,102 @@ +[project] +name = "Jinja2" +description = "A very fast and expressive template engine." +readme = "README.md" +license = {file = "LICENSE.txt"} +maintainers = [{name = "Pallets", email = "contact@palletsprojects.com"}] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Text Processing :: Markup :: HTML", + "Typing :: Typed", +] +requires-python = ">=3.7" +dependencies = ["MarkupSafe>=2.0"] +dynamic = ["version"] + +[project.urls] +Donate = "https://palletsprojects.com/donate" +Documentation = "https://jinja.palletsprojects.com/" +Changes = "https://jinja.palletsprojects.com/changes/" +Source = "https://github.com/pallets/jinja/" +Chat = "https://discord.gg/pallets" + +[project.optional-dependencies] +i18n = ["Babel>=2.7"] + +[project.entry-points."babel.extractors"] +jinja2 = "jinja2.ext:babel_extract[i18n]" + +[build-system] +requires = ["flit_core<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "jinja2" + +[tool.flit.sdist] +include = [ + "docs/", + "requirements/", + "tests/", + "CHANGES.md", + "tox.ini", +] +exclude = [ + "docs/_build/", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +filterwarnings = [ + "error", +] + +[tool.coverage.run] +branch = true +source = ["jinja2", "tests"] + +[tool.coverage.paths] +source = ["src", "*/site-packages"] + +[tool.mypy] +python_version = "3.8" +files = ["src/jinja2"] +show_error_codes = true +pretty = true +strict = true + +[tool.pyright] +pythonVersion = "3.8" +include = ["src/jinja2"] +typeCheckingMode = "basic" + +[tool.ruff] +src = ["src"] +fix = true +show-fixes = true +output-format = "full" + +[tool.ruff.lint] +select = [ + "B", # flake8-bugbear + "E", # pycodestyle error + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "W", # pycodestyle warning +] + +[tool.ruff.lint.isort] +force-single-line = true +order-by-type = false + +[tool.gha-update] +tag-only = [ + "slsa-framework/slsa-github-generator", +] diff --git a/requirements/build.in b/requirements/build.in new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/requirements/build.in @@ -0,0 +1 @@ +build diff --git a/requirements/build.txt b/requirements/build.txt new file mode 100644 index 000000000..9d6dd1040 --- /dev/null +++ b/requirements/build.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile build.in +# +build==1.2.2.post1 + # via -r build.in +packaging==24.2 + # via build +pyproject-hooks==1.2.0 + # via build diff --git a/requirements/dev.in b/requirements/dev.in new file mode 100644 index 000000000..99f5942f8 --- /dev/null +++ b/requirements/dev.in @@ -0,0 +1,6 @@ +-r docs.in +-r tests.in +-r typing.in +pip-compile-multi +pre-commit +tox diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 000000000..c90a78168 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,151 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile dev.in +# +alabaster==1.0.0 + # via sphinx +attrs==24.3.0 + # via + # outcome + # trio +babel==2.16.0 + # via sphinx +build==1.2.2.post1 + # via pip-tools +cachetools==5.5.0 + # via tox +certifi==2024.12.14 + # via requests +cfgv==3.4.0 + # via pre-commit +chardet==5.2.0 + # via tox +charset-normalizer==3.4.0 + # via requests +click==8.1.7 + # via + # pip-compile-multi + # pip-tools +colorama==0.4.6 + # via tox +distlib==0.3.9 + # via virtualenv +docutils==0.21.2 + # via sphinx +filelock==3.16.1 + # via + # tox + # virtualenv +identify==2.6.3 + # via pre-commit +idna==3.10 + # via + # requests + # trio +imagesize==1.4.1 + # via sphinx +iniconfig==2.0.0 + # via pytest +jinja2==3.1.4 + # via sphinx +markupsafe==3.0.2 + # via jinja2 +mypy==1.14.0 + # via -r /Users/david/Projects/jinja/requirements/typing.in +mypy-extensions==1.0.0 + # via mypy +nodeenv==1.9.1 + # via pre-commit +outcome==1.3.0.post0 + # via trio +packaging==24.2 + # via + # build + # pallets-sphinx-themes + # pyproject-api + # pytest + # sphinx + # tox +pallets-sphinx-themes==2.3.0 + # via -r /Users/david/Projects/jinja/requirements/docs.in +pip-compile-multi==2.7.1 + # via -r dev.in +pip-tools==7.4.1 + # via pip-compile-multi +platformdirs==4.3.6 + # via + # tox + # virtualenv +pluggy==1.5.0 + # via + # pytest + # tox +pre-commit==4.0.1 + # via -r dev.in +pygments==2.18.0 + # via sphinx +pyproject-api==1.8.0 + # via tox +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pytest==8.3.4 + # via -r /Users/david/Projects/jinja/requirements/tests.in +pyyaml==6.0.2 + # via pre-commit +requests==2.32.3 + # via sphinx +sniffio==1.3.1 + # via trio +snowballstemmer==2.2.0 + # via sphinx +sortedcontainers==2.4.0 + # via trio +sphinx==8.1.3 + # via + # -r /Users/david/Projects/jinja/requirements/docs.in + # pallets-sphinx-themes + # sphinx-issues + # sphinx-notfound-page + # sphinxcontrib-log-cabinet +sphinx-issues==5.0.0 + # via -r /Users/david/Projects/jinja/requirements/docs.in +sphinx-notfound-page==1.0.4 + # via pallets-sphinx-themes +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-log-cabinet==1.0.1 + # via -r /Users/david/Projects/jinja/requirements/docs.in +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +toposort==1.10 + # via pip-compile-multi +tox==4.23.2 + # via -r dev.in +trio==0.27.0 + # via -r /Users/david/Projects/jinja/requirements/tests.in +typing-extensions==4.12.2 + # via mypy +urllib3==2.2.3 + # via requests +virtualenv==20.28.0 + # via + # pre-commit + # tox +wheel==0.45.1 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements/docs.in b/requirements/docs.in new file mode 100644 index 000000000..7ec501b6d --- /dev/null +++ b/requirements/docs.in @@ -0,0 +1,4 @@ +Pallets-Sphinx-Themes +Sphinx +sphinx-issues +sphinxcontrib-log-cabinet diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 000000000..2283fa9b5 --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,63 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile docs.in +# +alabaster==1.0.0 + # via sphinx +babel==2.16.0 + # via sphinx +certifi==2024.12.14 + # via requests +charset-normalizer==3.4.0 + # via requests +docutils==0.21.2 + # via sphinx +idna==3.10 + # via requests +imagesize==1.4.1 + # via sphinx +jinja2==3.1.4 + # via sphinx +markupsafe==3.0.2 + # via jinja2 +packaging==24.2 + # via + # pallets-sphinx-themes + # sphinx +pallets-sphinx-themes==2.3.0 + # via -r docs.in +pygments==2.18.0 + # via sphinx +requests==2.32.3 + # via sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==8.1.3 + # via + # -r docs.in + # pallets-sphinx-themes + # sphinx-issues + # sphinx-notfound-page + # sphinxcontrib-log-cabinet +sphinx-issues==5.0.0 + # via -r docs.in +sphinx-notfound-page==1.0.4 + # via pallets-sphinx-themes +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-log-cabinet==1.0.1 + # via -r docs.in +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +urllib3==2.2.3 + # via requests diff --git a/requirements/tests.in b/requirements/tests.in new file mode 100644 index 000000000..5669c6ecd --- /dev/null +++ b/requirements/tests.in @@ -0,0 +1,2 @@ +pytest +trio diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 000000000..71dad37da --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile tests.in +# +attrs==24.3.0 + # via + # outcome + # trio +idna==3.10 + # via trio +iniconfig==2.0.0 + # via pytest +outcome==1.3.0.post0 + # via trio +packaging==24.2 + # via pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.4 + # via -r tests.in +sniffio==1.3.1 + # via trio +sortedcontainers==2.4.0 + # via trio +trio==0.27.0 + # via -r tests.in diff --git a/requirements/tests37.txt b/requirements/tests37.txt new file mode 100644 index 000000000..e8e3492fe --- /dev/null +++ b/requirements/tests37.txt @@ -0,0 +1,43 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --output-file=tests37.txt tests.in +# +attrs==24.2.0 + # via + # outcome + # trio +exceptiongroup==1.2.2 + # via + # pytest + # trio +idna==3.10 + # via trio +importlib-metadata==6.7.0 + # via + # attrs + # pluggy + # pytest +iniconfig==2.0.0 + # via pytest +outcome==1.3.0.post0 + # via trio +packaging==24.0 + # via pytest +pluggy==1.2.0 + # via pytest +pytest==7.4.4 + # via -r tests.in +sniffio==1.3.1 + # via trio +sortedcontainers==2.4.0 + # via trio +tomli==2.0.1 + # via pytest +trio==0.22.2 + # via -r tests.in +typing-extensions==4.7.1 + # via importlib-metadata +zipp==3.15.0 + # via importlib-metadata diff --git a/requirements/typing.in b/requirements/typing.in new file mode 100644 index 000000000..f0aa93ac8 --- /dev/null +++ b/requirements/typing.in @@ -0,0 +1 @@ +mypy diff --git a/requirements/typing.txt b/requirements/typing.txt new file mode 100644 index 000000000..f50d6d667 --- /dev/null +++ b/requirements/typing.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile typing.in +# +mypy==1.14.0 + # via -r typing.in +mypy-extensions==1.0.0 + # via mypy +typing-extensions==4.12.2 + # via mypy diff --git a/scripts/generate_identifier_pattern.py b/scripts/generate_identifier_pattern.py index 581319981..baf035eae 100755 --- a/scripts/generate_identifier_pattern.py +++ b/scripts/generate_identifier_pattern.py @@ -1,12 +1,8 @@ -#!/usr/bin/env python3 import itertools import os import re import sys -if sys.version_info[0] < 3: - raise RuntimeError("This needs to run on Python 3.") - def get_characters(): """Find every Unicode character that is valid in a Python `identifier`_ but @@ -33,8 +29,8 @@ def collapse_ranges(data): Source: https://stackoverflow.com/a/4629241/400617 """ - for _, b in itertools.groupby(enumerate(data), lambda x: ord(x[1]) - x[0]): - b = list(b) + for _, g in itertools.groupby(enumerate(data), lambda x: ord(x[1]) - x[0]): + b = list(g) yield b[0][1], b[-1][1] @@ -52,7 +48,7 @@ def build_pattern(ranges): out.append(a) out.append(b) else: - out.append("{}-{}".format(a, b)) + out.append(f"{a}-{b}") return "".join(out) @@ -70,7 +66,7 @@ def main(): f.write("import re\n\n") f.write("# generated by scripts/generate_identifier_pattern.py\n") f.write("pattern = re.compile(\n") - f.write(' r"[\\w{}]+" # noqa: B950\n'.format(pattern)) + f.write(f' r"[\\w{pattern}]+" # noqa: B950\n') f.write(")\n") diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2769e96ad..000000000 --- a/setup.cfg +++ /dev/null @@ -1,45 +0,0 @@ -[metadata] -license_file = LICENSE.rst -long_description_content_type = text/x-rst - -[bdist_wheel] -universal = true - -[tool:pytest] -testpaths = tests -filterwarnings = - error - ignore:the sets module:DeprecationWarning:jinja2.sandbox - -[coverage:run] -branch = True -source = - jinja2 - tests - -[coverage:paths] -source = - src - */site-packages - -[flake8] -# B = bugbear -# E = pycodestyle errors -# F = flake8 pyflakes -# W = pycodestyle warnings -# B9 = bugbear opinions -select = B, E, F, W, B9 -ignore = - # slice notation whitespace, invalid - E203 - # line length, handled by bugbear B950 - E501 - # bare except, handled by bugbear B001 - E722 - # bin op line break, invalid - W503 -# up to 88 allowed by bugbear B950 -max-line-length = 80 -per-file-ignores = - # __init__ module exports names - src/jinja2/__init__.py: F401 diff --git a/setup.py b/setup.py deleted file mode 100644 index 7d94cd3ae..000000000 --- a/setup.py +++ /dev/null @@ -1,56 +0,0 @@ -import io -import re - -from setuptools import find_packages -from setuptools import setup - -with io.open("README.rst", "rt", encoding="utf8") as f: - readme = f.read() - -with io.open("src/jinja2/__init__.py", "rt", encoding="utf8") as f: - version = re.search(r'__version__ = "(.*?)"', f.read(), re.M).group(1) - -setup( - name="Jinja2", - version=version, - url="https://palletsprojects.com/p/jinja/", - project_urls={ - "Documentation": "https://jinja.palletsprojects.com/", - "Code": "https://github.com/pallets/jinja", - "Issue tracker": "https://github.com/pallets/jinja/issues", - }, - license="BSD-3-Clause", - author="Armin Ronacher", - author_email="armin.ronacher@active-4.com", - maintainer="Pallets", - maintainer_email="contact@palletsprojects.com", - description="A very fast and expressive template engine.", - long_description=readme, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP :: Dynamic Content", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Text Processing :: Markup :: HTML", - ], - packages=find_packages("src"), - package_dir={"": "src"}, - include_package_data=True, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", - install_requires=["MarkupSafe>=0.23"], - extras_require={"i18n": ["Babel>=0.8"]}, - entry_points={"babel.extractors": ["jinja2 = jinja2.ext:babel_extract[i18n]"]}, -) diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py index f17866f6c..d669f295b 100644 --- a/src/jinja2/__init__.py +++ b/src/jinja2/__init__.py @@ -1,44 +1,38 @@ -# -*- coding: utf-8 -*- """Jinja is a template engine written in pure Python. It provides a non-XML syntax that supports inline expressions and an optional sandboxed environment. """ -from markupsafe import escape -from markupsafe import Markup -from .bccache import BytecodeCache -from .bccache import FileSystemBytecodeCache -from .bccache import MemcachedBytecodeCache -from .environment import Environment -from .environment import Template -from .exceptions import TemplateAssertionError -from .exceptions import TemplateError -from .exceptions import TemplateNotFound -from .exceptions import TemplateRuntimeError -from .exceptions import TemplatesNotFound -from .exceptions import TemplateSyntaxError -from .exceptions import UndefinedError -from .filters import contextfilter -from .filters import environmentfilter -from .filters import evalcontextfilter -from .loaders import BaseLoader -from .loaders import ChoiceLoader -from .loaders import DictLoader -from .loaders import FileSystemLoader -from .loaders import FunctionLoader -from .loaders import ModuleLoader -from .loaders import PackageLoader -from .loaders import PrefixLoader -from .runtime import ChainableUndefined -from .runtime import DebugUndefined -from .runtime import make_logging_undefined -from .runtime import StrictUndefined -from .runtime import Undefined -from .utils import clear_caches -from .utils import contextfunction -from .utils import environmentfunction -from .utils import evalcontextfunction -from .utils import is_undefined -from .utils import select_autoescape +from .bccache import BytecodeCache as BytecodeCache +from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache +from .environment import Environment as Environment +from .environment import Template as Template +from .exceptions import TemplateAssertionError as TemplateAssertionError +from .exceptions import TemplateError as TemplateError +from .exceptions import TemplateNotFound as TemplateNotFound +from .exceptions import TemplateRuntimeError as TemplateRuntimeError +from .exceptions import TemplatesNotFound as TemplatesNotFound +from .exceptions import TemplateSyntaxError as TemplateSyntaxError +from .exceptions import UndefinedError as UndefinedError +from .loaders import BaseLoader as BaseLoader +from .loaders import ChoiceLoader as ChoiceLoader +from .loaders import DictLoader as DictLoader +from .loaders import FileSystemLoader as FileSystemLoader +from .loaders import FunctionLoader as FunctionLoader +from .loaders import ModuleLoader as ModuleLoader +from .loaders import PackageLoader as PackageLoader +from .loaders import PrefixLoader as PrefixLoader +from .runtime import ChainableUndefined as ChainableUndefined +from .runtime import DebugUndefined as DebugUndefined +from .runtime import make_logging_undefined as make_logging_undefined +from .runtime import StrictUndefined as StrictUndefined +from .runtime import Undefined as Undefined +from .utils import clear_caches as clear_caches +from .utils import is_undefined as is_undefined +from .utils import pass_context as pass_context +from .utils import pass_environment as pass_environment +from .utils import pass_eval_context as pass_eval_context +from .utils import select_autoescape as select_autoescape -__version__ = "2.11.3" +__version__ = "3.1.5" diff --git a/src/jinja2/_compat.py b/src/jinja2/_compat.py deleted file mode 100644 index 1f044954a..000000000 --- a/src/jinja2/_compat.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import marshal -import sys - -PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, "pypy_translation_info") -_identity = lambda x: x - -if not PY2: - unichr = chr - range_type = range - text_type = str - string_types = (str,) - integer_types = (int,) - - iterkeys = lambda d: iter(d.keys()) - itervalues = lambda d: iter(d.values()) - iteritems = lambda d: iter(d.items()) - - import pickle - from io import BytesIO, StringIO - - NativeStringIO = StringIO - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - - ifilter = filter - imap = map - izip = zip - intern = sys.intern - - implements_iterator = _identity - implements_to_string = _identity - encode_filename = _identity - - marshal_dump = marshal.dump - marshal_load = marshal.load - -else: - unichr = unichr - text_type = unicode - range_type = xrange - string_types = (str, unicode) - integer_types = (int, long) - - iterkeys = lambda d: d.iterkeys() - itervalues = lambda d: d.itervalues() - iteritems = lambda d: d.iteritems() - - import cPickle as pickle - from cStringIO import StringIO as BytesIO, StringIO - - NativeStringIO = BytesIO - - exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") - - from itertools import imap, izip, ifilter - - intern = intern - - def implements_iterator(cls): - cls.next = cls.__next__ - del cls.__next__ - return cls - - def implements_to_string(cls): - cls.__unicode__ = cls.__str__ - cls.__str__ = lambda x: x.__unicode__().encode("utf-8") - return cls - - def encode_filename(filename): - if isinstance(filename, unicode): - return filename.encode("utf-8") - return filename - - def marshal_dump(code, f): - if isinstance(f, file): - marshal.dump(code, f) - else: - f.write(marshal.dumps(code)) - - def marshal_load(f): - if isinstance(f, file): - return marshal.load(f) - return marshal.loads(f.read()) - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a - # dummy metaclass for one level of class instantiation that replaces - # itself with the actual metaclass. - class metaclass(type): - def __new__(cls, name, this_bases, d): - return meta(name, bases, d) - - return type.__new__(metaclass, "temporary_class", (), {}) - - -try: - from urllib.parse import quote_from_bytes as url_quote -except ImportError: - from urllib import quote as url_quote - - -try: - from collections import abc -except ImportError: - import collections as abc - - -try: - from os import fspath -except ImportError: - try: - from pathlib import PurePath - except ImportError: - PurePath = None - - def fspath(path): - if hasattr(path, "__fspath__"): - return path.__fspath__() - - # Python 3.5 doesn't have __fspath__ yet, use str. - if PurePath is not None and isinstance(path, PurePath): - return str(path) - - return path diff --git a/src/jinja2/_identifier.py b/src/jinja2/_identifier.py index 224d5449d..928c1503c 100644 --- a/src/jinja2/_identifier.py +++ b/src/jinja2/_identifier.py @@ -2,5 +2,5 @@ # generated by scripts/generate_identifier_pattern.py pattern = re.compile( - r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߽߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛࣓-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣ৾ਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣૺ-૿ଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఄా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഀ-ഃ഻഼ാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳷-᳹᷀-᷹᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꣿꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𐴤-𐽆𐴧-𐽐𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑅅𑅆𑅳𑆀-𑆂𑆳-𑇀𑇉-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌻𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑑞𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑠬-𑠺𑨁-𑨊𑨳-𑨹𑨻-𑨾𑩇𑩑-𑩛𑪊-𑪙𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𑴱-𑴶𑴺𑴼𑴽𑴿-𑵅𑵇𑶊-𑶎𑶐𑶑𑶓-𑶗𑻳-𑻶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 ) diff --git a/src/jinja2/async_utils.py b/src/jinja2/async_utils.py new file mode 100644 index 000000000..f0c140205 --- /dev/null +++ b/src/jinja2/async_utils.py @@ -0,0 +1,99 @@ +import inspect +import typing as t +from functools import WRAPPER_ASSIGNMENTS +from functools import wraps + +from .utils import _PassArg +from .utils import pass_eval_context + +if t.TYPE_CHECKING: + import typing_extensions as te + +V = t.TypeVar("V") + + +def async_variant(normal_func): # type: ignore + def decorator(async_func): # type: ignore + pass_arg = _PassArg.from_obj(normal_func) + need_eval_context = pass_arg is None + + if pass_arg is _PassArg.environment: + + def is_async(args: t.Any) -> bool: + return t.cast(bool, args[0].is_async) + + else: + + def is_async(args: t.Any) -> bool: + return t.cast(bool, args[0].environment.is_async) + + # Take the doc and annotations from the sync function, but the + # name from the async function. Pallets-Sphinx-Themes + # build_function_directive expects __wrapped__ to point to the + # sync function. + async_func_attrs = ("__module__", "__name__", "__qualname__") + normal_func_attrs = tuple(set(WRAPPER_ASSIGNMENTS).difference(async_func_attrs)) + + @wraps(normal_func, assigned=normal_func_attrs) + @wraps(async_func, assigned=async_func_attrs, updated=()) + def wrapper(*args, **kwargs): # type: ignore + b = is_async(args) + + if need_eval_context: + args = args[1:] + + if b: + return async_func(*args, **kwargs) + + return normal_func(*args, **kwargs) + + if need_eval_context: + wrapper = pass_eval_context(wrapper) + + wrapper.jinja_async_variant = True # type: ignore[attr-defined] + return wrapper + + return decorator + + +_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)} + + +async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": + # Avoid a costly call to isawaitable + if type(value) in _common_primitives: + return t.cast("V", value) + + if inspect.isawaitable(value): + return await t.cast("t.Awaitable[V]", value) + + return value + + +class _IteratorToAsyncIterator(t.Generic[V]): + def __init__(self, iterator: "t.Iterator[V]"): + self._iterator = iterator + + def __aiter__(self) -> "te.Self": + return self + + async def __anext__(self) -> V: + try: + return next(self._iterator) + except StopIteration as e: + raise StopAsyncIteration(e.value) from e + + +def auto_aiter( + iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", +) -> "t.AsyncIterator[V]": + if hasattr(iterable, "__aiter__"): + return iterable.__aiter__() + else: + return _IteratorToAsyncIterator(iter(iterable)) + + +async def auto_to_list( + value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", +) -> t.List["V"]: + return [x async for x in auto_aiter(value)] diff --git a/src/jinja2/asyncfilters.py b/src/jinja2/asyncfilters.py deleted file mode 100644 index 3d98dbcc0..000000000 --- a/src/jinja2/asyncfilters.py +++ /dev/null @@ -1,158 +0,0 @@ -from functools import wraps - -from . import filters -from .asyncsupport import auto_aiter -from .asyncsupport import auto_await - - -async def auto_to_seq(value): - seq = [] - if hasattr(value, "__aiter__"): - async for item in value: - seq.append(item) - else: - for item in value: - seq.append(item) - return seq - - -async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): - seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) - if seq: - async for item in auto_aiter(seq): - if func(item): - yield item - - -def dualfilter(normal_filter, async_filter): - wrap_evalctx = False - if getattr(normal_filter, "environmentfilter", False) is True: - - def is_async(args): - return args[0].is_async - - wrap_evalctx = False - else: - has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True - has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True - wrap_evalctx = not has_evalctxfilter and not has_ctxfilter - - def is_async(args): - return args[0].environment.is_async - - @wraps(normal_filter) - def wrapper(*args, **kwargs): - b = is_async(args) - if wrap_evalctx: - args = args[1:] - if b: - return async_filter(*args, **kwargs) - return normal_filter(*args, **kwargs) - - if wrap_evalctx: - wrapper.evalcontextfilter = True - - wrapper.asyncfiltervariant = True - - return wrapper - - -def asyncfiltervariant(original): - def decorator(f): - return dualfilter(original, f) - - return decorator - - -@asyncfiltervariant(filters.do_first) -async def do_first(environment, seq): - try: - return await auto_aiter(seq).__anext__() - except StopAsyncIteration: - return environment.undefined("No first item, sequence was empty.") - - -@asyncfiltervariant(filters.do_groupby) -async def do_groupby(environment, value, attribute): - expr = filters.make_attrgetter(environment, attribute) - return [ - filters._GroupTuple(key, await auto_to_seq(values)) - for key, values in filters.groupby( - sorted(await auto_to_seq(value), key=expr), expr - ) - ] - - -@asyncfiltervariant(filters.do_join) -async def do_join(eval_ctx, value, d=u"", attribute=None): - return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) - - -@asyncfiltervariant(filters.do_list) -async def do_list(value): - return await auto_to_seq(value) - - -@asyncfiltervariant(filters.do_reject) -async def do_reject(*args, **kwargs): - return async_select_or_reject(args, kwargs, lambda x: not x, False) - - -@asyncfiltervariant(filters.do_rejectattr) -async def do_rejectattr(*args, **kwargs): - return async_select_or_reject(args, kwargs, lambda x: not x, True) - - -@asyncfiltervariant(filters.do_select) -async def do_select(*args, **kwargs): - return async_select_or_reject(args, kwargs, lambda x: x, False) - - -@asyncfiltervariant(filters.do_selectattr) -async def do_selectattr(*args, **kwargs): - return async_select_or_reject(args, kwargs, lambda x: x, True) - - -@asyncfiltervariant(filters.do_map) -async def do_map(*args, **kwargs): - seq, func = filters.prepare_map(args, kwargs) - if seq: - async for item in auto_aiter(seq): - yield await auto_await(func(item)) - - -@asyncfiltervariant(filters.do_sum) -async def do_sum(environment, iterable, attribute=None, start=0): - rv = start - if attribute is not None: - func = filters.make_attrgetter(environment, attribute) - else: - - def func(x): - return x - - async for item in auto_aiter(iterable): - rv += func(item) - return rv - - -@asyncfiltervariant(filters.do_slice) -async def do_slice(value, slices, fill_with=None): - return filters.do_slice(await auto_to_seq(value), slices, fill_with) - - -ASYNC_FILTERS = { - "first": do_first, - "groupby": do_groupby, - "join": do_join, - "list": do_list, - # we intentionally do not support do_last because that would be - # ridiculous - "reject": do_reject, - "rejectattr": do_rejectattr, - "map": do_map, - "select": do_select, - "selectattr": do_selectattr, - "sum": do_sum, - "slice": do_slice, -} diff --git a/src/jinja2/asyncsupport.py b/src/jinja2/asyncsupport.py deleted file mode 100644 index 78ba3739d..000000000 --- a/src/jinja2/asyncsupport.py +++ /dev/null @@ -1,264 +0,0 @@ -# -*- coding: utf-8 -*- -"""The code for async support. Importing this patches Jinja on supported -Python versions. -""" -import asyncio -import inspect -from functools import update_wrapper - -from markupsafe import Markup - -from .environment import TemplateModule -from .runtime import LoopContext -from .utils import concat -from .utils import internalcode -from .utils import missing - - -async def concat_async(async_gen): - rv = [] - - async def collect(): - async for event in async_gen: - rv.append(event) - - await collect() - return concat(rv) - - -async def generate_async(self, *args, **kwargs): - vars = dict(*args, **kwargs) - try: - async for event in self.root_render_func(self.new_context(vars)): - yield event - except Exception: - yield self.environment.handle_exception() - - -def wrap_generate_func(original_generate): - def _convert_generator(self, loop, args, kwargs): - async_gen = self.generate_async(*args, **kwargs) - try: - while 1: - yield loop.run_until_complete(async_gen.__anext__()) - except StopAsyncIteration: - pass - - def generate(self, *args, **kwargs): - if not self.environment.is_async: - return original_generate(self, *args, **kwargs) - return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) - - return update_wrapper(generate, original_generate) - - -async def render_async(self, *args, **kwargs): - if not self.environment.is_async: - raise RuntimeError("The environment was not created with async mode enabled.") - - vars = dict(*args, **kwargs) - ctx = self.new_context(vars) - - try: - return await concat_async(self.root_render_func(ctx)) - except Exception: - return self.environment.handle_exception() - - -def wrap_render_func(original_render): - def render(self, *args, **kwargs): - if not self.environment.is_async: - return original_render(self, *args, **kwargs) - loop = asyncio.get_event_loop() - return loop.run_until_complete(self.render_async(*args, **kwargs)) - - return update_wrapper(render, original_render) - - -def wrap_block_reference_call(original_call): - @internalcode - async def async_call(self): - rv = await concat_async(self._stack[self._depth](self._context)) - if self._context.eval_ctx.autoescape: - rv = Markup(rv) - return rv - - @internalcode - def __call__(self): - if not self._context.environment.is_async: - return original_call(self) - return async_call(self) - - return update_wrapper(__call__, original_call) - - -def wrap_macro_invoke(original_invoke): - @internalcode - async def async_invoke(self, arguments, autoescape): - rv = await self._func(*arguments) - if autoescape: - rv = Markup(rv) - return rv - - @internalcode - def _invoke(self, arguments, autoescape): - if not self._environment.is_async: - return original_invoke(self, arguments, autoescape) - return async_invoke(self, arguments, autoescape) - - return update_wrapper(_invoke, original_invoke) - - -@internalcode -async def get_default_module_async(self): - if self._module is not None: - return self._module - self._module = rv = await self.make_module_async() - return rv - - -def wrap_default_module(original_default_module): - @internalcode - def _get_default_module(self): - if self.environment.is_async: - raise RuntimeError("Template module attribute is unavailable in async mode") - return original_default_module(self) - - return _get_default_module - - -async def make_module_async(self, vars=None, shared=False, locals=None): - context = self.new_context(vars, shared, locals) - body_stream = [] - async for item in self.root_render_func(context): - body_stream.append(item) - return TemplateModule(self, context, body_stream) - - -def patch_template(): - from . import Template - - Template.generate = wrap_generate_func(Template.generate) - Template.generate_async = update_wrapper(generate_async, Template.generate_async) - Template.render_async = update_wrapper(render_async, Template.render_async) - Template.render = wrap_render_func(Template.render) - Template._get_default_module = wrap_default_module(Template._get_default_module) - Template._get_default_module_async = get_default_module_async - Template.make_module_async = update_wrapper( - make_module_async, Template.make_module_async - ) - - -def patch_runtime(): - from .runtime import BlockReference, Macro - - BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__) - Macro._invoke = wrap_macro_invoke(Macro._invoke) - - -def patch_filters(): - from .filters import FILTERS - from .asyncfilters import ASYNC_FILTERS - - FILTERS.update(ASYNC_FILTERS) - - -def patch_all(): - patch_template() - patch_runtime() - patch_filters() - - -async def auto_await(value): - if inspect.isawaitable(value): - return await value - return value - - -async def auto_aiter(iterable): - if hasattr(iterable, "__aiter__"): - async for item in iterable: - yield item - return - for item in iterable: - yield item - - -class AsyncLoopContext(LoopContext): - _to_iterator = staticmethod(auto_aiter) - - @property - async def length(self): - if self._length is not None: - return self._length - - try: - self._length = len(self._iterable) - except TypeError: - iterable = [x async for x in self._iterator] - self._iterator = self._to_iterator(iterable) - self._length = len(iterable) + self.index + (self._after is not missing) - - return self._length - - @property - async def revindex0(self): - return await self.length - self.index - - @property - async def revindex(self): - return await self.length - self.index0 - - async def _peek_next(self): - if self._after is not missing: - return self._after - - try: - self._after = await self._iterator.__anext__() - except StopAsyncIteration: - self._after = missing - - return self._after - - @property - async def last(self): - return await self._peek_next() is missing - - @property - async def nextitem(self): - rv = await self._peek_next() - - if rv is missing: - return self._undefined("there is no next item") - - return rv - - def __aiter__(self): - return self - - async def __anext__(self): - if self._after is not missing: - rv = self._after - self._after = missing - else: - rv = await self._iterator.__anext__() - - self.index0 += 1 - self._before = self._current - self._current = rv - return rv, self - - -async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): - import warnings - - warnings.warn( - "This template must be recompiled with at least Jinja 2.11, or" - " it will fail in 3.0.", - DeprecationWarning, - stacklevel=2, - ) - return AsyncLoopContext(iterable, undefined, recurse, depth0) - - -patch_all() diff --git a/src/jinja2/bccache.py b/src/jinja2/bccache.py index 9c0661030..ada8b099f 100644 --- a/src/jinja2/bccache.py +++ b/src/jinja2/bccache.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The optional bytecode cache system. This is useful if you have very complex template situations and the compilation of all those templates slows down your application too much. @@ -6,24 +5,34 @@ Situations where this is useful are often forking web applications that are initialized on the first request. """ + import errno import fnmatch +import marshal import os +import pickle import stat import sys import tempfile +import typing as t from hashlib import sha1 -from os import listdir -from os import path +from io import BytesIO +from types import CodeType + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + + class _MemcachedClient(te.Protocol): + def get(self, key: str) -> bytes: ... + + def set( + self, key: str, value: bytes, timeout: t.Optional[int] = None + ) -> None: ... -from ._compat import BytesIO -from ._compat import marshal_dump -from ._compat import marshal_load -from ._compat import pickle -from ._compat import text_type -from .utils import open_if_exists -bc_version = 4 +bc_version = 5 # Magic bytes to identify Jinja bytecode cache files. Contains the # Python major and minor version to avoid loading incompatible bytecode # if a project upgrades its Python version. @@ -34,7 +43,7 @@ ) -class Bucket(object): +class Bucket: """Buckets are used to store the bytecode for one template. It's created and initialized by the bytecode cache and passed to the loading functions. @@ -43,17 +52,17 @@ class Bucket(object): cache subclasses don't have to care about cache invalidation. """ - def __init__(self, environment, key, checksum): + def __init__(self, environment: "Environment", key: str, checksum: str) -> None: self.environment = environment self.key = key self.checksum = checksum self.reset() - def reset(self): + def reset(self) -> None: """Resets the bucket (unloads the bytecode).""" - self.code = None + self.code: t.Optional[CodeType] = None - def load_bytecode(self, f): + def load_bytecode(self, f: t.BinaryIO) -> None: """Loads bytecode from a file or file like object.""" # make sure the magic header is correct magic = f.read(len(bc_magic)) @@ -67,31 +76,31 @@ def load_bytecode(self, f): return # if marshal_load fails then we need to reload try: - self.code = marshal_load(f) + self.code = marshal.load(f) except (EOFError, ValueError, TypeError): self.reset() return - def write_bytecode(self, f): + def write_bytecode(self, f: t.IO[bytes]) -> None: """Dump the bytecode into the file or file like object passed.""" if self.code is None: raise TypeError("can't write empty bucket") f.write(bc_magic) pickle.dump(self.checksum, f, 2) - marshal_dump(self.code, f) + marshal.dump(self.code, f) - def bytecode_from_string(self, string): - """Load bytecode from a string.""" + def bytecode_from_string(self, string: bytes) -> None: + """Load bytecode from bytes.""" self.load_bytecode(BytesIO(string)) - def bytecode_to_string(self): - """Return the bytecode as string.""" + def bytecode_to_string(self) -> bytes: + """Return the bytecode as bytes.""" out = BytesIO() self.write_bytecode(out) return out.getvalue() -class BytecodeCache(object): +class BytecodeCache: """To implement your own bytecode cache you have to subclass this class and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of these methods are passed a :class:`~jinja2.bccache.Bucket`. @@ -120,41 +129,48 @@ def dump_bytecode(self, bucket): Jinja. """ - def load_bytecode(self, bucket): + def load_bytecode(self, bucket: Bucket) -> None: """Subclasses have to override this method to load bytecode into a bucket. If they are not able to find code in the cache for the bucket, it must not do anything. """ raise NotImplementedError() - def dump_bytecode(self, bucket): + def dump_bytecode(self, bucket: Bucket) -> None: """Subclasses have to override this method to write the bytecode from a bucket back to the cache. If it unable to do so it must not fail silently but raise an exception. """ raise NotImplementedError() - def clear(self): + def clear(self) -> None: """Clears the cache. This method is not used by Jinja but should be implemented to allow applications to clear the bytecode cache used by a particular environment. """ - def get_cache_key(self, name, filename=None): + def get_cache_key( + self, name: str, filename: t.Optional[t.Union[str]] = None + ) -> str: """Returns the unique hash key for this template name.""" hash = sha1(name.encode("utf-8")) + if filename is not None: - filename = "|" + filename - if isinstance(filename, text_type): - filename = filename.encode("utf-8") - hash.update(filename) + hash.update(f"|{filename}".encode()) + return hash.hexdigest() - def get_source_checksum(self, source): + def get_source_checksum(self, source: str) -> str: """Returns a checksum for the source.""" return sha1(source.encode("utf-8")).hexdigest() - def get_bucket(self, environment, name, filename, source): + def get_bucket( + self, + environment: "Environment", + name: str, + filename: t.Optional[str], + source: str, + ) -> Bucket: """Return a cache bucket for the given template. All arguments are mandatory but filename may be `None`. """ @@ -164,7 +180,7 @@ def get_bucket(self, environment, name, filename, source): self.load_bytecode(bucket) return bucket - def set_bucket(self, bucket): + def set_bucket(self, bucket: Bucket) -> None: """Put the bucket into the cache.""" self.dump_bytecode(bucket) @@ -187,14 +203,16 @@ class FileSystemBytecodeCache(BytecodeCache): This bytecode cache supports clearing of the cache using the clear method. """ - def __init__(self, directory=None, pattern="__jinja2_%s.cache"): + def __init__( + self, directory: t.Optional[str] = None, pattern: str = "__jinja2_%s.cache" + ) -> None: if directory is None: directory = self._get_default_cache_dir() self.directory = directory self.pattern = pattern - def _get_default_cache_dir(self): - def _unsafe_dir(): + def _get_default_cache_dir(self) -> str: + def _unsafe_dir() -> "te.NoReturn": raise RuntimeError( "Cannot determine safe temp directory. You " "need to explicitly provide one." @@ -209,7 +227,7 @@ def _unsafe_dir(): if not hasattr(os, "getuid"): _unsafe_dir() - dirname = "_jinja2-cache-%d" % os.getuid() + dirname = f"_jinja2-cache-{os.getuid()}" actual_dir = os.path.join(tmpdir, dirname) try: @@ -240,34 +258,72 @@ def _unsafe_dir(): return actual_dir - def _get_cache_filename(self, bucket): - return path.join(self.directory, self.pattern % bucket.key) + def _get_cache_filename(self, bucket: Bucket) -> str: + return os.path.join(self.directory, self.pattern % (bucket.key,)) - def load_bytecode(self, bucket): - f = open_if_exists(self._get_cache_filename(bucket), "rb") - if f is not None: + def load_bytecode(self, bucket: Bucket) -> None: + filename = self._get_cache_filename(bucket) + + # Don't test for existence before opening the file, since the + # file could disappear after the test before the open. + try: + f = open(filename, "rb") + except (FileNotFoundError, IsADirectoryError, PermissionError): + # PermissionError can occur on Windows when an operation is + # in progress, such as calling clear(). + return + + with f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket: Bucket) -> None: + # Write to a temporary file, then rename to the real name after + # writing. This avoids another process reading the file before + # it is fully written. + name = self._get_cache_filename(bucket) + f = tempfile.NamedTemporaryFile( + mode="wb", + dir=os.path.dirname(name), + prefix=os.path.basename(name), + suffix=".tmp", + delete=False, + ) + + def remove_silent() -> None: try: - bucket.load_bytecode(f) - finally: - f.close() + os.remove(f.name) + except OSError: + # Another process may have called clear(). On Windows, + # another program may be holding the file open. + pass - def dump_bytecode(self, bucket): - f = open(self._get_cache_filename(bucket), "wb") try: - bucket.write_bytecode(f) - finally: - f.close() + with f: + bucket.write_bytecode(f) + except BaseException: + remove_silent() + raise - def clear(self): + try: + os.replace(f.name, name) + except OSError: + # Another process may have called clear(). On Windows, + # another program may be holding the file open. + remove_silent() + except BaseException: + remove_silent() + raise + + def clear(self) -> None: # imported lazily here because google app-engine doesn't support # write access on the file system and the function does not exist # normally. from os import remove - files = fnmatch.filter(listdir(self.directory), self.pattern % "*") + files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",)) for filename in files: try: - remove(path.join(self.directory, filename)) + remove(os.path.join(self.directory, filename)) except OSError: pass @@ -284,7 +340,7 @@ class MemcachedBytecodeCache(BytecodeCache): - `python-memcached `_ (Unfortunately the django cache interface is not compatible because it - does not support storing binary data, only unicode. You can however pass + does not support storing binary data, only text. You can however pass the underlying cache client to the bytecode cache which is available as `django.core.cache.cache._client`.) @@ -319,32 +375,34 @@ class MemcachedBytecodeCache(BytecodeCache): def __init__( self, - client, - prefix="jinja2/bytecode/", - timeout=None, - ignore_memcache_errors=True, + client: "_MemcachedClient", + prefix: str = "jinja2/bytecode/", + timeout: t.Optional[int] = None, + ignore_memcache_errors: bool = True, ): self.client = client self.prefix = prefix self.timeout = timeout self.ignore_memcache_errors = ignore_memcache_errors - def load_bytecode(self, bucket): + def load_bytecode(self, bucket: Bucket) -> None: try: code = self.client.get(self.prefix + bucket.key) except Exception: if not self.ignore_memcache_errors: raise - code = None - if code is not None: + else: bucket.bytecode_from_string(code) - def dump_bytecode(self, bucket): - args = (self.prefix + bucket.key, bucket.bytecode_to_string()) - if self.timeout is not None: - args += (self.timeout,) + def dump_bytecode(self, bucket: Bucket) -> None: + key = self.prefix + bucket.key + value = bucket.bytecode_to_string() + try: - self.client.set(*args) + if self.timeout is not None: + self.client.set(key, value, self.timeout) + else: + self.client.set(key, value) except Exception: if not self.ignore_memcache_errors: raise diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 63297b42c..a4ff6a1b1 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1,7 +1,9 @@ -# -*- coding: utf-8 -*- """Compiles nodes from the parser into Python code.""" -from collections import namedtuple + +import typing as t +from contextlib import contextmanager from functools import update_wrapper +from io import StringIO from itertools import chain from keyword import iskeyword as is_python_keyword @@ -9,13 +11,6 @@ from markupsafe import Markup from . import nodes -from ._compat import imap -from ._compat import iteritems -from ._compat import izip -from ._compat import NativeStringIO -from ._compat import range_type -from ._compat import string_types -from ._compat import text_type from .exceptions import TemplateAssertionError from .idtracking import Symbols from .idtracking import VAR_LOAD_ALIAS @@ -24,9 +19,17 @@ from .idtracking import VAR_LOAD_UNDEFINED from .nodes import EvalContext from .optimizer import Optimizer +from .utils import _PassArg from .utils import concat from .visitor import NodeVisitor +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) + operators = { "eq": "==", "ne": "!=", @@ -38,79 +41,107 @@ "notin": "not in", } -# what method to iterate over items do we want to use for dict iteration -# in generated code? on 2.x let's go with iteritems, on 3.x with items -if hasattr(dict, "iteritems"): - dict_item_iter = "iteritems" -else: - dict_item_iter = "items" - -code_features = ["division"] - -# does this python version support generator stops? (PEP 0479) -try: - exec("from __future__ import generator_stop") - code_features.append("generator_stop") -except SyntaxError: - pass - -# does this python version support yield from? -try: - exec("def f(): yield from x()") -except SyntaxError: - supports_yield_from = False -else: - supports_yield_from = True - - -def optimizeconst(f): - def new_func(self, node, frame, **kwargs): + +def optimizeconst(f: F) -> F: + def new_func( + self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any + ) -> t.Any: # Only optimize if the frame is not volatile - if self.optimized and not frame.eval_ctx.volatile: + if self.optimizer is not None and not frame.eval_ctx.volatile: new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) - return update_wrapper(new_func, f) + return update_wrapper(new_func, f) # type: ignore[return-value] + + +def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: + @optimizeconst + def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None: + if ( + self.environment.sandboxed and op in self.environment.intercepted_binops # type: ignore + ): + self.write(f"environment.call_binop(context, {op!r}, ") + self.visit(node.left, frame) + self.write(", ") + self.visit(node.right, frame) + else: + self.write("(") + self.visit(node.left, frame) + self.write(f" {op} ") + self.visit(node.right, frame) + + self.write(")") + + return visitor + + +def _make_unop( + op: str, +) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]: + @optimizeconst + def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None: + if ( + self.environment.sandboxed and op in self.environment.intercepted_unops # type: ignore + ): + self.write(f"environment.call_unop(context, {op!r}, ") + self.visit(node.node, frame) + else: + self.write("(" + op) + self.visit(node.node, frame) + + self.write(")") + + return visitor def generate( - node, environment, name, filename, stream=None, defer_init=False, optimized=True -): + node: nodes.Template, + environment: "Environment", + name: t.Optional[str], + filename: t.Optional[str], + stream: t.Optional[t.TextIO] = None, + defer_init: bool = False, + optimized: bool = True, +) -> t.Optional[str]: """Generate the python source for a node tree.""" if not isinstance(node, nodes.Template): raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( environment, name, filename, stream, defer_init, optimized ) generator.visit(node) + if stream is None: - return generator.stream.getvalue() + return generator.stream.getvalue() # type: ignore + return None -def has_safe_repr(value): + +def has_safe_repr(value: t.Any) -> bool: """Does the node have a safe representation?""" if value is None or value is NotImplemented or value is Ellipsis: return True - if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: - return True - if type(value) in (tuple, list, set, frozenset): - for item in value: - if not has_safe_repr(item): - return False - return True - elif type(value) is dict: - for key, value in iteritems(value): - if not has_safe_repr(key): - return False - if not has_safe_repr(value): - return False + + if type(value) in {bool, int, float, complex, range, str, Markup}: return True + + if type(value) in {tuple, list, set, frozenset}: + return all(has_safe_repr(v) for v in value) + + if type(value) is dict: # noqa E721 + return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) + return False -def find_undeclared(nodes, names): +def find_undeclared( + nodes: t.Iterable[nodes.Node], names: t.Iterable[str] +) -> t.Set[str]: """Check if the names passed are accessed undeclared. The return value is a set of all the undeclared names from the sequence of names found. """ @@ -123,20 +154,49 @@ def find_undeclared(nodes, names): return visitor.undeclared -class MacroRef(object): - def __init__(self, node): +class MacroRef: + def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None: self.node = node self.accesses_caller = False self.accesses_kwargs = False self.accesses_varargs = False -class Frame(object): +class Frame: """Holds compile time information for us.""" - def __init__(self, eval_ctx, parent=None, level=None): + def __init__( + self, + eval_ctx: EvalContext, + parent: t.Optional["Frame"] = None, + level: t.Optional[int] = None, + ) -> None: self.eval_ctx = eval_ctx - self.symbols = Symbols(parent and parent.symbols or None, level=level) + + # the parent of this frame + self.parent = parent + + if parent is None: + self.symbols = Symbols(level=level) + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = False + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer: t.Optional[str] = None + + # the name of the block we're in, otherwise None. + self.block: t.Optional[str] = None + + else: + self.symbols = Symbols(parent.symbols, level=level) + self.require_output_check = parent.require_output_check + self.buffer = parent.buffer + self.block = parent.block # a toplevel frame is the root + soft frames such as if conditions. self.toplevel = False @@ -146,47 +206,40 @@ def __init__(self, eval_ctx, parent=None, level=None): # situations. self.rootlevel = False - # in some dynamic inheritance situations the compiler needs to add - # write tests around output statements. - self.require_output_check = parent and parent.require_output_check - - # inside some tags we are using a buffer rather than yield statements. - # this for example affects {% filter %} or {% macro %}. If a frame - # is buffered this variable points to the name of the list used as - # buffer. - self.buffer = None + # variables set inside of loops and blocks should not affect outer frames, + # but they still needs to be kept track of as part of the active context. + self.loop_frame = False + self.block_frame = False - # the name of the block we're in, otherwise None. - self.block = parent and parent.block or None + # track whether the frame is being used in an if-statement or conditional + # expression as it determines which errors should be raised during runtime + # or compile time. + self.soft_frame = False - # the parent of this frame - self.parent = parent - - if parent is not None: - self.buffer = parent.buffer - - def copy(self): + def copy(self) -> "te.Self": """Create a copy of the current one.""" rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) rv.symbols = self.symbols.copy() return rv - def inner(self, isolated=False): + def inner(self, isolated: bool = False) -> "Frame": """Return an inner frame.""" if isolated: return Frame(self.eval_ctx, level=self.symbols.level + 1) return Frame(self.eval_ctx, self) - def soft(self): + def soft(self) -> "te.Self": """Return a soft frame. A soft frame may not be modified as standalone thing as it shares the resources with the frame it was created of, but it's not a rootlevel frame any longer. - This is only used to implement if-statements. + This is only used to implement if-statements and conditional + expressions. """ rv = self.copy() rv.rootlevel = False + rv.soft_frame = True return rv __copy__ = copy @@ -199,19 +252,19 @@ class VisitorExit(RuntimeError): class DependencyFinderVisitor(NodeVisitor): """A visitor that collects filter and test calls.""" - def __init__(self): - self.filters = set() - self.tests = set() + def __init__(self) -> None: + self.filters: t.Set[str] = set() + self.tests: t.Set[str] = set() - def visit_Filter(self, node): + def visit_Filter(self, node: nodes.Filter) -> None: self.generic_visit(node) self.filters.add(node.name) - def visit_Test(self, node): + def visit_Test(self, node: nodes.Test) -> None: self.generic_visit(node) self.tests.add(node.name) - def visit_Block(self, node): + def visit_Block(self, node: nodes.Block) -> None: """Stop visiting at blocks.""" @@ -221,11 +274,11 @@ class UndeclaredNameVisitor(NodeVisitor): not stop at closure frames. """ - def __init__(self, names): + def __init__(self, names: t.Iterable[str]) -> None: self.names = set(names) - self.undeclared = set() + self.undeclared: t.Set[str] = set() - def visit_Name(self, node): + def visit_Name(self, node: nodes.Name) -> None: if node.ctx == "load" and node.name in self.names: self.undeclared.add(node.name) if self.undeclared == self.names: @@ -233,7 +286,7 @@ def visit_Name(self, node): else: self.names.discard(node.name) - def visit_Block(self, node): + def visit_Block(self, node: nodes.Block) -> None: """Stop visiting a blocks.""" @@ -246,26 +299,33 @@ class CompilerExit(Exception): class CodeGenerator(NodeVisitor): def __init__( - self, environment, name, filename, stream=None, defer_init=False, optimized=True - ): + self, + environment: "Environment", + name: t.Optional[str], + filename: t.Optional[str], + stream: t.Optional[t.TextIO] = None, + defer_init: bool = False, + optimized: bool = True, + ) -> None: if stream is None: - stream = NativeStringIO() + stream = StringIO() self.environment = environment self.name = name self.filename = filename self.stream = stream self.created_block_context = False self.defer_init = defer_init - self.optimized = optimized + self.optimizer: t.Optional[Optimizer] = None + if optimized: self.optimizer = Optimizer(environment) # aliases for imports - self.import_aliases = {} + self.import_aliases: t.Dict[str, str] = {} # a registry for all blocks. Because blocks are moved out # into the global python scope they are registered here - self.blocks = {} + self.blocks: t.Dict[str, nodes.Block] = {} # the number of extends statements so far self.extends_so_far = 0 @@ -279,12 +339,12 @@ def __init__( self.code_lineno = 1 # registry of all filters and tests (global, not block local) - self.tests = {} - self.filters = {} + self.tests: t.Dict[str, str] = {} + self.filters: t.Dict[str, str] = {} # the debug information - self.debug_info = [] - self._write_debug_info = None + self.debug_info: t.List[t.Tuple[int, int]] = [] + self._write_debug_info: t.Optional[int] = None # the number of new lines before the next write() self._new_lines = 0 @@ -303,75 +363,83 @@ def __init__( self._indentation = 0 # Tracks toplevel assignments - self._assign_stack = [] + self._assign_stack: t.List[t.Set[str]] = [] # Tracks parameter definition blocks - self._param_def_block = [] + self._param_def_block: t.List[t.Set[str]] = [] # Tracks the current context. self._context_reference_stack = ["context"] + @property + def optimized(self) -> bool: + return self.optimizer is not None + # -- Various compilation helpers - def fail(self, msg, lineno): + def fail(self, msg: str, lineno: int) -> "te.NoReturn": """Fail with a :exc:`TemplateAssertionError`.""" raise TemplateAssertionError(msg, lineno, self.name, self.filename) - def temporary_identifier(self): + def temporary_identifier(self) -> str: """Get a new unique identifier.""" self._last_identifier += 1 - return "t_%d" % self._last_identifier + return f"t_{self._last_identifier}" - def buffer(self, frame): + def buffer(self, frame: Frame) -> None: """Enable buffering for the frame from that point onwards.""" frame.buffer = self.temporary_identifier() - self.writeline("%s = []" % frame.buffer) + self.writeline(f"{frame.buffer} = []") - def return_buffer_contents(self, frame, force_unescaped=False): + def return_buffer_contents( + self, frame: Frame, force_unescaped: bool = False + ) -> None: """Return the buffer contents of the frame.""" if not force_unescaped: if frame.eval_ctx.volatile: self.writeline("if context.eval_ctx.autoescape:") self.indent() - self.writeline("return Markup(concat(%s))" % frame.buffer) + self.writeline(f"return Markup(concat({frame.buffer}))") self.outdent() self.writeline("else:") self.indent() - self.writeline("return concat(%s)" % frame.buffer) + self.writeline(f"return concat({frame.buffer})") self.outdent() return elif frame.eval_ctx.autoescape: - self.writeline("return Markup(concat(%s))" % frame.buffer) + self.writeline(f"return Markup(concat({frame.buffer}))") return - self.writeline("return concat(%s)" % frame.buffer) + self.writeline(f"return concat({frame.buffer})") - def indent(self): + def indent(self) -> None: """Indent by one.""" self._indentation += 1 - def outdent(self, step=1): + def outdent(self, step: int = 1) -> None: """Outdent by step.""" self._indentation -= step - def start_write(self, frame, node=None): + def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None: """Yield or write into the frame buffer.""" if frame.buffer is None: self.writeline("yield ", node) else: - self.writeline("%s.append(" % frame.buffer, node) + self.writeline(f"{frame.buffer}.append(", node) - def end_write(self, frame): + def end_write(self, frame: Frame) -> None: """End the writing process started by `start_write`.""" if frame.buffer is not None: self.write(")") - def simple_write(self, s, frame, node=None): + def simple_write( + self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None + ) -> None: """Simple shortcut for start_write + write + end_write.""" self.start_write(frame, node) self.write(s) self.end_write(frame) - def blockvisit(self, nodes, frame): + def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None: """Visit a list of nodes as block in a frame. If the current frame is no buffer a dummy ``if 0: yield None`` is written automatically. """ @@ -382,7 +450,7 @@ def blockvisit(self, nodes, frame): except CompilerExit: pass - def write(self, x): + def write(self, x: str) -> None: """Write a string into the output stream.""" if self._new_lines: if not self._first_write: @@ -396,19 +464,26 @@ def write(self, x): self._new_lines = 0 self.stream.write(x) - def writeline(self, x, node=None, extra=0): + def writeline( + self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0 + ) -> None: """Combination of newline and write.""" self.newline(node, extra) self.write(x) - def newline(self, node=None, extra=0): + def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None: """Add one or more newlines before the next write.""" self._new_lines = max(self._new_lines, 1 + extra) if node is not None and node.lineno != self._last_line: self._write_debug_info = node.lineno self._last_line = node.lineno - def signature(self, node, frame, extra_kwargs=None): + def signature( + self, + node: t.Union[nodes.Call, nodes.Filter, nodes.Test], + frame: Frame, + extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + ) -> None: """Writes a function call to the stream for the current node. A leading comma is added automatically. The extra keyword arguments may not include python keywords otherwise a syntax @@ -417,11 +492,10 @@ def signature(self, node, frame, extra_kwargs=None): """ # if any of the given keyword arguments is a python keyword # we have to make sure that no invalid call is created. - kwarg_workaround = False - for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): - if is_python_keyword(kwarg): - kwarg_workaround = True - break + kwarg_workaround = any( + is_python_keyword(t.cast(str, k)) + for k in chain((x.key for x in node.kwargs), extra_kwargs or ()) + ) for arg in node.args: self.write(", ") @@ -432,8 +506,8 @@ def signature(self, node, frame, extra_kwargs=None): self.write(", ") self.visit(kwarg, frame) if extra_kwargs is not None: - for key, value in iteritems(extra_kwargs): - self.write(", %s=%s" % (key, value)) + for key, value in extra_kwargs.items(): + self.write(f", {key}={value}") if node.dyn_args: self.write(", *") self.visit(node.dyn_args, frame) @@ -444,12 +518,12 @@ def signature(self, node, frame, extra_kwargs=None): else: self.write(", **{") for kwarg in node.kwargs: - self.write("%r: " % kwarg.key) + self.write(f"{kwarg.key!r}: ") self.visit(kwarg.value, frame) self.write(", ") if extra_kwargs is not None: - for key, value in iteritems(extra_kwargs): - self.write("%r: %s, " % (key, value)) + for key, value in extra_kwargs.items(): + self.write(f"{key!r}: {value}, ") if node.dyn_kwargs is not None: self.write("}, **") self.visit(node.dyn_kwargs, frame) @@ -461,50 +535,85 @@ def signature(self, node, frame, extra_kwargs=None): self.write(", **") self.visit(node.dyn_kwargs, frame) - def pull_dependencies(self, nodes): - """Pull all the dependencies.""" + def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None: + """Find all filter and test names used in the template and + assign them to variables in the compiled namespace. Checking + that the names are registered with the environment is done when + compiling the Filter and Test nodes. If the node is in an If or + CondExpr node, the check is done at runtime instead. + + .. versionchanged:: 3.0 + Filters and tests in If and CondExpr nodes are checked at + runtime instead of compile time. + """ visitor = DependencyFinderVisitor() + for node in nodes: visitor.visit(node) - for dependency in "filters", "tests": - mapping = getattr(self, dependency) - for name in getattr(visitor, dependency): - if name not in mapping: - mapping[name] = self.temporary_identifier() + + for id_map, names, dependency in ( + (self.filters, visitor.filters, "filters"), + ( + self.tests, + visitor.tests, + "tests", + ), + ): + for name in sorted(names): + if name not in id_map: + id_map[name] = self.temporary_identifier() + + # add check during runtime that dependencies used inside of executed + # blocks are defined, as this step may be skipped during compile time + self.writeline("try:") + self.indent() + self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]") + self.outdent() + self.writeline("except KeyError:") + self.indent() + self.writeline("@internalcode") + self.writeline(f"def {id_map[name]}(*unused):") + self.indent() self.writeline( - "%s = environment.%s[%r]" % (mapping[name], dependency, name) + f'raise TemplateRuntimeError("No {dependency[:-1]}' + f' named {name!r} found.")' ) + self.outdent() + self.outdent() - def enter_frame(self, frame): + def enter_frame(self, frame: Frame) -> None: undefs = [] - for target, (action, param) in iteritems(frame.symbols.loads): + for target, (action, param) in frame.symbols.loads.items(): if action == VAR_LOAD_PARAMETER: pass elif action == VAR_LOAD_RESOLVE: - self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param)) + self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") elif action == VAR_LOAD_ALIAS: - self.writeline("%s = %s" % (target, param)) + self.writeline(f"{target} = {param}") elif action == VAR_LOAD_UNDEFINED: undefs.append(target) else: raise NotImplementedError("unknown load instruction") if undefs: - self.writeline("%s = missing" % " = ".join(undefs)) + self.writeline(f"{' = '.join(undefs)} = missing") - def leave_frame(self, frame, with_python_scope=False): + def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None: if not with_python_scope: undefs = [] - for target, _ in iteritems(frame.symbols.loads): + for target in frame.symbols.loads: undefs.append(target) if undefs: - self.writeline("%s = missing" % " = ".join(undefs)) + self.writeline(f"{' = '.join(undefs)} = missing") - def func(self, name): - if self.environment.is_async: - return "async def %s" % name - return "def %s" % name + def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str: + return async_value if self.environment.is_async else sync_value - def macro_body(self, node, frame): + def func(self, name: str) -> str: + return f"{self.choose_async()}def {name}" + + def macro_body( + self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame + ) -> t.Tuple[Frame, MacroRef]: """Dump the function def of a macro or call block.""" frame = frame.inner() frame.symbols.analyze_node(node) @@ -513,6 +622,7 @@ def macro_body(self, node, frame): explicit_caller = None skip_special_params = set() args = [] + for idx, arg in enumerate(node.args): if arg.name == "caller": explicit_caller = idx @@ -552,7 +662,7 @@ def macro_body(self, node, frame): # macros are delayed, they never require output checks frame.require_output_check = False frame.symbols.analyze_node(node) - self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node) + self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) self.indent() self.buffer(frame) @@ -561,17 +671,17 @@ def macro_body(self, node, frame): self.push_parameter_definitions(frame) for idx, arg in enumerate(node.args): ref = frame.symbols.ref(arg.name) - self.writeline("if %s is missing:" % ref) + self.writeline(f"if {ref} is missing:") self.indent() try: default = node.defaults[idx - len(node.args)] except IndexError: self.writeline( - "%s = undefined(%r, name=%r)" - % (ref, "parameter %r was not provided" % arg.name, arg.name) + f'{ref} = undefined("parameter {arg.name!r} was not provided",' + f" name={arg.name!r})" ) else: - self.writeline("%s = " % ref) + self.writeline(f"{ref} = ") self.visit(default, frame) self.mark_parameter_stored(ref) self.outdent() @@ -584,50 +694,46 @@ def macro_body(self, node, frame): return frame, macro_ref - def macro_def(self, macro_ref, frame): + def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None: """Dump the macro definition for the def created by macro_body.""" arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) name = getattr(macro_ref.node, "name", None) if len(macro_ref.node.args) == 1: arg_tuple += "," self.write( - "Macro(environment, macro, %r, (%s), %r, %r, %r, " - "context.eval_ctx.autoescape)" - % ( - name, - arg_tuple, - macro_ref.accesses_kwargs, - macro_ref.accesses_varargs, - macro_ref.accesses_caller, - ) + f"Macro(environment, macro, {name!r}, ({arg_tuple})," + f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," + f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" ) - def position(self, node): + def position(self, node: nodes.Node) -> str: """Return a human readable position for the node.""" - rv = "line %d" % node.lineno + rv = f"line {node.lineno}" if self.name is not None: - rv += " in " + repr(self.name) + rv = f"{rv} in {self.name!r}" return rv - def dump_local_context(self, frame): - return "{%s}" % ", ".join( - "%r: %s" % (name, target) - for name, target in iteritems(frame.symbols.dump_stores()) + def dump_local_context(self, frame: Frame) -> str: + items_kv = ", ".join( + f"{name!r}: {target}" + for name, target in frame.symbols.dump_stores().items() ) + return f"{{{items_kv}}}" - def write_commons(self): + def write_commons(self) -> None: """Writes a common preamble that is used by root and block functions. Primarily this sets up common local helpers and enforces a generator through a dead branch. """ self.writeline("resolve = context.resolve_or_missing") self.writeline("undefined = environment.undefined") + self.writeline("concat = environment.concat") # always use the standard Undefined class for the implicit else of # conditional expressions self.writeline("cond_expr_undefined = Undefined") self.writeline("if 0: yield None") - def push_parameter_definitions(self, frame): + def push_parameter_definitions(self, frame: Frame) -> None: """Pushes all parameter targets from the given frame into a local stack that permits tracking of yet to be assigned parameters. In particular this enables the optimization from `visit_Name` to skip @@ -636,97 +742,109 @@ def push_parameter_definitions(self, frame): """ self._param_def_block.append(frame.symbols.dump_param_targets()) - def pop_parameter_definitions(self): + def pop_parameter_definitions(self) -> None: """Pops the current parameter definitions set.""" self._param_def_block.pop() - def mark_parameter_stored(self, target): + def mark_parameter_stored(self, target: str) -> None: """Marks a parameter in the current parameter definitions as stored. This will skip the enforced undefined checks. """ if self._param_def_block: self._param_def_block[-1].discard(target) - def push_context_reference(self, target): + def push_context_reference(self, target: str) -> None: self._context_reference_stack.append(target) - def pop_context_reference(self): + def pop_context_reference(self) -> None: self._context_reference_stack.pop() - def get_context_ref(self): + def get_context_ref(self) -> str: return self._context_reference_stack[-1] - def get_resolve_func(self): + def get_resolve_func(self) -> str: target = self._context_reference_stack[-1] if target == "context": return "resolve" - return "%s.resolve" % target + return f"{target}.resolve" - def derive_context(self, frame): - return "%s.derived(%s)" % ( - self.get_context_ref(), - self.dump_local_context(frame), - ) + def derive_context(self, frame: Frame) -> str: + return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" - def parameter_is_undeclared(self, target): + def parameter_is_undeclared(self, target: str) -> bool: """Checks if a given target is an undeclared parameter.""" if not self._param_def_block: return False return target in self._param_def_block[-1] - def push_assign_tracking(self): + def push_assign_tracking(self) -> None: """Pushes a new layer for assignment tracking.""" self._assign_stack.append(set()) - def pop_assign_tracking(self, frame): + def pop_assign_tracking(self, frame: Frame) -> None: """Pops the topmost level for assignment tracking and updates the context variables if necessary. """ vars = self._assign_stack.pop() - if not frame.toplevel or not vars: + if ( + not frame.block_frame + and not frame.loop_frame + and not frame.toplevel + or not vars + ): return public_names = [x for x in vars if x[:1] != "_"] if len(vars) == 1: name = next(iter(vars)) ref = frame.symbols.ref(name) - self.writeline("context.vars[%r] = %s" % (name, ref)) + if frame.loop_frame: + self.writeline(f"_loop_vars[{name!r}] = {ref}") + return + if frame.block_frame: + self.writeline(f"_block_vars[{name!r}] = {ref}") + return + self.writeline(f"context.vars[{name!r}] = {ref}") else: - self.writeline("context.vars.update({") - for idx, name in enumerate(vars): + if frame.loop_frame: + self.writeline("_loop_vars.update({") + elif frame.block_frame: + self.writeline("_block_vars.update({") + else: + self.writeline("context.vars.update({") + for idx, name in enumerate(sorted(vars)): if idx: self.write(", ") ref = frame.symbols.ref(name) - self.write("%r: %s" % (name, ref)) + self.write(f"{name!r}: {ref}") self.write("})") - if public_names: + if not frame.block_frame and not frame.loop_frame and public_names: if len(public_names) == 1: - self.writeline("context.exported_vars.add(%r)" % public_names[0]) + self.writeline(f"context.exported_vars.add({public_names[0]!r})") else: - self.writeline( - "context.exported_vars.update((%s))" - % ", ".join(imap(repr, public_names)) - ) + names_str = ", ".join(map(repr, sorted(public_names))) + self.writeline(f"context.exported_vars.update(({names_str}))") # -- Statement Visitors - def visit_Template(self, node, frame=None): + def visit_Template( + self, node: nodes.Template, frame: t.Optional[Frame] = None + ) -> None: assert frame is None, "no root frame allowed" eval_ctx = EvalContext(self.environment, self.name) + from .runtime import async_exported from .runtime import exported - self.writeline("from __future__ import %s" % ", ".join(code_features)) - self.writeline("from jinja2.runtime import " + ", ".join(exported)) - if self.environment.is_async: - self.writeline( - "from jinja2.asyncsupport import auto_await, " - "auto_aiter, AsyncLoopContext" - ) + exported_names = sorted(exported + async_exported) + else: + exported_names = sorted(exported) + + self.writeline("from jinja2.runtime import " + ", ".join(exported_names)) # if we want a deferred initialization we cannot move the # environment into a local name - envenv = not self.defer_init and ", environment=environment" or "" + envenv = "" if self.defer_init else ", environment=environment" # do we have an extends tag at all? If not, we can save some # overhead by just not processing any inheritance code. @@ -735,7 +853,7 @@ def visit_Template(self, node, frame=None): # find all blocks for block in node.find_all(nodes.Block): if block.name in self.blocks: - self.fail("block %r defined twice" % block.name, block.lineno) + self.fail(f"block {block.name!r} defined twice", block.lineno) self.blocks[block.name] = block # find all imports and import them @@ -745,16 +863,16 @@ def visit_Template(self, node, frame=None): self.import_aliases[imp] = alias = self.temporary_identifier() if "." in imp: module, obj = imp.rsplit(".", 1) - self.writeline("from %s import %s as %s" % (module, obj, alias)) + self.writeline(f"from {module} import {obj} as {alias}") else: - self.writeline("import %s as %s" % (imp, alias)) + self.writeline(f"import {imp} as {alias}") # add the load name - self.writeline("name = %r" % self.name) + self.writeline(f"name = {self.name!r}") # generate the root render function. self.writeline( - "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1 + f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 ) self.indent() self.write_commons() @@ -763,7 +881,7 @@ def visit_Template(self, node, frame=None): frame = Frame(eval_ctx) if "self" in find_undeclared(node.body, ("self",)): ref = frame.symbols.declare_parameter("self") - self.writeline("%s = TemplateReference(context)" % ref) + self.writeline(f"{ref} = TemplateReference(context)") frame.symbols.analyze_node(node) frame.toplevel = frame.rootlevel = True frame.require_output_check = have_extends and not self.has_known_extends @@ -781,24 +899,24 @@ def visit_Template(self, node, frame=None): self.indent() self.writeline("if parent_template is not None:") self.indent() - if supports_yield_from and not self.environment.is_async: + if not self.environment.is_async: self.writeline("yield from parent_template.root_render_func(context)") else: - self.writeline( - "%sfor event in parent_template." - "root_render_func(context):" - % (self.environment.is_async and "async " or "") - ) + self.writeline("agen = parent_template.root_render_func(context)") + self.writeline("try:") + self.indent() + self.writeline("async for event in agen:") self.indent() self.writeline("yield event") self.outdent() + self.outdent() + self.writeline("finally: await agen.aclose()") self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. - for name, block in iteritems(self.blocks): + for name, block in self.blocks.items(): self.writeline( - "%s(context, missing=missing%s):" - % (self.func("block_" + name), envenv), + f"{self.func('block_' + name)}(context, missing=missing{envenv}):", block, 1, ) @@ -808,32 +926,29 @@ def visit_Template(self, node, frame=None): # toplevel template. This would cause a variety of # interesting issues with identifier tracking. block_frame = Frame(eval_ctx) + block_frame.block_frame = True undeclared = find_undeclared(block.body, ("self", "super")) if "self" in undeclared: ref = block_frame.symbols.declare_parameter("self") - self.writeline("%s = TemplateReference(context)" % ref) + self.writeline(f"{ref} = TemplateReference(context)") if "super" in undeclared: ref = block_frame.symbols.declare_parameter("super") - self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name)) + self.writeline(f"{ref} = context.super({name!r}, block_{name})") block_frame.symbols.analyze_node(block) block_frame.block = name + self.writeline("_block_vars = {}") self.enter_frame(block_frame) self.pull_dependencies(block.body) self.blockvisit(block.body, block_frame) self.leave_frame(block_frame, with_python_scope=True) self.outdent() - self.writeline( - "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks), - extra=1, - ) - - # add a function that returns the debug info - self.writeline( - "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info) - ) + blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) + self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) + debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) + self.writeline(f"debug_info = {debug_kv_str!r}") - def visit_Block(self, node, frame): + def visit_Block(self, node: nodes.Block, frame: Frame) -> None: """Call a block and register it for the template.""" level = 0 if frame.toplevel: @@ -851,27 +966,38 @@ def visit_Block(self, node, frame): else: context = self.get_context_ref() - if ( - supports_yield_from - and not self.environment.is_async - and frame.buffer is None - ): + if node.required: + self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node) + self.indent() self.writeline( - "yield from context.blocks[%r][0](%s)" % (node.name, context), node + f'raise TemplateRuntimeError("Required block {node.name!r} not found")', + node, + ) + self.outdent() + + if not self.environment.is_async and frame.buffer is None: + self.writeline( + f"yield from context.blocks[{node.name!r}][0]({context})", node ) else: - loop = self.environment.is_async and "async for" or "for" + self.writeline(f"gen = context.blocks[{node.name!r}][0]({context})") + self.writeline("try:") + self.indent() self.writeline( - "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context), + f"{self.choose_async()}for event in gen:", node, ) self.indent() self.simple_write("event", frame) self.outdent() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" + ) self.outdent(level) - def visit_Extends(self, node, frame): + def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None: """Calls the extender.""" if not frame.toplevel: self.fail("cannot use extend from a non top-level scope", node.lineno) @@ -880,7 +1006,6 @@ def visit_Extends(self, node, frame): # far, we don't have to add a check if something extended # the template before this one. if self.extends_so_far > 0: - # if we have a known extends we just add a template runtime # error into the generated code. We could catch that at compile # time too, but i welcome it not to confuse users by throwing the @@ -888,7 +1013,7 @@ def visit_Extends(self, node, frame): if not self.has_known_extends: self.writeline("if parent_template is not None:") self.indent() - self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times") + self.writeline('raise TemplateRuntimeError("extended multiple times")') # if we have a known extends already we don't need that code here # as we know that the template execution will end here. @@ -899,10 +1024,8 @@ def visit_Extends(self, node, frame): self.writeline("parent_template = environment.get_template(", node) self.visit(node.template, frame) - self.write(", %r)" % self.name) - self.writeline( - "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter - ) + self.write(f", {self.name!r})") + self.writeline("for name, parent_block in parent_template.blocks.items():") self.indent() self.writeline("context.blocks.setdefault(name, []).append(parent_block)") self.outdent() @@ -916,7 +1039,7 @@ def visit_Extends(self, node, frame): # and now we have one more self.extends_so_far += 1 - def visit_Include(self, node, frame): + def visit_Include(self, node: nodes.Include, frame: Frame) -> None: """Handles includes.""" if node.ignore_missing: self.writeline("try:") @@ -924,16 +1047,16 @@ def visit_Include(self, node, frame): func_name = "get_or_select_template" if isinstance(node.template, nodes.Const): - if isinstance(node.template.value, string_types): + if isinstance(node.template.value, str): func_name = "get_template" elif isinstance(node.template.value, (tuple, list)): func_name = "select_template" elif isinstance(node.template, (nodes.Tuple, nodes.List)): func_name = "select_template" - self.writeline("template = environment.%s(" % func_name, node) + self.writeline(f"template = environment.{func_name}(", node) self.visit(node.template, frame) - self.write(", %r)" % self.name) + self.write(f", {self.name!r})") if node.ignore_missing: self.outdent() self.writeline("except TemplateNotFound:") @@ -943,84 +1066,68 @@ def visit_Include(self, node, frame): self.writeline("else:") self.indent() - skip_event_yield = False + def loop_body() -> None: + self.indent() + self.simple_write("event", frame) + self.outdent() + if node.with_context: - loop = self.environment.is_async and "async for" or "for" self.writeline( - "%s event in template.root_render_func(" - "template.new_context(context.get_all(), True, " - "%s)):" % (loop, self.dump_local_context(frame)) + f"gen = template.root_render_func(" + "template.new_context(context.get_all(), True," + f" {self.dump_local_context(frame)}))" + ) + self.writeline("try:") + self.indent() + self.writeline(f"{self.choose_async()}for event in gen:") + loop_body() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" ) elif self.environment.is_async: self.writeline( - "for event in (await " - "template._get_default_module_async())" + "for event in (await template._get_default_module_async())" "._body_stream:" ) + loop_body() else: - if supports_yield_from: - self.writeline("yield from template._get_default_module()._body_stream") - skip_event_yield = True - else: - self.writeline( - "for event in template._get_default_module()._body_stream:" - ) - - if not skip_event_yield: - self.indent() - self.simple_write("event", frame) - self.outdent() + self.writeline("yield from template._get_default_module()._body_stream") if node.ignore_missing: self.outdent() - def visit_Import(self, node, frame): - """Visit regular imports.""" - self.writeline("%s = " % frame.symbols.ref(node.target), node) - if frame.toplevel: - self.write("context.vars[%r] = " % node.target) - if self.environment.is_async: - self.write("await ") - self.write("environment.get_template(") + def _import_common( + self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame + ) -> None: + self.write(f"{self.choose_async('await ')}environment.get_template(") self.visit(node.template, frame) - self.write(", %r)." % self.name) + self.write(f", {self.name!r}).") + if node.with_context: + f_name = f"make_module{self.choose_async('_async')}" self.write( - "make_module%s(context.get_all(), True, %s)" - % ( - self.environment.is_async and "_async" or "", - self.dump_local_context(frame), - ) + f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})" ) - elif self.environment.is_async: - self.write("_get_default_module_async()") else: - self.write("_get_default_module()") + self.write(f"_get_default_module{self.choose_async('_async')}(context)") + + def visit_Import(self, node: nodes.Import, frame: Frame) -> None: + """Visit regular imports.""" + self.writeline(f"{frame.symbols.ref(node.target)} = ", node) + if frame.toplevel: + self.write(f"context.vars[{node.target!r}] = ") + + self._import_common(node, frame) + if frame.toplevel and not node.target.startswith("_"): - self.writeline("context.exported_vars.discard(%r)" % node.target) + self.writeline(f"context.exported_vars.discard({node.target!r})") - def visit_FromImport(self, node, frame): + def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: """Visit named imports.""" self.newline(node) - self.write( - "included_template = %senvironment.get_template(" - % (self.environment.is_async and "await " or "") - ) - self.visit(node.template, frame) - self.write(", %r)." % self.name) - if node.with_context: - self.write( - "make_module%s(context.get_all(), True, %s)" - % ( - self.environment.is_async and "_async" or "", - self.dump_local_context(frame), - ) - ) - elif self.environment.is_async: - self.write("_get_default_module_async()") - else: - self.write("_get_default_module()") - + self.write("included_template = ") + self._import_common(node, frame) var_names = [] discarded_names = [] for name in node.names: @@ -1029,22 +1136,23 @@ def visit_FromImport(self, node, frame): else: alias = name self.writeline( - "%s = getattr(included_template, " - "%r, missing)" % (frame.symbols.ref(alias), name) + f"{frame.symbols.ref(alias)} =" + f" getattr(included_template, {name!r}, missing)" ) - self.writeline("if %s is missing:" % frame.symbols.ref(alias)) + self.writeline(f"if {frame.symbols.ref(alias)} is missing:") self.indent() + # The position will contain the template name, and will be formatted + # into a string that will be compiled into an f-string. Curly braces + # in the name must be replaced with escapes so that they will not be + # executed as part of the f-string. + position = self.position(node).replace("{", "{{").replace("}", "}}") + message = ( + "the template {included_template.__name__!r}" + f" (imported on {position})" + f" does not export the requested name {name!r}" + ) self.writeline( - "%s = undefined(%r %% " - "included_template.__name__, " - "name=%r)" - % ( - frame.symbols.ref(alias), - "the template %%r (imported on %s) does " - "not export the requested name %s" - % (self.position(node), repr(name)), - name, - ) + f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" ) self.outdent() if frame.toplevel: @@ -1055,35 +1163,35 @@ def visit_FromImport(self, node, frame): if var_names: if len(var_names) == 1: name = var_names[0] - self.writeline( - "context.vars[%r] = %s" % (name, frame.symbols.ref(name)) - ) + self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") else: - self.writeline( - "context.vars.update({%s})" - % ", ".join( - "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names - ) + names_kv = ", ".join( + f"{name!r}: {frame.symbols.ref(name)}" for name in var_names ) + self.writeline(f"context.vars.update({{{names_kv}}})") if discarded_names: if len(discarded_names) == 1: - self.writeline("context.exported_vars.discard(%r)" % discarded_names[0]) + self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") else: + names_str = ", ".join(map(repr, discarded_names)) self.writeline( - "context.exported_vars.difference_" - "update((%s))" % ", ".join(imap(repr, discarded_names)) + f"context.exported_vars.difference_update(({names_str}))" ) - def visit_For(self, node, frame): + def visit_For(self, node: nodes.For, frame: Frame) -> None: loop_frame = frame.inner() + loop_frame.loop_frame = True test_frame = frame.inner() else_frame = frame.inner() # try to figure out if we have an extended loop. An extended loop # is necessary if the loop is in recursive mode if the special loop - # variable is accessed in the body. - extended_loop = node.recursive or "loop" in find_undeclared( - node.iter_child_nodes(only=("body",)), ("loop",) + # variable is accessed in the body if the body is a scoped block. + extended_loop = ( + node.recursive + or "loop" + in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",)) + or any(block.scoped for block in node.find_all(nodes.Block)) ) loop_ref = None @@ -1097,13 +1205,13 @@ def visit_For(self, node, frame): if node.test: loop_filter_func = self.temporary_identifier() test_frame.symbols.analyze_node(node, for_branch="test") - self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test) + self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) self.indent() self.enter_frame(test_frame) - self.writeline(self.environment.is_async and "async for " or "for ") + self.writeline(self.choose_async("async for ", "for ")) self.visit(node.target, loop_frame) self.write(" in ") - self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter") + self.write(self.choose_async("auto_aiter(fiter)", "fiter")) self.write(":") self.indent() self.writeline("if ", node.test) @@ -1120,7 +1228,7 @@ def visit_For(self, node, frame): # variable is a special one we have to enforce aliasing for it. if node.recursive: self.writeline( - "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node + f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node ) self.indent() self.buffer(loop_frame) @@ -1131,7 +1239,7 @@ def visit_For(self, node, frame): # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: - self.writeline("%s = missing" % loop_ref) + self.writeline(f"{loop_ref} = missing") for name in node.find_all(nodes.Name): if name.ctx == "store" and name.name == "loop": @@ -1142,20 +1250,17 @@ def visit_For(self, node, frame): if node.else_: iteration_indicator = self.temporary_identifier() - self.writeline("%s = 1" % iteration_indicator) + self.writeline(f"{iteration_indicator} = 1") - self.writeline(self.environment.is_async and "async for " or "for ", node) + self.writeline(self.choose_async("async for ", "for "), node) self.visit(node.target, loop_frame) if extended_loop: - if self.environment.is_async: - self.write(", %s in AsyncLoopContext(" % loop_ref) - else: - self.write(", %s in LoopContext(" % loop_ref) + self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(") else: self.write(" in ") if node.test: - self.write("%s(" % loop_filter_func) + self.write(f"{loop_filter_func}(") if node.recursive: self.write("reciter") else: @@ -1170,21 +1275,22 @@ def visit_For(self, node, frame): if node.recursive: self.write(", undefined, loop_render_func, depth):") else: - self.write(extended_loop and ", undefined):" or ":") + self.write(", undefined):" if extended_loop else ":") self.indent() self.enter_frame(loop_frame) + self.writeline("_loop_vars = {}") self.blockvisit(node.body, loop_frame) if node.else_: - self.writeline("%s = 0" % iteration_indicator) + self.writeline(f"{iteration_indicator} = 0") self.outdent() self.leave_frame( loop_frame, with_python_scope=node.recursive and not node.else_ ) if node.else_: - self.writeline("if %s:" % iteration_indicator) + self.writeline(f"if {iteration_indicator}:") self.indent() self.enter_frame(else_frame) self.blockvisit(node.else_, else_frame) @@ -1197,9 +1303,7 @@ def visit_For(self, node, frame): self.return_buffer_contents(loop_frame) self.outdent() self.start_write(frame, node) - if self.environment.is_async: - self.write("await ") - self.write("loop(") + self.write(f"{self.choose_async('await ')}loop(") if self.environment.is_async: self.write("auto_aiter(") self.visit(node.iter, frame) @@ -1208,7 +1312,12 @@ def visit_For(self, node, frame): self.write(", loop)") self.end_write(frame) - def visit_If(self, node, frame): + # at the end of the iteration, clear any assignments made in the + # loop from the top level + if self._assign_stack: + self._assign_stack[-1].difference_update(loop_frame.symbols.stores) + + def visit_If(self, node: nodes.If, frame: Frame) -> None: if_frame = frame.soft() self.writeline("if ", node) self.visit(node.test, if_frame) @@ -1229,17 +1338,17 @@ def visit_If(self, node, frame): self.blockvisit(node.else_, if_frame) self.outdent() - def visit_Macro(self, node, frame): + def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None: macro_frame, macro_ref = self.macro_body(node, frame) self.newline() if frame.toplevel: if not node.name.startswith("_"): - self.write("context.exported_vars.add(%r)" % node.name) - self.writeline("context.vars[%r] = " % node.name) - self.write("%s = " % frame.symbols.ref(node.name)) + self.write(f"context.exported_vars.add({node.name!r})") + self.writeline(f"context.vars[{node.name!r}] = ") + self.write(f"{frame.symbols.ref(node.name)} = ") self.macro_def(macro_ref, macro_frame) - def visit_CallBlock(self, node, frame): + def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None: call_frame, macro_ref = self.macro_body(node, frame) self.writeline("caller = ") self.macro_def(macro_ref, call_frame) @@ -1247,7 +1356,7 @@ def visit_CallBlock(self, node, frame): self.visit_Call(node.call, frame, forward_caller=True) self.end_write(frame) - def visit_FilterBlock(self, node, frame): + def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None: filter_frame = frame.inner() filter_frame.symbols.analyze_node(node) self.enter_frame(filter_frame) @@ -1258,11 +1367,11 @@ def visit_FilterBlock(self, node, frame): self.end_write(frame) self.leave_frame(filter_frame) - def visit_With(self, node, frame): + def visit_With(self, node: nodes.With, frame: Frame) -> None: with_frame = frame.inner() with_frame.symbols.analyze_node(node) self.enter_frame(with_frame) - for target, expr in izip(node.targets, node.values): + for target, expr in zip(node.targets, node.values): self.newline() self.visit(target, with_frame) self.write(" = ") @@ -1270,18 +1379,25 @@ def visit_With(self, node, frame): self.blockvisit(node.body, with_frame) self.leave_frame(with_frame) - def visit_ExprStmt(self, node, frame): + def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None: self.newline(node) self.visit(node.node, frame) - _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) - #: The default finalize function if the environment isn't configured - #: with one. Or if the environment has one, this is called on that - #: function's output for constants. - _default_finalize = text_type - _finalize = None + class _FinalizeInfo(t.NamedTuple): + const: t.Optional[t.Callable[..., str]] + src: t.Optional[str] + + @staticmethod + def _default_finalize(value: t.Any) -> t.Any: + """The default finalize function if the environment isn't + configured with one. Or, if the environment has one, this is + called on that function's output for constants. + """ + return str(value) + + _finalize: t.Optional[_FinalizeInfo] = None - def _make_finalize(self): + def _make_finalize(self) -> _FinalizeInfo: """Build the finalize function to be used on constants and at runtime. Cached so it's only created once for all output nodes. @@ -1297,39 +1413,48 @@ def _make_finalize(self): if self._finalize is not None: return self._finalize + finalize: t.Optional[t.Callable[..., t.Any]] finalize = default = self._default_finalize src = None if self.environment.finalize: src = "environment.finalize(" env_finalize = self.environment.finalize + pass_arg = { + _PassArg.context: "context", + _PassArg.eval_context: "context.eval_ctx", + _PassArg.environment: "environment", + }.get( + _PassArg.from_obj(env_finalize) # type: ignore + ) + finalize = None + + if pass_arg is None: - def finalize(value): - return default(env_finalize(value)) + def finalize(value: t.Any) -> t.Any: # noqa: F811 + return default(env_finalize(value)) + + else: + src = f"{src}{pass_arg}, " - if getattr(env_finalize, "contextfunction", False) is True: - src += "context, " - finalize = None # noqa: F811 - elif getattr(env_finalize, "evalcontextfunction", False) is True: - src += "context.eval_ctx, " - finalize = None - elif getattr(env_finalize, "environmentfunction", False) is True: - src += "environment, " + if pass_arg == "environment": - def finalize(value): - return default(env_finalize(self.environment, value)) + def finalize(value: t.Any) -> t.Any: # noqa: F811 + return default(env_finalize(self.environment, value)) self._finalize = self._FinalizeInfo(finalize, src) return self._finalize - def _output_const_repr(self, group): + def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: """Given a group of constant values converted from ``Output`` child nodes, produce a string to write to the template module source. """ return repr(concat(group)) - def _output_child_to_const(self, node, frame, finalize): + def _output_child_to_const( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> str: """Try to optimize a child of an ``Output`` node by trying to convert it to constant, finalized data at compile time. @@ -1344,25 +1469,29 @@ def _output_child_to_const(self, node, frame, finalize): # Template data doesn't go through finalize. if isinstance(node, nodes.TemplateData): - return text_type(const) + return str(const) - return finalize.const(const) + return finalize.const(const) # type: ignore - def _output_child_pre(self, node, frame, finalize): + def _output_child_pre( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> None: """Output extra source code before visiting a child of an ``Output`` node. """ if frame.eval_ctx.volatile: - self.write("(escape if context.eval_ctx.autoescape else to_string)(") + self.write("(escape if context.eval_ctx.autoescape else str)(") elif frame.eval_ctx.autoescape: self.write("escape(") else: - self.write("to_string(") + self.write("str(") if finalize.src is not None: self.write(finalize.src) - def _output_child_post(self, node, frame, finalize): + def _output_child_post( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> None: """Output extra source code after visiting a child of an ``Output`` node. """ @@ -1371,7 +1500,7 @@ def _output_child_post(self, node, frame, finalize): if finalize.src is not None: self.write(")") - def visit_Output(self, node, frame): + def visit_Output(self, node: nodes.Output, frame: Frame) -> None: # If an extends is active, don't render outside a block. if frame.require_output_check: # A top-level extends is known to exist at compile time. @@ -1382,7 +1511,7 @@ def visit_Output(self, node, frame): self.indent() finalize = self._make_finalize() - body = [] + body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = [] # Evaluate constants at compile time if possible. Each item in # body will be either a list of static data or a node to be @@ -1414,9 +1543,9 @@ def visit_Output(self, node, frame): if frame.buffer is not None: if len(body) == 1: - self.writeline("%s.append(" % frame.buffer) + self.writeline(f"{frame.buffer}.append(") else: - self.writeline("%s.extend((" % frame.buffer) + self.writeline(f"{frame.buffer}.extend((") self.indent() @@ -1450,15 +1579,38 @@ def visit_Output(self, node, frame): if frame.require_output_check: self.outdent() - def visit_Assign(self, node, frame): + def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() + + # ``a.b`` is allowed for assignment, and is parsed as an NSRef. However, + # it is only valid if it references a Namespace object. Emit a check for + # that for each ref here, before assignment code is emitted. This can't + # be done in visit_NSRef as the ref could be in the middle of a tuple. + seen_refs: t.Set[str] = set() + + for nsref in node.find_all(nodes.NSRef): + if nsref.name in seen_refs: + # Only emit the check for each reference once, in case the same + # ref is used multiple times in a tuple, `ns.a, ns.b = c, d`. + continue + + seen_refs.add(nsref.name) + ref = frame.symbols.ref(nsref.name) + self.writeline(f"if not isinstance({ref}, Namespace):") + self.indent() + self.writeline( + "raise TemplateRuntimeError" + '("cannot assign attribute on non-namespace object")' + ) + self.outdent() + self.newline(node) self.visit(node.target, frame) self.write(" = ") self.visit(node.node, frame) self.pop_assign_tracking(frame) - def visit_AssignBlock(self, node, frame): + def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None: self.push_assign_tracking() block_frame = frame.inner() # This is a special case. Since a set block always captures we @@ -1475,15 +1627,17 @@ def visit_AssignBlock(self, node, frame): if node.filter is not None: self.visit_Filter(node.filter, block_frame) else: - self.write("concat(%s)" % block_frame.buffer) + self.write(f"concat({block_frame.buffer})") self.write(")") self.pop_assign_tracking(frame) self.leave_frame(block_frame) # -- Expression Visitors - def visit_Name(self, node, frame): - if node.ctx == "store" and frame.toplevel: + def visit_Name(self, node: nodes.Name, frame: Frame) -> None: + if node.ctx == "store" and ( + frame.toplevel or frame.loop_frame or frame.block_frame + ): if self._assign_stack: self._assign_stack[-1].add(node.name) ref = frame.symbols.ref(node.name) @@ -1499,52 +1653,45 @@ def visit_Name(self, node, frame): and not self.parameter_is_undeclared(ref) ): self.write( - "(undefined(name=%r) if %s is missing else %s)" - % (node.name, ref, ref) + f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" ) return self.write(ref) - def visit_NSRef(self, node, frame): - # NSRefs can only be used to store values; since they use the normal - # `foo.bar` notation they will be parsed as a normal attribute access - # when used anywhere but in a `set` context + def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: + # NSRef is a dotted assignment target a.b=c, but uses a[b]=c internally. + # visit_Assign emits code to validate that each ref is to a Namespace + # object only. That can't be emitted here as the ref could be in the + # middle of a tuple assignment. ref = frame.symbols.ref(node.name) - self.writeline("if not isinstance(%s, Namespace):" % ref) - self.indent() - self.writeline( - "raise TemplateRuntimeError(%r)" - % "cannot assign attribute on non-namespace object" - ) - self.outdent() - self.writeline("%s[%r]" % (ref, node.attr)) + self.writeline(f"{ref}[{node.attr!r}]") - def visit_Const(self, node, frame): + def visit_Const(self, node: nodes.Const, frame: Frame) -> None: val = node.as_const(frame.eval_ctx) if isinstance(val, float): self.write(str(val)) else: self.write(repr(val)) - def visit_TemplateData(self, node, frame): + def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None: try: self.write(repr(node.as_const(frame.eval_ctx))) except nodes.Impossible: self.write( - "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data + f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" ) - def visit_Tuple(self, node, frame): + def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None: self.write("(") idx = -1 for idx, item in enumerate(node.items): if idx: self.write(", ") self.visit(item, frame) - self.write(idx == 0 and ",)" or ")") + self.write(",)" if idx == 0 else ")") - def visit_List(self, node, frame): + def visit_List(self, node: nodes.List, frame: Frame) -> None: self.write("[") for idx, item in enumerate(node.items): if idx: @@ -1552,7 +1699,7 @@ def visit_List(self, node, frame): self.visit(item, frame) self.write("]") - def visit_Dict(self, node, frame): + def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None: self.write("{") for idx, item in enumerate(node.items): if idx: @@ -1562,96 +1709,59 @@ def visit_Dict(self, node, frame): self.visit(item.value, frame) self.write("}") - def binop(operator, interceptable=True): # noqa: B902 - @optimizeconst - def visitor(self, node, frame): - if ( - self.environment.sandboxed - and operator in self.environment.intercepted_binops - ): - self.write("environment.call_binop(context, %r, " % operator) - self.visit(node.left, frame) - self.write(", ") - self.visit(node.right, frame) - else: - self.write("(") - self.visit(node.left, frame) - self.write(" %s " % operator) - self.visit(node.right, frame) - self.write(")") - - return visitor - - def uaop(operator, interceptable=True): # noqa: B902 - @optimizeconst - def visitor(self, node, frame): - if ( - self.environment.sandboxed - and operator in self.environment.intercepted_unops - ): - self.write("environment.call_unop(context, %r, " % operator) - self.visit(node.node, frame) - else: - self.write("(" + operator) - self.visit(node.node, frame) - self.write(")") - - return visitor - - visit_Add = binop("+") - visit_Sub = binop("-") - visit_Mul = binop("*") - visit_Div = binop("/") - visit_FloorDiv = binop("//") - visit_Pow = binop("**") - visit_Mod = binop("%") - visit_And = binop("and", interceptable=False) - visit_Or = binop("or", interceptable=False) - visit_Pos = uaop("+") - visit_Neg = uaop("-") - visit_Not = uaop("not ", interceptable=False) - del binop, uaop + visit_Add = _make_binop("+") + visit_Sub = _make_binop("-") + visit_Mul = _make_binop("*") + visit_Div = _make_binop("/") + visit_FloorDiv = _make_binop("//") + visit_Pow = _make_binop("**") + visit_Mod = _make_binop("%") + visit_And = _make_binop("and") + visit_Or = _make_binop("or") + visit_Pos = _make_unop("+") + visit_Neg = _make_unop("-") + visit_Not = _make_unop("not ") @optimizeconst - def visit_Concat(self, node, frame): + def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None: if frame.eval_ctx.volatile: - func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" + func_name = "(markup_join if context.eval_ctx.volatile else str_join)" elif frame.eval_ctx.autoescape: func_name = "markup_join" else: - func_name = "unicode_join" - self.write("%s((" % func_name) + func_name = "str_join" + self.write(f"{func_name}((") for arg in node.nodes: self.visit(arg, frame) self.write(", ") self.write("))") @optimizeconst - def visit_Compare(self, node, frame): + def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None: self.write("(") self.visit(node.expr, frame) for op in node.ops: self.visit(op, frame) self.write(")") - def visit_Operand(self, node, frame): - self.write(" %s " % operators[node.op]) + def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None: + self.write(f" {operators[node.op]} ") self.visit(node.expr, frame) @optimizeconst - def visit_Getattr(self, node, frame): + def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None: if self.environment.is_async: self.write("(await auto_await(") self.write("environment.getattr(") self.visit(node.node, frame) - self.write(", %r)" % node.attr) + self.write(f", {node.attr!r})") if self.environment.is_async: self.write("))") @optimizeconst - def visit_Getitem(self, node, frame): + def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None: # slices bypass the environment getitem method. if isinstance(node.arg, nodes.Slice): self.visit(node.node, frame) @@ -1671,7 +1781,7 @@ def visit_Getitem(self, node, frame): if self.environment.is_async: self.write("))") - def visit_Slice(self, node, frame): + def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None: if node.start is not None: self.visit(node.start, frame) self.write(":") @@ -1681,60 +1791,83 @@ def visit_Slice(self, node, frame): self.write(":") self.visit(node.step, frame) - @optimizeconst - def visit_Filter(self, node, frame): + @contextmanager + def _filter_test_common( + self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool + ) -> t.Iterator[None]: if self.environment.is_async: - self.write("await auto_await(") - self.write(self.filters[node.name] + "(") - func = self.environment.filters.get(node.name) - if func is None: - self.fail("no filter named %r" % node.name, node.lineno) - if getattr(func, "contextfilter", False) is True: - self.write("context, ") - elif getattr(func, "evalcontextfilter", False) is True: - self.write("context.eval_ctx, ") - elif getattr(func, "environmentfilter", False) is True: - self.write("environment, ") - - # if the filter node is None we are inside a filter block - # and want to write to the current buffer - if node.node is not None: - self.visit(node.node, frame) - elif frame.eval_ctx.volatile: - self.write( - "(context.eval_ctx.autoescape and" - " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer) - ) - elif frame.eval_ctx.autoescape: - self.write("Markup(concat(%s))" % frame.buffer) + self.write("(await auto_await(") + + if is_filter: + self.write(f"{self.filters[node.name]}(") + func = self.environment.filters.get(node.name) else: - self.write("concat(%s)" % frame.buffer) + self.write(f"{self.tests[node.name]}(") + func = self.environment.tests.get(node.name) + + # When inside an If or CondExpr frame, allow the filter to be + # undefined at compile time and only raise an error if it's + # actually called at runtime. See pull_dependencies. + if func is None and not frame.soft_frame: + type_name = "filter" if is_filter else "test" + self.fail(f"No {type_name} named {node.name!r}.", node.lineno) + + pass_arg = { + _PassArg.context: "context", + _PassArg.eval_context: "context.eval_ctx", + _PassArg.environment: "environment", + }.get( + _PassArg.from_obj(func) # type: ignore + ) + + if pass_arg is not None: + self.write(f"{pass_arg}, ") + + # Back to the visitor function to handle visiting the target of + # the filter or test. + yield + self.signature(node, frame) self.write(")") + if self.environment.is_async: - self.write(")") + self.write("))") @optimizeconst - def visit_Test(self, node, frame): - self.write(self.tests[node.name] + "(") - if node.name not in self.environment.tests: - self.fail("no test named %r" % node.name, node.lineno) - self.visit(node.node, frame) - self.signature(node, frame) - self.write(")") + def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None: + with self._filter_test_common(node, frame, True): + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + f"(Markup(concat({frame.buffer}))" + f" if context.eval_ctx.autoescape else concat({frame.buffer}))" + ) + elif frame.eval_ctx.autoescape: + self.write(f"Markup(concat({frame.buffer}))") + else: + self.write(f"concat({frame.buffer})") @optimizeconst - def visit_CondExpr(self, node, frame): - def write_expr2(): + def visit_Test(self, node: nodes.Test, frame: Frame) -> None: + with self._filter_test_common(node, frame, False): + self.visit(node.node, frame) + + @optimizeconst + def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None: + frame = frame.soft() + + def write_expr2() -> None: if node.expr2 is not None: - return self.visit(node.expr2, frame) + self.visit(node.expr2, frame) + return + self.write( - "cond_expr_undefined(%r)" - % ( - "the inline if-" - "expression on %s evaluated to false and " - "no else section was defined." % self.position(node) - ) + f'cond_expr_undefined("the inline if-expression on' + f" {self.position(node)} evaluated to false and no else" + f' section was defined.")' ) self.write("(") @@ -1746,71 +1879,89 @@ def write_expr2(): self.write(")") @optimizeconst - def visit_Call(self, node, frame, forward_caller=False): + def visit_Call( + self, node: nodes.Call, frame: Frame, forward_caller: bool = False + ) -> None: if self.environment.is_async: - self.write("await auto_await(") + self.write("(await auto_await(") if self.environment.sandboxed: self.write("environment.call(context, ") else: self.write("context.call(") self.visit(node.node, frame) - extra_kwargs = forward_caller and {"caller": "caller"} or None + extra_kwargs = {"caller": "caller"} if forward_caller else None + loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {} + block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {} + if extra_kwargs: + extra_kwargs.update(loop_kwargs, **block_kwargs) + elif loop_kwargs or block_kwargs: + extra_kwargs = dict(loop_kwargs, **block_kwargs) self.signature(node, frame, extra_kwargs) self.write(")") if self.environment.is_async: - self.write(")") + self.write("))") - def visit_Keyword(self, node, frame): + def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None: self.write(node.key + "=") self.visit(node.value, frame) # -- Unused nodes for extensions - def visit_MarkSafe(self, node, frame): + def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None: self.write("Markup(") self.visit(node.expr, frame) self.write(")") - def visit_MarkSafeIfAutoescape(self, node, frame): - self.write("(context.eval_ctx.autoescape and Markup or identity)(") + def visit_MarkSafeIfAutoescape( + self, node: nodes.MarkSafeIfAutoescape, frame: Frame + ) -> None: + self.write("(Markup if context.eval_ctx.autoescape else identity)(") self.visit(node.expr, frame) self.write(")") - def visit_EnvironmentAttribute(self, node, frame): + def visit_EnvironmentAttribute( + self, node: nodes.EnvironmentAttribute, frame: Frame + ) -> None: self.write("environment." + node.name) - def visit_ExtensionAttribute(self, node, frame): - self.write("environment.extensions[%r].%s" % (node.identifier, node.name)) + def visit_ExtensionAttribute( + self, node: nodes.ExtensionAttribute, frame: Frame + ) -> None: + self.write(f"environment.extensions[{node.identifier!r}].{node.name}") - def visit_ImportedName(self, node, frame): + def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None: self.write(self.import_aliases[node.importname]) - def visit_InternalName(self, node, frame): + def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None: self.write(node.name) - def visit_ContextReference(self, node, frame): + def visit_ContextReference( + self, node: nodes.ContextReference, frame: Frame + ) -> None: self.write("context") - def visit_DerivedContextReference(self, node, frame): + def visit_DerivedContextReference( + self, node: nodes.DerivedContextReference, frame: Frame + ) -> None: self.write(self.derive_context(frame)) - def visit_Continue(self, node, frame): + def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None: self.writeline("continue", node) - def visit_Break(self, node, frame): + def visit_Break(self, node: nodes.Break, frame: Frame) -> None: self.writeline("break", node) - def visit_Scope(self, node, frame): + def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None: scope_frame = frame.inner() scope_frame.symbols.analyze_node(node) self.enter_frame(scope_frame) self.blockvisit(node.body, scope_frame) self.leave_frame(scope_frame) - def visit_OverlayScope(self, node, frame): + def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None: ctx = self.temporary_identifier() - self.writeline("%s = %s" % (ctx, self.derive_context(frame))) - self.writeline("%s.vars = " % ctx) + self.writeline(f"{ctx} = {self.derive_context(frame)}") + self.writeline(f"{ctx}.vars = ") self.visit(node.context, frame) self.push_context_reference(ctx) @@ -1821,9 +1972,11 @@ def visit_OverlayScope(self, node, frame): self.leave_frame(scope_frame) self.pop_context_reference() - def visit_EvalContextModifier(self, node, frame): + def visit_EvalContextModifier( + self, node: nodes.EvalContextModifier, frame: Frame + ) -> None: for keyword in node.options: - self.writeline("context.eval_ctx.%s = " % keyword.key) + self.writeline(f"context.eval_ctx.{keyword.key} = ") self.visit(keyword.value, frame) try: val = keyword.value.as_const(frame.eval_ctx) @@ -1832,12 +1985,14 @@ def visit_EvalContextModifier(self, node, frame): else: setattr(frame.eval_ctx, keyword.key, val) - def visit_ScopedEvalContextModifier(self, node, frame): + def visit_ScopedEvalContextModifier( + self, node: nodes.ScopedEvalContextModifier, frame: Frame + ) -> None: old_ctx_name = self.temporary_identifier() saved_ctx = frame.eval_ctx.save() - self.writeline("%s = context.eval_ctx.save()" % old_ctx_name) + self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") self.visit_EvalContextModifier(node, frame) for child in node.body: self.visit(child, frame) frame.eval_ctx.revert(saved_ctx) - self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name) + self.writeline(f"context.eval_ctx.revert({old_ctx_name})") diff --git a/src/jinja2/constants.py b/src/jinja2/constants.py index bf7f2ca72..41a1c23b0 100644 --- a/src/jinja2/constants.py +++ b/src/jinja2/constants.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- #: list of lorem ipsum words used by the lipsum() helper function -LOREM_IPSUM_WORDS = u"""\ +LOREM_IPSUM_WORDS = """\ a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at auctor augue bibendum blandit class commodo condimentum congue consectetuer consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus diff --git a/src/jinja2/debug.py b/src/jinja2/debug.py index 5d8aec31d..eeeeee78b 100644 --- a/src/jinja2/debug.py +++ b/src/jinja2/debug.py @@ -1,38 +1,37 @@ import sys +import typing as t from types import CodeType +from types import TracebackType -from . import TemplateSyntaxError -from ._compat import PYPY +from .exceptions import TemplateSyntaxError from .utils import internal_code from .utils import missing +if t.TYPE_CHECKING: + from .runtime import Context -def rewrite_traceback_stack(source=None): + +def rewrite_traceback_stack(source: t.Optional[str] = None) -> BaseException: """Rewrite the current exception to replace any tracebacks from within compiled template code with tracebacks that look like they came from the template source. This must be called within an ``except`` block. - :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, - the current ``exc_info`` is used. :param source: For ``TemplateSyntaxError``, the original source if known. - :return: A :meth:`sys.exc_info` tuple that can be re-raised. + :return: The original exception with the rewritten traceback. """ - exc_type, exc_value, tb = sys.exc_info() + _, exc_value, tb = sys.exc_info() + exc_value = t.cast(BaseException, exc_value) + tb = t.cast(TracebackType, tb) if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: exc_value.translated = True exc_value.source = source - - try: - # Remove the old traceback on Python 3, otherwise the frames - # from the compiler still show up. - exc_value.with_traceback(None) - except AttributeError: - pass - + # Remove the old traceback, otherwise the frames from the + # compiler still show up. + exc_value.with_traceback(None) # Outside of runtime, so the frame isn't executing template # code, but it still needs to point at the template. tb = fake_traceback( @@ -68,12 +67,15 @@ def rewrite_traceback_stack(source=None): # Assign tb_next in reverse to avoid circular references. for tb in reversed(stack): - tb_next = tb_set_next(tb, tb_next) + tb.tb_next = tb_next + tb_next = tb - return exc_type, exc_value, tb_next + return exc_value.with_traceback(tb_next) -def fake_traceback(exc_value, tb, filename, lineno): +def fake_traceback( # type: ignore + exc_value: BaseException, tb: t.Optional[TracebackType], filename: str, lineno: int +) -> TracebackType: """Produce a new traceback object that looks like it came from the template source instead of the compiled code. The filename, line number, and location name will point to the template, and the local @@ -100,79 +102,60 @@ def fake_traceback(exc_value, tb, filename, lineno): "__jinja_exception__": exc_value, } # Raise an exception at the correct line number. - code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + code: CodeType = compile( + "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec" + ) # Build a new code object that points to the template file and # replaces the location with a block name. - try: - location = "template" - - if tb is not None: - function = tb.tb_frame.f_code.co_name - - if function == "root": - location = "top-level template code" - elif function.startswith("block_"): - location = 'block "%s"' % function[6:] - - # Collect arguments for the new code object. CodeType only - # accepts positional arguments, and arguments were inserted in - # new Python versions. - code_args = [] - - for attr in ( - "argcount", - "posonlyargcount", # Python 3.8 - "kwonlyargcount", # Python 3 - "nlocals", - "stacksize", - "flags", - "code", # codestring - "consts", # constants - "names", - "varnames", - ("filename", filename), - ("name", location), - "firstlineno", - "lnotab", - "freevars", - "cellvars", - ): - if isinstance(attr, tuple): - # Replace with given value. - code_args.append(attr[1]) - continue - - try: - # Copy original value if it exists. - code_args.append(getattr(code, "co_" + attr)) - except AttributeError: - # Some arguments were added later. - continue - - code = CodeType(*code_args) - except Exception: - # Some environments such as Google App Engine don't support - # modifying code objects. - pass + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = f"block {function[6:]!r}" + + if sys.version_info >= (3, 8): + code = code.replace(co_name=location) + else: + code = CodeType( + code.co_argcount, + code.co_kwonlyargcount, + code.co_nlocals, + code.co_stacksize, + code.co_flags, + code.co_code, + code.co_consts, + code.co_names, + code.co_varnames, + code.co_filename, + location, + code.co_firstlineno, + code.co_lnotab, + code.co_freevars, + code.co_cellvars, + ) # Execute the new code, which is guaranteed to raise, and return # the new traceback without this frame. try: exec(code, globals, locals) except BaseException: - return sys.exc_info()[2].tb_next + return sys.exc_info()[2].tb_next # type: ignore -def get_template_locals(real_locals): +def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any]: """Based on the runtime locals, get the context that would be available at that point in the template. """ # Start with the current template context. - ctx = real_locals.get("context") + ctx: t.Optional[Context] = real_locals.get("context") - if ctx: - data = ctx.get_all().copy() + if ctx is not None: + data: t.Dict[str, t.Any] = ctx.get_all().copy() else: data = {} @@ -180,7 +163,7 @@ def get_template_locals(real_locals): # rather than pushing a context. Local variables follow the scheme # l_depth_name. Find the highest-depth local that has a value for # each name. - local_overrides = {} + local_overrides: t.Dict[str, t.Tuple[int, t.Any]] = {} for name, value in real_locals.items(): if not name.startswith("l_") or value is missing: @@ -188,8 +171,8 @@ def get_template_locals(real_locals): continue try: - _, depth, name = name.split("_", 2) - depth = int(depth) + _, depth_str, name = name.split("_", 2) + depth = int(depth_str) except ValueError: continue @@ -206,63 +189,3 @@ def get_template_locals(real_locals): data[name] = value return data - - -if sys.version_info >= (3, 7): - # tb_next is directly assignable as of Python 3.7 - def tb_set_next(tb, tb_next): - tb.tb_next = tb_next - return tb - - -elif PYPY: - # PyPy might have special support, and won't work with ctypes. - try: - import tputil - except ImportError: - # Without tproxy support, use the original traceback. - def tb_set_next(tb, tb_next): - return tb - - else: - # With tproxy support, create a proxy around the traceback that - # returns the new tb_next. - def tb_set_next(tb, tb_next): - def controller(op): - if op.opname == "__getattribute__" and op.args[0] == "tb_next": - return tb_next - - return op.delegate() - - return tputil.make_proxy(controller, obj=tb) - - -else: - # Use ctypes to assign tb_next at the C level since it's read-only - # from Python. - import ctypes - - class _CTraceback(ctypes.Structure): - _fields_ = [ - # Extra PyObject slots when compiled with Py_TRACE_REFS. - ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), - # Only care about tb_next as an object, not a traceback. - ("tb_next", ctypes.py_object), - ] - - def tb_set_next(tb, tb_next): - c_tb = _CTraceback.from_address(id(tb)) - - # Clear out the old tb_next. - if tb.tb_next is not None: - c_tb_next = ctypes.py_object(tb.tb_next) - c_tb.tb_next = ctypes.py_object() - ctypes.pythonapi.Py_DecRef(c_tb_next) - - # Assign the new tb_next. - if tb_next is not None: - c_tb_next = ctypes.py_object(tb_next) - ctypes.pythonapi.Py_IncRef(c_tb_next) - c_tb.tb_next = c_tb_next - - return tb diff --git a/src/jinja2/defaults.py b/src/jinja2/defaults.py index 8e0e7d771..638cad3d2 100644 --- a/src/jinja2/defaults.py +++ b/src/jinja2/defaults.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- -from ._compat import range_type +import typing as t + from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 from .tests import TESTS as DEFAULT_TESTS # noqa: F401 from .utils import Cycler @@ -7,6 +7,9 @@ from .utils import Joiner from .utils import Namespace +if t.TYPE_CHECKING: + import typing_extensions as te + # defaults for the parser / lexer BLOCK_START_STRING = "{%" BLOCK_END_STRING = "%}" @@ -14,17 +17,17 @@ VARIABLE_END_STRING = "}}" COMMENT_START_STRING = "{#" COMMENT_END_STRING = "#}" -LINE_STATEMENT_PREFIX = None -LINE_COMMENT_PREFIX = None +LINE_STATEMENT_PREFIX: t.Optional[str] = None +LINE_COMMENT_PREFIX: t.Optional[str] = None TRIM_BLOCKS = False LSTRIP_BLOCKS = False -NEWLINE_SEQUENCE = "\n" +NEWLINE_SEQUENCE: "te.Literal['\\n', '\\r\\n', '\\r']" = "\n" KEEP_TRAILING_NEWLINE = False # default filters, tests and namespace DEFAULT_NAMESPACE = { - "range": range_type, + "range": range, "dict": dict, "lipsum": generate_lorem_ipsum, "cycler": Cycler, @@ -33,10 +36,11 @@ } # default policies -DEFAULT_POLICIES = { +DEFAULT_POLICIES: t.Dict[str, t.Any] = { "compiler.ascii_str": True, "urlize.rel": "noopener", "urlize.target": None, + "urlize.extra_schemes": None, "truncate.leeway": 5, "json.dumps_function": None, "json.dumps_kwargs": {"sort_keys": True}, diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 8430390ee..0fc6e5be8 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -1,35 +1,30 @@ -# -*- coding: utf-8 -*- """Classes for managing templates and their runtime and compile time options. """ + import os -import sys +import typing +import typing as t import weakref +from collections import ChainMap +from functools import lru_cache from functools import partial from functools import reduce +from types import CodeType from markupsafe import Markup from . import nodes -from ._compat import encode_filename -from ._compat import implements_iterator -from ._compat import implements_to_string -from ._compat import iteritems -from ._compat import PY2 -from ._compat import PYPY -from ._compat import reraise -from ._compat import string_types -from ._compat import text_type from .compiler import CodeGenerator from .compiler import generate from .defaults import BLOCK_END_STRING from .defaults import BLOCK_START_STRING from .defaults import COMMENT_END_STRING from .defaults import COMMENT_START_STRING -from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_FILTERS # type: ignore[attr-defined] from .defaults import DEFAULT_NAMESPACE from .defaults import DEFAULT_POLICIES -from .defaults import DEFAULT_TESTS +from .defaults import DEFAULT_TESTS # type: ignore[attr-defined] from .defaults import KEEP_TRAILING_NEWLINE from .defaults import LINE_COMMENT_PREFIX from .defaults import LINE_STATEMENT_PREFIX @@ -44,25 +39,34 @@ from .exceptions import TemplateSyntaxError from .exceptions import UndefinedError from .lexer import get_lexer +from .lexer import Lexer from .lexer import TokenStream from .nodes import EvalContext from .parser import Parser from .runtime import Context from .runtime import new_context from .runtime import Undefined +from .utils import _PassArg from .utils import concat from .utils import consume -from .utils import have_async_gen from .utils import import_string from .utils import internalcode from .utils import LRUCache from .utils import missing -# for direct template usage we have up to ten living environments -_spontaneous_environments = LRUCache(10) +if t.TYPE_CHECKING: + import typing_extensions as te + + from .bccache import BytecodeCache + from .ext import Extension + from .loaders import BaseLoader +_env_bound = t.TypeVar("_env_bound", bound="Environment") -def get_spontaneous_environment(cls, *args): + +# for direct template usage we have up to ten living environments +@lru_cache(maxsize=10) +def get_spontaneous_environment(cls: t.Type[_env_bound], *args: t.Any) -> _env_bound: """Return a new spontaneous environment. A spontaneous environment is used for templates created directly rather than through an existing environment. @@ -70,75 +74,74 @@ def get_spontaneous_environment(cls, *args): :param cls: Environment class to create. :param args: Positional arguments passed to environment. """ - key = (cls, args) + env = cls(*args) + env.shared = True + return env - try: - return _spontaneous_environments[key] - except KeyError: - _spontaneous_environments[key] = env = cls(*args) - env.shared = True - return env - -def create_cache(size): +def create_cache( + size: int, +) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: """Return the cache class for the given size.""" if size == 0: return None + if size < 0: return {} - return LRUCache(size) + + return LRUCache(size) # type: ignore -def copy_cache(cache): +def copy_cache( + cache: t.Optional[t.MutableMapping[t.Any, t.Any]], +) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: """Create an empty copy of the given cache.""" if cache is None: return None - elif type(cache) is dict: + + if type(cache) is dict: # noqa E721 return {} - return LRUCache(cache.capacity) + + return LRUCache(cache.capacity) # type: ignore -def load_extensions(environment, extensions): +def load_extensions( + environment: "Environment", + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]], +) -> t.Dict[str, "Extension"]: """Load the extensions from the list and bind it to the environment. - Returns a dict of instantiated environments. + Returns a dict of instantiated extensions. """ result = {} + for extension in extensions: - if isinstance(extension, string_types): - extension = import_string(extension) - result[extension.identifier] = extension(environment) - return result + if isinstance(extension, str): + extension = t.cast(t.Type["Extension"], import_string(extension)) + result[extension.identifier] = extension(environment) -def fail_for_missing_callable(string, name): - msg = string % name - if isinstance(name, Undefined): - try: - name._fail_with_undefined_error() - except Exception as e: - msg = "%s (%s; did you forget to quote the callable name?)" % (msg, e) - raise TemplateRuntimeError(msg) + return result -def _environment_sanity_check(environment): +def _environment_config_check(environment: _env_bound) -> _env_bound: """Perform a sanity check on the environment.""" assert issubclass( environment.undefined, Undefined - ), "undefined must be a subclass of undefined because filters depend on it." + ), "'undefined' must be a subclass of 'jinja2.Undefined'." assert ( environment.block_start_string != environment.variable_start_string != environment.comment_start_string - ), "block, variable and comment start strings must be different" - assert environment.newline_sequence in ( + ), "block, variable and comment start strings must be different." + assert environment.newline_sequence in { "\r", "\r\n", "\n", - ), "newline_sequence set to unknown line ending string." + }, "'newline_sequence' must be one of '\\n', '\\r\\n', or '\\r'." return environment -class Environment(object): +class Environment: r"""The core component of Jinja is the `Environment`. It contains important shared variables like configuration, filters, tests, globals and others. Instances of this class may be modified if @@ -256,9 +259,8 @@ class Environment(object): See :ref:`bytecode-cache` for more information. `enable_async` - If set to true this enables async template execution which allows - you to take advantage of newer Python features. This requires - Python 3.6 or later. + If set to true this enables async template execution which + allows using async functions and generators. """ #: if this environment is sandboxed. Modifying this variable won't make @@ -271,7 +273,7 @@ class Environment(object): overlayed = False #: the environment this environment is linked to if it is an overlay - linked_to = None + linked_to: t.Optional["Environment"] = None #: shared environments have this set to `True`. A shared environment #: must not be modified @@ -279,36 +281,40 @@ class Environment(object): #: the class that is used for code generation. See #: :class:`~jinja2.compiler.CodeGenerator` for more information. - code_generator_class = CodeGenerator + code_generator_class: t.Type["CodeGenerator"] = CodeGenerator + + concat = "".join - #: the context class thatis used for templates. See + #: the context class that is used for templates. See #: :class:`~jinja2.runtime.Context` for more information. - context_class = Context + context_class: t.Type[Context] = Context + + template_class: t.Type["Template"] def __init__( self, - block_start_string=BLOCK_START_STRING, - block_end_string=BLOCK_END_STRING, - variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING, - comment_start_string=COMMENT_START_STRING, - comment_end_string=COMMENT_END_STRING, - line_statement_prefix=LINE_STATEMENT_PREFIX, - line_comment_prefix=LINE_COMMENT_PREFIX, - trim_blocks=TRIM_BLOCKS, - lstrip_blocks=LSTRIP_BLOCKS, - newline_sequence=NEWLINE_SEQUENCE, - keep_trailing_newline=KEEP_TRAILING_NEWLINE, - extensions=(), - optimized=True, - undefined=Undefined, - finalize=None, - autoescape=False, - loader=None, - cache_size=400, - auto_reload=True, - bytecode_cache=None, - enable_async=False, + block_start_string: str = BLOCK_START_STRING, + block_end_string: str = BLOCK_END_STRING, + variable_start_string: str = VARIABLE_START_STRING, + variable_end_string: str = VARIABLE_END_STRING, + comment_start_string: str = COMMENT_START_STRING, + comment_end_string: str = COMMENT_END_STRING, + line_statement_prefix: t.Optional[str] = LINE_STATEMENT_PREFIX, + line_comment_prefix: t.Optional[str] = LINE_COMMENT_PREFIX, + trim_blocks: bool = TRIM_BLOCKS, + lstrip_blocks: bool = LSTRIP_BLOCKS, + newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = NEWLINE_SEQUENCE, + keep_trailing_newline: bool = KEEP_TRAILING_NEWLINE, + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = (), + optimized: bool = True, + undefined: t.Type[Undefined] = Undefined, + finalize: t.Optional[t.Callable[..., t.Any]] = None, + autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = False, + loader: t.Optional["BaseLoader"] = None, + cache_size: int = 400, + auto_reload: bool = True, + bytecode_cache: t.Optional["BytecodeCache"] = None, + enable_async: bool = False, ): # !!Important notice!! # The constructor accepts quite a few arguments that should be @@ -336,7 +342,7 @@ def __init__( self.keep_trailing_newline = keep_trailing_newline # runtime information - self.undefined = undefined + self.undefined: t.Type[Undefined] = undefined self.optimized = optimized self.finalize = finalize self.autoescape = autoescape @@ -358,52 +364,50 @@ def __init__( # load extensions self.extensions = load_extensions(self, extensions) - self.enable_async = enable_async - self.is_async = self.enable_async and have_async_gen - if self.is_async: - # runs patch_all() to enable async support - from . import asyncsupport # noqa: F401 - - _environment_sanity_check(self) + self.is_async = enable_async + _environment_config_check(self) - def add_extension(self, extension): + def add_extension(self, extension: t.Union[str, t.Type["Extension"]]) -> None: """Adds an extension after the environment was created. .. versionadded:: 2.5 """ self.extensions.update(load_extensions(self, [extension])) - def extend(self, **attributes): + def extend(self, **attributes: t.Any) -> None: """Add the items to the instance of the environment if they do not exist yet. This is used by :ref:`extensions ` to register callbacks and configuration values without breaking inheritance. """ - for key, value in iteritems(attributes): + for key, value in attributes.items(): if not hasattr(self, key): setattr(self, key, value) def overlay( self, - block_start_string=missing, - block_end_string=missing, - variable_start_string=missing, - variable_end_string=missing, - comment_start_string=missing, - comment_end_string=missing, - line_statement_prefix=missing, - line_comment_prefix=missing, - trim_blocks=missing, - lstrip_blocks=missing, - extensions=missing, - optimized=missing, - undefined=missing, - finalize=missing, - autoescape=missing, - loader=missing, - cache_size=missing, - auto_reload=missing, - bytecode_cache=missing, - ): + block_start_string: str = missing, + block_end_string: str = missing, + variable_start_string: str = missing, + variable_end_string: str = missing, + comment_start_string: str = missing, + comment_end_string: str = missing, + line_statement_prefix: t.Optional[str] = missing, + line_comment_prefix: t.Optional[str] = missing, + trim_blocks: bool = missing, + lstrip_blocks: bool = missing, + newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = missing, + keep_trailing_newline: bool = missing, + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = missing, + optimized: bool = missing, + undefined: t.Type[Undefined] = missing, + finalize: t.Optional[t.Callable[..., t.Any]] = missing, + autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = missing, + loader: t.Optional["BaseLoader"] = missing, + cache_size: int = missing, + auto_reload: bool = missing, + bytecode_cache: t.Optional["BytecodeCache"] = missing, + enable_async: bool = missing, + ) -> "te.Self": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed @@ -414,16 +418,23 @@ def overlay( up completely. Not all attributes are truly linked, some are just copied over so modifications on the original environment may not shine through. + + .. versionchanged:: 3.1.5 + ``enable_async`` is applied correctly. + + .. versionchanged:: 3.1.2 + Added the ``newline_sequence``, ``keep_trailing_newline``, + and ``enable_async`` parameters to match ``__init__``. """ args = dict(locals()) - del args["self"], args["cache_size"], args["extensions"] + del args["self"], args["cache_size"], args["extensions"], args["enable_async"] rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) rv.overlayed = True rv.linked_to = self - for key, value in iteritems(args): + for key, value in args.items(): if value is not missing: setattr(rv, key, value) @@ -433,25 +444,33 @@ def overlay( rv.cache = copy_cache(self.cache) rv.extensions = {} - for key, value in iteritems(self.extensions): + for key, value in self.extensions.items(): rv.extensions[key] = value.bind(rv) if extensions is not missing: rv.extensions.update(load_extensions(rv, extensions)) - return _environment_sanity_check(rv) + if enable_async is not missing: + rv.is_async = enable_async - lexer = property(get_lexer, doc="The lexer for this environment.") + return _environment_config_check(rv) + + @property + def lexer(self) -> Lexer: + """The lexer for this environment.""" + return get_lexer(self) - def iter_extensions(self): + def iter_extensions(self) -> t.Iterator["Extension"]: """Iterates over the extensions by priority.""" return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) - def getitem(self, obj, argument): + def getitem( + self, obj: t.Any, argument: t.Union[str, t.Any] + ) -> t.Union[t.Any, Undefined]: """Get an item or attribute of an object but prefer the item.""" try: return obj[argument] except (AttributeError, TypeError, LookupError): - if isinstance(argument, string_types): + if isinstance(argument, str): try: attr = str(argument) except Exception: @@ -463,9 +482,9 @@ def getitem(self, obj, argument): pass return self.undefined(obj=obj, name=argument) - def getattr(self, obj, attribute): + def getattr(self, obj: t.Any, attribute: str) -> t.Any: """Get an item or attribute of an object but prefer the attribute. - Unlike :meth:`getitem` the attribute *must* be a bytestring. + Unlike :meth:`getitem` the attribute *must* be a string. """ try: return getattr(obj, attribute) @@ -476,51 +495,113 @@ def getattr(self, obj, attribute): except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) - def call_filter( - self, name, value, args=None, kwargs=None, context=None, eval_ctx=None - ): - """Invokes a filter on a value the same way the compiler does it. + def _filter_test_common( + self, + name: t.Union[str, Undefined], + value: t.Any, + args: t.Optional[t.Sequence[t.Any]], + kwargs: t.Optional[t.Mapping[str, t.Any]], + context: t.Optional[Context], + eval_ctx: t.Optional[EvalContext], + is_filter: bool, + ) -> t.Any: + if is_filter: + env_map = self.filters + type_name = "filter" + else: + env_map = self.tests + type_name = "test" - Note that on Python 3 this might return a coroutine in case the - filter is running from an environment in async mode and the filter - supports async execution. It's your responsibility to await this - if needed. + func = env_map.get(name) # type: ignore - .. versionadded:: 2.7 - """ - func = self.filters.get(name) if func is None: - fail_for_missing_callable("no filter named %r", name) - args = [value] + list(args or ()) - if getattr(func, "contextfilter", False) is True: + msg = f"No {type_name} named {name!r}." + + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = f"{msg} ({e}; did you forget to quote the callable name?)" + + raise TemplateRuntimeError(msg) + + args = [value, *(args if args is not None else ())] + kwargs = kwargs if kwargs is not None else {} + pass_arg = _PassArg.from_obj(func) + + if pass_arg is _PassArg.context: if context is None: raise TemplateRuntimeError( - "Attempted to invoke context filter without context" + f"Attempted to invoke a context {type_name} without context." ) + args.insert(0, context) - elif getattr(func, "evalcontextfilter", False) is True: + elif pass_arg is _PassArg.eval_context: if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) - elif getattr(func, "environmentfilter", False) is True: + elif pass_arg is _PassArg.environment: args.insert(0, self) - return func(*args, **(kwargs or {})) - def call_test(self, name, value, args=None, kwargs=None): - """Invokes a test on a value the same way the compiler does it. + return func(*args, **kwargs) + + def call_filter( + self, + name: str, + value: t.Any, + args: t.Optional[t.Sequence[t.Any]] = None, + kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + context: t.Optional[Context] = None, + eval_ctx: t.Optional[EvalContext] = None, + ) -> t.Any: + """Invoke a filter on a value the same way the compiler does. + + This might return a coroutine if the filter is running from an + environment in async mode and the filter supports async + execution. It's your responsibility to await this if needed. .. versionadded:: 2.7 """ - func = self.tests.get(name) - if func is None: - fail_for_missing_callable("no test named %r", name) - return func(value, *(args or ()), **(kwargs or {})) + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, True + ) + + def call_test( + self, + name: str, + value: t.Any, + args: t.Optional[t.Sequence[t.Any]] = None, + kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + context: t.Optional[Context] = None, + eval_ctx: t.Optional[EvalContext] = None, + ) -> t.Any: + """Invoke a test on a value the same way the compiler does. + + This might return a coroutine if the test is running from an + environment in async mode and the test supports async execution. + It's your responsibility to await this if needed. + + .. versionchanged:: 3.0 + Tests support ``@pass_context``, etc. decorators. Added + the ``context`` and ``eval_ctx`` parameters. + + .. versionadded:: 2.7 + """ + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, False + ) @internalcode - def parse(self, source, name=None, filename=None): + def parse( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> nodes.Template: """Parse the sourcecode and return the abstract syntax tree. This tree of nodes is used by the compiler to convert the template into executable source- or bytecode. This is useful for debugging or to @@ -534,11 +615,18 @@ def parse(self, source, name=None, filename=None): except TemplateSyntaxError: self.handle_exception(source=source) - def _parse(self, source, name, filename): + def _parse( + self, source: str, name: t.Optional[str], filename: t.Optional[str] + ) -> nodes.Template: """Internal parsing function used by `parse` and `compile`.""" - return Parser(self, source, name, encode_filename(filename)).parse() + return Parser(self, source, name, filename).parse() - def lex(self, source, name=None, filename=None): + def lex( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> t.Iterator[t.Tuple[int, str, str]]: """Lex the given sourcecode and return a generator that yields tokens as tuples in the form ``(lineno, token_type, value)``. This can be useful for :ref:`extension development ` @@ -548,13 +636,18 @@ def lex(self, source, name=None, filename=None): of the extensions to be applied you have to filter source through the :meth:`preprocess` method. """ - source = text_type(source) + source = str(source) try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: self.handle_exception(source=source) - def preprocess(self, source, name=None, filename=None): + def preprocess( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> str: """Preprocesses the source with all extensions. This is automatically called for all parsing and compiling methods but *not* for :meth:`lex` because there you usually only want the actual source tokenized. @@ -562,28 +655,43 @@ def preprocess(self, source, name=None, filename=None): return reduce( lambda s, e: e.preprocess(s, name, filename), self.iter_extensions(), - text_type(source), + str(source), ) - def _tokenize(self, source, name, filename=None, state=None): + def _tokenize( + self, + source: str, + name: t.Optional[str], + filename: t.Optional[str] = None, + state: t.Optional[str] = None, + ) -> TokenStream: """Called by the parser to do the preprocessing and filtering for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. """ source = self.preprocess(source, name, filename) stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): - stream = ext.filter_stream(stream) + stream = ext.filter_stream(stream) # type: ignore + if not isinstance(stream, TokenStream): stream = TokenStream(stream, name, filename) + return stream - def _generate(self, source, name, filename, defer_init=False): + def _generate( + self, + source: nodes.Template, + name: t.Optional[str], + filename: t.Optional[str], + defer_init: bool = False, + ) -> str: """Internal hook that can be overridden to hook a different generate method in. .. versionadded:: 2.5 """ - return generate( + return generate( # type: ignore source, self, name, @@ -592,7 +700,7 @@ def _generate(self, source, name, filename, defer_init=False): optimized=self.optimized, ) - def _compile(self, source, filename): + def _compile(self, source: str, filename: str) -> CodeType: """Internal hook that can be overridden to hook a different compile method in. @@ -600,8 +708,35 @@ def _compile(self, source, filename): """ return compile(source, filename, "exec") + @typing.overload + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: "te.Literal[False]" = False, + defer_init: bool = False, + ) -> CodeType: ... + + @typing.overload + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: "te.Literal[True]" = ..., + defer_init: bool = False, + ) -> str: ... + @internalcode - def compile(self, source, name=None, filename=None, raw=False, defer_init=False): + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: bool = False, + defer_init: bool = False, + ) -> t.Union[str, CodeType]: """Compile a node or template source code. The `name` parameter is the load name of the template after it was joined using :meth:`join_path` if necessary, not the filename on the file system. @@ -623,7 +758,7 @@ def compile(self, source, name=None, filename=None, raw=False, defer_init=False) """ source_hint = None try: - if isinstance(source, string_types): + if isinstance(source, str): source_hint = source source = self._parse(source, name, filename) source = self._generate(source, name, filename, defer_init=defer_init) @@ -631,13 +766,13 @@ def compile(self, source, name=None, filename=None, raw=False, defer_init=False) return source if filename is None: filename = "