diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 0a86c1a75..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,98 +0,0 @@ -# UNUSED, only for reference. If windows testing is needed, please add that to github actions -# CI on Windows via appveyor -environment: - GIT_DAEMON_PATH: "C:\\Program Files\\Git\\mingw64\\libexec\\git-core" - CYGWIN_GIT_PATH: "C:\\cygwin\\bin;%GIT_DAEMON_PATH%" - CYGWIN64_GIT_PATH: "C:\\cygwin64\\bin;%GIT_DAEMON_PATH%" - - matrix: - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - GIT_PATH: "%GIT_DAEMON_PATH%" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - GIT_PATH: "%GIT_DAEMON_PATH%" - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6" - GIT_PATH: "%GIT_DAEMON_PATH%" - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7" - GIT_PATH: "%GIT_DAEMON_PATH%" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - MAYFAIL: "yes" - GIT_PATH: "%GIT_DAEMON_PATH%" - ## Cygwin - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - IS_CYGWIN: "yes" - MAYFAIL: "yes" - GIT_PATH: "%CYGWIN64_GIT_PATH%" - -matrix: - allow_failures: - - MAYFAIL: "yes" -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - git --version - where git git-daemon python pip pip3 pip34 sh - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%" == "yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install -r requirements.txt - - pip install -r test-requirements.txt - - pip install codecov - - ## Copied from `init-tests-after-clone.sh`. - # - - | - git submodule update --init --recursive - git fetch --tags - git tag __testing_point__ - git checkout master || git checkout -b master - git reset --hard HEAD~1 - git reset --hard HEAD~1 - git reset --hard HEAD~1 - git reset --hard __testing_point__ - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%IS_CYGWIN%" == "yes" ( - nosetests -v - ) ELSE ( - IF "%PYTHON_VERSION%" == "3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) - ) - -on_success: - - IF "%PYTHON_VERSION%" == "3.5" IF NOT "%IS_CYGWIN%" == "yes" (codecov) - -# Enable this to be able to login to the build worker. You can use the -# `remmina` program in Ubuntu, use the login information that the line below -# prints into the log. -#on_finish: -# - | -# echo "Running on_finish to establish connection back to the instance" -# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index e658e6785..000000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -engines: - duplication: - enabled: true - config: - languages: - - python - pep8: - enabled: true - radon: - enabled: true -ratings: - paths: - - "**.py" -exclude_paths: diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index e2b6256e9..000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -source = git - -; to make nosetests happy -[report] -include = */git/* -omit = */git/ext/* diff --git a/.deepsource.toml b/.deepsource.toml deleted file mode 100644 index d55288b87..000000000 --- a/.deepsource.toml +++ /dev/null @@ -1,15 +0,0 @@ -version = 1 - -test_patterns = [ - 'test/**/test_*.py' -] - -exclude_patterns = [ - 'doc/**', - 'etc/sublime-text' -] - -[[analyzers]] -name = 'python' -enabled = true -runtime_version = '3.x.x' diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index b59962d21..000000000 --- a/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -.git/ -.tox/ diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..08001ffac --- /dev/null +++ b/.flake8 @@ -0,0 +1,38 @@ +[flake8] +show-source = True +count= True +statistics = True +# E265 = comment blocks like @{ section, which it can't handle +# E266 = too many leading '#' for block comment +# E731 = do not assign a lambda expression, use a def +# W293 = Blank line contains whitespace +# W504 = Line break after operator +# E704 = multiple statements in one line - used for @override +# TC002 = move third party import to TYPE_CHECKING +# ANN = flake8-annotations +# TC, TC2 = flake8-type-checking +# D = flake8-docstrings + +# select = C,E,F,W ANN, TC, TC2 # to enable code. Disabled if not listed, including builtin codes +enable-extensions = TC, TC2 # only needed for extensions not enabled by default + +ignore = E265,E266,E731,E704, + W293, W504, + ANN0 ANN1 ANN2, + TC002, + TC0, TC1, TC2 + # B, + A, + D, + RST, RST3 + +exclude = .tox,.venv,build,dist,doc,git/ext/,test + +rst-roles = # for flake8-RST-docstrings + attr,class,func,meth,mod,obj,ref,term,var # used by sphinx + +min-python-version = 3.7.0 + +# for `black` compatibility +max-line-length = 120 +extend-ignore = E203,W503 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..80819f5d8 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: byron diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..203f3c889 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml new file mode 100644 index 000000000..16b42f89c --- /dev/null +++ b/.github/workflows/cygwin-test.yml @@ -0,0 +1,59 @@ +name: test-cygwin + +on: + push: + branches: + main + pull_request: + branches: + main + +jobs: + build: + runs-on: windows-latest + env: + CHERE_INVOKING: 1 + SHELLOPTS: igncr + TMP: "/tmp" + TEMP: "/tmp" + + steps: + - name: Force LF line endings + run: git config --global core.autocrlf input + - uses: actions/checkout@v3 + with: + fetch-depth: 9999 + - uses: cygwin/cygwin-install-action@v3 + with: + packages: python39 python39-pip python39-virtualenv git + - name: Tell git to trust this repo + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + /usr/bin/git config --global --add safe.directory $(pwd) + /usr/bin/git config --global protocol.file.allow always + - name: Install dependencies and prepare tests + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + set -x + /usr/bin/python -m pip install --upgrade pip setuptools wheel + /usr/bin/python --version; /usr/bin/git --version + /usr/bin/git submodule update --init --recursive + /usr/bin/git fetch --tags + /usr/bin/python -m pip install -r requirements.txt + /usr/bin/python -m pip install -r test-requirements.txt + TRAVIS=yes ./init-tests-after-clone.sh + /usr/bin/git config --global user.email "travis@ci.com" + /usr/bin/git config --global user.name "Travis Runner" + # If we rewrite the user's config by accident, we will mess it up + # and cause subsequent tests to fail + cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Lint with flake8 + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + set -x + /usr/bin/python -m flake8 + - name: Test with pytest + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + /usr/bin/python -m pytest + continue-on-error: false diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a4f765220..5373dace6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,9 +5,12 @@ name: Python package on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] + +permissions: + contents: read jobs: build: @@ -15,45 +18,61 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 9999 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies and prepare tests run: | set -x - python -m pip install --upgrade pip + + python -m pip install --upgrade pip setuptools wheel python --version; git --version git submodule update --init --recursive git fetch --tags - + + pip install -r requirements.txt pip install -r test-requirements.txt TRAVIS=yes ./init-tests-after-clone.sh - + git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Lint with flake8 run: | set -x - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 --ignore=W293,E265,E266,W503,W504,E731 --count --show-source --statistics - - name: Test with nose + flake8 + + - name: Check types with mypy + # With new versions of pypi new issues might arise. This is a problem if there is nobody able to fix them, + # so we have to ignore errors until that changes. + continue-on-error: true run: | set -x - pip install nose - nosetests -v --with-coverage + mypy -p git + + - name: Tell git to trust this repo + run: | + /usr/bin/git config --global --add safe.directory $(pwd) + /usr/bin/git config --global protocol.file.allow always + + - name: Test with pytest + run: | + set -x + pytest + continue-on-error: false + - name: Documentation run: | set -x pip install -r doc/requirements.txt - make -C doc html \ No newline at end of file + make -C doc html diff --git a/.gitignore b/.gitignore index 369657525..72da84eee 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,8 @@ nbproject /.vscode/ .idea/ .cache/ +.mypy_cache/ +.pytest_cache/ +monkeytype.sqlite3 +output.txt +tox.ini diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1fbb1ddb8..000000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -# UNUSED, only for reference. If adjustments are needed, please see github actions -language: python -python: - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "nightly" - # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) -matrix: - allow_failures: - - python: "nightly" -git: - # a higher depth is needed for most of the tests - must be high enough to not actually be shallow - # as we clone our own repository in the process - depth: 99999 -install: - - python --version; git --version - - git submodule update --init --recursive - - git fetch --tags - - pip install -r test-requirements.txt - - pip install -r doc/requirements.txt - - pip install codecov - - # generate some reflog as git-python tests need it (in master) - - ./init-tests-after-clone.sh - - # as commits are performed with the default user, it needs to be set for travis too - - git config --global user.email "travis@ci.com" - - git config --global user.name "Travis Runner" - # If we rewrite the user's config by accident, we will mess it up - # and cause subsequent tests to fail - - cat git/test/fixtures/.gitconfig >> ~/.gitconfig -script: - # Make sure we limit open handles to see if we are leaking them - - ulimit -n 128 - - ulimit -n - - coverage run --omit="test/*" -m unittest --buffer - - coverage report - - if [ "$TRAVIS_PYTHON_VERSION" == '3.5' ]; then cd doc && make html; fi - - if [ "$TRAVIS_PYTHON_VERSION" == '3.6' ]; then flake8 --ignore=W293,E265,E266,W503,W504,E731; fi -after_success: - - codecov diff --git a/AUTHORS b/AUTHORS index 7b21b2b26..8ccc09fc0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -43,4 +43,12 @@ Contributors are: -Liam Beguin -Ram Rachum -Alba Mendez +-Robert Westman +-Hugo van Kemenade +-Hiroki Tokunaga +-Julien Mauroy +-Patrick Gerard +-Luke Twist +-Joseph Hale +-Santos Gallegos Portions derived from other open source works and are clearly marked. diff --git a/CHANGES b/CHANGES index aa8116b23..9796566ae 100644 --- a/CHANGES +++ b/CHANGES @@ -1,2 +1,2 @@ Please see the online documentation for the latest changelog: -https://github.com/gitpython-developers/GitPython/blob/master/doc/source/changes.rst +https://github.com/gitpython-developers/GitPython/blob/main/doc/source/changes.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4217cbaf9..56af0df2a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,10 @@ -### How to contribute +# How to contribute The following is a short step-by-step rundown of what one typically would do to contribute. -* [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. -* For setting up the environment to run the self tests, please look at `.travis.yml`. -* Please try to **write a test that fails unless the contribution is present.** -* Feel free to add yourself to AUTHORS file. -* Create a pull request. - +- [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. +- For setting up the environment to run the self tests, please run `init-tests-after-clone.sh`. +- Please try to **write a test that fails unless the contribution is present.** +- Try to avoid massive commits and prefer to take small steps, with one commit for each. +- Feel free to add yourself to AUTHORS file. +- Create a pull request. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f2d7e22f5..000000000 --- a/Dockerfile +++ /dev/null @@ -1,84 +0,0 @@ -# -# Contributed by: James E. King III (@jeking3) -# -# This Dockerfile creates an Ubuntu Xenial build environment -# that can run the same test suite as Travis CI. -# - -FROM ubuntu:xenial - -# Metadata -LABEL maintainer="jking@apache.org" -LABEL description="CI environment for testing GitPython" - -ENV CONTAINER_USER=user -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - add-apt-key \ - apt \ - apt-transport-https \ - apt-utils \ - ca-certificates \ - curl \ - git \ - net-tools \ - openssh-client \ - sudo \ - vim \ - wget - -RUN add-apt-key -v 6A755776 -k keyserver.ubuntu.com && \ - add-apt-key -v E1DF1F24 -k keyserver.ubuntu.com && \ - echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \ - echo "deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y --install-recommends git python2.7 python3.4 python3.5 python3.6 python3.7 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python2.7 27 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.4 34 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 35 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 36 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 37 - -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3 get-pip.py && \ - pip3 install tox - -# Clean up -RUN rm -rf /var/cache/apt/* && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /tmp/* && \ - rm -rf /var/tmp/* - -################################################################# -# Build as a regular user -# Credit: https://github.com/delcypher/docker-ubuntu-cxx-dev/blob/master/Dockerfile -# License: None specified at time of import -# Add non-root user for container but give it sudo access. -# Password is the same as the username -RUN useradd -m ${CONTAINER_USER} && \ - echo ${CONTAINER_USER}:${CONTAINER_USER} | chpasswd && \ - echo "${CONTAINER_USER} ALL=(root) ALL" >> /etc/sudoers -RUN chsh --shell /bin/bash ${CONTAINER_USER} -USER ${CONTAINER_USER} -################################################################# - -# The test suite will not tolerate running against a branch that isn't "master", so -# check out the project to a well-known location that can be used by the test suite. -# This has the added benefit of protecting the local repo fed into the container -# as a volume from getting destroyed by a bug exposed by the test suite. :) -ENV TRAVIS=ON -RUN git clone --recursive https://github.com/gitpython-developers/GitPython.git /home/${CONTAINER_USER}/testrepo && \ - cd /home/${CONTAINER_USER}/testrepo && \ - ./init-tests-after-clone.sh -ENV GIT_PYTHON_TEST_GIT_REPO_BASE=/home/${CONTAINER_USER}/testrepo -ENV TRAVIS= - -# Ensure any local pip installations get on the path -ENV PATH=/home/${CONTAINER_USER}/.local/bin:${PATH} - -# Set the global default git user to be someone non-descript -RUN git config --global user.email ci@gitpython.org && \ - git config --global user.name "GitPython CI User" - diff --git a/MANIFEST.in b/MANIFEST.in index 5fd771db3..eac2a1514 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,12 @@ -include VERSION -include LICENSE -include CHANGES include AUTHORS +include CHANGES include CONTRIBUTING.md +include LICENSE include README.md +include VERSION include requirements.txt +include test-requirements.txt +include git/py.typed recursive-include doc * recursive-exclude test * diff --git a/Makefile b/Makefile index 709813ff2..2af8de084 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean release force_release docker-build test nose-pdb +.PHONY: all clean release force_release all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all @@ -15,20 +15,6 @@ release: clean make force_release force_release: clean - git push --tags origin master + git push --tags origin main python3 setup.py sdist bdist_wheel - twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* - -docker-build: - docker build --quiet -t gitpython:xenial -f Dockerfile . - -test: docker-build - # NOTE!!! - # NOTE!!! If you are not running from master or have local changes then tests will fail - # NOTE!!! - docker run --rm -v ${CURDIR}:/src -w /src -t gitpython:xenial tox - -nose-pdb: docker-build - # run tests under nose and break on error or failure into python debugger - # HINT: set PYVER to "pyXX" to change from the default of py37 to pyXX for nose tests - docker run --rm --env PYVER=${PYVER} -v ${CURDIR}:/src -w /src -it gitpython:xenial /bin/bash dockernose.sh + twine upload dist/* diff --git a/README.md b/README.md index befb2afb5..54a735e53 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +![Python package](https://github.com/gitpython-developers/GitPython/workflows/Python%20package/badge.svg) +[![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) +[![Packaging status](https://repology.org/badge/tiny-repos/python:gitpython.svg)](https://repology.org/metapackage/python:gitpython/versions) + ## [Gitoxide](https://github.com/Byron/gitoxide): A peek into the future… I started working on GitPython in 2009, back in the days when Python was 'my thing' and I had great plans with it. @@ -20,11 +24,20 @@ or low-level like git-plumbing. It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, -or the faster, but more resource intensive *git command* implementation. +or the faster, but more resource intensive _git command_ implementation. The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. +### DEVELOPMENT STATUS + +This project is in **maintenance mode**, which means that + +- …there will be no feature development, unless these are contributed +- …there will be no bug fixes, unless they are relevant to the safety of users, or contributed +- …issues will be responded to with waiting times of up to a month + +The project is open to contributions of all kinds, as well as new maintainers. ### REQUIREMENTS @@ -33,8 +46,8 @@ in your `PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. -* Git (1.7.x or newer) -* Python >= 3.4 +- Git (1.7.x or newer) +- Python >= 3.7 The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. The installer takes care of installing them for you. @@ -83,26 +96,29 @@ See [Issue #525](https://github.com/gitpython-developers/GitPython/issues/525). ### RUNNING TESTS -*Important*: Right after cloning this repository, please be sure to have executed +_Important_: Right after cloning this repository, please be sure to have executed the `./init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. -On *Windows*, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` +On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine with MINGW's. -The easiest way to run tests is by using [tox](https://pypi.python.org/pypi/tox) -a wrapper around virtualenv. It will take care of setting up environments with the proper -dependencies installed and execute test commands. To install it simply: +Ensure testing libraries are installed. +In the root directory, run: `pip install -r test-requirements.txt` + +To lint, run: `flake8` - pip install tox +To typecheck, run: `mypy -p git` -Then run: +To test, run: `pytest` - tox +Configuration for flake8 is in the ./.flake8 file. +Configurations for mypy, pytest and coverage.py are in ./pyproject.toml. -For more fine-grained control, you can use `unittest`. +The same linting and testing will also be performed against different supported python versions +upon submitting a pull request (or on each push if you have a fork with a "main" branch and actions enabled). ### Contributions @@ -110,30 +126,35 @@ Please have a look at the [contributions file][contributing]. ### INFRASTRUCTURE -* [User Documentation](http://gitpython.readthedocs.org) -* [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) - * Please post on stackoverflow and use the `gitpython` tag -* [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) - * Post reproducible bugs and feature requests as a new issue. +- [User Documentation](http://gitpython.readthedocs.org) +- [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) +- Please post on stackoverflow and use the `gitpython` tag +- [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) + - Post reproducible bugs and feature requests as a new issue. Please be sure to provide the following information if posting bugs: - * GitPython version (e.g. `import git; git.__version__`) - * Python version (e.g. `python --version`) - * The encountered stack-trace, if applicable - * Enough information to allow reproducing the issue + - GitPython version (e.g. `import git; git.__version__`) + - Python version (e.g. `python --version`) + - The encountered stack-trace, if applicable + - Enough information to allow reproducing the issue ### How to make a new release -* Update/verify the **version** in the `VERSION` file -* Update/verify that the `doc/source/changes.rst` changelog file was updated -* Commit everything -* Run `git tag -s ` to tag the version in Git -* Run `make release` -* Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. -* set the upcoming version in the `VERSION` file, usually be +- Update/verify the **version** in the `VERSION` file +- Update/verify that the `doc/source/changes.rst` changelog file was updated +- Commit everything +- Run `git tag -s ` to tag the version in Git +- Run `make release` +- Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. +- set the upcoming version in the `VERSION` file, usually be incrementing the patch level, and possibly by appending `-dev`. Probably you want to `git push` once more. -### How to verify a release +### How to verify a release (DEPRECATED) + +Note that what follows is deprecated and future releases won't be signed anymore. +More details about how it came to that can be found [in this issue](https://github.com/gitpython-developers/gitdb/issues/77). + +---- Please only use releases from `pypi` as you can verify the respective source tarballs. @@ -181,36 +202,22 @@ gpg --edit-key 4C08421980C9 ### Projects using GitPython -* [PyDriller](https://github.com/ishepard/pydriller) -* [Kivy Designer](https://github.com/kivy/kivy-designer) -* [Prowl](https://github.com/nettitude/Prowl) -* [Python Taint](https://github.com/python-security/pyt) -* [Buster](https://github.com/axitkhurana/buster) -* [git-ftp](https://github.com/ezyang/git-ftp) -* [Git-Pandas](https://github.com/wdm0006/git-pandas) -* [PyGitUp](https://github.com/msiemens/PyGitUp) -* [PyJFuzz](https://github.com/mseclab/PyJFuzz) -* [Loki](https://github.com/Neo23x0/Loki) -* [Omniwallet](https://github.com/OmniLayer/omniwallet) -* [GitViper](https://github.com/BeayemX/GitViper) -* [Git Gud](https://github.com/bthayer2365/git-gud) +- [PyDriller](https://github.com/ishepard/pydriller) +- [Kivy Designer](https://github.com/kivy/kivy-designer) +- [Prowl](https://github.com/nettitude/Prowl) +- [Python Taint](https://github.com/python-security/pyt) +- [Buster](https://github.com/axitkhurana/buster) +- [git-ftp](https://github.com/ezyang/git-ftp) +- [Git-Pandas](https://github.com/wdm0006/git-pandas) +- [PyGitUp](https://github.com/msiemens/PyGitUp) +- [PyJFuzz](https://github.com/mseclab/PyJFuzz) +- [Loki](https://github.com/Neo23x0/Loki) +- [Omniwallet](https://github.com/OmniLayer/omniwallet) +- [GitViper](https://github.com/BeayemX/GitViper) +- [Git Gud](https://github.com/bthayer2365/git-gud) ### LICENSE -New BSD License. See the LICENSE file. - -### DEVELOPMENT STATUS - -![Python package](https://github.com/gitpython-developers/GitPython/workflows/Python%20package/badge.svg) -[![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) -[![Packaging status](https://repology.org/badge/tiny-repos/python:gitpython.svg)](https://repology.org/metapackage/python:gitpython/versions) - -This project is in **maintenance mode**, which means that - -* …there will be no feature development, unless these are contributed -* …there will be no bug fixes, unless they are relevant to the safety of users, or contributed -* …issues will be responded to with waiting times of up to a month - -The project is open to contributions of all kinds, as well as new maintainers. +New BSD License. See the LICENSE file. [contributing]: https://github.com/gitpython-developers/GitPython/blob/master/CONTRIBUTING.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..cf25c09ea --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security Policy + +## Supported Versions + +Only the latest version of GitPython can receive security updates. If a vulnerability is discovered, a fix can be issued in a new release, while older releases +are likely to be yanked. + +| Version | Supported | +| ------- | ------------------ | +| 3.x.x | :white_check_mark: | +| < 3.0 | :x: | + +## Reporting a Vulnerability + +Please report private portions of a vulnerability to sebastian.thiel@icloud.com that would help to reproduce and fix it. To receive updates on progress and provide +general information to the public, you can create an issue [on the issue tracker](https://github.com/gitpython-developers/GitPython/issues). diff --git a/VERSION b/VERSION index b48ce58fd..51b450da3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.12 +3.1.30 diff --git a/doc/requirements.txt b/doc/requirements.txt index 98e5c06a0..41a7c90f1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,2 +1,3 @@ -sphinx<2.0 +sphinx==4.3.0 sphinx_rtd_theme +sphinx-autodoc-typehints diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 16c206197..7cd09a1c5 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,11 +2,196 @@ Changelog ========= +3.1.30 +====== + +- Make injections of command-invocations harder or impossible for clone and others. + See https://github.com/gitpython-developers/GitPython/pull/1518 for details. + Note that this might constitute a breaking change for some users, and if so please + let us know and we add an opt-out to this. +- Prohibit insecure options and protocols by default, which is potentially a breaking change, + but a necessary fix for https://github.com/gitpython-developers/GitPython/issues/1515. + Please take a look at the PR for more information and how to bypass these protections + in case they cause breakage: https://github.com/gitpython-developers/GitPython/pull/1521. + + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/60?closed=1 + + +3.1.29 +====== + +- Make the git.__version__ re-appear. + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/59?closed=1 + +3.1.28 +====== + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/58?closed=1 + +3.1.27 +====== + +- Reduced startup time due to optimized imports. +- Fix a vulenerability that could cause great slowdowns when encountering long remote path names + when pulling/fetching. + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/57?closed=1 + +3.1.26 +====== + +- Fixes a leaked file descriptor when reading the index, which would cause make writing a previously + read index on windows impossible. + See https://github.com/gitpython-developers/GitPython/issues/1395 for details. + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/56?closed=1 + + +3.1.25 +====== + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/55?closed=1 + + +3.1.24 +====== + +* Newly added timeout flag is not be enabled by default, and was renamed to kill_after_timeout + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/54?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/53?closed=1 + +3.1.23 (YANKED) +=============== + +* This is the second typed release with a lot of improvements under the hood. + +* General: + + - Remove python 3.6 support + + - Remove distutils ahead of deprecation in standard library. + + - Update sphinx to 4.1.12 and use autodoc-typehints. + + - Include README as long_description on PyPI + + - Test against earliest and latest minor version available on Github Actions (e.g. 3.9.0 and 3.9.7) + + +* Typing: + + - Add types to ALL functions. + + - Ensure py.typed is collected. + + - Increase mypy strictness with disallow_untyped_defs, warn_redundant_casts, warn_unreachable. + + - Use typing.NamedTuple and typing.OrderedDict now 3.6 dropped. + + - Make Protocol classes ABCs at runtime due to new behaviour/bug in 3.9.7 & 3.10.0-rc1 + + - Remove use of typing.TypeGuard until later release, to allow dependent libs time to update. + + - Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095 + +* Runtime improvements: + + - Add clone_multi_options support to submodule.add() + + - Delay calling get_user_id() unless essential, to support sand-boxed environments. + + - Add timeout to handle_process_output(), in case thread.join() hangs. + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/53?closed=1 + + +3.1.20 (YANKED) +=============== + +* This is the second typed release with a lot of improvements under the hood. + * Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095 + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/52?closed=1 + + +3.1.19 (YANKED) +=============== + +* This is the second typed release with a lot of improvements under the hood. + * Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095 + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/51?closed=1 + +3.1.18 +====== + +* drop support for python 3.5 to reduce maintenance burden on typing. Lower patch levels of python 3.5 would break, too. + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/50?closed=1 + +3.1.17 +====== + +* Fix issues from 3.1.16 (see https://github.com/gitpython-developers/GitPython/issues/1238) +* Fix issues from 3.1.15 (see https://github.com/gitpython-developers/GitPython/issues/1223) +* Add more static typing information + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/49?closed=1 + +3.1.16 (YANKED) +=============== + +* Fix issues from 3.1.15 (see https://github.com/gitpython-developers/GitPython/issues/1223) +* Add more static typing information + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/48?closed=1 + +3.1.15 (YANKED) +=============== + +* add deprecation warning for python 3.5 + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/47?closed=1 + +3.1.14 +====== + +* git.Commit objects now have a ``replace`` method that will return a + copy of the commit with modified attributes. +* Add python 3.9 support +* Drop python 3.4 support + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/46?closed=1 + +3.1.13 +====== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/45?closed=1 + 3.1.12 ====== See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/44?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/44?closed=1 3.1.11 ====== @@ -14,20 +199,20 @@ https://github.com/gitpython-developers/gitpython/milestone/44?closed=1 Fixes regression of 3.1.10. See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/43?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/43?closed=1 3.1.10 ====== See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/42?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/42?closed=1 3.1.9 ===== See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/41?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/41?closed=1 3.1.8 @@ -38,7 +223,7 @@ https://github.com/gitpython-developers/gitpython/milestone/41?closed=1 See the following for more details: -https://github.com/gitpython-developers/gitpython/milestone/40?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/40?closed=1 3.1.7 @@ -64,13 +249,13 @@ https://github.com/gitpython-developers/gitpython/milestone/40?closed=1 * package size was reduced significantly not placing tests into the package anymore. See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/39?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/39?closed=1 3.1.3 ===== See the following for details: -https://github.com/gitpython-developers/gitpython/milestone/38?closed=1 +https://github.com/gitpython-developers/gitpython/milestone/38?closed=1 3.1.2 ===== @@ -119,7 +304,7 @@ Bugfixes Bugfixes -------- -* Fixed Repo.__repr__ when subclassed +* Fixed Repo.__repr__ when subclassed (`#968 `_) * Removed compatibility shims for Python < 3.4 and old mock library * Replaced usage of deprecated unittest aliases and Logger.warn @@ -142,7 +327,7 @@ Bugfixes -------- * Fixed warning for usage of environment variables for paths containing ``$`` or ``%`` - (`#832 `_, + (`#832 `_, `#961 `_) * Added support for parsing Git internal date format (@ ) (`#965 `_) @@ -300,7 +485,7 @@ Notable fixes * The `GIT_DIR` environment variable does not override the `path` argument when initializing a `Repo` object anymore. However, if said `path` unset, `GIT_DIR` will be used to fill the void. - + All issues and PRs can be viewed in all detail when following this URL: https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone%3A%22v2.1.0+-+proper+windows+support%22 @@ -330,7 +515,7 @@ https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone 2.0.7 - New Features ==================== -* `IndexFile.commit(...,skip_hooks=False)` added. This parameter emulates the +* `IndexFile.commit(...,skip_hooks=False)` added. This parameter emulates the behaviour of `--no-verify` on the command-line. 2.0.6 - Fixes and Features @@ -370,7 +555,7 @@ https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone commit messages contained ``\r`` characters * Fix: progress handler exceptions are not caught anymore, which would usually just hide bugs previously. -* Fix: The `Git.execute` method will now redirect `stdout` to `devnull` if `with_stdout` is false, +* Fix: The `Git.execute` method will now redirect `stdout` to `devnull` if `with_stdout` is false, which is the intended behaviour based on the parameter's documentation. 2.0.2 - Fixes @@ -441,7 +626,7 @@ It follows the `semantic version scheme `_, and thus will not - Renamed `ignore_tree_extension_data` keyword argument in `IndexFile.write(...)` to `ignore_extension_data` * If the git command executed during `Remote.push(...)|fetch(...)` returns with an non-zero exit code and GitPython didn't obtain any head-information, the corresponding `GitCommandError` will be raised. This may break previous code which expected - these operations to never raise. However, that behavious is undesirable as it would effectively hide the fact that there + these operations to never raise. However, that behaviour is undesirable as it would effectively hide the fact that there was an error. See `this issue `__ for more information. * If the git executable can't be found in the PATH or at the path provided by `GIT_PYTHON_GIT_EXECUTABLE`, this is made diff --git a/doc/source/conf.py b/doc/source/conf.py index 0ec64179e..54f1f4723 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -14,43 +14,44 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If your extensions are in another directory, add it here. If the directory # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. -#sys.path.append(os.path.abspath('.')) -sys.path.insert(0, os.path.abspath('../..')) +# sys.path.append(os.path.abspath('.')) +sys.path.insert(0, os.path.abspath("../..")) # General configuration # --------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] # Add any paths that contain templates here, relative to this directory. templates_path = [] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8' +# source_encoding = 'utf-8' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'GitPython' -copyright = 'Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel' +project = "GitPython" +copyright = "Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -with open(os.path.join(os.path.dirname(__file__),"..", "..", 'VERSION')) as fd: +with open(os.path.join(os.path.dirname(__file__), "..", "..", "VERSION")) as fd: VERSION = fd.readline().strip() version = VERSION # The full version, including alpha/beta/rc tags. @@ -58,61 +59,60 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. -#unused_docs = [] +# unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. -exclude_trees = ['build'] +exclude_trees = ["build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # Options for HTML output # ----------------------- -html_theme = 'sphinx_rtd_theme' -html_theme_options = { -} +html_theme = "sphinx_rtd_theme" +html_theme_options = {} # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -121,72 +121,71 @@ # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_use_modindex = True +# html_use_modindex = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, the reST sources are included in the HTML build as _sources/. -#html_copy_source = True +# html_copy_source = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' +# html_file_suffix = '' # Output file base name for HTML help builder. -htmlhelp_basename = 'gitpythondoc' +htmlhelp_basename = "gitpythondoc" # Options for LaTeX output # ------------------------ # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'GitPython.tex', ur'GitPython Documentation', - ur'Michael Trier', 'manual'), + ("index", "GitPython.tex", r"GitPython Documentation", r"Michael Trier", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_use_modindex = True +# latex_use_modindex = True diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4b18ccfcb..4f22a0942 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -13,15 +13,17 @@ The object database implementation is optimized for handling large quantities of Requirements ============ -* `Python`_ >= 3.4 +* `Python`_ >= 3.7 * `Git`_ 1.7.0 or newer It should also work with older versions, but it may be that some operations involving remotes will not work as expected. * `GitDB`_ - a pure python git database implementation +* `typing_extensions`_ >= 3.7.3.4 (if python < 3.10) .. _Python: https://www.python.org .. _Git: https://git-scm.com/ .. _GitDB: https://pypi.python.org/pypi/gitdb +.. _typing_extensions: https://pypi.org/project/typing-extensions/ Installing GitPython ==================== @@ -32,7 +34,7 @@ installed, just run the following from the command-line: .. sourcecode:: none - # pip install gitpython + # pip install GitPython This command will download the latest version of GitPython from the `Python Package Index `_ and install it @@ -60,7 +62,7 @@ Leakage of System Resources --------------------------- GitPython is not suited for long-running processes (like daemons) as it tends to -leak system resources. It was written in a time where destructors (as implemented +leak system resources. It was written in a time where destructors (as implemented in the `__del__` method) still ran deterministically. In case you still want to use it in such a context, you will want to search the diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index d548f8829..fcbc18bff 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -8,9 +8,9 @@ GitPython Tutorial ================== -GitPython provides object model access to your git repository. This tutorial is composed of multiple sections, most of which explains a real-life usecase. +GitPython provides object model access to your git repository. This tutorial is composed of multiple sections, most of which explain a real-life use case. -All code presented here originated from `test_docs.py `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes, all you need is a developer installation of git-python. +All code presented here originated from `test_docs.py `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes. All you need is a developer installation of git-python. Meet the Repo type ****************** @@ -66,7 +66,7 @@ Archive the repository contents to a tar file. Advanced Repo Usage =================== -And of course, there is much more you can do with this type, most of the following will be explained in greater detail in specific tutorials. Don't worry if you don't understand some of these examples right away, as they may require a thorough understanding of gits inner workings. +And of course, there is much more you can do with this type, most of the following will be explained in greater detail in specific tutorials. Don't worry if you don't understand some of these examples right away, as they may require a thorough understanding of git's inner workings. Query relevant repository paths ... @@ -363,7 +363,7 @@ Handling Remotes :start-after: # [25-test_references_and_objects] :end-before: # ![25-test_references_and_objects] -You can easily access configuration information for a remote by accessing options as if they where attributes. The modification of remote configuration is more explicit though. +You can easily access configuration information for a remote by accessing options as if they were attributes. The modification of remote configuration is more explicit though. .. literalinclude:: ../../test/test_docs.py :language: python @@ -391,7 +391,7 @@ Here's an example executable that can be used in place of the `ssh_executable` a ID_RSA=/var/lib/openshift/5562b947ecdd5ce939000038/app-deployments/id_rsa exec /usr/bin/ssh -o StrictHostKeyChecking=no -i $ID_RSA "$@" -Please note that the script must be executable (i.e. `chomd +x script.sh`). `StrictHostKeyChecking=no` is used to avoid prompts asking to save the hosts key to `~/.ssh/known_hosts`, which happens in case you run this as daemon. +Please note that the script must be executable (i.e. `chmod +x script.sh`). `StrictHostKeyChecking=no` is used to avoid prompts asking to save the hosts key to `~/.ssh/known_hosts`, which happens in case you run this as daemon. You might also have a look at `Git.update_environment(...)` in case you want to setup a changed environment more permanently. @@ -509,14 +509,14 @@ The type of the database determines certain performance characteristics, such as GitDB ===== -The GitDB is a pure-python implementation of the git object database. It is the default database to use in GitPython 0.3. Its uses less memory when handling huge files, but will be 2 to 5 times slower when extracting large quantities small of objects from densely packed repositories:: +The GitDB is a pure-python implementation of the git object database. It is the default database to use in GitPython 0.3. It uses less memory when handling huge files, but will be 2 to 5 times slower when extracting large quantities of small objects from densely packed repositories:: repo = Repo("path/to/repo", odbt=GitDB) GitCmdObjectDB ============== -The git command database uses persistent git-cat-file instances to read repository information. These operate very fast under all conditions, but will consume additional memory for the process itself. When extracting large files, memory usage will be much higher than the one of the ``GitDB``:: +The git command database uses persistent git-cat-file instances to read repository information. These operate very fast under all conditions, but will consume additional memory for the process itself. When extracting large files, memory usage will be much higher than ``GitDB``:: repo = Repo("path/to/repo", odbt=GitCmdObjectDB) diff --git a/dockernose.sh b/dockernose.sh deleted file mode 100755 index c9227118a..000000000 --- a/dockernose.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -ex -if [ -z "${PYVER}" ]; then - PYVER=py37 -fi - -# remember to use "-s" if you inject pdb.set_trace() as this disables nosetests capture of streams - -tox -e ${PYVER} --notest -PYTHONPATH=/src/.tox/${PYVER}/lib/python*/site-packages /src/.tox/${PYVER}/bin/nosetests --pdb $* diff --git a/git/__init__.py b/git/__init__.py index 534408308..f746e1fca 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -4,22 +4,24 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php # flake8: noqa -#@PydevCodeAnalysisIgnore +# @PydevCodeAnalysisIgnore +from git.exc import * # @NoMove @IgnorePep8 import inspect import os import sys - import os.path as osp +from typing import Optional +from git.types import PathLike -__version__ = 'git' +__version__ = "git" -#{ Initialization -def _init_externals(): +# { Initialization +def _init_externals() -> None: """Initialize external projects by putting them into the path""" - if __version__ == 'git' and 'PYOXIDIZER' not in os.environ: - sys.path.insert(1, osp.join(osp.dirname(__file__), 'ext', 'gitdb')) + if __version__ == "git" and "PYOXIDIZER" not in os.environ: + sys.path.insert(1, osp.join(osp.dirname(__file__), "ext", "gitdb")) try: import gitdb @@ -27,26 +29,27 @@ def _init_externals(): raise ImportError("'gitdb' could not be found in your PYTHONPATH") from e # END verify import -#} END initialization + +# } END initialization + ################# _init_externals() ################# -#{ Imports +# { Imports -from git.exc import * # @NoMove @IgnorePep8 try: from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # @NoMove @IgnorePep8 - from git.refs import * # @NoMove @IgnorePep8 - from git.diff import * # @NoMove @IgnorePep8 - from git.db import * # @NoMove @IgnorePep8 - from git.cmd import Git # @NoMove @IgnorePep8 - from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # @NoMove @IgnorePep8 - from git.index import * # @NoMove @IgnorePep8 - from git.util import ( # @NoMove @IgnorePep8 + from git.objects import * # @NoMove @IgnorePep8 + from git.refs import * # @NoMove @IgnorePep8 + from git.diff import * # @NoMove @IgnorePep8 + from git.db import * # @NoMove @IgnorePep8 + from git.cmd import Git # @NoMove @IgnorePep8 + from git.repo import Repo # @NoMove @IgnorePep8 + from git.remote import * # @NoMove @IgnorePep8 + from git.index import * # @NoMove @IgnorePep8 + from git.util import ( # @NoMove @IgnorePep8 LockFile, BlockingLockFile, Stats, @@ -54,18 +57,18 @@ def _init_externals(): rmtree, ) except GitError as exc: - raise ImportError('%s: %s' % (exc.__class__.__name__, exc)) from exc + raise ImportError("%s: %s" % (exc.__class__.__name__, exc)) from exc -#} END imports +# } END imports -__all__ = [name for name, obj in locals().items() - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] -#{ Initialize git executable path +# { Initialize git executable path GIT_OK = None -def refresh(path=None): + +def refresh(path: Optional[PathLike] = None) -> None: """Convenience method for setting the git executable path.""" global GIT_OK GIT_OK = False @@ -76,11 +79,14 @@ def refresh(path=None): return GIT_OK = True -#} END initialize git executable path + + +# } END initialize git executable path + ################# try: refresh() except Exception as exc: - raise ImportError('Failed to initialize: {0}'.format(exc)) from exc + raise ImportError("Failed to initialize: {0}".format(exc)) from exc ################# diff --git a/git/cmd.py b/git/cmd.py index 836aafffb..9ef1e3a65 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -3,21 +3,16 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - +from __future__ import annotations +import re from contextlib import contextmanager import io import logging import os import signal -from subprocess import ( - call, - Popen, - PIPE -) +from subprocess import call, Popen, PIPE, DEVNULL import subprocess -import sys import threading -from collections import OrderedDict from textwrap import dedent from git.compat import ( @@ -28,26 +23,64 @@ is_win, ) from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path +from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present -from .exc import ( - GitCommandError, - GitCommandNotFound -) +from .exc import GitCommandError, GitCommandNotFound, UnsafeOptionError, UnsafeProtocolError from .util import ( LazyMixin, stream_copy, ) -execute_kwargs = {'istream', 'with_extended_output', - 'with_exceptions', 'as_process', 'stdout_as_string', - 'output_stream', 'with_stdout', 'kill_after_timeout', - 'universal_newlines', 'shell', 'env', 'max_chunk_size'} +# typing --------------------------------------------------------------------------- + +from typing import ( + Any, + AnyStr, + BinaryIO, + Callable, + Dict, + IO, + Iterator, + List, + Mapping, + Sequence, + TYPE_CHECKING, + TextIO, + Tuple, + Union, + cast, + overload, +) + +from git.types import PathLike, Literal, TBD + +if TYPE_CHECKING: + from git.repo.base import Repo + from git.diff import DiffIndex + + +# --------------------------------------------------------------------------------- + +execute_kwargs = { + "istream", + "with_extended_output", + "with_exceptions", + "as_process", + "stdout_as_string", + "output_stream", + "with_stdout", + "kill_after_timeout", + "universal_newlines", + "shell", + "env", + "max_chunk_size", + "strip_newline_in_stdout", +} log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) -__all__ = ('Git',) +__all__ = ("Git",) # ============================================================================== @@ -56,8 +89,20 @@ # Documentation ## @{ -def handle_process_output(process, stdout_handler, stderr_handler, - finalizer=None, decode_streams=True): + +def handle_process_output( + process: "Git.AutoInterrupt" | Popen, + stdout_handler: Union[ + None, + Callable[[AnyStr], None], + Callable[[List[AnyStr]], None], + Callable[[bytes, "Repo", "DiffIndex"], None], + ], + stderr_handler: Union[None, Callable[[AnyStr], None], Callable[[List[AnyStr]], None]], + finalizer: Union[None, Callable[[Union[subprocess.Popen, "Git.AutoInterrupt"]], None]] = None, + decode_streams: bool = True, + kill_after_timeout: Union[None, float] = None, +) -> None: """Registers for notifications to learn that process output is ready to read, and dispatches lines to the respective line handlers. This function returns once the finalizer returns @@ -72,63 +117,110 @@ def handle_process_output(process, stdout_handler, stderr_handler, their contents to handlers. Set it to False if `universal_newline == True` (then streams are in text-mode) or if decoding must happen later (i.e. for Diffs). + :param kill_after_timeout: + float or None, Default = None + To specify a timeout in seconds for the git command, after which the process + should be killed. """ # Use 2 "pump" threads and wait for both to finish. - def pump_stream(cmdline, name, stream, is_decode, handler): + def pump_stream( + cmdline: List[str], + name: str, + stream: Union[BinaryIO, TextIO], + is_decode: bool, + handler: Union[None, Callable[[Union[bytes, str]], None]], + ) -> None: try: for line in stream: if handler: if is_decode: - line = line.decode(defenc) - handler(line) + assert isinstance(line, bytes) + line_str = line.decode(defenc) + handler(line_str) + else: + handler(line) + except Exception as ex: - log.error("Pumping %r of cmd(%s) failed due to: %r", name, cmdline, ex) - raise CommandError(['<%s-pump>' % name] + cmdline, ex) from ex + log.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}") + if "I/O operation on closed file" not in str(ex): + # Only reraise if the error was not due to the stream closing + raise CommandError([f"<{name}-pump>"] + remove_password_if_present(cmdline), ex) from ex finally: stream.close() - cmdline = getattr(process, 'args', '') # PY3+ only + if hasattr(process, "proc"): + process = cast("Git.AutoInterrupt", process) + cmdline: str | Tuple[str, ...] | List[str] = getattr(process.proc, "args", "") + p_stdout = process.proc.stdout if process.proc else None + p_stderr = process.proc.stderr if process.proc else None + else: + process = cast(Popen, process) + cmdline = getattr(process, "args", "") + p_stdout = process.stdout + p_stderr = process.stderr + if not isinstance(cmdline, (tuple, list)): cmdline = cmdline.split() - pumps = [] - if process.stdout: - pumps.append(('stdout', process.stdout, stdout_handler)) - if process.stderr: - pumps.append(('stderr', process.stderr, stderr_handler)) + pumps: List[Tuple[str, IO, Callable[..., None] | None]] = [] + if p_stdout: + pumps.append(("stdout", p_stdout, stdout_handler)) + if p_stderr: + pumps.append(("stderr", p_stderr, stderr_handler)) - threads = [] + threads: List[threading.Thread] = [] for name, stream, handler in pumps: - t = threading.Thread(target=pump_stream, - args=(cmdline, name, stream, decode_streams, handler)) - t.setDaemon(True) + t = threading.Thread(target=pump_stream, args=(cmdline, name, stream, decode_streams, handler)) + t.daemon = True t.start() threads.append(t) ## FIXME: Why Join?? Will block if `stdin` needs feeding... # for t in threads: - t.join() + t.join(timeout=kill_after_timeout) + if t.is_alive(): + if isinstance(process, Git.AutoInterrupt): + process._terminate() + else: # Don't want to deal with the other case + raise RuntimeError( + "Thread join() timed out in cmd.handle_process_output()." + f" kill_after_timeout={kill_after_timeout} seconds" + ) + if stderr_handler: + error_str: Union[str, bytes] = ( + "error: process killed because it timed out." f" kill_after_timeout={kill_after_timeout} seconds" + ) + if not decode_streams and isinstance(p_stderr, BinaryIO): + # Assume stderr_handler needs binary input + error_str = cast(str, error_str) + error_str = error_str.encode() + # We ignore typing on the next line because mypy does not like + # the way we inferred that stderr takes str or bytes + stderr_handler(error_str) # type: ignore if finalizer: return finalizer(process) + else: + return None -def dashify(string): - return string.replace('_', '-') +def dashify(string: str) -> str: + return string.replace("_", "-") -def slots_to_dict(self, exclude=()): +def slots_to_dict(self: object, exclude: Sequence[str] = ()) -> Dict[str, Any]: return {s: getattr(self, s) for s in self.__slots__ if s not in exclude} -def dict_to_slots_and__excluded_are_none(self, d, excluded=()): +def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], excluded: Sequence[str] = ()) -> None: for k, v in d.items(): setattr(self, k, v) for k in excluded: setattr(self, k, None) + ## -- End Utilities -- @} @@ -137,8 +229,9 @@ def dict_to_slots_and__excluded_are_none(self, d, excluded=()): ## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, # see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal -PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP - if is_win else 0) +PROC_CREATIONFLAGS = ( + CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP if is_win else 0 # type: ignore[attr-defined] +) # mypy error if not windows class Git(LazyMixin): @@ -157,20 +250,30 @@ class Git(LazyMixin): of the command to stdout. Set its value to 'full' to see details about the returned values. """ - __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", - "_git_options", "_persistent_git_options", "_environment") - _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') + __slots__ = ( + "_working_dir", + "cat_file_all", + "cat_file_header", + "_version_info", + "_git_options", + "_persistent_git_options", + "_environment", + ) + + _excluded_ = ("cat_file_all", "cat_file_header", "_version_info") - def __getstate__(self): + re_unsafe_protocol = re.compile("(.+)::.+") + + def __getstate__(self) -> Dict[str, Any]: return slots_to_dict(self, exclude=self._excluded_) - def __setstate__(self, d): + def __setstate__(self, d: Dict[str, Any]) -> None: dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) # CONFIGURATION - git_exec_name = "git" # default that should work on linux and windows + git_exec_name = "git" # default that should work on linux and windows # Enables debugging of GitPython's git commands GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) @@ -189,7 +292,7 @@ def __setstate__(self, d): # the top level __init__ @classmethod - def refresh(cls, path=None): + def refresh(cls, path: Union[None, PathLike] = None) -> bool: """This gets called by the refresh function (see the top level __init__). """ @@ -209,7 +312,7 @@ def refresh(cls, path=None): # - a GitCommandNotFound error is spawned by ourselves # - a PermissionError is spawned if the git executable provided # cannot be executed for whatever reason - + has_git = False try: cls().version() @@ -219,13 +322,18 @@ def refresh(cls, path=None): # warn or raise exception if test failed if not has_git: - err = dedent("""\ + err = ( + dedent( + """\ Bad git executable. The git executable must be specified in one of the following ways: - be included in your $PATH - be set via $%s - explicitly set via git.refresh() - """) % cls._git_exec_env_var + """ + ) + % cls._git_exec_env_var + ) # revert to whatever the old_git was cls.GIT_PYTHON_GIT_EXECUTABLE = old_git @@ -251,7 +359,9 @@ def refresh(cls, path=None): if mode in quiet: pass elif mode in warn or mode in error: - err = dedent("""\ + err = ( + dedent( + """\ %s All git commands will error until this is rectified. @@ -263,32 +373,42 @@ def refresh(cls, path=None): Example: export %s=%s - """) % ( - err, - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - cls._refresh_env_var, - quiet[0]) + """ + ) + % ( + err, + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), + cls._refresh_env_var, + quiet[0], + ) + ) if mode in warn: print("WARNING: %s" % err) else: raise ImportError(err) else: - err = dedent("""\ + err = ( + dedent( + """\ %s environment variable has been set but it has been set with an invalid value. Use only the following values: - %s: for no warning or exception - %s: for a printed warning - %s: for a raised exception - """) % ( - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error)) + """ + ) + % ( + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), + ) + ) raise ImportError(err) # we get here if this was the init refresh and the refresh mode @@ -304,11 +424,21 @@ def refresh(cls, path=None): return has_git @classmethod - def is_cygwin(cls): + def is_cygwin(cls) -> bool: return is_cygwin_git(cls.GIT_PYTHON_GIT_EXECUTABLE) + @overload @classmethod - def polish_url(cls, url, is_cygwin=None): + def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: + ... + + @overload + @classmethod + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: + ... + + @classmethod + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: if is_cygwin is None: is_cygwin = cls.is_cygwin() @@ -322,12 +452,53 @@ def polish_url(cls, url, is_cygwin=None): Hence we undo the escaping just to be sure. """ url = os.path.expandvars(url) - if url.startswith('~'): + if url.startswith("~"): url = os.path.expanduser(url) url = url.replace("\\\\", "\\").replace("\\", "/") - return url + @classmethod + def check_unsafe_protocols(cls, url: str) -> None: + """ + Check for unsafe protocols. + + Apart from the usual protocols (http, git, ssh), + Git allows "remote helpers" that have the form `::
`, + one of these helpers (`ext::`) can be used to invoke any arbitrary command. + + See: + + - https://git-scm.com/docs/gitremote-helpers + - https://git-scm.com/docs/git-remote-ext + """ + match = cls.re_unsafe_protocol.match(url) + if match: + protocol = match.group(1) + raise UnsafeProtocolError( + f"The `{protocol}::` protocol looks suspicious, use `allow_unsafe_protocols=True` to allow it." + ) + + @classmethod + def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: + """ + Check for unsafe options. + + Some options that are passed to `git ` can be used to execute + arbitrary commands, this are blocked by default. + """ + # Options can be of the form `foo` or `--foo bar` `--foo=bar`, + # so we need to check if they start with "--foo" or if they are equal to "foo". + bare_unsafe_options = [ + option.lstrip("-") + for option in unsafe_options + ] + for option in options: + for unsafe_option, bare_option in zip(unsafe_options, bare_unsafe_options): + if option.startswith(unsafe_option) or option == bare_option: + raise UnsafeOptionError( + f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." + ) + class AutoInterrupt(object): """Kill/Interrupt the stored process instance once this instance goes out of scope. It is used to prevent processes piling up in case iterators stop reading. @@ -336,13 +507,19 @@ class AutoInterrupt(object): The wait method was overridden to perform automatic status code checking and possibly raise.""" - __slots__ = ("proc", "args") + __slots__ = ("proc", "args", "status") + + # If this is non-zero it will override any status code during + # _terminate, used to prevent race conditions in testing + _status_code_if_terminate: int = 0 - def __init__(self, proc, args): + def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: self.proc = proc self.args = args + self.status: Union[int, None] = None - def __del__(self): + def _terminate(self) -> None: + """Terminate the underlying process""" if self.proc is None: return @@ -354,22 +531,24 @@ def __del__(self): proc.stdout.close() if proc.stderr: proc.stderr.close() - # did the process finish already so we have a return code ? try: if proc.poll() is not None: - return + self.status = self._status_code_if_terminate or proc.poll() + return None except OSError as ex: log.info("Ignored error after process had died: %r", ex) # can be that nothing really exists anymore ... - if os is None or getattr(os, 'kill', None) is None: - return + if os is None or getattr(os, "kill", None) is None: + return None # try to kill it try: proc.terminate() - proc.wait() # ensure process goes away + status = proc.wait() # ensure process goes away + + self.status = self._status_code_if_terminate or status except OSError as ex: log.info("Ignored error after process had died: %r", ex) except AttributeError: @@ -378,36 +557,53 @@ def __del__(self): # we simply use the shell and redirect to nul. Its slower than CreateProcess, question # is whether we really want to see all these messages. Its annoying no matter what. if is_win: - call(("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), shell=True) + call( + ("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), + shell=True, + ) # END exception handling - def __getattr__(self, attr): + def __del__(self) -> None: + self._terminate() + + def __getattr__(self, attr: str) -> Any: return getattr(self.proc, attr) - def wait(self, stderr=b''): # TODO: Bad choice to mimic `proc.wait()` but with different args. + # TODO: Bad choice to mimic `proc.wait()` but with different args. + def wait(self, stderr: Union[None, str, bytes] = b"") -> int: """Wait for the process and return its status code. :param stderr: Previously read value of stderr, in case stderr is already closed. :warn: may deadlock if output or error pipes are used and not handled separately. :raise GitCommandError: if the return status is not 0""" if stderr is None: - stderr = b'' - stderr = force_bytes(data=stderr, encoding='utf-8') - - status = self.proc.wait() + stderr_b = b"" + stderr_b = force_bytes(data=stderr, encoding="utf-8") + status: Union[int, None] + if self.proc is not None: + status = self.proc.wait() + p_stderr = self.proc.stderr + else: # Assume the underlying proc was killed earlier or never existed + status = self.status + p_stderr = None + + def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: + if stream: + try: + return stderr_b + force_bytes(stream.read()) + except (OSError, ValueError): + return stderr_b or b"" + else: + return stderr_b or b"" - def read_all_from_possibly_closed_stream(stream): - try: - return stderr + force_bytes(stream.read()) - except ValueError: - return stderr or b'' + # END status handling if status != 0: - errstr = read_all_from_possibly_closed_stream(self.proc.stderr) - log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) - raise GitCommandError(self.args, status, errstr) - # END status handling + errstr = read_all_from_possibly_closed_stream(p_stderr) + log.debug("AutoInterrupt wait stderr: %r" % (errstr,)) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) return status + # END auto interrupt class CatFileContentStream(object): @@ -419,12 +615,12 @@ class CatFileContentStream(object): If not all data is read to the end of the objects's lifetime, we read the rest to assure the underlying stream continues to work""" - __slots__ = ('_stream', '_nbr', '_size') + __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") - def __init__(self, size, stream): + def __init__(self, size: int, stream: IO[bytes]) -> None: self._stream = stream self._size = size - self._nbr = 0 # num bytes read + self._nbr = 0 # num bytes read # special case: if the object is empty, has null bytes, get the # final newline right away. @@ -432,10 +628,10 @@ def __init__(self, size, stream): stream.read(1) # END handle empty streams - def read(self, size=-1): + def read(self, size: int = -1) -> bytes: bytes_left = self._size - self._nbr if bytes_left == 0: - return b'' + return b"" if size > -1: # assure we don't try to read past our limit size = min(bytes_left, size) @@ -448,13 +644,13 @@ def read(self, size=-1): # check for depletion, read our final byte to make the stream usable by others if self._size - self._nbr == 0: - self._stream.read(1) # final newline + self._stream.read(1) # final newline # END finish reading return data - def readline(self, size=-1): + def readline(self, size: int = -1) -> bytes: if self._nbr == self._size: - return b'' + return b"" # clamp size to lowest allowed value bytes_left = self._size - self._nbr @@ -474,7 +670,7 @@ def readline(self, size=-1): return data - def readlines(self, size=-1): + def readlines(self, size: int = -1) -> List[bytes]: if self._nbr == self._size: return [] @@ -495,17 +691,20 @@ def readlines(self, size=-1): return out # skipcq: PYL-E0301 - def __iter__(self): + def __iter__(self) -> "Git.CatFileContentStream": return self - def next(self): + def __next__(self) -> bytes: + return next(self) + + def next(self) -> bytes: line = self.readline() if not line: raise StopIteration return line - def __del__(self): + def __del__(self) -> None: bytes_left = self._size - self._nbr if bytes_left: # read and discard - seeking is impossible within a stream @@ -513,7 +712,7 @@ def __del__(self): self._stream.read(bytes_left + 1) # END handle incomplete read - def __init__(self, working_dir=None): + def __init__(self, working_dir: Union[None, PathLike] = None): """Initialize this instance with: :param working_dir: @@ -523,25 +722,25 @@ def __init__(self, working_dir=None): .git directory in case of bare repositories.""" super(Git, self).__init__() self._working_dir = expand_path(working_dir) - self._git_options = () - self._persistent_git_options = [] + self._git_options: Union[List[str], Tuple[str, ...]] = () + self._persistent_git_options: List[str] = [] # Extra environment variables to pass to git commands - self._environment = {} + self._environment: Dict[str, str] = {} # cached command slots - self.cat_file_header = None - self.cat_file_all = None + self.cat_file_header: Union[None, TBD] = None + self.cat_file_all: Union[None, TBD] = None - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: """A convenience method as it allows to call the command as if it was an object. :return: Callable object that will execute call _call_process with your arguments.""" - if name[0] == '_': + if name[0] == "_": return LazyMixin.__getattr__(self, name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) - def set_persistent_git_options(self, **kwargs): + def set_persistent_git_options(self, **kwargs: Any) -> None: """Specify command line options to the git executable for subsequent subcommand calls @@ -552,46 +751,99 @@ def set_persistent_git_options(self, **kwargs): the subcommand. """ - self._persistent_git_options = self.transform_kwargs( - split_single_char_options=True, **kwargs) + self._persistent_git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) - def _set_cache_(self, attr): - if attr == '_version_info': + def _set_cache_(self, attr: str) -> None: + if attr == "_version_info": # We only use the first 4 numbers, as everything else could be strings in fact (on windows) - version_numbers = self._call_process('version').split(' ')[2] - self._version_info = tuple(int(n) for n in version_numbers.split('.')[:4] if n.isdigit()) + process_version = self._call_process("version") # should be as default *args and **kwargs used + version_numbers = process_version.split(" ")[2] + + self._version_info = cast( + Tuple[int, int, int, int], + tuple(int(n) for n in version_numbers.split(".")[:4] if n.isdigit()), + ) else: super(Git, self)._set_cache_(attr) # END handle version info @property - def working_dir(self): + def working_dir(self) -> Union[None, PathLike]: """:return: Git directory we are working on""" return self._working_dir @property - def version_info(self): + def version_info(self) -> Tuple[int, int, int, int]: """ :return: tuple(int, int, int, int) tuple with integers representing the major, minor and additional version numbers as parsed from git version. This value is generated on demand and is cached""" return self._version_info - def execute(self, command, - istream=None, - with_extended_output=False, - with_exceptions=True, - as_process=False, - output_stream=None, - stdout_as_string=True, - kill_after_timeout=None, - with_stdout=True, - universal_newlines=False, - shell=None, - env=None, - max_chunk_size=io.DEFAULT_BUFFER_SIZE, - **subprocess_kwargs - ): + @overload + def execute(self, command: Union[str, Sequence[Any]], *, as_process: Literal[True]) -> "AutoInterrupt": + ... + + @overload + def execute( + self, + command: Union[str, Sequence[Any]], + *, + as_process: Literal[False] = False, + stdout_as_string: Literal[True], + ) -> Union[str, Tuple[int, str, str]]: + ... + + @overload + def execute( + self, + command: Union[str, Sequence[Any]], + *, + as_process: Literal[False] = False, + stdout_as_string: Literal[False] = False, + ) -> Union[bytes, Tuple[int, bytes, str]]: + ... + + @overload + def execute( + self, + command: Union[str, Sequence[Any]], + *, + with_extended_output: Literal[False], + as_process: Literal[False], + stdout_as_string: Literal[True], + ) -> str: + ... + + @overload + def execute( + self, + command: Union[str, Sequence[Any]], + *, + with_extended_output: Literal[False], + as_process: Literal[False], + stdout_as_string: Literal[False], + ) -> bytes: + ... + + def execute( + self, + command: Union[str, Sequence[Any]], + istream: Union[None, BinaryIO] = None, + with_extended_output: bool = False, + with_exceptions: bool = True, + as_process: bool = False, + output_stream: Union[None, BinaryIO] = None, + stdout_as_string: bool = True, + kill_after_timeout: Union[None, float] = None, + with_stdout: bool = True, + universal_newlines: bool = False, + shell: Union[None, bool] = None, + env: Union[None, Mapping[str, str]] = None, + max_chunk_size: int = io.DEFAULT_BUFFER_SIZE, + strip_newline_in_stdout: bool = True, + **subprocess_kwargs: Any, + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]: """Handles executing the command on the shell and consumes and returns the returned information (stdout) @@ -635,7 +887,7 @@ def execute(self, command, :param env: A dictionary of environment variables to be passed to `subprocess.Popen`. - + :param max_chunk_size: Maximum number of bytes in one chunk of data passed to the output_stream in one invocation of write() method. If the given number is not positive then @@ -662,7 +914,8 @@ def execute(self, command, effects on a repository. For example, stale locks in case of git gc could render the repository incapable of accepting changes until the lock is manually removed. - + :param strip_newline_in_stdout: + Whether to strip the trailing `\n` of the command stdout. :return: * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True @@ -679,11 +932,18 @@ def execute(self, command, :note: If you add additional keyword arguments to the signature of this method, you must update the execute_kwargs tuple housed in this module.""" - if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process): - log.info(' '.join(command)) + # Remove password for the command if present + redacted_command = remove_password_if_present(command) + if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != "full" or as_process): + log.info(" ".join(redacted_command)) # Allow the user to have the command executed in their working dir. - cwd = self._working_dir or os.getcwd() + try: + cwd = self._working_dir or os.getcwd() # type: Union[None, str] + if not os.access(str(cwd), os.X_OK): + cwd = None + except FileNotFoundError: + cwd = None # Start the process inline_env = env @@ -700,94 +960,114 @@ def execute(self, command, if is_win: cmd_not_found_exception = OSError - if kill_after_timeout: - raise GitCommandError(command, '"kill_after_timeout" feature is not supported on Windows.') + if kill_after_timeout is not None: + raise GitCommandError( + redacted_command, + '"kill_after_timeout" feature is not supported on Windows.', + ) else: - if sys.version_info[0] > 2: - cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable - else: - cmd_not_found_exception = OSError + cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable # end handle - stdout_sink = (PIPE - if with_stdout - else getattr(subprocess, 'DEVNULL', None) or open(os.devnull, 'wb')) + stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") istream_ok = "None" if istream: istream_ok = "" - log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", - command, cwd, universal_newlines, shell, istream_ok) + log.debug( + "Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", + redacted_command, + cwd, + universal_newlines, + shell, + istream_ok, + ) try: - proc = Popen(command, - env=env, - cwd=cwd, - bufsize=-1, - stdin=istream, - stderr=PIPE, - stdout=stdout_sink, - shell=shell is not None and shell or self.USE_SHELL, - close_fds=is_posix, # unsupported on windows - universal_newlines=universal_newlines, - creationflags=PROC_CREATIONFLAGS, - **subprocess_kwargs - ) + proc = Popen( + command, + env=env, + cwd=cwd, + bufsize=-1, + stdin=istream or DEVNULL, + stderr=PIPE, + stdout=stdout_sink, + shell=shell is not None and shell or self.USE_SHELL, + close_fds=is_posix, # unsupported on windows + universal_newlines=universal_newlines, + creationflags=PROC_CREATIONFLAGS, + **subprocess_kwargs, + ) + except cmd_not_found_exception as err: - raise GitCommandNotFound(command, err) from err + raise GitCommandNotFound(redacted_command, err) from err + else: + # replace with a typeguard for Popen[bytes]? + proc.stdout = cast(BinaryIO, proc.stdout) + proc.stderr = cast(BinaryIO, proc.stderr) if as_process: return self.AutoInterrupt(proc, command) - def _kill_process(pid): - """ Callback method to kill a process. """ - p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE, - creationflags=PROC_CREATIONFLAGS) + def _kill_process(pid: int) -> None: + """Callback method to kill a process.""" + p = Popen( + ["ps", "--ppid", str(pid)], + stdout=PIPE, + creationflags=PROC_CREATIONFLAGS, + ) child_pids = [] - for line in p.stdout: - if len(line.split()) > 0: - local_pid = (line.split())[0] - if local_pid.isdigit(): - child_pids.append(int(local_pid)) + if p.stdout is not None: + for line in p.stdout: + if len(line.split()) > 0: + local_pid = (line.split())[0] + if local_pid.isdigit(): + child_pids.append(int(local_pid)) try: # Windows does not have SIGKILL, so use SIGTERM instead - sig = getattr(signal, 'SIGKILL', signal.SIGTERM) + sig = getattr(signal, "SIGKILL", signal.SIGTERM) os.kill(pid, sig) for child_pid in child_pids: try: os.kill(child_pid, sig) except OSError: pass - kill_check.set() # tell the main routine that the process was killed + kill_check.set() # tell the main routine that the process was killed except OSError: # It is possible that the process gets completed in the duration after timeout # happens and before we try to kill the process. pass return + # end - if kill_after_timeout: + if kill_after_timeout is not None: kill_check = threading.Event() watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,)) # Wait for the process to return status = 0 - stdout_value = b'' - stderr_value = b'' + stdout_value: Union[str, bytes] = b"" + stderr_value: Union[str, bytes] = b"" newline = "\n" if universal_newlines else b"\n" try: if output_stream is None: - if kill_after_timeout: + if kill_after_timeout is not None: watchdog.start() stdout_value, stderr_value = proc.communicate() - if kill_after_timeout: + if kill_after_timeout is not None: watchdog.cancel() - if kill_check.isSet(): - stderr_value = ('Timeout: the command "%s" did not complete in %d ' - 'secs.' % (" ".join(command), kill_after_timeout)).encode(defenc) + if kill_check.is_set(): + stderr_value = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( + " ".join(redacted_command), + kill_after_timeout, + ) + if not universal_newlines: + stderr_value = stderr_value.encode(defenc) # strip trailing "\n" - if stdout_value.endswith(newline): + if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore stdout_value = stdout_value[:-1] - if stderr_value.endswith(newline): + if stderr_value.endswith(newline): # type: ignore stderr_value = stderr_value[:-1] + status = proc.returncode else: max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE @@ -795,7 +1075,7 @@ def _kill_process(pid): stdout_value = proc.stdout.read() stderr_value = proc.stderr.read() # strip trailing "\n" - if stderr_value.endswith(newline): + if stderr_value.endswith(newline): # type: ignore stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling @@ -803,16 +1083,22 @@ def _kill_process(pid): proc.stdout.close() proc.stderr.close() - if self.GIT_PYTHON_TRACE == 'full': - cmdstr = " ".join(command) + if self.GIT_PYTHON_TRACE == "full": + cmdstr = " ".join(redacted_command) + + def as_text(stdout_value: Union[bytes, str]) -> str: + return not output_stream and safe_decode(stdout_value) or "" - def as_text(stdout_value): - return not output_stream and safe_decode(stdout_value) or '' # end if stderr_value: - log.info("%s -> %d; stdout: '%s'; stderr: '%s'", - cmdstr, status, as_text(stdout_value), safe_decode(stderr_value)) + log.info( + "%s -> %d; stdout: '%s'; stderr: '%s'", + cmdstr, + status, + as_text(stdout_value), + safe_decode(stderr_value), + ) elif stdout_value: log.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value)) else: @@ -820,7 +1106,7 @@ def as_text(stdout_value): # END handle debug printing if with_exceptions and status != 0: - raise GitCommandError(command, status, stderr_value, stdout_value) + raise GitCommandError(redacted_command, status, stderr_value, stdout_value) if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream stdout_value = safe_decode(stdout_value) @@ -831,10 +1117,10 @@ def as_text(stdout_value): else: return stdout_value - def environment(self): + def environment(self) -> Dict[str, str]: return self._environment - def update_environment(self, **kwargs): + def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: """ Set environment variables for future git invocations. Return all changed values in a format that can be passed back into this function to revert @@ -861,7 +1147,7 @@ def update_environment(self, **kwargs): return old_env @contextmanager - def custom_environment(self, **kwargs): + def custom_environment(self, **kwargs: Any) -> Iterator[None]: """ A context manager around the above ``update_environment`` method to restore the environment back to its previous state after operation. @@ -879,7 +1165,7 @@ def custom_environment(self, **kwargs): finally: self.update_environment(**old_env) - def transform_kwarg(self, name, value, split_single_char_options): + def transform_kwarg(self, name: str, value: Any, split_single_char_options: bool) -> List[str]: if len(name) == 1: if value is True: return ["-%s" % name] @@ -895,10 +1181,9 @@ def transform_kwarg(self, name, value, split_single_char_options): return ["--%s=%s" % (dashify(name), value)] return [] - def transform_kwargs(self, split_single_char_options=True, **kwargs): + def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any) -> List[str]: """Transforms Python style kwargs into git command line options.""" args = [] - kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) for k, v in kwargs.items(): if isinstance(v, (list, tuple)): for value in v: @@ -908,21 +1193,18 @@ def transform_kwargs(self, split_single_char_options=True, **kwargs): return args @classmethod - def __unpack_args(cls, arg_list): - if not isinstance(arg_list, (list, tuple)): - return [str(arg_list)] + def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: outlist = [] - for arg in arg_list: - if isinstance(arg_list, (list, tuple)): - outlist.extend(cls.__unpack_args(arg)) - # END recursion - else: - outlist.append(str(arg)) - # END for each arg + if isinstance(arg_list, (list, tuple)): + for arg in arg_list: + outlist.extend(cls._unpack_args(arg)) + else: + outlist.append(str(arg_list)) + return outlist - def __call__(self, **kwargs): + def __call__(self, **kwargs: Any) -> "Git": """Specify command line options to the git executable for a subcommand call @@ -934,11 +1216,33 @@ def __call__(self, **kwargs): ``Examples``:: git(work_tree='/tmp').difftool()""" - self._git_options = self.transform_kwargs( - split_single_char_options=True, **kwargs) + self._git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) return self - def _call_process(self, method, *args, **kwargs): + @overload + def _call_process(self, method: str, *args: None, **kwargs: None) -> str: + ... # if no args given, execute called with all defaults + + @overload + def _call_process( + self, + method: str, + istream: int, + as_process: Literal[True], + *args: Any, + **kwargs: Any, + ) -> "Git.AutoInterrupt": + ... + + @overload + def _call_process( + self, method: str, *args: Any, **kwargs: Any + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: + ... + + def _call_process( + self, method: str, *args: Any, **kwargs: Any + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: """Run the given git command with the specified arguments and return the result as a String @@ -955,8 +1259,8 @@ def _call_process(self, method, *args, **kwargs): It contains key-values for the following: - the :meth:`execute()` kwds, as listed in :var:`execute_kwargs`; - "command options" to be converted by :meth:`transform_kwargs()`; - - the `'insert_kwargs_after'` key which its value must match one of ``*args``, - and any cmd-options will be appended after the matched arg. + - the `'insert_kwargs_after'` key which its value must match one of ``*args`` + and any cmd-options will be appended after the matched arg. Examples:: @@ -966,28 +1270,33 @@ def _call_process(self, method, *args, **kwargs): git rev-list max-count 10 --header master - :return: Same as ``execute``""" + :return: Same as ``execute`` + if no args given used execute default (esp. as_process = False, stdout_as_string = True) + and return str""" # Handle optional arguments prior to calling transform_kwargs # otherwise these'll end up in args, which is bad. exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} - insert_after_this_arg = opts_kwargs.pop('insert_kwargs_after', None) + insert_after_this_arg = opts_kwargs.pop("insert_kwargs_after", None) # Prepare the argument list + opt_args = self.transform_kwargs(**opts_kwargs) - ext_args = self.__unpack_args([a for a in args if a is not None]) + ext_args = self._unpack_args([a for a in args if a is not None]) if insert_after_this_arg is None: - args = opt_args + ext_args + args_list = opt_args + ext_args else: try: index = ext_args.index(insert_after_this_arg) except ValueError as err: - raise ValueError("Couldn't find argument '%s' in args %s to insert cmd options after" - % (insert_after_this_arg, str(ext_args))) from err + raise ValueError( + "Couldn't find argument '%s' in args %s to insert cmd options after" + % (insert_after_this_arg, str(ext_args)) + ) from err # end handle error - args = ext_args[:index + 1] + opt_args + ext_args[index + 1:] + args_list = ext_args[: index + 1] + opt_args + ext_args[index + 1 :] # end handle opts_kwargs call = [self.GIT_PYTHON_GIT_EXECUTABLE] @@ -1001,11 +1310,11 @@ def _call_process(self, method, *args, **kwargs): self._git_options = () call.append(dashify(method)) - call.extend(args) + call.extend(args_list) return self.execute(call, **exec_kwargs) - def _parse_object_header(self, header_line): + def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: """ :param header_line: type_string size_as_int @@ -1027,20 +1336,21 @@ def _parse_object_header(self, header_line): raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) - def _prepare_ref(self, ref): + def _prepare_ref(self, ref: AnyStr) -> bytes: # required for command to separate refs on stdin, as bytes - refstr = ref if isinstance(ref, bytes): # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text - refstr = ref.decode('ascii') + refstr: str = ref.decode("ascii") elif not isinstance(ref, str): - refstr = str(ref) # could be ref-object + refstr = str(ref) # could be ref-object + else: + refstr = ref if not refstr.endswith("\n"): refstr += "\n" return refstr.encode(defenc) - def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): + def _get_persistent_cmd(self, attr_name: str, cmd_name: str, *args: Any, **kwargs: Any) -> "Git.AutoInterrupt": cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val @@ -1050,15 +1360,19 @@ def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): cmd = self._call_process(cmd_name, *args, **options) setattr(self, attr_name, cmd) + cmd = cast("Git.AutoInterrupt", cmd) return cmd - def __get_object_header(self, cmd, ref): - cmd.stdin.write(self._prepare_ref(ref)) - cmd.stdin.flush() - return self._parse_object_header(cmd.stdout.readline()) + def __get_object_header(self, cmd: "Git.AutoInterrupt", ref: AnyStr) -> Tuple[str, str, int]: + if cmd.stdin and cmd.stdout: + cmd.stdin.write(self._prepare_ref(ref)) + cmd.stdin.flush() + return self._parse_object_header(cmd.stdout.readline()) + else: + raise ValueError("cmd stdin was empty") - def get_object_header(self, ref): - """ Use this method to quickly examine the type and size of the object behind + def get_object_header(self, ref: str) -> Tuple[str, str, int]: + """Use this method to quickly examine the type and size of the object behind the given ref. :note: The method will only suffer from the costs of command invocation @@ -1068,25 +1382,26 @@ def get_object_header(self, ref): cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) - def get_object_data(self, ref): - """ As get_object_header, but returns object data as well + def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: + """As get_object_header, but returns object data as well :return: (hexsha, type_string, size_as_int,data_string) :note: not threadsafe""" hexsha, typename, size, stream = self.stream_object_data(ref) data = stream.read(size) - del(stream) + del stream return (hexsha, typename, size, data) - def stream_object_data(self, ref): - """ As get_object_header, but returns the data as a stream + def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileContentStream"]: + """As get_object_header, but returns the data as a stream :return: (hexsha, type_string, size_as_int, stream) :note: This method is not threadsafe, you need one independent Command instance per thread to be safe !""" cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) - return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout)) + cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() + return (hexsha, typename, size, self.CatFileContentStream(size, cmd_stdout)) - def clear_cache(self): + def clear_cache(self) -> "Git": """Clear all kinds of internal caches to release resources. Currently persistent commands will be interrupted. diff --git a/git/compat.py b/git/compat.py index de8a238ba..e7ef28c30 100644 --- a/git/compat.py +++ b/git/compat.py @@ -11,57 +11,94 @@ import os import sys - from gitdb.utils.encoding import ( - force_bytes, # @UnusedImport - force_text # @UnusedImport + force_bytes, # @UnusedImport + force_text, # @UnusedImport +) + +# typing -------------------------------------------------------------------- + +from typing import ( + Any, + AnyStr, + Dict, + IO, + Optional, + Tuple, + Type, + Union, + overload, ) +# --------------------------------------------------------------------------- + -is_win = (os.name == 'nt') -is_posix = (os.name == 'posix') -is_darwin = (os.name == 'darwin') +is_win: bool = os.name == "nt" +is_posix = os.name == "posix" +is_darwin = os.name == "darwin" defenc = sys.getfilesystemencoding() -def safe_decode(s): +@overload +def safe_decode(s: None) -> None: + ... + + +@overload +def safe_decode(s: AnyStr) -> str: + ... + + +def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: """Safely decodes a binary string to unicode""" if isinstance(s, str): return s elif isinstance(s, bytes): - return s.decode(defenc, 'surrogateescape') - elif s is not None: - raise TypeError('Expected bytes or text, but got %r' % (s,)) + return s.decode(defenc, "surrogateescape") + elif s is None: + return None + else: + raise TypeError("Expected bytes or text, but got %r" % (s,)) -def safe_encode(s): - """Safely decodes a binary string to unicode""" +@overload +def safe_encode(s: None) -> None: + ... + + +@overload +def safe_encode(s: AnyStr) -> bytes: + ... + + +def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: + """Safely encodes a binary string to unicode""" if isinstance(s, str): return s.encode(defenc) elif isinstance(s, bytes): return s - elif s is not None: - raise TypeError('Expected bytes or text, but got %r' % (s,)) + elif s is None: + return None + else: + raise TypeError("Expected bytes or text, but got %r" % (s,)) + + +@overload +def win_encode(s: None) -> None: + ... -def win_encode(s): +@overload +def win_encode(s: AnyStr) -> bytes: + ... + + +def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: """Encode unicodes for process arguments on Windows.""" if isinstance(s, str): return s.encode(locale.getpreferredencoding(False)) elif isinstance(s, bytes): return s elif s is not None: - raise TypeError('Expected bytes or text, but got %r' % (s,)) - - -def with_metaclass(meta, *bases): - """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" - class metaclass(meta): - __call__ = type.__call__ - __init__ = type.__init__ - - def __new__(cls, name, nbases, d): - if nbases is None: - return type.__new__(cls, name, (), d) - return meta(name, bases, d) - return metaclass(meta.__name__ + 'Helper', None, {}) + raise TypeError("Expected bytes or text, but got %r" % (s,)) + return None diff --git a/git/config.py b/git/config.py index 9f09efe2b..71d7ea689 100644 --- a/git/config.py +++ b/git/config.py @@ -6,52 +6,92 @@ """Module containing module parser implementation able to properly read and write configuration files""" +import sys import abc from functools import wraps import inspect -from io import IOBase +from io import BufferedReader, IOBase import logging import os import re import fnmatch -from collections import OrderedDict from git.compat import ( defenc, force_text, - with_metaclass, is_win, ) + from git.util import LockFile import os.path as osp import configparser as cp +# typing------------------------------------------------------- + +from typing import ( + Any, + Callable, + Generic, + IO, + List, + Dict, + Sequence, + TYPE_CHECKING, + Tuple, + TypeVar, + Union, + cast, +) + +from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T + +if TYPE_CHECKING: + from git.repo.base import Repo + from io import BytesIO + +T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser") +T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool) -__all__ = ('GitConfigParser', 'SectionConstraint') +if sys.version_info[:3] < (3, 7, 2): + # typing.Ordereddict not added until py 3.7.2 + from collections import OrderedDict + OrderedDict_OMD = OrderedDict +else: + from typing import OrderedDict -log = logging.getLogger('git.config') + OrderedDict_OMD = OrderedDict[str, List[T_OMD_value]] # type: ignore[assignment, misc] + +# ------------------------------------------------------------- + +__all__ = ("GitConfigParser", "SectionConstraint") + + +log = logging.getLogger("git.config") log.addHandler(logging.NullHandler()) # invariants # represents the configuration level of a configuration file -CONFIG_LEVELS = ("system", "user", "global", "repository") + + +CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") + # Section pattern to detect conditional includes. # https://git-scm.com/docs/git-config#_conditional_includes CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") -class MetaParserBuilder(abc.ABCMeta): +class MetaParserBuilder(abc.ABCMeta): # noqa: B024 + """Utility class wrapping base-class methods into decorators that assure read-only properties""" - """Utlity class wrapping base-class methods into decorators that assure read-only properties""" - def __new__(cls, name, bases, clsdict): + def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": """ Equip all base-class methods with a needs_values decorator, and all non-const methods with a set_dirty_and_flush_changes decorator in addition to that.""" - kmm = '_mutating_methods_' + kmm = "_mutating_methods_" if kmm in clsdict: mutating_methods = clsdict[kmm] for base in bases: @@ -73,33 +113,35 @@ def __new__(cls, name, bases, clsdict): return new_type -def needs_values(func): +def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: """Returns method assuring we read values (on demand) before we try to access them""" @wraps(func) - def assure_data_present(self, *args, **kwargs): + def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: self.read() return func(self, *args, **kwargs) + # END wrapper method return assure_data_present -def set_dirty_and_flush_changes(non_const_func): +def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: """Return method that checks whether given non constant function may be called. If so, the instance will be set dirty. Additionally, we flush the changes right to disk""" - def flush_changes(self, *args, **kwargs): + def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: rval = non_const_func(self, *args, **kwargs) self._dirty = True self.write() return rval + # END wrapper method flush_changes.__name__ = non_const_func.__name__ return flush_changes -class SectionConstraint(object): +class SectionConstraint(Generic[T_ConfigParser]): """Constrains a ConfigParser to only option commands which are constrained to always use the section we have been initialized with. @@ -108,70 +150,81 @@ class SectionConstraint(object): :note: If used as a context manager, will release the wrapped ConfigParser.""" - __slots__ = ("_config", "_section_name") - _valid_attrs_ = ("get_value", "set_value", "get", "set", "getint", "getfloat", "getboolean", "has_option", - "remove_section", "remove_option", "options") - def __init__(self, config, section): + __slots__ = ("_config", "_section_name") + _valid_attrs_ = ( + "get_value", + "set_value", + "get", + "set", + "getint", + "getfloat", + "getboolean", + "has_option", + "remove_section", + "remove_option", + "options", + ) + + def __init__(self, config: T_ConfigParser, section: str) -> None: self._config = config self._section_name = section - def __del__(self): + def __del__(self) -> None: # Yes, for some reason, we have to call it explicitly for it to work in PY3 ! # Apparently __del__ doesn't get call anymore if refcount becomes 0 # Ridiculous ... . self._config.release() - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> Any: if attr in self._valid_attrs_: return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) return super(SectionConstraint, self).__getattribute__(attr) - def _call_config(self, method, *args, **kwargs): + def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: """Call the configuration at the given method which must take a section name as first argument""" return getattr(self._config, method)(self._section_name, *args, **kwargs) @property - def config(self): + def config(self) -> T_ConfigParser: """return: Configparser instance we constrain""" return self._config - def release(self): + def release(self) -> None: """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" return self._config.release() - def __enter__(self): + def __enter__(self) -> "SectionConstraint[T_ConfigParser]": self._config.__enter__() return self - def __exit__(self, exception_type, exception_value, traceback): + def __exit__(self, exception_type: str, exception_value: str, traceback: str) -> None: self._config.__exit__(exception_type, exception_value, traceback) -class _OMD(OrderedDict): +class _OMD(OrderedDict_OMD): """Ordered multi-dict.""" - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: _T) -> None: super(_OMD, self).__setitem__(key, [value]) - def add(self, key, value): + def add(self, key: str, value: Any) -> None: if key not in self: super(_OMD, self).__setitem__(key, [value]) - return - + return None super(_OMD, self).__getitem__(key).append(value) - def setall(self, key, values): + def setall(self, key: str, values: List[_T]) -> None: super(_OMD, self).__setitem__(key, values) - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return super(_OMD, self).__getitem__(key)[-1] - def getlast(self, key): + def getlast(self, key: str) -> Any: return super(_OMD, self).__getitem__(key)[-1] - def setlast(self, key, value): + def setlast(self, key: str, value: Any) -> None: if key not in self: super(_OMD, self).__setitem__(key, [value]) return @@ -179,22 +232,22 @@ def setlast(self, key, value): prior = super(_OMD, self).__getitem__(key) prior[-1] = value - def get(self, key, default=None): + def get(self, key: str, default: Union[_T, None] = None) -> Union[_T, None]: return super(_OMD, self).get(key, [default])[-1] - def getall(self, key): + def getall(self, key: str) -> List[_T]: return super(_OMD, self).__getitem__(key) - def items(self): + def items(self) -> List[Tuple[str, _T]]: # type: ignore[override] """List of (key, last value for key).""" return [(k, self[k]) for k in self] - def items_all(self): + def items_all(self) -> List[Tuple[str, List[_T]]]: """List of (key, list of values for key).""" return [(k, self.getall(k)) for k in self] -def get_config_path(config_level): +def get_config_path(config_level: Lit_config_levels) -> str: # we do not support an absolute path of the gitconfig on windows , # use the global config instead @@ -204,17 +257,21 @@ def get_config_path(config_level): if config_level == "system": return "/etc/gitconfig" elif config_level == "user": - config_home = os.environ.get("XDG_CONFIG_HOME") or osp.join(os.environ.get("HOME", '~'), ".config") + config_home = os.environ.get("XDG_CONFIG_HOME") or osp.join(os.environ.get("HOME", "~"), ".config") return osp.normpath(osp.expanduser(osp.join(config_home, "git", "config"))) elif config_level == "global": return osp.normpath(osp.expanduser("~/.gitconfig")) elif config_level == "repository": raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") - - raise ValueError("Invalid configuration level: %r" % config_level) + else: + # Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs + assert_never( + config_level, # type: ignore[unreachable] + ValueError(f"Invalid configuration level: {config_level!r}"), + ) -class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): +class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """Implements specifics required to read git style configuration files. @@ -232,27 +289,34 @@ class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, obje must match perfectly. If used as a context manager, will release the locked file.""" - #{ Configuration + # { Configuration # The lock type determines the type of lock to use in new configuration readers. # They must be compatible to the LockFile interface. # A suitable alternative would be the BlockingLockFile t_lock = LockFile - re_comment = re.compile(r'^\s*[#;]') + re_comment = re.compile(r"^\s*[#;]") - #} END configuration + # } END configuration - optvalueonly_source = r'\s*(?P