From f498de9bfd67bcbb42d36dfb8ff9e59ec788825b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Jan 2015 05:49:18 +0100 Subject: [PATCH 01/38] Remote.update() didn't pass kwargs along to git command. Fixes #250 --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index d048f87bc..2267c2035 100644 --- a/git/remote.py +++ b/git/remote.py @@ -517,7 +517,7 @@ def update(self, **kwargs): Additional arguments passed to git-remote update :return: self """ - self.repo.git.remote("update", self.name) + self.repo.git.remote("update", self.name, **kwargs) return self def _get_fetch_info_from_stderr(self, proc, progress): From 47d9f1321c3a6da68a7909fcb1a66a209066d4c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Jan 2015 06:34:48 +0100 Subject: [PATCH 02/38] Added test to verify we can handle fetch prunes. They are just skipped. Fixes #249 --- git/test/test_remote.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/git/test/test_remote.py b/git/test/test_remote.py index 98d74d8be..b7bf98018 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -445,6 +445,19 @@ def test_base(self, rw_repo, remote_repo): origin = rw_repo.remote('origin') assert origin == rw_repo.remotes.origin + # Verify we can handle prunes when fetching + # stderr lines look like this: x [deleted] (none) -> origin/experiment-2012 + # These should just be skipped + num_deleted = False + for branch in remote_repo.heads: + if branch.name != 'master': + branch.delete(remote_repo, branch, force=True) + num_deleted += 1 + # end + # end for each branch + assert num_deleted > 0 + assert len(rw_repo.remotes.origin.fetch(prune=True)) == 1, "deleted everything but master" + @with_rw_repo('HEAD', bare=True) def test_creation_and_removal(self, bare_rw_repo): new_name = "test_new_one" From 91645aa87e79e54a56efb8686eb4ab6c8ba67087 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Jan 2015 06:54:49 +0100 Subject: [PATCH 03/38] This should fix the test failure on travis --- git/test/test_remote.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/git/test/test_remote.py b/git/test/test_remote.py index b7bf98018..2540e49b0 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -448,6 +448,11 @@ def test_base(self, rw_repo, remote_repo): # Verify we can handle prunes when fetching # stderr lines look like this: x [deleted] (none) -> origin/experiment-2012 # These should just be skipped + # If we don't have a manual checkout, we can't actually assume there are any non-master branches + remote_repo.create_head("myone_for_deletion") + # Get the branch - to be pruned later + origin.fetch() + num_deleted = False for branch in remote_repo.heads: if branch.name != 'master': From 91562247e0d11d28b4bc6d85ba50b7af2bd7224b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 2 Feb 2015 08:06:13 +0100 Subject: [PATCH 04/38] An attempt to better steer questions and answers. Currently, people put it onto the mailing list and on stack overflow [ci skip] --- README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0019dc179..1895635fd 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,11 @@ The object database implementation is optimized for handling large quantities of ### REQUIREMENTS -* Git ( tested with 1.8.3.4 ) -* Python Nose - used for running the tests - - Tested with nose 1.3.0 -* Mock by Michael Foord used for tests - - Tested with 1.0.1 -* Coverage - used for tests coverage +GitPython needs the `git` executable to be installed on the system and available in your `PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. -The list of dependencies are listed in /requirements.txt and /test-requirements.txt. The installer takes care of installing them for you though. +* Git (1.7.x or newer) + +The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. The installer takes care of installing them for you. ### INSTALL @@ -62,8 +59,16 @@ You can watch me fix issues or implement new features [live on Twitch][twitch-ch ### INFRASTRUCTURE * [User Documentation](http://gitpython.readthedocs.org) +* [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) + * Please post on stackoverflow and use the `gitpython` tag * [Mailing List](http://groups.google.com/group/git-python) + * Please use it for everything that doesn't fit Stackoverflow. * [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) + * Post reproducible bugs and feature requests as a new issue. Please be sure to provide the following information if posting bugs: + * GitPython version (e.g. `import git; git.__version__`) + * Python version (e.g. `python --version`) + * The encountered stack-trace, if applicable + * Enough information to allow reproducing the issue ### LICENSE From d8bbfea4cdcb36ce0e9ee7d7cad4c41096d4d54f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 5 Feb 2015 06:10:42 +0100 Subject: [PATCH 05/38] Updated copyright information. Fixes #246 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 942996b0b..f86f08ac8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -43,7 +43,7 @@ # General information about the project. project = u'GitPython' -copyright = u'Copyright (C) 2008, 2009 Michael Trier and contributors, 2010 Sebastian Thiel' +copyright = u'Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From d7729245060f9aecfef4544f91e2656aa8d483ec Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 7 Feb 2015 11:14:03 -0500 Subject: [PATCH 06/38] ENH: respect GIT_PYTHON_TEST_GIT_REPO_BASE env var in tests --- git/test/lib/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 8300f2722..5efb7d9d9 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -15,7 +15,7 @@ from git import Repo, Remote, GitCommandError, Git from git.compat import string_types -GIT_REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) __all__ = ( 'fixture_path', 'fixture', 'absolute_project_path', 'StringProcessAdapter', From 756b7ad43d0ad18858075f90ce5eaec2896d439c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 7 Feb 2015 11:14:42 -0500 Subject: [PATCH 07/38] BF: skip unicode filename test in env not supporting unicode encodings --- git/test/test_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/git/test/test_base.py b/git/test/test_base.py index 91b9d0053..72f2d5542 100644 --- a/git/test/test_base.py +++ b/git/test/test_base.py @@ -5,6 +5,7 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os +import sys import tempfile import git.objects.base as base @@ -116,6 +117,14 @@ def test_add_unicode(self, rw_repo): filename = u"שלום.txt" file_path = os.path.join(rw_repo.working_dir, filename) + + # verify first that we could encode file name in this environment + try: + _ = file_path.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + from nose import SkipTest + raise SkipTest("Environment doesn't support unicode filenames") + open(file_path, "wb").write(b'something') if os.name == 'nt': From 873823f39275ceb8d65ebfae74206c7347e07123 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 8 Feb 2015 21:47:55 -0500 Subject: [PATCH 08/38] BF: run commit hook with repo.working_dir as cwd Otherwise commit hook might rightfully fail, as happens if repository is e.g. git-annex repository. See e.g. now failing https://travis-ci.org/datalad/datalad/builds/49802394\#L1590 which seems to pass tests nicely with patch as this --- git/index/fun.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/index/fun.py b/git/index/fun.py index f07cf7dc3..c2fe912e6 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -72,6 +72,7 @@ def run_commit_hook(name, index): env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=index.repo.working_dir, close_fds=(os.name == 'posix')) stdout, stderr = cmd.communicate() cmd.stdout.close() From fe426d404b727d0567f21871f61cc6dc881e8bf0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 9 Feb 2015 19:22:18 +0100 Subject: [PATCH 09/38] Minor Flake8 fixes. Latest version of it is required to show the issues travis shows as well --- git/index/fun.py | 6 +++--- git/test/lib/helper.py | 10 ++++++---- git/test/test_base.py | 2 +- tox.ini | 4 +++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index f07cf7dc3..f8bd9d894 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -10,8 +10,6 @@ S_IFREG, ) -S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule - from io import BytesIO import os import subprocess @@ -33,7 +31,6 @@ CE_NAMEMASK, CE_STAGESHIFT ) -CE_NAMEMASK_INV = ~CE_NAMEMASK from .util import ( pack, @@ -47,6 +44,9 @@ force_text ) +S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule +CE_NAMEMASK_INV = ~CE_NAMEMASK + __all__ = ('write_cache', 'read_cache', 'write_tree_from_cache', 'entry_key', 'stat_mode_to_index_mode', 'S_IFGITLINK', 'run_commit_hook', 'hook_path') diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 5efb7d9d9..31bee78f6 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -15,7 +15,9 @@ from git import Repo, Remote, GitCommandError, Git from git.compat import string_types -GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +osp = os.path.dirname + +GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", osp(osp(osp(osp(__file__))))) __all__ = ( 'fixture_path', 'fixture', 'absolute_project_path', 'StringProcessAdapter', @@ -26,7 +28,7 @@ def fixture_path(name): - test_dir = os.path.dirname(os.path.dirname(__file__)) + test_dir = osp(osp(__file__)) return os.path.join(test_dir, "fixtures", name) @@ -35,7 +37,7 @@ def fixture(name): def absolute_project_path(): - return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + return os.path.abspath(os.path.join(osp(__file__), "..", "..")) #} END routines @@ -195,7 +197,7 @@ def remote_repo_creator(self): d_remote.config_writer.set('url', remote_repo_url) - temp_dir = os.path.dirname(_mktemp()) + temp_dir = osp(_mktemp()) # On windows, this will fail ... we deal with failures anyway and default to telling the user to do it try: gd = Git().daemon(temp_dir, enable='receive-pack', as_process=True) diff --git a/git/test/test_base.py b/git/test/test_base.py index 72f2d5542..94379ca34 100644 --- a/git/test/test_base.py +++ b/git/test/test_base.py @@ -120,7 +120,7 @@ def test_add_unicode(self, rw_repo): # verify first that we could encode file name in this environment try: - _ = file_path.encode(sys.getfilesystemencoding()) + file_path.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: from nose import SkipTest raise SkipTest("Environment doesn't support unicode filenames") diff --git a/tox.ini b/tox.ini index bb1e9a85b..da624b5ee 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,9 @@ commands = {posargs} [flake8] #show-source = True # E265 = comment blocks like @{ section, which it can't handle +# E266 = too many leading '#' for block comment +# E731 = do not assign a lambda expression, use a def # W293 = Blank line contains whitespace -ignore = E265,W293 +ignore = E265,W293,E266,E731 max-line-length = 120 exclude = .tox,.venv,build,dist,doc,git/ext/ From f51fe3e66d358e997f4af4e91a894a635f7cb601 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 9 Feb 2015 21:05:41 +0100 Subject: [PATCH 10/38] Added previously missing parameter documentation for Repo.__init__ . Related to #255 --- git/repo/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/git/repo/base.py b/git/repo/base.py index b7a9e29dc..e59cb0c7d 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -121,6 +121,11 @@ def __init__(self, path=None, odbt=DefaultDBType, search_parent_directories=Fals Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. .git/objects. It will be used to access all object data + :param search_parent_directories: + if True, all parent directories will be searched for a valid repo as well. + + Please note that this was the default behaviour in older versions of GitPython, + which is considered a bug though. :raise InvalidGitRepositoryError: :raise NoSuchPathError: :return: git.Repo """ From 158bc981130bfbe214190cac19da228d1f321fe1 Mon Sep 17 00:00:00 2001 From: Jonas Trappenberg Date: Mon, 9 Feb 2015 15:15:15 -0800 Subject: [PATCH 11/38] Replace GIT_SSH with GIT_SSH_COMMAND for SSH key management. Also move untestable documentation out of test. Related: #234, #242 --- doc/source/tutorial.rst | 9 ++++----- git/test/test_docs.py | 9 --------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 0d60f0aae..e86fd8d57 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -331,12 +331,11 @@ You can easily access configuration information for a remote by accessing option :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] -You can also specify per-call custom environments using a new context manager on the Git command +You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. -.. literalinclude:: ../../git/test/test_docs.py - :language: python - :start-after: # [32-test_references_and_objects] - :end-before: # ![32-test_references_and_objects] + ssh_cmd = 'ssh -i id_deployment_key' + with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + repo.remotes.origin.fetch() Submodule Handling ****************** diff --git a/git/test/test_docs.py b/git/test/test_docs.py index 8dfef1c6d..586f0ce4a 100644 --- a/git/test/test_docs.py +++ b/git/test/test_docs.py @@ -437,15 +437,6 @@ def test_references_and_objects(self, rw_dir): git.for_each_ref() # '-' becomes '_' when calling it # ![31-test_references_and_objects] - # [32-test_references_and_objects] - ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh') - with repo.git.custom_environment(GIT_SSH=ssh_executable): - # Note that we don't actually make the call here, as our test-setup doesn't permit it to - # succeed. - # It will in your case :) - repo.remotes.origin.fetch - # ![32-test_references_and_objects] - def test_submodules(self): # [1-test_submodules] repo = self.rorepo From 5ac93b1d7e0694ceb303ee72c312921a9b1588f4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 19 Feb 2015 16:56:13 +0100 Subject: [PATCH 12/38] Use uuid instead of tempfile.mkdtmp, which created an actual directory. That, over time, could have caused slow downs due to file-system hassle. Fixes #258 --- git/objects/submodule/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index be243acc5..82c465acb 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -37,7 +37,7 @@ import os import logging -import tempfile +import uuid __all__ = ["Submodule", "UpdateProgress"] @@ -992,7 +992,7 @@ def rename(self, new_name): source_dir = mod.git_dir # Let's be sure the submodule name is not so obviously tied to a directory if destination_module_abspath.startswith(mod.git_dir): - tmp_dir = self._module_abspath(self.repo, self.path, os.path.basename(tempfile.mkdtemp())) + tmp_dir = self._module_abspath(self.repo, self.path, str(uuid.uuid4())) os.renames(source_dir, tmp_dir) source_dir = tmp_dir # end handle self-containment From 0f5320e8171324a002d3769824152cf5166a21a2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 19 Feb 2015 17:12:15 +0100 Subject: [PATCH 13/38] Fix flake8 issue. It's new in the latest version of flake - thanks travis for letting me know. --- git/objects/submodule/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 82c465acb..9c1dc6471 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -578,11 +578,13 @@ def update(self, recursive=False, init=True, to_latest_revision=False, progress= base_commit = mrepo.merge_base(mrepo.head.commit, hexsha) if len(base_commit) == 0 or base_commit[0].hexsha == hexsha: if force: - log.debug("Will force checkout or reset on local branch that is possibly in the future of" - + "the commit it will be checked out to, effectively 'forgetting' new commits") + msg = "Will force checkout or reset on local branch that is possibly in the future of" + msg += "the commit it will be checked out to, effectively 'forgetting' new commits" + log.debug(msg) else: - log.info("Skipping %s on branch '%s' of submodule repo '%s' as it contains " - + "un-pushed commits", is_detached and "checkout" or "reset", mrepo.head, mrepo) + msg = "Skipping %s on branch '%s' of submodule repo '%s' as it contains un-pushed commits" + msg %= (is_detached and "checkout" or "reset", mrepo.head, mrepo) + log.info(msg) may_reset = False # end handle force # end handle if we are in the future From 346424daaaf2bb3936b5f4c2891101763dc2bdc0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 19 Feb 2015 18:02:35 +0100 Subject: [PATCH 14/38] Assure to not iterate packed-refs file, ever. Related to #252 --- git/refs/symbolic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index fed7006e6..d884250f7 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -140,6 +140,7 @@ def _get_ref_info(cls, repo, ref_path): # Don't only split on spaces, but on whitespace, which allows to parse lines like # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo tokens = value.split() + assert(len(tokens) != 0) except (OSError, IOError): # Probably we are just packed, find our entry in the packed refs file # NOTE: We are not a symbolic ref if we are in a packed file, as these @@ -582,6 +583,8 @@ def _iter_items(cls, repo, common_path=None): # END prune non-refs folders for f in files: + if f == 'packed-refs': + continue abs_path = to_native_path_linux(join_path(root, f)) rela_paths.add(abs_path.replace(to_native_path_linux(repo.git_dir) + '/', "")) # END for each file in root directory From 3d74543a545a9468cabec5d20519db025952efed Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 19 Feb 2015 18:29:10 +0100 Subject: [PATCH 15/38] Adjust minimum git version with git-file support. As I am pretty sure to have tested it with 1.7.0, I assume they added the git file feature somewhere between .0 .10. Fixes #252 --- git/objects/submodule/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 9c1dc6471..4347df55d 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -136,7 +136,8 @@ def _get_intermediate_items(self, item): @classmethod def _need_gitfile_submodules(cls, git): - return git.version_info[:3] >= (1, 8, 0) + # on debian wheezy, it seems to already support git files at this version. Maybe even earler, who knows + return git.version_info[:3] >= (1, 7, 10) def __eq__(self, other): """Compare with another submodule""" From 38fc944390d57399d393dc34b4d1c5c81241fb87 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Feb 2015 07:09:18 +0100 Subject: [PATCH 16/38] It seems something within our environment changed ... ... as we are now running out of file handles. Previously, it worked ... and gitpython didn't change --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1fda20189..069527ebf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ install: - git config --global user.name "Travis Runner" script: # Make sure we limit open handles to see if we are leaking them - - ulimit -n 64 + - ulimit -n 96 - ulimit -n - nosetests -v --with-coverage - flake8 From e2feb62c17acd1dddb6cd125d8b90933c32f89e1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Feb 2015 18:11:13 +0100 Subject: [PATCH 17/38] Apparently, git 1.7.9 supports git-files too, lets assume it's starting at 1.7.0 --- git/objects/submodule/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 4347df55d..7e8dd053e 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -136,8 +136,7 @@ def _get_intermediate_items(self, item): @classmethod def _need_gitfile_submodules(cls, git): - # on debian wheezy, it seems to already support git files at this version. Maybe even earler, who knows - return git.version_info[:3] >= (1, 7, 10) + return git.version_info[:3] >= (1, 7, 0) def __eq__(self, other): """Compare with another submodule""" From c00567d3025016550d55a7abaf94cbb82a5c44fb Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Feb 2015 15:06:30 -0500 Subject: [PATCH 18/38] BF: do not checkout master -- that ruins testing of PRs. "reset" master to original HEAD --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 069527ebf..b53228ca1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,12 @@ install: - pip install coveralls flake8 sphinx # generate some reflog as git-python tests need it (in master) + - git tag __testing_point__ - git checkout master - git reset --hard HEAD~1 - git reset --hard HEAD~1 - git reset --hard HEAD~1 - - git reset --hard origin/master + - git reset --hard __testing_point__ # as commits are performed with the default user, it needs to be set for travis too - git config --global user.email "travis@ci.com" From e0acb8371bb2b68c2bda04db7cb2746ba3f9da86 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 21 Feb 2015 10:12:11 +0100 Subject: [PATCH 19/38] Added 'insert_kwargs_after' flag for consumption by _call_process. While at it, all other invocations of .git in remote.py were reviewed Fixes #262 --- git/cmd.py | 14 +++++++++++++- git/remote.py | 8 ++++++-- git/test/test_git.py | 4 ++++ git/test/test_remote.py | 3 +++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7e15d4eaa..911bb9f36 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -753,11 +753,23 @@ def _call_process(self, method, *args, **kwargs): except KeyError: pass + insert_after_this_arg = kwargs.pop('insert_kwargs_after', None) + # Prepare the argument list opt_args = self.transform_kwargs(**kwargs) ext_args = self.__unpack_args([a for a in args if a is not None]) - args = opt_args + ext_args + if insert_after_this_arg is None: + args = opt_args + ext_args + else: + try: + index = ext_args.index(insert_after_this_arg) + except ValueError: + raise ValueError("Couldn't find argument '%s' in args %s to insert kwargs after" + % (insert_after_this_arg, str(ext_args))) + # end handle error + args = ext_args[:index + 1] + opt_args + ext_args[index + 1:] + # end handle kwargs def make_call(): call = [self.GIT_PYTHON_GIT_EXECUTABLE] diff --git a/git/remote.py b/git/remote.py index 2267c2035..8911aaa43 100644 --- a/git/remote.py +++ b/git/remote.py @@ -477,7 +477,9 @@ def create(cls, repo, name, url, **kwargs): :param kwargs: Additional arguments to be passed to the git-remote add command :return: New Remote instance :raise GitCommandError: in case an origin with that name already exists""" - repo.git.remote("add", name, url, **kwargs) + scmd = 'add' + kwargs['insert_kwargs_after'] = scmd + repo.git.remote(scmd, name, url, **kwargs) return cls(repo, name) # add is an alias @@ -517,7 +519,9 @@ def update(self, **kwargs): Additional arguments passed to git-remote update :return: self """ - self.repo.git.remote("update", self.name, **kwargs) + scmd = 'update' + kwargs['insert_kwargs_after'] = scmd + self.repo.git.remote(scmd, self.name, **kwargs) return self def _get_fetch_info_from_stderr(self, proc, progress): diff --git a/git/test/test_git.py b/git/test/test_git.py index 8087bc454..ef6947552 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -155,6 +155,10 @@ def test_single_char_git_options_are_passed_to_git(self): def test_change_to_transform_kwargs_does_not_break_command_options(self): self.git.log(n=1) + def test_insert_after_kwarg_raises(self): + # This isn't a complete add command, which doesn't matter here + self.failUnlessRaises(ValueError, self.git.remote, 'add', insert_kwargs_after='foo') + def test_env_vars_passed_to_git(self): editor = 'non_existant_editor' with mock.patch.dict('os.environ', {'GIT_EDITOR': editor}): diff --git a/git/test/test_remote.py b/git/test/test_remote.py index 2540e49b0..8500a295a 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -486,6 +486,9 @@ def test_creation_and_removal(self, bare_rw_repo): # END if deleted remote matches existing remote's name # END for each remote + # Issue #262 - the next call would fail if bug wasn't fixed + bare_rw_repo.create_remote('bogus', '/bogus/path', mirror='push') + def test_fetch_info(self): # assure we can handle remote-tracking branches fetch_info_line_fmt = "c437ee5deb8d00cf02f03720693e4c802e99f390 not-for-merge %s '0.3' of " From 3236f8b966061b23ba063f3f7e1652764fee968f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 21 Feb 2015 11:14:53 +0100 Subject: [PATCH 20/38] `stale_refs()` may now also handle other kinds of references, like tags. Fixes #260 --- git/remote.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/git/remote.py b/git/remote.py index 8911aaa43..bb2165f22 100644 --- a/git/remote.py +++ b/git/remote.py @@ -455,7 +455,13 @@ def stale_refs(self): remote side, but are still available locally. The IterableList is prefixed, hence the 'origin' must be omitted. See - 'refs' property for an example.""" + 'refs' property for an example. + + To make things more complicated, it can be possble for the list to include + other kinds of references, for example, tag references, if these are stale + as well. This is a fix for the issue described here: + https://github.com/gitpython-developers/GitPython/issues/260 + """ out_refs = IterableList(RemoteReference._id_attribute_, "%s/" % self.name) for line in self.repo.git.remote("prune", "--dry-run", self).splitlines()[2:]: # expecting @@ -463,8 +469,14 @@ def stale_refs(self): token = " * [would prune] " if not line.startswith(token): raise ValueError("Could not parse git-remote prune result: %r" % line) - fqhn = "%s/%s" % (RemoteReference._common_path_default, line.replace(token, "")) - out_refs.append(RemoteReference(self.repo, fqhn)) + ref_name = line.replace(token, "") + # sometimes, paths start with a full ref name, like refs/tags/foo, see #260 + if ref_name.startswith(Reference._common_path_default + '/'): + out_refs.append(SymbolicReference.from_path(self.repo, ref_name)) + else: + fqhn = "%s/%s" % (RemoteReference._common_path_default, ref_name) + out_refs.append(RemoteReference(self.repo, fqhn)) + # end special case handlin # END for each line return out_refs From 9d1a489931ecbdd652111669c0bd86bcd6f5abbe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 21 Feb 2015 11:21:55 +0100 Subject: [PATCH 21/38] Fixed trailing white space! Think about how expensive this single invisible character was, with all the time and energy spent on it ! --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index bb2165f22..6a15c9c4f 100644 --- a/git/remote.py +++ b/git/remote.py @@ -458,7 +458,7 @@ def stale_refs(self): 'refs' property for an example. To make things more complicated, it can be possble for the list to include - other kinds of references, for example, tag references, if these are stale + other kinds of references, for example, tag references, if these are stale as well. This is a fix for the issue described here: https://github.com/gitpython-developers/GitPython/issues/260 """ From 630d030058c234e50d87196b624adc2049834472 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Feb 2015 08:33:54 +0100 Subject: [PATCH 22/38] Improved documentation on IndexFile.add(...) Related to #224 [ci skip] --- git/index/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/index/base.py b/git/index/base.py index b73edd6f3..733d4a63b 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -625,6 +625,10 @@ def add(self, items, force=True, fprogress=lambda *args: None, path_rewriter=Non strings denote a relative or absolute path into the repository pointing to an existing file, i.e. CHANGES, lib/myfile.ext, '/home/gitrepo/lib/myfile.ext'. + Absolute paths must start with working tree directory of this index's repository + to be considered valid. For example, if it was initialized with a non-normalized path, like + `/root/repo/../repo`, absolute paths to be added must start with `/root/repo/../repo`. + Paths provided like this must exist. When added, they will be written into the object database. From a5e607e8aa62ca3778f1026c27a927ee5c79749b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 2 Mar 2015 14:19:05 +0100 Subject: [PATCH 23/38] fix(iter-commit): ambiguous argument error In repositories like > git branch -a * test > ls test `repo.iter_commits` failed due to an ambigous argument (`'git rev-list test`). Now this cannot happen anymore. fixes #264 --- git/objects/commit.py | 7 +++++-- git/test/test_commit.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index b9718694f..f13760fdb 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -189,9 +189,12 @@ def iter_items(cls, repo, rev, paths='', **kwargs): if 'pretty' in kwargs: raise ValueError("--pretty cannot be used as parsing expects single sha's only") # END handle pretty - args = list() + + # use -- in any case, to prevent possibility of ambiguous arguments + # see https://github.com/gitpython-developers/GitPython/issues/264 + args = ['--'] if paths: - args.extend(('--', paths)) + args.extend((paths, )) # END if paths proc = repo.git.rev_list(rev, args, as_process=True, **kwargs) diff --git a/git/test/test_commit.py b/git/test/test_commit.py index 1f0f8c561..3e958edf2 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -19,15 +19,19 @@ Actor, ) from gitdb import IStream +from gitdb.test.lib import with_rw_directory from git.compat import ( string_types, text_type ) +from git import Repo +from git.repo.fun import touch from io import BytesIO import time import sys import re +import os def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -219,6 +223,15 @@ def test_rev_list_bisect_all(self): for sha1, commit in zip(expected_ids, commits): assert_equal(sha1, commit.hexsha) + @with_rw_directory + def test_ambiguous_arg_iteration(self, rw_dir): + rw_repo = Repo.init(os.path.join(rw_dir, 'test_ambiguous_arg')) + path = os.path.join(rw_repo.working_tree_dir, 'master') + touch(path) + rw_repo.index.add([path]) + rw_repo.index.commit('initial commit') + list(rw_repo.iter_commits(rw_repo.head.ref)) # should fail unless bug is fixed + def test_count(self): assert self.rorepo.tag('refs/tags/0.1.5').commit.count() == 143 From 50f763cfe38a1d69a3a04e41a36741545885f1d8 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 2 Mar 2015 09:25:27 -0800 Subject: [PATCH 24/38] Store path attribute on Diff object If a file in a commit contains no changes (for example, if only the file mode is changed) there will be no blob attached. This is usually where the filename is stored, so without it, the calling context can not tell what file was changed. Instead, always store a_path and b_path on the Diff object so that information is available. --- git/diff.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/git/diff.py b/git/diff.py index 378823696..dc53f3f71 100644 --- a/git/diff.py +++ b/git/diff.py @@ -168,11 +168,13 @@ class Diff(object): a_mode is None a_blob is None + a_path is None ``Deleted File``:: b_mode is None b_blob is None + b_path is None ``Working Tree Blobs`` @@ -200,8 +202,8 @@ class Diff(object): NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 - __slots__ = ("a_blob", "b_blob", "a_mode", "b_mode", "new_file", "deleted_file", - "rename_from", "rename_to", "diff") + __slots__ = ("a_blob", "b_blob", "a_mode", "b_mode", "a_path", "b_path", + "new_file", "deleted_file", "rename_from", "rename_to", "diff") def __init__(self, repo, a_path, b_path, a_blob_id, b_blob_id, a_mode, b_mode, new_file, deleted_file, rename_from, @@ -210,6 +212,9 @@ def __init__(self, repo, a_path, b_path, a_blob_id, b_blob_id, a_mode, self.a_mode = a_mode self.b_mode = b_mode + self.a_path = a_path + self.b_path = b_path + if self.a_mode: self.a_mode = mode_str_to_int(self.a_mode) if self.b_mode: From 21a6cb7336b61f904198f1d48526dcbe9cba6eec Mon Sep 17 00:00:00 2001 From: "Kyle P. Johnson" Date: Fri, 27 Mar 2015 10:04:58 -0400 Subject: [PATCH 25/38] Fix typo --- git/objects/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/util.py b/git/objects/util.py index cefef862d..567b1d5b2 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -216,7 +216,7 @@ def __getattr__(self, attr): class Traversable(object): - """Simple interface to perforam depth-first or breadth-first traversals + """Simple interface to perform depth-first or breadth-first traversals into one direction. Subclasses only need to implement one function. Instances of the Subclass must be hashable""" From 1c2502ee83927437442b13b83f3a7976e4146a01 Mon Sep 17 00:00:00 2001 From: Markus Siemens Date: Tue, 7 Apr 2015 14:33:39 +0200 Subject: [PATCH 26/38] Fix problem with submodules on Windows On Windows, `repo.create_submodule(...)` failed because git didn't recognize the worktree path set in `.git/modules/sub/config` (`fatal: bad config file line 6 in ./config`). This commit makes `_write_git_file_and_module_config` convert the worktree path to the linux format (forward slashes) which git recognizes. --- git/objects/submodule/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 7e8dd053e..f9b0b6ad6 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -293,7 +293,8 @@ def _write_git_file_and_module_config(cls, working_tree_dir, module_abspath): fp.close() writer = GitConfigParser(os.path.join(module_abspath, 'config'), read_only=False, merge_includes=False) - writer.set_value('core', 'worktree', os.path.relpath(working_tree_dir, start=module_abspath)) + writer.set_value('core', 'worktree', + to_native_path_linux(os.path.relpath(working_tree_dir, start=module_abspath))) writer.release() #{ Edit Interface From 54e27f7bbb412408bbf0d2735b07d57193869ea6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 7 Apr 2015 17:19:22 +0200 Subject: [PATCH 27/38] fix(docs): sphinx docs build in latest version --- doc/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index f86f08ac8..add686d3f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -94,7 +94,6 @@ # ----------------------- html_theme_options = { - "stickysidebar": "true" } # The style sheet to use for HTML and HTML Help pages. A file of that name From 1fd07a0a7a6300db1db8b300a3f567b31b335570 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 7 Apr 2015 18:27:16 +0200 Subject: [PATCH 28/38] test(index): test for #265 However, it doesn't reproduce on the latest version of GitPython. Maybe it's on an older one. --- git/test/test_index.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/git/test/test_index.py b/git/test/test_index.py index 63f99f109..8124ec164 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -44,8 +44,11 @@ BaseIndexEntry, IndexEntry ) +from gitdb.test.lib import with_rw_directory from git.index.fun import hook_path +import git + class TestIndex(TestBase): @@ -765,3 +768,15 @@ def test_index_bare_add(self, rw_bare_repo): except InvalidGitRepositoryError: asserted = True assert asserted, "Adding using a filename is not correctly asserted." + + @with_rw_directory + def test_index_add_corruption(self, rw_dir): + # Test for https://github.com/gitpython-developers/GitPython/issues/265 + repo = git.Repo.clone_from("git://pkgs.fedoraproject.org/GitPython", rw_dir) + assert not repo.is_dirty() + file_path = os.path.join(rw_dir, "GitPython.spec") + open(file_path, 'wb').close() + assert repo.is_dirty() + repo.index.add(['0001-GPG-signature-support-on-commit-object.patch', 'GitPython.spec', '.gitignore', 'sources']) + repo.git.commit(m="committing file") + assert not repo.is_dirty() From ab7c3223076306ca71f692ed5979199863cf45a7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 7 Apr 2015 18:38:42 +0200 Subject: [PATCH 29/38] fix(externals): init external in dev mode only Previously it would always adjust your system path, which is bad behaviour. --- git/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/__init__.py b/git/__init__.py index 5630b91d0..e8dae2723 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -15,7 +15,8 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'gitdb')) + if __version__ == 'git': + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'ext', 'gitdb')) try: import gitdb From 8f043d8a1d7f4076350ff0c778bfa60f7aa2f7aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 09:50:26 +0200 Subject: [PATCH 30/38] fix(index): don't write extension data by default It turned out that the index is not actually corrupted, which is good news. What happens is that `git` writes `TREE` extension data into the index, which causes it to write out the given tree *as is* next time a `git commit` is executed. When using `git add`, this extension data is maintained automatically. However, GitPython doesn't do that ... . Usually this is no problem at all, as you are supposed to use `IndexFile.commit(...)` along with `IndexFile.add(...)`. Thanks to a shortcoming in the GitPython API, the index was automatically written out whenever files have been added, without providing control over whether or not *extension data* will be written along with it. My fix consists of an additional flag in `IndexFile.add(...)`, which causes extension data not to be written by default, so commits can be safely done via `git commit` or `IndexFile.commit(...)`. However, this might introduce new subtle bugs in case someone is relying on extension data to be written. As this can be controlled through the said flag though, a fix is easily done in that case. Fixes #265 --- doc/source/changes.rst | 6 ++++++ git/index/base.py | 37 +++++++++++++++++++++++++------------ git/test/test_index.py | 15 --------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 5b85e56ab..52d70c0f1 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +0.3.7 - Fixes +============= +* `IndexFile.add()` will now write the index without any extension data by default. However, you may override this behaviour with the new `write_extension_data` keyword argument. + + - Renamed `ignore_tree_extension_data` keyword argument in `IndexFile.write(...)` to `ignore_extension_data` + 0.3.6 - Features ================ * **DOCS** diff --git a/git/index/base.py b/git/index/base.py index 733d4a63b..0553f418a 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -172,16 +172,17 @@ def _entries_sorted(self): """:return: list of entries, in a sorted fashion, first by path, then by stage""" return sorted(self.entries.values(), key=lambda e: (e.path, e.stage)) - def _serialize(self, stream, ignore_tree_extension_data=False): + def _serialize(self, stream, ignore_extension_data=False): entries = self._entries_sorted() - write_cache(entries, - stream, - (ignore_tree_extension_data and None) or self._extension_data) + extension_data = self._extension_data + if ignore_extension_data: + extension_data = None + write_cache(entries, stream, extension_data) return self #} END serializable interface - def write(self, file_path=None, ignore_tree_extension_data=False): + def write(self, file_path=None, ignore_extension_data=False): """Write the current state to our file path or to the given one :param file_path: @@ -190,9 +191,10 @@ def write(self, file_path=None, ignore_tree_extension_data=False): Please note that this will change the file_path of this index to the one you gave. - :param ignore_tree_extension_data: + :param ignore_extension_data: If True, the TREE type extension data read in the index will not - be written to disk. Use this if you have altered the index and + be written to disk. NOTE that no extension data is actually written. + Use this if you have altered the index and would like to use git-write-tree afterwards to create a tree representing your written changes. If this data is present in the written index, git-write-tree @@ -208,7 +210,7 @@ def write(self, file_path=None, ignore_tree_extension_data=False): lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) - self._serialize(stream, ignore_tree_extension_data) + self._serialize(stream, ignore_extension_data) lfd.commit() @@ -612,7 +614,7 @@ def _entries_for_paths(self, paths, path_rewriter, fprogress, entries): return entries_added def add(self, items, force=True, fprogress=lambda *args: None, path_rewriter=None, - write=True): + write=True, write_extension_data=False): """Add files from the working tree, specific blobs or BaseIndexEntries to the index. @@ -689,8 +691,19 @@ def add(self, items, force=True, fprogress=lambda *args: None, path_rewriter=Non Please note that entry.path is relative to the git repository. :param write: - If True, the index will be written once it was altered. Otherwise - the changes only exist in memory and are not available to git commands. + If True, the index will be written once it was altered. Otherwise + the changes only exist in memory and are not available to git commands. + + :param write_extension_data: + If True, extension data will be written back to the index. This can lead to issues in case + it is containing the 'TREE' extension, which will cause the `git commit` command to write an + old tree, instead of a new one representing the now changed index. + This doesn't matter if you use `IndexFile.commit()`, which ignores the `TREE` extension altogether. + You should set it to True if you intend to use `IndexFile.commit()` exclusively while maintaining + support for third-party extensions. Besides that, you can usually safely ignore the built-in + extensions when using GitPython on repositories that are not handled manually at all. + All current built-in extensions are listed here: + http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/technical/index-format.txt :return: List(BaseIndexEntries) representing the entries just actually added. @@ -763,7 +776,7 @@ def handle_null_entries(self): self.entries[(entry.path, 0)] = IndexEntry.from_base(entry) if write: - self.write() + self.write(ignore_extension_data=not write_extension_data) # END handle write return entries_added diff --git a/git/test/test_index.py b/git/test/test_index.py index 8124ec164..63f99f109 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -44,11 +44,8 @@ BaseIndexEntry, IndexEntry ) -from gitdb.test.lib import with_rw_directory from git.index.fun import hook_path -import git - class TestIndex(TestBase): @@ -768,15 +765,3 @@ def test_index_bare_add(self, rw_bare_repo): except InvalidGitRepositoryError: asserted = True assert asserted, "Adding using a filename is not correctly asserted." - - @with_rw_directory - def test_index_add_corruption(self, rw_dir): - # Test for https://github.com/gitpython-developers/GitPython/issues/265 - repo = git.Repo.clone_from("git://pkgs.fedoraproject.org/GitPython", rw_dir) - assert not repo.is_dirty() - file_path = os.path.join(rw_dir, "GitPython.spec") - open(file_path, 'wb').close() - assert repo.is_dirty() - repo.index.add(['0001-GPG-signature-support-on-commit-object.patch', 'GitPython.spec', '.gitignore', 'sources']) - repo.git.commit(m="committing file") - assert not repo.is_dirty() From 98f6995bdcbd10ea0387d0c55cb6351b81a857fd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 10:13:50 +0200 Subject: [PATCH 31/38] fix(index): _store_path() now closes it's stream This should prevent a resource warning given in py3 Fixes #263 --- git/index/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/index/base.py b/git/index/base.py index 0553f418a..10de3358a 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -579,6 +579,7 @@ def _store_path(self, filepath, fprogress): fprogress(filepath, False, filepath) istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) fprogress(filepath, True, filepath) + stream.close() return BaseIndexEntry((stat_mode_to_index_mode(st.st_mode), istream.binsha, 0, to_native_path_linux(filepath))) From 723f100a422577235e06dc024a73285710770fca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 10:41:18 +0200 Subject: [PATCH 32/38] fix(docs): be clear about exit code handling When pushing/pulling, we ignore errors unless it's exit code 128. The reason for this is now made explicit to make clear that issues are handled by PushInfo flags accordingly. Related #271 --- git/test/lib/helper.py | 14 +++++++++++--- git/util.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 31bee78f6..541b972de 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -232,7 +232,13 @@ def remote_repo_creator(self): prev_cwd = os.getcwd() os.chdir(rw_repo.working_dir) try: - return func(self, rw_repo, rw_remote_repo) + try: + return func(self, rw_repo, rw_remote_repo) + except: + print("Keeping repos after failure: repo_dir = %s, remote_repo_dir = %s" + % (repo_dir, remote_repo_dir), file=sys.stderr) + repo_dir = remote_repo_dir = None + raise finally: # gd.proc.kill() ... no idea why that doesn't work if gd is not None: @@ -241,8 +247,10 @@ def remote_repo_creator(self): os.chdir(prev_cwd) rw_repo.git.clear_cache() rw_remote_repo.git.clear_cache() - shutil.rmtree(repo_dir, onerror=_rmtree_onerror) - shutil.rmtree(remote_repo_dir, onerror=_rmtree_onerror) + if repo_dir: + shutil.rmtree(repo_dir, onerror=_rmtree_onerror) + if remote_repo_dir: + shutil.rmtree(remote_repo_dir, onerror=_rmtree_onerror) if gd is not None: gd.proc.wait() diff --git a/git/util.py b/git/util.py index 02c54bc3e..f41f20fb4 100644 --- a/git/util.py +++ b/git/util.py @@ -159,6 +159,23 @@ def finalize_process(proc): except GitCommandError: # if a push has rejected items, the command has non-zero return status # a return status of 128 indicates a connection error - reraise the previous one + # Everything else will still be parsed and made available through PushInfo flags + # Estimated error results look like this: + # ```bash + # To /var/folders/xp/m48gs2tx2vg95tmtzw7tprs40000gn/T/tmpk5jeBeremote_repo_test_base + # ! refs/heads/master:refs/heads/master [rejected] (non-fast-forward) + # Done + # error: failed to push some refs to + # '/var/folders/xp/m48gs2tx2vg95tmtzw7tprs40000gn/T/tmpk5jeBeremote_repo_test_base' + # hint: Updates were rejected because the tip of your current branch is behind + # hint: its remote counterpart. Integrate the remote changes (e.g. + # hint: 'git pull ...') before pushing again. + # hint: See the 'Note about fast-forwards' in 'git push --help' for details. + # ``` + # See https://github.com/gitpython-developers/GitPython/blob/master/git/test/test_remote.py#L305 + # on how to check for these kinds of errors. + # Also see this issue for a reason for this verbosity: + # https://github.com/gitpython-developers/GitPython/issues/271 if proc.poll() == 128: raise pass From e9f8f159ebad405b2c08aa75f735146bb8e216ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 11:00:32 +0200 Subject: [PATCH 33/38] fix(remote): allow to raise during push/fetch Do not swallow non-zero exit status during push and fetch unless we managed to parse head information. This behaviour will effetively handle cases were no work was done due to invalid refspecs or insufficient permissions. Fixes #271 --- doc/source/changes.rst | 4 ++++ git/remote.py | 13 +++++++++++-- git/test/test_remote.py | 3 +-- git/util.py | 32 ++------------------------------ 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 52d70c0f1..35c2f15c3 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -7,6 +7,10 @@ Changelog * `IndexFile.add()` will now write the index without any extension data by default. However, you may override this behaviour with the new `write_extension_data` keyword argument. - Renamed `ignore_tree_extension_data` keyword argument in `IndexFile.write(...)` to `ignore_extension_data` +* If the git command executed during `Remote.push(...)|fetch(...)` returns with an non-zero exit code and GitPython didn't + obtain any head-information, the corresponding `GitCommandError` will be raised. This may break previous code which expected + these operations to never raise. However, that behavious is undesirable as it would effectively hide the fact that there + was an error. See `this issue `_ for more information. 0.3.6 - Features ================ diff --git a/git/remote.py b/git/remote.py index 6a15c9c4f..4baa2838c 100644 --- a/git/remote.py +++ b/git/remote.py @@ -568,7 +568,12 @@ def _get_fetch_info_from_stderr(self, proc, progress): # end # We are only interested in stderr here ... - finalize_process(proc) + try: + finalize_process(proc) + except Exception: + if len(fetch_info_lines) == 0: + raise + # end exception handler # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') @@ -601,7 +606,11 @@ def stdout_handler(line): # END exception handling # END for each line - handle_process_output(proc, stdout_handler, progress_handler, finalize_process) + try: + handle_process_output(proc, stdout_handler, progress_handler, finalize_process) + except Exception: + if len(output) == 0: + raise return output def fetch(self, refspec=None, progress=None, **kwargs): diff --git a/git/test/test_remote.py b/git/test/test_remote.py index 8500a295a..c419ecee9 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -313,8 +313,7 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): self._do_test_push_result(res, remote) # invalid refspec - res = remote.push("hellothere") - assert len(res) == 0 + self.failUnlessRaises(GitCommandError, remote.push, "hellothere") # push new tags progress = TestRemoteProgress() diff --git a/git/util.py b/git/util.py index f41f20fb4..1147cb535 100644 --- a/git/util.py +++ b/git/util.py @@ -16,10 +16,7 @@ # NOTE: Some of the unused imports might be used/imported by others. # Handle once test-cases are back up and running. -from .exc import ( - GitCommandError, - InvalidGitRepositoryError -) +from .exc import InvalidGitRepositoryError from .compat import ( MAXSIZE, @@ -154,32 +151,7 @@ def get_user_id(): def finalize_process(proc): """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" - try: - proc.wait() - except GitCommandError: - # if a push has rejected items, the command has non-zero return status - # a return status of 128 indicates a connection error - reraise the previous one - # Everything else will still be parsed and made available through PushInfo flags - # Estimated error results look like this: - # ```bash - # To /var/folders/xp/m48gs2tx2vg95tmtzw7tprs40000gn/T/tmpk5jeBeremote_repo_test_base - # ! refs/heads/master:refs/heads/master [rejected] (non-fast-forward) - # Done - # error: failed to push some refs to - # '/var/folders/xp/m48gs2tx2vg95tmtzw7tprs40000gn/T/tmpk5jeBeremote_repo_test_base' - # hint: Updates were rejected because the tip of your current branch is behind - # hint: its remote counterpart. Integrate the remote changes (e.g. - # hint: 'git pull ...') before pushing again. - # hint: See the 'Note about fast-forwards' in 'git push --help' for details. - # ``` - # See https://github.com/gitpython-developers/GitPython/blob/master/git/test/test_remote.py#L305 - # on how to check for these kinds of errors. - # Also see this issue for a reason for this verbosity: - # https://github.com/gitpython-developers/GitPython/issues/271 - if proc.poll() == 128: - raise - pass - # END exception handling + proc.wait() #} END utilities From 1c2dd54358dd526d1d08a8e4a977f041aff74174 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 11:43:10 +0200 Subject: [PATCH 34/38] fix(cmd): throw GitCommandNotFoundError ... ... if it is not found. Previously, especially on windows, this wasn't explicit. Fixes #248, affects #126 --- doc/source/changes.rst | 5 +++++ git/cmd.py | 46 ++++++++++++++++++++++++++++++------------ git/exc.py | 6 ++++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 35c2f15c3..99b578bd8 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -12,6 +12,11 @@ Changelog these operations to never raise. However, that behavious is undesirable as it would effectively hide the fact that there was an error. See `this issue `_ for more information. +* If the git executable can't be found in the PATH or at the path provided by `GIT_PYTHON_GIT_EXECUTABLE`, this is made + obvious by throwing `GitCommandNotFound`, both on unix and on windows. + + - Those who support **GUI on windows** will now have to set `git.Git.USE_SHELL = True` to get the previous behaviour. + 0.3.6 - Features ================ * **DOCS** diff --git a/git/cmd.py b/git/cmd.py index 911bb9f36..429046be1 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -26,7 +26,10 @@ stream_copy, WaitGroup ) -from .exc import GitCommandError +from .exc import ( + GitCommandError, + GitCommandNotFound +) from git.compat import ( string_types, defenc, @@ -241,6 +244,12 @@ class Git(LazyMixin): _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) + # If True, a shell will be used when executing git commands. + # This should only be desirable on windows, see https://github.com/gitpython-developers/GitPython/pull/126 + # for more information + # Override this value using `Git.USE_SHELL = True` + USE_SHELL = False + class AutoInterrupt(object): """Kill/Interrupt the stored process instance once this instance goes out of scope. It is @@ -543,18 +552,29 @@ def execute(self, command, env["LC_MESSAGES"] = "C" env.update(self._environment) - proc = Popen(command, - env=env, - cwd=cwd, - stdin=istream, - stderr=PIPE, - stdout=PIPE, - # Prevent cmd prompt popups on windows by using a shell ... . - # See https://github.com/gitpython-developers/GitPython/pull/126 - shell=sys.platform == 'win32', - close_fds=(os.name == 'posix'), # unsupported on windows - **subprocess_kwargs - ) + if sys.platform == 'win32': + cmd_not_found_exception = WindowsError + else: + if sys.version_info[0] > 2: + cmd_not_found_exception = FileNotFoundError # NOQA # this is defined, but flake8 doesn't know + else: + cmd_not_found_exception = OSError + # end handle + + try: + proc = Popen(command, + env=env, + cwd=cwd, + stdin=istream, + stderr=PIPE, + stdout=PIPE, + shell=self.USE_SHELL, + close_fds=(os.name == 'posix'), # unsupported on windows + **subprocess_kwargs + ) + except cmd_not_found_exception as err: + raise GitCommandNotFound(str(err)) + if as_process: return self.AutoInterrupt(proc, command) diff --git a/git/exc.py b/git/exc.py index 7ee6726e8..f5b52374b 100644 --- a/git/exc.py +++ b/git/exc.py @@ -18,6 +18,12 @@ class NoSuchPathError(OSError): """ Thrown if a path could not be access by the system. """ +class GitCommandNotFound(Exception): + """Thrown if we cannot find the `git` executable in the PATH or at the path given by + the GIT_PYTHON_GIT_EXECUTABLE environment variable""" + pass + + class GitCommandError(Exception): """ Thrown if execution of the git command fails with non-zero status code. """ From a08733d76a8254a20a28f4c3875a173dcf0ad129 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 12:08:54 +0200 Subject: [PATCH 35/38] fix(test_cmd): handle GitCommandNotFound in test Related to #248 --- git/test/test_git.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/git/test/test_git.py b/git/test/test_git.py index ef6947552..742c842db 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -19,6 +19,7 @@ from git import ( Git, GitCommandError, + GitCommandNotFound, Repo ) from gitdb.test.lib import with_rw_directory @@ -127,11 +128,7 @@ def test_version(self): def test_cmd_override(self): prev_cmd = self.git.GIT_PYTHON_GIT_EXECUTABLE - if os.name == 'nt': - exc = GitCommandError - else: - exc = OSError - # end handle windows case + exc = GitCommandNotFound try: # set it to something that doens't exist, assure it raises type(self.git).GIT_PYTHON_GIT_EXECUTABLE = os.path.join( From 6cc1e7e08094494916db1aadda17e03ce695d049 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 14:58:33 +0200 Subject: [PATCH 36/38] docs(tutorial): add pre-v2.3 GIT_SSH example It goes along with the new one advertising the GIT_SSH_COMMAND environment variable. Related to #256 --- doc/source/tutorial.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index e86fd8d57..84b3b5f50 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -331,11 +331,17 @@ You can easily access configuration information for a remote by accessing option :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] -You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. +You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. The following example works with `git` starting at *v2.3*. ssh_cmd = 'ssh -i id_deployment_key' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): repo.remotes.origin.fetch() + +This one sets a custom script to be executed in place of `ssh`, and can be used in `git` prior to *v2.3*. + + ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh') + with repo.git.custom_environment(GIT_SSH=ssh_executable): + repo.remotes.origin.fetch() Submodule Handling ****************** From 545ac2574cfb75b02e407e814e10f76bc485926d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 15:05:23 +0200 Subject: [PATCH 37/38] docs(tutorial): fix GIT_SSH examples They didn't show up as code-block Related to #256 --- doc/source/tutorial.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 84b3b5f50..632d2d0cc 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -331,17 +331,19 @@ You can easily access configuration information for a remote by accessing option :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] -You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. The following example works with `git` starting at *v2.3*. +You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. The following example works with `git` starting at *v2.3*:: ssh_cmd = 'ssh -i id_deployment_key' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): repo.remotes.origin.fetch() -This one sets a custom script to be executed in place of `ssh`, and can be used in `git` prior to *v2.3*. +This one sets a custom script to be executed in place of `ssh`, and can be used in `git` prior to *v2.3*:: ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh') with repo.git.custom_environment(GIT_SSH=ssh_executable): repo.remotes.origin.fetch() + +You might also have a look at `Git.update_environment(...)` in case you want to setup a changed environment more permanently. Submodule Handling ****************** From 3470e269bcdc9091d0c5e25e7c09ce175c7cee77 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Apr 2015 15:14:37 +0200 Subject: [PATCH 38/38] fix(version-up): v0.3.7 * milestone URL: http://goo.gl/HFaeZ4 --- VERSION | 2 +- doc/source/changes.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 449d7e73a..0f8268533 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.6 +0.3.7 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 99b578bd8..d9f44a86c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -16,6 +16,9 @@ Changelog obvious by throwing `GitCommandNotFound`, both on unix and on windows. - Those who support **GUI on windows** will now have to set `git.Git.USE_SHELL = True` to get the previous behaviour. + +* A list of all issues can be found `on github `_ + 0.3.6 - Features ================