Skip to content

Navigation Menu

Sign in

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 85f38a1

Browse filesBrowse files
ankostisankostis
committed
remote, #525: pump fetch-infos instead of GIL-read stderr
+ `handle_process_output()` accepts null-finalizer, to pump completely stderr before raising any errors. + test: Enable `TestGit.test_environment()` on Windows (to checks stderr consumption).
1 parent 8364597 commit 85f38a1
Copy full SHA for 85f38a1

File tree

4 files changed

+38
-40
lines changed
Filter options

4 files changed

+38
-40
lines changed

‎git/cmd.py

Copy file name to clipboardExpand all lines: git/cmd.py
+6-2Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@
5959
# Documentation
6060
## @{
6161

62-
def handle_process_output(process, stdout_handler, stderr_handler, finalizer, decode_streams=True):
62+
def handle_process_output(process, stdout_handler, stderr_handler,
63+
finalizer=None, decode_streams=True):
6364
"""Registers for notifications to lean that process output is ready to read, and dispatches lines to
6465
the respective line handlers.
6566
This function returns once the finalizer returns
@@ -108,10 +109,13 @@ def pump_stream(cmdline, name, stream, is_decode, handler):
108109
t.start()
109110
threads.append(t)
110111

112+
## FIXME: Why Join?? Will block if `stdin` needs feeding...
113+
#
111114
for t in threads:
112115
t.join()
113116

114-
return finalizer(process)
117+
if finalizer:
118+
return finalizer(process)
115119

116120

117121
def dashify(string):

‎git/remote.py

Copy file name to clipboardExpand all lines: git/remote.py
+10-16Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -630,25 +630,19 @@ def _get_fetch_info_from_stderr(self, proc, progress):
630630
cmds = set(PushInfo._flag_map.keys()) & set(FetchInfo._flag_map.keys())
631631

632632
progress_handler = progress.new_message_handler()
633+
handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False)
633634

634-
stderr_text = None
635+
stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or ''
636+
proc.wait(stderr=stderr_text)
637+
if stderr_text:
638+
log.warning("Error lines received while fetching: %s", stderr_text)
635639

636-
for line in proc.stderr:
640+
for line in progress.other_lines:
637641
line = force_text(line)
638-
for pline in progress_handler(line):
639-
# END handle special messages
640-
for cmd in cmds:
641-
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
642-
fetch_info_lines.append(line)
643-
continue
644-
# end find command code
645-
# end for each comand code we know
646-
# end for each line progress didn't handle
647-
# end
648-
if progress.error_lines():
649-
stderr_text = '\n'.join(progress.error_lines())
650-
651-
finalize_process(proc, stderr=stderr_text)
642+
for cmd in cmds:
643+
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
644+
fetch_info_lines.append(line)
645+
continue
652646

653647
# read head information
654648
with open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') as fp:

‎git/test/test_git.py

Copy file name to clipboardExpand all lines: git/test/test_git.py
+9-12Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -207,18 +207,15 @@ def test_environment(self, rw_dir):
207207
rw_repo = Repo.init(os.path.join(rw_dir, 'repo'))
208208
remote = rw_repo.create_remote('ssh-origin', "ssh://git@server/foo")
209209

210-
# This only works if we are not evaluating git-push/pull output in a thread !
211-
import select
212-
if hasattr(select, 'poll'):
213-
with rw_repo.git.custom_environment(GIT_SSH=path):
214-
try:
215-
remote.fetch()
216-
except GitCommandError as err:
217-
if sys.version_info[0] < 3 and is_darwin:
218-
self.assertIn('ssh-orig, ' in str(err))
219-
self.assertEqual(err.status, 128)
220-
else:
221-
self.assertIn('FOO', str(err))
210+
with rw_repo.git.custom_environment(GIT_SSH=path):
211+
try:
212+
remote.fetch()
213+
except GitCommandError as err:
214+
if sys.version_info[0] < 3 and is_darwin:
215+
self.assertIn('ssh-orig, ' in str(err))
216+
self.assertEqual(err.status, 128)
217+
else:
218+
self.assertIn('FOO', str(err))
222219

223220
def test_handle_process_output(self):
224221
from git.cmd import handle_process_output

‎git/util.py

Copy file name to clipboardExpand all lines: git/util.py
+13-10Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -199,33 +199,34 @@ class RemoteProgress(object):
199199
DONE_TOKEN = 'done.'
200200
TOKEN_SEPARATOR = ', '
201201

202-
__slots__ = ("_cur_line", "_seen_ops", "_error_lines")
202+
__slots__ = ('_cur_line',
203+
'_seen_ops',
204+
'error_lines', # Lines that started with 'error:' or 'fatal:'.
205+
'other_lines') # Lines not denoting progress (i.e.g. push-infos).
203206
re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
204207
re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")
205208

206209
def __init__(self):
207210
self._seen_ops = list()
208211
self._cur_line = None
209-
self._error_lines = []
210-
211-
def error_lines(self):
212-
"""Returns all lines that started with error: or fatal:"""
213-
return self._error_lines
212+
self.error_lines = []
213+
self.other_lines = []
214214

215215
def _parse_progress_line(self, line):
216216
"""Parse progress information from the given line as retrieved by git-push
217217
or git-fetch.
218218
219-
Lines that seem to contain an error (i.e. start with error: or fatal:) are stored
220-
separately and can be queried using `error_lines()`.
219+
- Lines that do not contain progress info are stored in :attr:`other_lines`.
220+
- Lines that seem to contain an error (i.e. start with error: or fatal:) are stored
221+
in :attr:`error_lines`.
221222
222223
:return: list(line, ...) list of lines that could not be processed"""
223224
# handle
224225
# Counting objects: 4, done.
225226
# Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done.
226227
self._cur_line = line
227-
if len(self._error_lines) > 0 or self._cur_line.startswith(('error:', 'fatal:')):
228-
self._error_lines.append(self._cur_line)
228+
if len(self.error_lines) > 0 or self._cur_line.startswith(('error:', 'fatal:')):
229+
self.error_lines.append(self._cur_line)
229230
return []
230231

231232
sub_lines = line.split('\r')
@@ -284,6 +285,7 @@ def _parse_progress_line(self, line):
284285
self.line_dropped(sline)
285286
# Note: Don't add this line to the failed lines, as we have to silently
286287
# drop it
288+
self.other_lines.extend(failed_lines)
287289
return failed_lines
288290
# END handle op code
289291

@@ -309,6 +311,7 @@ def _parse_progress_line(self, line):
309311
max_count and float(max_count),
310312
message)
311313
# END for each sub line
314+
self.other_lines.extend(failed_lines)
312315
return failed_lines
313316

314317
def new_message_handler(self):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.