Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 93b49c5

Browse filesBrowse files
tipabutipabu
committed
py3: Be able to parse non-RFC-compliant request lines
There's a bug in CPython [1] that causes servers to mis-parse request lines that include the bytes \x85 or \xa0. Naturally, we have functional tests that (with high probability) will send such request lines. There's a fix proposed, but the earliest it's likely to land would be for 3.8, and we need to be able to target an earlier Python. So, intercept the request line immediately before parsing and re-write it to be RFC-compliant. Note that this is done for Python 2 as well, though there should be no change in the request environment that eventlet eventually hands to us. [1] https://bugs.python.org/issue33973 Change-Id: Ie648f5c04d4415f3b620fb196fa567ce7575d522
1 parent f55167a commit 93b49c5
Copy full SHA for 93b49c5

2 files changed

+60-7Lines changed: 60 additions & 7 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎swift/common/wsgi.py‎

Copy file name to clipboardExpand all lines: swift/common/wsgi.py
+32-1Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535

3636
from swift.common import utils, constraints
3737
from swift.common.storage_policy import BindPortsCache
38-
from swift.common.swob import Request, wsgi_unquote
38+
from swift.common.swob import Request, wsgi_quote, wsgi_unquote, \
39+
wsgi_quote_plus, wsgi_unquote_plus, wsgi_to_bytes, bytes_to_wsgi
3940
from swift.common.utils import capture_stdio, disable_fallocate, \
4041
drop_privileges, get_logger, NullLogger, config_true_value, \
4142
validate_configuration, get_hub, config_auto_int_value, \
@@ -433,6 +434,36 @@ def get_default_type(self):
433434
'''If the client didn't provide a content type, leave it blank.'''
434435
return ''
435436

437+
def parse_request(self):
438+
if not six.PY2:
439+
# request lines *should* be ascii per the RFC, but historically
440+
# we've allowed (and even have func tests that use) arbitrary
441+
# bytes. This breaks on py3 (see https://bugs.python.org/issue33973
442+
# ) but the work-around is simple: munge the request line to be
443+
# properly quoted. py2 will do the right thing without this, but it
444+
# doesn't hurt to re-write the request line like this and it
445+
# simplifies testing.
446+
if self.raw_requestline.count(b' ') >= 2:
447+
parts = self.raw_requestline.split(b' ', 2)
448+
path, q, query = parts[1].partition(b'?')
449+
# unquote first, so we don't over-quote something
450+
# that was *correctly* quoted
451+
path = wsgi_to_bytes(wsgi_quote(wsgi_unquote(
452+
bytes_to_wsgi(path))))
453+
query = b'&'.join(
454+
sep.join([
455+
wsgi_to_bytes(wsgi_quote_plus(wsgi_unquote_plus(
456+
bytes_to_wsgi(key)))),
457+
wsgi_to_bytes(wsgi_quote_plus(wsgi_unquote_plus(
458+
bytes_to_wsgi(val))))
459+
])
460+
for part in query.split(b'&')
461+
for key, sep, val in (part.partition(b'='), ))
462+
parts[1] = path + q + query
463+
self.raw_requestline = b' '.join(parts)
464+
# else, mangled protocol, most likely; let base class deal with it
465+
return wsgi.HttpProtocol.parse_request(self)
466+
436467

437468
class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
438469
"""
Collapse file

‎test/unit/common/test_wsgi.py‎

Copy file name to clipboardExpand all lines: test/unit/common/test_wsgi.py
+28-6Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import eventlet.wsgi
2929

30+
import six
3031
from six import BytesIO
3132
from six.moves.urllib.parse import quote
3233

@@ -984,11 +985,6 @@ def test_make_env_keeps_infocache(self):
984985

985986

986987
class TestSwiftHttpProtocol(unittest.TestCase):
987-
def setUp(self):
988-
patcher = mock.patch('swift.common.wsgi.wsgi.HttpProtocol')
989-
self.mock_super = patcher.start()
990-
self.addCleanup(patcher.stop)
991-
992988
def _proto_obj(self):
993989
# Make an object we can exercise... note the base class's __init__()
994990
# does a bunch of work, so we just new up an object like eventlet.wsgi
@@ -1041,12 +1037,38 @@ def test_swift_http_protocol_parse_request_no_proxy(self):
10411037

10421038
self.assertEqual(False, proto_obj.parse_request())
10431039

1044-
self.assertEqual([], self.mock_super.mock_calls)
10451040
self.assertEqual([
10461041
mock.call(400, "Bad HTTP/0.9 request type ('jimmy')"),
10471042
], proto_obj.send_error.mock_calls)
10481043
self.assertEqual(('a', '123'), proto_obj.client_address)
10491044

1045+
def test_request_line_cleanup(self):
1046+
def do_test(line_from_socket, expected_line=None):
1047+
if expected_line is None:
1048+
expected_line = line_from_socket
1049+
1050+
proto_obj = self._proto_obj()
1051+
proto_obj.raw_requestline = line_from_socket
1052+
with mock.patch('swift.common.wsgi.wsgi.HttpProtocol') \
1053+
as mock_super:
1054+
proto_obj.parse_request()
1055+
1056+
self.assertEqual([mock.call.parse_request(proto_obj)],
1057+
mock_super.mock_calls)
1058+
self.assertEqual(proto_obj.raw_requestline, expected_line)
1059+
1060+
do_test(b'GET / HTTP/1.1')
1061+
do_test(b'GET /%FF HTTP/1.1')
1062+
1063+
if not six.PY2:
1064+
do_test(b'GET /\xff HTTP/1.1', b'GET /%FF HTTP/1.1')
1065+
do_test(b'PUT /Here%20Is%20A%20SnowMan:\xe2\x98\x83 HTTP/1.0',
1066+
b'PUT /Here%20Is%20A%20SnowMan%3A%E2%98%83 HTTP/1.0')
1067+
do_test(
1068+
b'POST /?and%20it=fixes+params&'
1069+
b'PALMTREE=\xf0%9f\x8c%b4 HTTP/1.1',
1070+
b'POST /?and+it=fixes+params&PALMTREE=%F0%9F%8C%B4 HTTP/1.1')
1071+
10501072

10511073
class TestProxyProtocol(unittest.TestCase):
10521074
def _run_bytes_through_protocol(self, bytes_from_client, protocol_class):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.