Address Code Review Comments.

python · orsenthil · Apr 29, 2021 · Apr 25, 2021 · Apr 28, 2021 · Apr 28, 2021
commit ee779d61c8f70f4fd68847f9820cf25e06306f94
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
@@ -312,9 +312,8 @@ or on combining URL components into a URL string.
   ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
   decomposed before parsing, no error will be raised.

-   Following the specification in WHATWG which updates RFC 3986, ASCII newline
-   ``\n``, ``\r`` or ``\r\n`` and tab ``\t`` characters are stripped from
-   the url.
+   Following the specification in `WHATWG`_ which updates RFC 3986, ASCII newline
+   ``\n``, ``\r`` and tab ``\t`` characters are stripped from the url.

   .. versionchanged:: 3.6
      Out-of-range port numbers now raise :exc:`ValueError`, instead of
@@ -680,6 +679,10 @@ task isn't already covered by the URL parsing functions above.

 .. seealso::

+   `WHATWG`_ -  URL Living standard
+      Working Group for the URL Standard that defines URLs, domains, IP addresses, the
+      application/x-www-form-urlencoded format, and their API.
+
   :rfc:`3986` - Uniform Resource Identifiers
      This is the current standard (STD66). Any changes to urllib.parse module
      should conform to this. Certain deviations could be observed, which are
@@ -703,3 +706,5 @@ task isn't already covered by the URL parsing functions above.

   :rfc:`1738` - Uniform Resource Locators (URL)
      This specifies the formal syntax and semantics of absolute URLs.
+
+.. _WHATWG: https://url.spec.whatwg.org/#concept-basic-url-parser
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
@@ -566,21 +566,6 @@ def test_urlsplit_attributes(self):
        self.assertEqual(p.port, 80)
        self.assertEqual(p.geturl(), url)

-        # Remove ASCII tabs and newlines from input
-
-        url = "http://www.python.org/java\nscript:\talert('msg\r\n')/#frag"
-        p = urllib.parse.urlsplit(url)
-        self.assertEqual(p.scheme, "http")
-        self.assertEqual(p.netloc, "www.python.org")
-        self.assertEqual(p.path, "/javascript:alert('msg')/")
-        self.assertEqual(p.query, "")
-        self.assertEqual(p.fragment, "frag")
-        self.assertEqual(p.username, None)
-        self.assertEqual(p.password, None)
-        self.assertEqual(p.hostname, "www.python.org")
-        self.assertEqual(p.port, None)
-        self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/#frag")
-
        # And check them all again, only with bytes this time
        url = b"HTTP://WWW.PYTHON.ORG/doc/#frag"
        p = urllib.parse.urlsplit(url)
@@ -621,6 +606,28 @@ def test_urlsplit_attributes(self):
        self.assertEqual(p.port, 80)
        self.assertEqual(p.geturl(), url)

+        # Verify an illegal port raises ValueError
+        url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag"
+        p = urllib.parse.urlsplit(url)
+        with self.assertRaisesRegex(ValueError, "out of range"):
+            p.port
+
+    def test_urlsplit_remove_unsafe_bytes(self):
+        # Remove ASCII tabs and newlines from input
+        url = "http://www.python.org/java\nscript:\talert('msg\r\n')/#frag"
+        p = urllib.parse.urlsplit(url)
+        self.assertEqual(p.scheme, "http")
+        self.assertEqual(p.netloc, "www.python.org")
+        self.assertEqual(p.path, "/javascript:alert('msg')/")
+        self.assertEqual(p.query, "")
+        self.assertEqual(p.fragment, "frag")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, "www.python.org")
+        self.assertEqual(p.port, None)
+        self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/#frag")
+
+        # Remove ASCII tabs and newlines from input as bytes.
        url = b"http://www.python.org/java\nscript:\talert('msg\r\n')/#frag"
        p = urllib.parse.urlsplit(url)
        self.assertEqual(p.scheme, b"http")
@@ -634,12 +641,6 @@ def test_urlsplit_attributes(self):
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/#frag")

-        # Verify an illegal port raises ValueError
-        url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag"
-        p = urllib.parse.urlsplit(url)
-        with self.assertRaisesRegex(ValueError, "out of range"):
-            p.port
-
    def test_attributes_bad_port(self):
        """Check handling of invalid ports."""
        for bytes in (False, True):

diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
@@ -78,6 +78,9 @@
                '0123456789'
                '+-.')

+# Unsafe bytes to be removed per WHATWG spec
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
+
 # XXX: Consider replacing with functools.lru_cache
 MAX_CACHE_SIZE = 20
 _parse_cache = {}
@@ -469,8 +472,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
        else:
            scheme, url = url[:i].lower(), url[i+1:]

-    _unsafe_chars_to_remove = ['\t', '\r', '\n']
-    url = url.translate({ord(c): None for c in _unsafe_chars_to_remove})
+    for b in _UNSAFE_URL_BYTES_TO_REMOVE:
+        url = url.replace(b, "")

    if url[:2] == '//':
        netloc, url = _splitnetloc(url, 2)