From 4ed41e92145b79837a66d82c67b749541b7ed600 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sat, 4 Aug 2018 12:31:29 +0000 Subject: [PATCH 01/38] uuid: correct the field seperator value of ethernet adapter MAC address on AIX --- Lib/test/test_uuid.py | 21 +++++++++++++++++-- Lib/uuid.py | 9 +++++--- .../2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 1 + 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 7af1d7aec797c5..6553a4887ee8d2 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -6,11 +6,13 @@ import os import shutil import subprocess +import sys + +_notAIX = not sys.platform.startswith("aix") py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) - def importable(name): try: __import__(name) @@ -516,10 +518,17 @@ class BaseTestInternals: @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_find_mac(self): - data = ''' + if _notAIX: + data = ''' fake hwaddr cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab +''' + else: + data = ''' +fake hwaddr +cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 +eth0 Link encap:Ethernet HWaddr 12.34.56.78.90.ab ''' popen = unittest.mock.MagicMock() @@ -549,6 +558,10 @@ def check_node(self, node, requires=None): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): + if _notAIX: + node = uuid._ifconfig_getnode() + else: + self.skipTest('because AIX "ifconfig" does not provide macaddr') node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @@ -559,6 +572,10 @@ def test_ip_getnode(self): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): + if _notAIX: + node = uuid._ifconfig_getnode() + else: + self.skipTest('because AIX "arp" does not provide macaddr') node = self.uuid._arp_getnode() self.check_node(node, 'arp') diff --git a/Lib/uuid.py b/Lib/uuid.py index 66383218e70c0d..127d82576d3667 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -52,6 +52,9 @@ __author__ = 'Ka-Ping Yee ' +_notAIX = not sys.platform.startswith("aix") +_mac_delim = b':' if _notAIX else b'.' + RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', 'reserved for Microsoft compatibility', 'reserved for future definition'] @@ -373,7 +376,7 @@ def _find_mac(command, args, hw_identifiers, get_index): if words[i] in hw_identifiers: try: word = words[get_index(i)] - mac = int(word.replace(b':', b''), 16) + mac = int(word.replace(_mac_delim, b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -455,8 +458,8 @@ def _netstat_getnode(): try: words = line.rstrip().split() word = words[i] - if len(word) == 17 and word.count(b':') == 5: - mac = int(word.replace(b':', b''), 16) + if len(word) <= 17 and len(word) >= 11 and word.count(_mac_delim) == 5: + mac = int(word.replace(_mac_delim, b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst new file mode 100644 index 00000000000000..1e89f9c4ac3cec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -0,0 +1 @@ +uuid: correct the field seperator (_mac_field) value of an ethernet adapter MAC address for AIX From c6029a473a1d7bca577ec450b41aea5eca9d6ec6 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sat, 4 Aug 2018 13:25:37 +0000 Subject: [PATCH 02/38] correct test logic, cut/paste error --- Lib/test/test_uuid.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 6553a4887ee8d2..987a2b19bb137b 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -559,10 +559,9 @@ def check_node(self, node, requires=None): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): if _notAIX: - node = uuid._ifconfig_getnode() + node = self.uuid._ifconfig_getnode() else: self.skipTest('because AIX "ifconfig" does not provide macaddr') - node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') @@ -573,10 +572,9 @@ def test_ip_getnode(self): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): if _notAIX: - node = uuid._ifconfig_getnode() + node = self.uuid._arp_getnode() else: self.skipTest('because AIX "arp" does not provide macaddr') - node = self.uuid._arp_getnode() self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') From 7e1874d81e2a62690bfbcc875fd52b25fde5b02b Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 5 Aug 2018 10:14:05 +0000 Subject: [PATCH 03/38] specify shorter getters list per https://bugs.python.org/issue28009#msg306562 also, calculate correct value when the fields are not all two characters --- Lib/uuid.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 127d82576d3667..f8891bac5f581f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -458,8 +458,15 @@ def _netstat_getnode(): try: words = line.rstrip().split() word = words[i] - if len(word) <= 17 and len(word) >= 11 and word.count(_mac_delim) == 5: - mac = int(word.replace(_mac_delim, b''), 16) + if word.count(_mac_delim) == 5: + if len(word) == 17: + mac = int(word.replace(_mac_delim, b''), 16) + elif len(word) < 17 and len(word) >= 11: + mac = 0 + hexs = word.split(_mac_delim) + for hex in hexs: + mac <<= 8 + mac += int(hex, 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -666,6 +673,8 @@ def _random_getnode(): _NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] +_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] + def getnode(*, getters=None): """Get the hardware address as a 48-bit positive integer. @@ -681,7 +690,7 @@ def getnode(*, getters=None): if sys.platform == 'win32': getters = _NODE_GETTERS_WIN32 else: - getters = _NODE_GETTERS_UNIX + getters = _NODE_GETTERS_UNIX if _notAIX else _NODE_GETTERS_AIX for getter in getters + [_random_getnode]: try: From 6a5bccb68d8c415163fb0645c0b86645f48f9c3c Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 11:55:54 +0000 Subject: [PATCH 04/38] write _notAIX as non-negative constant and use 'not AIX' --- Lib/test/test_uuid.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 987a2b19bb137b..3ff599d18055aa 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -8,7 +8,7 @@ import subprocess import sys -_notAIX = not sys.platform.startswith("aix") +AIX = sys.platform.startswith("aix") py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) @@ -518,7 +518,7 @@ class BaseTestInternals: @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_find_mac(self): - if _notAIX: + if not AIX: data = ''' fake hwaddr cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 @@ -558,7 +558,7 @@ def check_node(self, node, requires=None): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): - if _notAIX: + if not AIX: node = self.uuid._ifconfig_getnode() else: self.skipTest('because AIX "ifconfig" does not provide macaddr') @@ -571,7 +571,7 @@ def test_ip_getnode(self): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): - if _notAIX: + if not AIX: node = self.uuid._arp_getnode() else: self.skipTest('because AIX "arp" does not provide macaddr') From a98309b18a11a2b091a92cab0cc55e49c958c8b0 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 11:57:56 +0000 Subject: [PATCH 05/38] simplify NEWS entry --- .../next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst index 1e89f9c4ac3cec..0d809eaf3d65c8 100644 --- a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -1 +1 @@ -uuid: correct the field seperator (_mac_field) value of an ethernet adapter MAC address for AIX +Fix uuid.uuid1() and uuid.get_node() on AIX From 53ef750de90589377eae55e6adddd8838ef05c35 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 12:20:55 +0000 Subject: [PATCH 06/38] skip a test the right way --- Lib/test/test_uuid.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 3ff599d18055aa..691c62dfdec526 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -557,11 +557,8 @@ def check_node(self, node, requires=None): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(AIX, 'because AIX "ifconfig" does not provide macaddr') def test_ifconfig_getnode(self): - if not AIX: - node = self.uuid._ifconfig_getnode() - else: - self.skipTest('because AIX "ifconfig" does not provide macaddr') self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') From 6fd5f8e1e6f7bad86f1fb9282ef64834af112995 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 12:21:56 +0000 Subject: [PATCH 07/38] capitalize constants --- Lib/uuid.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index f8891bac5f581f..33b4c3df373c48 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,7 +53,7 @@ __author__ = 'Ka-Ping Yee ' _notAIX = not sys.platform.startswith("aix") -_mac_delim = b':' if _notAIX else b'.' +_MAC_DELIM = b':' if _notAIX else b'.' RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', @@ -376,7 +376,7 @@ def _find_mac(command, args, hw_identifiers, get_index): if words[i] in hw_identifiers: try: word = words[get_index(i)] - mac = int(word.replace(_mac_delim, b''), 16) + mac = int(word.replace(_MAC_DELIM, b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -458,12 +458,15 @@ def _netstat_getnode(): try: words = line.rstrip().split() word = words[i] - if word.count(_mac_delim) == 5: + if word.count(_MAC_DELIM) == 5: if len(word) == 17: - mac = int(word.replace(_mac_delim, b''), 16) + mac = int(word.replace(_MAC_DELIM, b''), 16) + # the extracted hex string will not be a 12 hex digit + # string, so extract the fields and add them in + # piece by piece elif len(word) < 17 and len(word) >= 11: mac = 0 - hexs = word.split(_mac_delim) + hexs = word.split(_MAC_DELIM) for hex in hexs: mac <<= 8 mac += int(hex, 16) From 3bb1c08d48babc57a6538c7868580385233a39a0 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 12:39:57 +0000 Subject: [PATCH 08/38] oops - restore missing assignment --- Lib/test/test_uuid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 691c62dfdec526..4a854bde5b505f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -559,6 +559,7 @@ def check_node(self, node, requires=None): @unittest.skipUnless(os.name == 'posix', 'requires Posix') @unittest.skipIf(AIX, 'because AIX "ifconfig" does not provide macaddr') def test_ifconfig_getnode(self): + node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') From 1af8e3dbbe3a6c100b9cdcd2ba680e120f4e0161 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 12:46:37 +0000 Subject: [PATCH 09/38] skip both tests the right way --- Lib/test/test_uuid.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 4a854bde5b505f..3c542983368138 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -568,11 +568,9 @@ def test_ip_getnode(self): self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(AIX, 'because AIX "arp" does not provide macaddr') def test_arp_getnode(self): - if not AIX: - node = self.uuid._arp_getnode() - else: - self.skipTest('because AIX "arp" does not provide macaddr') + node = self.uuid._arp_getnode() self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') From d3eaab94876483de23ea11473d4df84c9b2d6e3a Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 12:49:44 +0000 Subject: [PATCH 10/38] write _notAIX as non-negative constant and use 'not _AIX' --- Lib/uuid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 33b4c3df373c48..03683b70cc6472 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -52,8 +52,8 @@ __author__ = 'Ka-Ping Yee ' -_notAIX = not sys.platform.startswith("aix") -_MAC_DELIM = b':' if _notAIX else b'.' +_AIX = sys.platform.startswith("aix") +_MAC_DELIM = b':' if not _AIX else b'.' RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', @@ -693,7 +693,7 @@ def getnode(*, getters=None): if sys.platform == 'win32': getters = _NODE_GETTERS_WIN32 else: - getters = _NODE_GETTERS_UNIX if _notAIX else _NODE_GETTERS_AIX + getters = _NODE_GETTERS_UNIX if not _AIX else _NODE_GETTERS_AIX for getter in getters + [_random_getnode]: try: From 3469a01cdf2501f7222c76b64b3806966afca58d Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 22 Aug 2018 15:29:57 +0000 Subject: [PATCH 11/38] added additional 'skip' entries for commands that do not exist on AIX --- Lib/test/test_uuid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 3c542983368138..ec95c2e0aa4401 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -563,6 +563,7 @@ def test_ifconfig_getnode(self): self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(AIX, 'requires command "ip"') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip') @@ -574,6 +575,7 @@ def test_arp_getnode(self): self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(AIX, 'requires command "lanscan"') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() self.check_node(node, 'lanscan') From 399e8ec61382b3f040822d26599e72de55c09114 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Fri, 24 Aug 2018 11:42:13 +0000 Subject: [PATCH 12/38] Use and set a single _NODE_GETTERS constant with platform test outside getnode() Move _netstat_getnode() find alg so that it can be tested using unittest.mock Add new test `test_uuid/find_mac_netstat` for AIX and netstat --- Lib/test/test_uuid.py | 41 ++++++++++++----- Lib/uuid.py | 104 ++++++++++++++++++++++-------------------- 2 files changed, 83 insertions(+), 62 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ec95c2e0aa4401..e569d03b96f5f8 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -325,8 +325,7 @@ def test_uuid1_eui64(self): with unittest.mock.patch.multiple( self.uuid, _node=None, # Ignore any cached node value. - _NODE_GETTERS_WIN32=[too_large_getter], - _NODE_GETTERS_UNIX=[too_large_getter], + _NODE_GETTERS=[too_large_getter], ): node = self.uuid.getnode() self.assertTrue(0 < node < (1 << 48), '%012x' % node) @@ -516,21 +515,39 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): class BaseTestInternals: uuid = None + @unittest.skipUnless(AIX, 'requires AIX') + def test_find_mac_netstat(self): + data = '''Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll +en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 + 01:00:5e:00:00:01 +en0 1500 192.168.129 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +en0 1500 192.168.90 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +''' + popen = unittest.mock.MagicMock() + popen.stdout = io.BytesIO(data.encode()) + + with unittest.mock.patch.object(shutil, 'which', + return_value='/usr/bin/netstat'): + with unittest.mock.patch.object(subprocess, 'Popen', + return_value=popen): + mac = self.uuid._find_mac_netstat( + command='netstat', + args='-ia', + hw_identifiers=b'Address', + get_index=lambda x: x * 1, + ) + + self.assertEqual(mac, 0xfead0c012304) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(AIX, 'because AIX "ifconfig" does not provide macaddr') def test_find_mac(self): - if not AIX: - data = ''' -fake hwaddr + data = '''fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab ''' - else: - data = ''' -fake hwaddr -cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 -eth0 Link encap:Ethernet HWaddr 12.34.56.78.90.ab -''' - popen = unittest.mock.MagicMock() popen.stdout = io.BytesIO(data.encode()) diff --git a/Lib/uuid.py b/Lib/uuid.py index 03683b70cc6472..f338f34f7ec922 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -52,6 +52,7 @@ __author__ = 'Ka-Ping Yee ' +_WIN32 = sys.platform == 'win32' _AIX = sys.platform.startswith("aix") _MAC_DELIM = b':' if not _AIX else b'.' @@ -391,6 +392,48 @@ def _find_mac(command, args, hw_identifiers, get_index): pass return first_local_mac or None +def _find_mac_netstat(command, args, hw_identifiers, get_index): + first_local_mac = None + mac = None + try: + proc = _popen(command, *args.split()) + if not proc: + return None + with proc: + words = proc.stdout.readline().rstrip().split() + try: + i = words.index(hw_identifiers) + except ValueError: + return None + for line in proc.stdout: + try: + words = line.rstrip().split() + word = words[get_index(i)] + if len(word) == 17 and word.count(_MAC_DELIM) == 5: + mac = int(word.replace(_MAC_DELIM, b''), 16) + elif _AIX and word.count(_MAC_DELIM) == 5: + # the extracted hex string is not a 12 hex digit + # string, so add the fields piece by piece + if len(word) < 17 and len(word) >= 11: + mac = 0 + fields = word.split(_MAC_DELIM) + for hex in fields: + mac <<= 8 + mac += int(hex, 16) + if mac and _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + except (ValueError, IndexError): + # Virtual interfaces, such as those provided by + # VPNs, do not have a colon-delimited MAC address + # as expected, but a 16-byte HWAddr separated by + # dashes. These should be ignored in favor of a + # real MAC address + pass + except OSError: + pass + return first_local_mac or None + def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. @@ -442,42 +485,9 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" - # This might work on AIX, Tru64 UNIX. - first_local_mac = None - try: - proc = _popen('netstat', '-ia') - if not proc: - return None - with proc: - words = proc.stdout.readline().rstrip().split() - try: - i = words.index(b'Address') - except ValueError: - return None - for line in proc.stdout: - try: - words = line.rstrip().split() - word = words[i] - if word.count(_MAC_DELIM) == 5: - if len(word) == 17: - mac = int(word.replace(_MAC_DELIM, b''), 16) - # the extracted hex string will not be a 12 hex digit - # string, so extract the fields and add them in - # piece by piece - elif len(word) < 17 and len(word) >= 11: - mac = 0 - hexs = word.split(_MAC_DELIM) - for hex in hexs: - mac <<= 8 - mac += int(hex, 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - except (ValueError, IndexError): - pass - except OSError: - pass - return first_local_mac or None + # This works on AIX and might work on Tru64 UNIX. + mac = _find_mac_netstat('netstat', '-ia', b'Address', lambda i: i+0) + return mac if mac else None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" @@ -669,15 +679,15 @@ def _random_getnode(): return random.getrandbits(48) | (1 << 40) +if _AIX: + _NODE_GETTERS = [_unix_getnode, _netstat_getnode] +elif _WIN32: + _NODE_GETTERS = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] +else: + _NODE_GETTERS = [_unix_getnode, _ifconfig_getnode, _ip_getnode, + _arp_getnode, _lanscan_getnode, _netstat_getnode] _node = None -_NODE_GETTERS_WIN32 = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] - -_NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, - _arp_getnode, _lanscan_getnode, _netstat_getnode] - -_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] - def getnode(*, getters=None): """Get the hardware address as a 48-bit positive integer. @@ -690,12 +700,7 @@ def getnode(*, getters=None): if _node is not None: return _node - if sys.platform == 'win32': - getters = _NODE_GETTERS_WIN32 - else: - getters = _NODE_GETTERS_UNIX if not _AIX else _NODE_GETTERS_AIX - - for getter in getters + [_random_getnode]: + for getter in _NODE_GETTERS + [_random_getnode]: try: _node = getter() except: @@ -704,7 +709,6 @@ def getnode(*, getters=None): return _node assert False, '_random_getnode() returned invalid value: {}'.format(_node) - _last_timestamp = None def uuid1(node=None, clock_seq=None): From 6e2a9bff54c7df1cc0051cfca3329a368af7d261 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 16 Sep 2018 15:46:45 +0000 Subject: [PATCH 13/38] modify the lambda function so it does not look like a bug --- Lib/test/test_uuid.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e569d03b96f5f8..584978bc2e840f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,11 +4,12 @@ import contextlib import io import os +import platform import shutil import subprocess import sys -AIX = sys.platform.startswith("aix") +AIX = platform.system() == 'AIX' py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) @@ -536,7 +537,7 @@ def test_find_mac_netstat(self): command='netstat', args='-ia', hw_identifiers=b'Address', - get_index=lambda x: x * 1, + get_index=lambda x: x, ) self.assertEqual(mac, 0xfead0c012304) From bb3a46002132c516c08de3d0168f876a5cbcd85f Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 16 Sep 2018 15:51:26 +0000 Subject: [PATCH 14/38] resolve differences in imports --- Lib/test/test_uuid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 584978bc2e840f..4386e7c81e8203 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,6 +4,7 @@ import contextlib import io import os +import pickle import platform import shutil import subprocess From db6767f1df2920e1af46f8d7842a3b2487c5115e Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 16 Sep 2018 15:54:16 +0000 Subject: [PATCH 15/38] Switch back to using sys.platform.startswith() to resolve conflicts --- Lib/test/test_uuid.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 4386e7c81e8203..da399e2fcfca72 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,13 +4,11 @@ import contextlib import io import os -import pickle -import platform import shutil import subprocess import sys -AIX = platform.system() == 'AIX' +AIX = sys.platform.startswith("aix") py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) From 25d3ef11ca70c1d9f5e3292d012b870ade7bd20d Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 16 Sep 2018 15:58:47 +0000 Subject: [PATCH 16/38] still trying to get differencs to resolve... --- Lib/test/test_uuid.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index da399e2fcfca72..8b158bc3f9bf29 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -8,10 +8,9 @@ import subprocess import sys -AIX = sys.platform.startswith("aix") - py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) +AIX = sys.platform.startswith("aix") def importable(name): try: From fa9b43bd6421b52976f854226aefe8ed31f01626 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 28 Oct 2018 12:10:28 +0000 Subject: [PATCH 17/38] changes per request. Thx for the review! --- Lib/test/test_uuid.py | 13 +++++++------ Lib/uuid.py | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 8b158bc3f9bf29..dbfb9e8ffae06f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,13 +4,13 @@ import contextlib import io import os +import platform import shutil import subprocess -import sys py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) -AIX = sys.platform.startswith("aix") +AIX = platform.system() == 'AIX' def importable(name): try: @@ -541,9 +541,10 @@ def test_find_mac_netstat(self): self.assertEqual(mac, 0xfead0c012304) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'because AIX "ifconfig" does not provide macaddr') + @unittest.skipIf(AIX, 'AIX "ifconfig" does not provide macaddr') def test_find_mac(self): - data = '''fake Link encap:UNSPEC hwaddr 00-00 + data = ''' +fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab ''' @@ -573,7 +574,7 @@ def check_node(self, node, requires=None): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'because AIX "ifconfig" does not provide macaddr') + @unittest.skipIf(AIX, 'AIX "ifconfig" does not provide macaddr') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @@ -585,7 +586,7 @@ def test_ip_getnode(self): self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'because AIX "arp" does not provide macaddr') + @unittest.skipIf(AIX, 'AIX "arp" does not provide macaddr') def test_arp_getnode(self): node = self.uuid._arp_getnode() self.check_node(node, 'arp') diff --git a/Lib/uuid.py b/Lib/uuid.py index f338f34f7ec922..e319842eb854ca 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -486,8 +486,8 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - mac = _find_mac_netstat('netstat', '-ia', b'Address', lambda i: i+0) - return mac if mac else None + mac = _find_mac_netstat('netstat', '-ia', b'Address', lambda i: i) + return mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" From 4a0c8f08b2dde60ef993f6061d3b7e7d6b06122b Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 28 Oct 2018 12:18:28 +0000 Subject: [PATCH 18/38] resolve merge conflict --- Lib/test/test_uuid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index dbfb9e8ffae06f..1c5f3925d89b54 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,6 +4,7 @@ import contextlib import io import os +import pickle import platform import shutil import subprocess From 64637071453e665a62902a51cfbadf1b9d939671 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 28 Oct 2018 12:23:12 +0000 Subject: [PATCH 19/38] resolve merge conflict, 2nd try --- Lib/test/test_uuid.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 1c5f3925d89b54..dfcaf810ce705f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -5,13 +5,16 @@ import io import os import pickle -import platform import shutil import subprocess +import sys py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) -AIX = platform.system() == 'AIX' +# would prefer using platform.system() but CI tests have a merge conflict with "import platform" +# removing it for the moment +# AIX = platform.system() == 'AIX' +AIX = sys.platform.startswith("aix") def importable(name): try: From a0ef760bf56ecebc1fad544ae0f08dbd7480291f Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 12 Nov 2018 09:56:03 +0200 Subject: [PATCH 20/38] remove unused added import --- Lib/test/test_uuid.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index dfcaf810ce705f..75106278aa5f69 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,7 +4,6 @@ import contextlib import io import os -import pickle import shutil import subprocess import sys From 095e221f1b2a9632e1f879af97f55056af6150aa Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Wed, 14 Nov 2018 21:41:16 +0000 Subject: [PATCH 21/38] Per requested changes --- Lib/test/test_uuid.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index dfcaf810ce705f..d90e10147edc28 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -11,10 +11,7 @@ py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) -# would prefer using platform.system() but CI tests have a merge conflict with "import platform" -# removing it for the moment -# AIX = platform.system() == 'AIX' -AIX = sys.platform.startswith("aix") +_AIX = sys.platform.startswith("aix") def importable(name): try: @@ -518,9 +515,10 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): class BaseTestInternals: uuid = None - @unittest.skipUnless(AIX, 'requires AIX') + @unittest.skipUnless(_AIX, 'requires AIX') def test_find_mac_netstat(self): - data = '''Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll + data = '''\ +Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 01:00:5e:00:00:01 en0 1500 192.168.129 x071 1714807956 0 711348489 0 0 @@ -545,7 +543,7 @@ def test_find_mac_netstat(self): self.assertEqual(mac, 0xfead0c012304) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'AIX "ifconfig" does not provide macaddr') + @unittest.skipIf(_AIX, 'AIX "ifconfig" does not provide macaddr') def test_find_mac(self): data = ''' fake Link encap:UNSPEC hwaddr 00-00 @@ -578,25 +576,25 @@ def check_node(self, node, requires=None): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'AIX "ifconfig" does not provide macaddr') + @unittest.skipIf(_AIX, 'AIX "ifconfig" does not provide macaddr') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'requires command "ip"') + @unittest.skipIf(_AIX, 'requires command "ip"') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'AIX "arp" does not provide macaddr') + @unittest.skipIf(_AIX, 'AIX "arp" does not provide macaddr') def test_arp_getnode(self): node = self.uuid._arp_getnode() self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(AIX, 'requires command "lanscan"') + @unittest.skipIf(_AIX, 'requires command "lanscan"') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() self.check_node(node, 'lanscan') From 8f0687a06051c79bd40e5b57042859f404821045 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Fri, 14 Jun 2019 12:48:16 +0000 Subject: [PATCH 22/38] rename find_mac_routines (in test_uuid.py) --- Lib/test/test_uuid.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 0143b9d1069221..f579ea91ae4601 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -674,8 +674,11 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): class BaseTestInternals: uuid = None + # data is specific to AIX - with '.' as _MAC_DELIM + # and strings shorter than 17 bytes (no leading 0) @unittest.skipUnless(_AIX, 'requires AIX') - def test_find_mac_netstat(self): + # key is on lineX, value is on lineX+1 aka 'nextline' + def test_find_mac_nextline(self): data = '''\ Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 @@ -692,7 +695,7 @@ def test_find_mac_netstat(self): return_value='/usr/bin/netstat'): with mock.patch.object(subprocess, 'Popen', return_value=popen): - mac = self.uuid._find_mac_netstat( + mac = self.uuid._find_mac_nextline( command='netstat', args='-ia', hw_identifiers=b'Address', @@ -702,7 +705,8 @@ def test_find_mac_netstat(self): self.assertEqual(mac, 0xfead0c012304) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - def test_find_mac(self): + # key and value are on the same line aka 'inline' + def test_find_mac_inline(self): data = ''' fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 @@ -715,7 +719,7 @@ def test_find_mac(self): return_value='/sbin/ifconfig'): with mock.patch.object(subprocess, 'Popen', return_value=popen): - mac = self.uuid._find_mac( + mac = self.uuid._find_mac_inline( command='ifconfig', args='', hw_identifiers=[b'hwaddr'], From 47566700652d7272c7dceeb0b1712b59fef3fa4f Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Fri, 14 Jun 2019 17:40:09 +0000 Subject: [PATCH 23/38] fix sync issues --- Lib/test/test_uuid.py | 4 +++- Lib/uuid.py | 20 +++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index f579ea91ae4601..3dc9a7494012ad 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -462,7 +462,8 @@ def test_uuid1_eui64(self): with mock.patch.multiple( self.uuid, _node=None, # Ignore any cached node value. - _NODE_GETTERS=[too_large_getter], + _NODE_GETTERS_WIN32=[too_large_getter], + _NODE_GETTERS_UNIX=[too_large_getter], ): node = self.uuid.getnode() self.assertTrue(0 < node < (1 << 48), '%012x' % node) @@ -705,6 +706,7 @@ def test_find_mac_nextline(self): self.assertEqual(mac, 0xfead0c012304) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(_AIX, 'AIX has different DELIM') # key and value are on the same line aka 'inline' def test_find_mac_inline(self): data = ''' diff --git a/Lib/uuid.py b/Lib/uuid.py index 3e3337c96450e9..f0a689791514ce 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -381,7 +381,8 @@ def _popen(command, *args): def _is_universal(mac): return not (mac & (1 << 41)) -def _find_mac(command, args, hw_identifiers, get_index): +# keyword and value are on the same line aka 'inline' +def _find_mac_inline(command, args, hw_identifiers, get_index): first_local_mac = None try: proc = _popen(command, *args.split()) @@ -409,7 +410,8 @@ def _find_mac(command, args, hw_identifiers, get_index): pass return first_local_mac or None -def _find_mac_netstat(command, args, hw_identifiers, get_index): +# value is on the line following the keyword - aka 'nextline' +def _find_mac_nextline(command, args, hw_identifiers, get_index): first_local_mac = None mac = None try: @@ -456,7 +458,7 @@ def _ifconfig_getnode(): # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): - mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) + mac = _find_mac_inline('ifconfig', args, keywords, lambda i: i+1) if mac: return mac return None @@ -464,7 +466,7 @@ def _ifconfig_getnode(): def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. - mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1) + mac = _find_mac_inline('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac return None @@ -478,17 +480,17 @@ def _arp_getnode(): return None # Try getting the MAC addr from arp based on our IP address (Solaris). - mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) + mac = _find_mac_inline('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) if mac: return mac # This works on OpenBSD - mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) + mac = _find_mac_inline('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) if mac: return mac # This works on Linux, FreeBSD and NetBSD - mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], + mac = _find_mac_inline('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. if mac: @@ -498,12 +500,12 @@ def _arp_getnode(): def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" # This might work on HP-UX. - return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) + return _find_mac_inline('lanscan', '-ai', [b'lan0'], lambda i: 0) def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - mac = _find_mac_netstat('netstat', '-ia', b'Address', lambda i: i) + mac = _find_mac_nextline('netstat', '-ia', b'Address', lambda i: i) return mac or None def _ipconfig_getnode(): From 10f272ed6e8c0c13d61c1cb0efda6ad14e95b9f3 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 17 Jun 2019 11:00:51 +0000 Subject: [PATCH 24/38] add inline comments, correct typos --- Lib/test/test_uuid.py | 8 ++-- Lib/uuid.py | 43 ++++++++++++------- .../2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 2 +- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index f647db256b4973..4dea9601f39438 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -678,7 +678,7 @@ class BaseTestInternals: # and strings shorter than 17 bytes (no leading 0) @unittest.skipUnless(_AIX, 'requires AIX') # key is on lineX, value is on lineX+1 aka 'nextline' - def test_find_mac_nextline(self): + def test_find_mac_nextlines(self): data = '''\ Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 @@ -695,11 +695,11 @@ def test_find_mac_nextline(self): return_value='/usr/bin/netstat'): with mock.patch.object(subprocess, 'Popen', return_value=popen): - mac = self.uuid._find_mac_nextline( + mac = self.uuid._find_mac_nextlines( command='netstat', args='-ia', hw_identifiers=b'Address', - get_index=lambda x: x, + f_index=lambda x: x, ) self.assertEqual(mac, 0xfead0c012304) @@ -724,7 +724,7 @@ def test_find_mac_inline(self): command='ifconfig', args='', hw_identifiers=[b'hwaddr'], - get_index=lambda x: x + 1, + f_index=lambda x: x + 1, ) self.assertEqual(mac, 0x1234567890ab) diff --git a/Lib/uuid.py b/Lib/uuid.py index 91d3e1cd728092..3e4d5009eea604 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,7 +53,6 @@ __author__ = 'Ka-Ping Yee ' -======= # The recognized platforms - known behaviors _AIX = platform.system() == 'AIX' _DARWIN = platform.system() == 'Darwin' @@ -386,8 +385,14 @@ def _popen(command, *args): def _is_universal(mac): return not (mac & (1 << 41)) + +# In the next two fucnctions: +# command: name of command to run +# args: arguments passed to command +# hw_identifers: keywords used to locate a value +# f_index: lambda function to modify, if needed, an index value # keyword and value are on the same line aka 'inline' -def _find_mac_inline(command, args, hw_identifiers, get_index): +def _find_mac_inline(command, args, hw_identifiers, f_index): first_local_mac = None try: proc = _popen(command, *args.split()) @@ -399,7 +404,7 @@ def _find_mac_inline(command, args, hw_identifiers, get_index): for i in range(len(words)): if words[i] in hw_identifiers: try: - word = words[get_index(i)] + word = words[f_index(i)] mac = int(word.replace(_MAC_DELIM, b''), 16) if _is_universal(mac): return mac @@ -415,8 +420,8 @@ def _find_mac_inline(command, args, hw_identifiers, get_index): pass return first_local_mac or None -# value is on the line following the keyword - aka 'nextline' -def _find_mac_nextline(command, args, hw_identifiers, get_index): +# Keyword is only on firstline - values on remaining lines +def _find_mac_nextlines(command, args, hw_identifiers, f_index): first_local_mac = None mac = None try: @@ -424,23 +429,28 @@ def _find_mac_nextline(command, args, hw_identifiers, get_index): if not proc: return None with proc: - words = proc.stdout.readline().rstrip().split() + keywords = proc.stdout.readline().rstrip().split() try: - i = words.index(hw_identifiers) + i = keywords.index(hw_identifiers) except ValueError: return None + # we have the index (i) into the data that follows for line in proc.stdout: try: - words = line.rstrip().split() - word = words[get_index(i)] - if len(word) == 17 and word.count(_MAC_DELIM) == 5: - mac = int(word.replace(_MAC_DELIM, b''), 16) - elif _AIX and word.count(_MAC_DELIM) == 5: + values = line.rstrip().split() + value = values[f_index(i)] + if len(value) == 17 and value.count(_MAC_DELIM) == 5: + mac = int(value.replace(_MAC_DELIM, b''), 16) + # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. + # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 + # not + # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 + elif _AIX and value.count(_MAC_DELIM) == 5: # the extracted hex string is not a 12 hex digit # string, so add the fields piece by piece - if len(word) < 17 and len(word) >= 11: + if len(value) < 17 and len(value) >= 11: mac = 0 - fields = word.split(_MAC_DELIM) + fields = value.split(_MAC_DELIM) for hex in fields: mac <<= 8 mac += int(hex, 16) @@ -458,6 +468,9 @@ def _find_mac_nextline(command, args, hw_identifiers, get_index): pass return first_local_mac or None + +# The following functions call external programs to 'get' a macaddr value to +# be used as basis for an uuid def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. @@ -510,7 +523,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - mac = _find_mac_nextline('netstat', '-ia', b'Address', lambda i: i) + mac = _find_mac_nextlines('netstat', '-ia', b'Address', lambda i: i) return mac or None def _ipconfig_getnode(): diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst index 0d809eaf3d65c8..cb6c20510751c0 100644 --- a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -1 +1 @@ -Fix uuid.uuid1() and uuid.get_node() on AIX +Fix uuid.getnode() on AIX - to use proper MAC_DELIM character From 27b6c320f603b8b5b7f62c9c894f6fba43ce019a Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Fri, 21 Jun 2019 14:19:15 +0000 Subject: [PATCH 25/38] Improve blurb --- .../next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst index cb6c20510751c0..707c31abd9d32e 100644 --- a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -1 +1 @@ -Fix uuid.getnode() on AIX - to use proper MAC_DELIM character +Fix uuid.getnode() on AIX to use proper MAC_DELIM character From 33969b984db78d400917290acb9538aa7c4ddd05 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 24 Jun 2019 13:56:17 +0300 Subject: [PATCH 26/38] refactor more of getting a command's stdout into the helper func --- Lib/uuid.py | 158 +++++++++++++++++++++++++++------------------------- 1 file changed, 81 insertions(+), 77 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 3e4d5009eea604..c226eb6a6ddb54 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -348,24 +348,33 @@ def version(self): if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) -def _popen(command, *args): - import os, shutil, subprocess - executable = shutil.which(command) - if executable is None: - path = os.pathsep.join(('/sbin', '/usr/sbin')) - executable = shutil.which(command, path=path) + +def _get_command_stdout(command, *args): + import io, os, shutil, subprocess + + try: + executable = shutil.which(command) if executable is None: + path = os.pathsep.join(('/sbin', '/usr/sbin')) + executable = shutil.which(command, path=path) + if executable is None: + return None + # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output + # on stderr (Note: we don't have an example where the words we search + # for are actually localized, but in theory some system could do so.) + env = dict(os.environ) + env['LC_ALL'] = 'C' + proc = subprocess.Popen((executable,) + args, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=env) + if not proc: return None - # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output - # on stderr (Note: we don't have an example where the words we search - # for are actually localized, but in theory some system could do so.) - env = dict(os.environ) - env['LC_ALL'] = 'C' - proc = subprocess.Popen((executable,) + args, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - env=env) - return proc + stdout, stderr = proc.communicate() + return io.BytesIO(stdout) + except (OSError, subprocess.SubprocessError): + return None + # For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant # bit of the first octet signifies whether the MAC address is universally (0) @@ -393,68 +402,19 @@ def _is_universal(mac): # f_index: lambda function to modify, if needed, an index value # keyword and value are on the same line aka 'inline' def _find_mac_inline(command, args, hw_identifiers, f_index): - first_local_mac = None - try: - proc = _popen(command, *args.split()) - if not proc: - return None - with proc: - for line in proc.stdout: - words = line.lower().rstrip().split() - for i in range(len(words)): - if words[i] in hw_identifiers: - try: - word = words[f_index(i)] - mac = int(word.replace(_MAC_DELIM, b''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - except (ValueError, IndexError): - # Virtual interfaces, such as those provided by - # VPNs, do not have a colon-delimited MAC address - # as expected, but a 16-byte HWAddr separated by - # dashes. These should be ignored in favor of a - # real MAC address - pass - except OSError: - pass - return first_local_mac or None + stdout = _get_command_stdout(command, args) + if stdout is None: + return None -# Keyword is only on firstline - values on remaining lines -def _find_mac_nextlines(command, args, hw_identifiers, f_index): first_local_mac = None - mac = None - try: - proc = _popen(command, *args.split()) - if not proc: - return None - with proc: - keywords = proc.stdout.readline().rstrip().split() - try: - i = keywords.index(hw_identifiers) - except ValueError: - return None - # we have the index (i) into the data that follows - for line in proc.stdout: + for line in stdout: + words = line.lower().rstrip().split() + for i in range(len(words)): + if words[i] in hw_identifiers: try: - values = line.rstrip().split() - value = values[f_index(i)] - if len(value) == 17 and value.count(_MAC_DELIM) == 5: - mac = int(value.replace(_MAC_DELIM, b''), 16) - # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. - # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 - # not - # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 - elif _AIX and value.count(_MAC_DELIM) == 5: - # the extracted hex string is not a 12 hex digit - # string, so add the fields piece by piece - if len(value) < 17 and len(value) >= 11: - mac = 0 - fields = value.split(_MAC_DELIM) - for hex in fields: - mac <<= 8 - mac += int(hex, 16) - if mac and _is_universal(mac): + word = words[f_index(i)] + mac = int(word.replace(_MAC_DELIM, b''), 16) + if _is_universal(mac): return mac first_local_mac = first_local_mac or mac except (ValueError, IndexError): @@ -464,8 +424,52 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): # dashes. These should be ignored in favor of a # real MAC address pass - except OSError: - pass + return first_local_mac or None + + +# Keyword is only on firstline - values on remaining lines +def _find_mac_nextlines(command, args, hw_identifiers, f_index): + stdout = _get_command_stdout(command, args) + if stdout is None: + return None + + first_local_mac = None + mac = None + keywords = stdout.readline().rstrip().split() + try: + i = keywords.index(hw_identifiers) + except ValueError: + return None + # we have the index (i) into the data that follows + for line in stdout: + try: + values = line.rstrip().split() + value = values[f_index(i)] + if len(value) == 17 and value.count(_MAC_DELIM) == 5: + mac = int(value.replace(_MAC_DELIM, b''), 16) + # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. + # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 + # not + # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 + elif _AIX and value.count(_MAC_DELIM) == 5: + # the extracted hex string is not a 12 hex digit + # string, so add the fields piece by piece + if len(value) < 17 and len(value) >= 11: + mac = 0 + fields = value.split(_MAC_DELIM) + for hex in fields: + mac <<= 8 + mac += int(hex, 16) + if mac and _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + except (ValueError, IndexError): + # Virtual interfaces, such as those provided by + # VPNs, do not have a colon-delimited MAC address + # as expected, but a 16-byte HWAddr separated by + # dashes. These should be ignored in favor of a + # real MAC address + pass return first_local_mac or None From 7a577343ab008c4489fe55686f27a2154981de57 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 24 Jun 2019 14:12:09 +0300 Subject: [PATCH 27/38] reduce scope of try/except blocks, for clarity --- Lib/uuid.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index c226eb6a6ddb54..a46545ca288172 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -414,9 +414,6 @@ def _find_mac_inline(command, args, hw_identifiers, f_index): try: word = words[f_index(i)] mac = int(word.replace(_MAC_DELIM, b''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -424,6 +421,10 @@ def _find_mac_inline(command, args, hw_identifiers, f_index): # dashes. These should be ignored in favor of a # real MAC address pass + else: + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac return first_local_mac or None @@ -460,9 +461,6 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): for hex in fields: mac <<= 8 mac += int(hex, 16) - if mac and _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -470,6 +468,10 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): # dashes. These should be ignored in favor of a # real MAC address pass + else: + if mac and _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac return first_local_mac or None From 4628cea99632f33e91d69ec5e99255aafd4ce63e Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 24 Jun 2019 21:39:41 +0300 Subject: [PATCH 28/38] make tests run regardless of host OS --- Lib/test/test_uuid.py | 68 ++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 4dea9601f39438..ce01edc5b608f4 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -674,11 +674,9 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): class BaseTestInternals: _uuid = py_uuid - # data is specific to AIX - with '.' as _MAC_DELIM - # and strings shorter than 17 bytes (no leading 0) - @unittest.skipUnless(_AIX, 'requires AIX') - # key is on lineX, value is on lineX+1 aka 'nextline' + def test_find_mac_nextlines(self): + # key is on line X, value is on line X+1 aka 'nextline' data = '''\ Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 @@ -688,44 +686,48 @@ def test_find_mac_nextlines(self): en0 1500 192.168.90 x071 1714807956 0 711348489 0 0 224.0.0.1 ''' - popen = mock.MagicMock() - popen.stdout = io.BytesIO(data.encode()) - - with mock.patch.object(shutil, 'which', - return_value='/usr/bin/netstat'): - with mock.patch.object(subprocess, 'Popen', - return_value=popen): - mac = self.uuid._find_mac_nextlines( - command='netstat', - args='-ia', - hw_identifiers=b'Address', - f_index=lambda x: x, - ) + + def mock_get_command_stdout(command, args): + return io.BytesIO(data.encode()) + + # The above data is specific to AIX - with '.' as _MAC_DELIM + # and strings shorter than 17 bytes (no leading 0). + # The above data will only be parsed properly on non-AIX unixes. + with mock.patch.multiple(self.uuid, + _AIX=True, + _MAC_DELIM=b'.', + _get_command_stdout=mock_get_command_stdout): + mac = self.uuid._find_mac_nextlines( + command='netstat', + args='-ia', + hw_identifiers=b'Address', + f_index=lambda x: x, + ) self.assertEqual(mac, 0xfead0c012304) - @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(_AIX, 'AIX has different DELIM') - # key and value are on the same line aka 'inline' def test_find_mac_inline(self): + # key and value are on the same line aka 'inline' data = ''' fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab ''' - popen = mock.MagicMock() - popen.stdout = io.BytesIO(data.encode()) - - with mock.patch.object(shutil, 'which', - return_value='/sbin/ifconfig'): - with mock.patch.object(subprocess, 'Popen', - return_value=popen): - mac = self.uuid._find_mac_inline( - command='ifconfig', - args='', - hw_identifiers=[b'hwaddr'], - f_index=lambda x: x + 1, - ) + + def mock_get_command_stdout(command, args): + return io.BytesIO(data.encode()) + + # The above data will only be parsed properly on non-AIX unixes. + with mock.patch.multiple(self.uuid, + _AIX=False, + _MAC_DELIM=b':', + _get_command_stdout=mock_get_command_stdout): + mac = self.uuid._find_mac_inline( + command='ifconfig', + args='', + hw_identifiers=[b'hwaddr'], + f_index=lambda x: x + 1, + ) self.assertEqual(mac, 0x1234567890ab) From d2830a2e090e58679c93fef50657fc8b7181170b Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Sun, 14 Jul 2019 22:06:01 +0300 Subject: [PATCH 29/38] remove no longer necessary _AIX definition in the test file --- Lib/test/test_uuid.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ce01edc5b608f4..e65b27d5ef2f70 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -14,7 +14,6 @@ py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) -_AIX = sys.platform.startswith("aix") def importable(name): try: From 068872733f725957a0aa8309586cb3e93c6fd1ec Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Sun, 14 Jul 2019 22:09:37 +0300 Subject: [PATCH 30/38] fix unittest import and indentation in one place --- Lib/test/test_uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e65b27d5ef2f70..f695147353a6d4 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,4 +1,4 @@ -import unittest.mock +import unittest from test import support import builtins import contextlib @@ -538,7 +538,7 @@ def mock_generate_time_safe(self, safe_value): if f is None: self.skipTest('need uuid._generate_time_safe') with mock.patch.object(self.uuid, '_generate_time_safe', - lambda: (f()[0], safe_value)): + lambda: (f()[0], safe_value)): yield @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') From ff1ee20f7bb36375a0f100ac54cd59166f87315d Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Sun, 14 Jul 2019 22:45:06 +0300 Subject: [PATCH 31/38] simplify macaddr parsing and control flow in _find_mac_nextlines() This brings it more in line with _find_mac_inline(). --- Lib/uuid.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index a46545ca288172..ef41cc534fa0b6 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -434,33 +434,26 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): if stdout is None: return None - first_local_mac = None - mac = None keywords = stdout.readline().rstrip().split() try: i = keywords.index(hw_identifiers) except ValueError: return None # we have the index (i) into the data that follows + + first_local_mac = None for line in stdout: try: - values = line.rstrip().split() - value = values[f_index(i)] - if len(value) == 17 and value.count(_MAC_DELIM) == 5: - mac = int(value.replace(_MAC_DELIM, b''), 16) + words = line.rstrip().split() + word = words[f_index(i)] # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 # not # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 - elif _AIX and value.count(_MAC_DELIM) == 5: - # the extracted hex string is not a 12 hex digit - # string, so add the fields piece by piece - if len(value) < 17 and len(value) >= 11: - mac = 0 - fields = value.split(_MAC_DELIM) - for hex in fields: - mac <<= 8 - mac += int(hex, 16) + parts = word.split(_MAC_DELIM) + if len(parts) == 6 and all(0 < len(p) <= 2 for p in parts): + hexstr = b''.join(p.rjust(2, b'0') for p in parts) + mac = int(hexstr, 16) except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -469,7 +462,7 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): # real MAC address pass else: - if mac and _is_universal(mac): + if _is_universal(mac): return mac first_local_mac = first_local_mac or mac return first_local_mac or None From 083e9c6175172176c583a6498f0b75d8b2efc486 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Sun, 14 Jul 2019 22:55:21 +0300 Subject: [PATCH 32/38] no need for extra return value check in _netstart_getnode() --- Lib/uuid.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index ef41cc534fa0b6..05b312e749e071 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -522,8 +522,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - mac = _find_mac_nextlines('netstat', '-ia', b'Address', lambda i: i) - return mac or None + return _find_mac_nextlines('netstat', '-ia', b'Address', lambda i: i) def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" From 30cd017a19b37488f36eb0cee0b746885fed8243 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Sun, 14 Jul 2019 23:18:51 +0300 Subject: [PATCH 33/38] improve wording of NEWS entry --- .../next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst index 707c31abd9d32e..e8cdc78929701e 100644 --- a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -1 +1,2 @@ -Fix uuid.getnode() on AIX to use proper MAC_DELIM character +Fix uuid.getnode() on AIX to properly parse MAC addresses in netstat's output. +Patch by Michael Felt. \ No newline at end of file From 27a972b8de167980b172251f4299f1be545cf52b Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 15 Jul 2019 11:26:26 +0300 Subject: [PATCH 34/38] make AIX formatted MAC address handling AIX-specific This as per Victor Stinner's explicit request. --- Lib/uuid.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 05b312e749e071..e02ac66079c063 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -446,14 +446,21 @@ def _find_mac_nextlines(command, args, hw_identifiers, f_index): try: words = line.rstrip().split() word = words[f_index(i)] - # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. - # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 - # not - # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 - parts = word.split(_MAC_DELIM) - if len(parts) == 6 and all(0 < len(p) <= 2 for p in parts): - hexstr = b''.join(p.rjust(2, b'0') for p in parts) - mac = int(hexstr, 16) + if len(word) == 17: + mac = int(word.replace(_MAC_DELIM, b''), 16) + elif _AIX: + # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. + # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 + # not + # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 + parts = word.split(_MAC_DELIM) + if len(parts) == 6 and all(0 < len(p) <= 2 for p in parts): + hexstr = b''.join(p.rjust(2, b'0') for p in parts) + mac = int(hexstr, 16) + else: + continue + else: + continue except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address From 543e66d0013346ab343002f8a63f316fef15b8f4 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 15 Jul 2019 17:02:37 +0000 Subject: [PATCH 35/38] Stop reverse DNS lookups with netstat --- Lib/test/test_uuid.py | 2 +- Lib/uuid.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index f695147353a6d4..5cf4cda896afda 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -698,7 +698,7 @@ def mock_get_command_stdout(command, args): _get_command_stdout=mock_get_command_stdout): mac = self.uuid._find_mac_nextlines( command='netstat', - args='-ia', + args='-ian', hw_identifiers=b'Address', f_index=lambda x: x, ) diff --git a/Lib/uuid.py b/Lib/uuid.py index e02ac66079c063..f73caf85afaa17 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -529,7 +529,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - return _find_mac_nextlines('netstat', '-ia', b'Address', lambda i: i) + return _find_mac_nextlines('netstat', '-ian', b'Address', lambda i: i) def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" From 588fda72a64a88a7adc616ad860af095e3b8ed47 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 17 Jul 2019 13:44:59 +0300 Subject: [PATCH 36/38] add _MAC_OMITS_LEADING_ZEROES, better function names and doc-strings --- Lib/test/test_uuid.py | 23 +++++++-------- Lib/uuid.py | 66 +++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5cf4cda896afda..ecc9dd38acf5e9 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -674,8 +674,7 @@ class BaseTestInternals: _uuid = py_uuid - def test_find_mac_nextlines(self): - # key is on line X, value is on line X+1 aka 'nextline' + def test_find_under_heading(self): data = '''\ Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 @@ -691,22 +690,20 @@ def mock_get_command_stdout(command, args): # The above data is specific to AIX - with '.' as _MAC_DELIM # and strings shorter than 17 bytes (no leading 0). - # The above data will only be parsed properly on non-AIX unixes. with mock.patch.multiple(self.uuid, - _AIX=True, _MAC_DELIM=b'.', + _MAC_OMITS_LEADING_ZEROES=True, _get_command_stdout=mock_get_command_stdout): - mac = self.uuid._find_mac_nextlines( + mac = self.uuid._find_mac_under_heading( command='netstat', args='-ian', - hw_identifiers=b'Address', - f_index=lambda x: x, + heading=b'Address', ) self.assertEqual(mac, 0xfead0c012304) - def test_find_mac_inline(self): - # key and value are on the same line aka 'inline' + def test_find_mac_near_keyword(self): + # key and value are on the same line data = ''' fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 @@ -718,14 +715,14 @@ def mock_get_command_stdout(command, args): # The above data will only be parsed properly on non-AIX unixes. with mock.patch.multiple(self.uuid, - _AIX=False, _MAC_DELIM=b':', + _MAC_OMITS_LEADING_ZEROES=False, _get_command_stdout=mock_get_command_stdout): - mac = self.uuid._find_mac_inline( + mac = self.uuid._find_mac_near_keyword( command='ifconfig', args='', - hw_identifiers=[b'hwaddr'], - f_index=lambda x: x + 1, + keywords=[b'hwaddr'], + get_word_index=lambda x: x + 1, ) self.assertEqual(mac, 0x1234567890ab) diff --git a/Lib/uuid.py b/Lib/uuid.py index f73caf85afaa17..596843cdcf7f14 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -58,7 +58,12 @@ _DARWIN = platform.system() == 'Darwin' _LINUX = platform.system() == 'Linux' _WINDOWS = platform.system() == 'Windows' -_MAC_DELIM = b':' if not _AIX else b'.' + +_MAC_DELIM = b':' +_MAC_OMITS_LEADING_ZEROES = False +if _AIX: + _MAC_DELIM = b'.' + _MAC_OMITS_LEADING_ZEROES = True RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', @@ -353,12 +358,11 @@ def _get_command_stdout(command, *args): import io, os, shutil, subprocess try: - executable = shutil.which(command) + path_dirs = os.environ.get('PATH', os.defpath).split(os.pathsep) + path_dirs.extend(['/sbin', '/usr/sbin']) + executable = shutil.which(command, path=os.pathsep.join(path_dirs)) if executable is None: - path = os.pathsep.join(('/sbin', '/usr/sbin')) - executable = shutil.which(command, path=path) - if executable is None: - return None + return None # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output # on stderr (Note: we don't have an example where the words we search # for are actually localized, but in theory some system could do so.) @@ -395,13 +399,15 @@ def _is_universal(mac): return not (mac & (1 << 41)) -# In the next two fucnctions: -# command: name of command to run -# args: arguments passed to command -# hw_identifers: keywords used to locate a value -# f_index: lambda function to modify, if needed, an index value -# keyword and value are on the same line aka 'inline' -def _find_mac_inline(command, args, hw_identifiers, f_index): +def _find_mac_near_keyword(command, args, keywords, get_word_index): + """Searches a command's output for a MAC address near a keyword. + + Each line of words in the output is case-insensitively searched for + any of the given keywords. Upon a match, get_word_index is invoked + to pick a word from the line, given the index of the match. For + example, lambda i: 0 would get the first word on the line, while + lambda i: i - 1 would get the word preceding the keyword. + """ stdout = _get_command_stdout(command, args) if stdout is None: return None @@ -410,9 +416,9 @@ def _find_mac_inline(command, args, hw_identifiers, f_index): for line in stdout: words = line.lower().rstrip().split() for i in range(len(words)): - if words[i] in hw_identifiers: + if words[i] in keywords: try: - word = words[f_index(i)] + word = words[get_word_index(i)] mac = int(word.replace(_MAC_DELIM, b''), 16) except (ValueError, IndexError): # Virtual interfaces, such as those provided by @@ -428,27 +434,31 @@ def _find_mac_inline(command, args, hw_identifiers, f_index): return first_local_mac or None -# Keyword is only on firstline - values on remaining lines -def _find_mac_nextlines(command, args, hw_identifiers, f_index): +def _find_mac_under_heading(command, args, heading): + """Looks for a MAC address under a heading in a command's output. + + The first line of words in the output is searched for the given + heading. Words at the same word index as the heading in subsequent + lines are then examined to see if they look like MAC addresses. + """ stdout = _get_command_stdout(command, args) if stdout is None: return None keywords = stdout.readline().rstrip().split() try: - i = keywords.index(hw_identifiers) + column_index = keywords.index(heading) except ValueError: return None - # we have the index (i) into the data that follows first_local_mac = None for line in stdout: try: words = line.rstrip().split() - word = words[f_index(i)] + word = words[column_index] if len(word) == 17: mac = int(word.replace(_MAC_DELIM, b''), 16) - elif _AIX: + elif _MAC_OMITS_LEADING_ZEROES: # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 # not @@ -482,7 +492,7 @@ def _ifconfig_getnode(): # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): - mac = _find_mac_inline('ifconfig', args, keywords, lambda i: i+1) + mac = _find_mac_near_keyword('ifconfig', args, keywords, lambda i: i+1) if mac: return mac return None @@ -490,7 +500,7 @@ def _ifconfig_getnode(): def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. - mac = _find_mac_inline('ip', 'link', [b'link/ether'], lambda i: i+1) + mac = _find_mac_near_keyword('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac return None @@ -504,17 +514,17 @@ def _arp_getnode(): return None # Try getting the MAC addr from arp based on our IP address (Solaris). - mac = _find_mac_inline('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) if mac: return mac # This works on OpenBSD - mac = _find_mac_inline('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) if mac: return mac # This works on Linux, FreeBSD and NetBSD - mac = _find_mac_inline('arp', '-an', [os.fsencode('(%s)' % ip_addr)], + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. if mac: @@ -524,12 +534,12 @@ def _arp_getnode(): def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" # This might work on HP-UX. - return _find_mac_inline('lanscan', '-ai', [b'lan0'], lambda i: 0) + return _find_mac_near_keyword('lanscan', '-ai', [b'lan0'], lambda i: 0) def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This works on AIX and might work on Tru64 UNIX. - return _find_mac_nextlines('netstat', '-ian', b'Address', lambda i: i) + return _find_mac_under_heading('netstat', '-ian', b'Address') def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" From 28f7a019c66ab598ef278485419b65d2e21915e1 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Fri, 30 Aug 2019 18:38:52 +0000 Subject: [PATCH 37/38] update blurb and comments in test_uuid.py --- Lib/test/test_uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ecc9dd38acf5e9..ddf7e6dc1b8e45 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -688,8 +688,8 @@ def test_find_under_heading(self): def mock_get_command_stdout(command, args): return io.BytesIO(data.encode()) - # The above data is specific to AIX - with '.' as _MAC_DELIM - # and strings shorter than 17 bytes (no leading 0). + # The above data is from AIX - with '.' as _MAC_DELIM and strings + # shorter than 17 bytes (no leading 0). (_MAC_OMITS_LEADING_ZEROES=True) with mock.patch.multiple(self.uuid, _MAC_DELIM=b'.', _MAC_OMITS_LEADING_ZEROES=True, From 6fc2129e92cca93bcaf4e8aa3370f9ad100da592 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Tue, 3 Sep 2019 20:20:18 +0000 Subject: [PATCH 38/38] Update News blurb --- .../next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst index e8cdc78929701e..40f638234a4586 100644 --- a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -1,2 +1,4 @@ -Fix uuid.getnode() on AIX to properly parse MAC addresses in netstat's output. -Patch by Michael Felt. \ No newline at end of file +Fix uuid.getnode() on platforms with '.' as MAC Addr delimiter as well +fix for MAC Addr format that omits a leading 0 in MAC Addr values. +Currently, AIX is the only know platform with these settings. +Patch by Michael Felt.