Python 3 fixes - Get it installing and running on Python3

Todo: - Tests still not ported - So far only tested MySQL collector and hostedgraphite handler
python-diamond · erkie · Apr 9, 2022 · Apr 9, 2022 · Nov 25, 2022 · Apr 9, 2022
commit e6c1ceec6568e642d816478c77e900f1c505dc0f
diff --git a/bin/diamond b/bin/diamond
@@ -148,7 +148,7 @@ def main():

            # Read existing pid file
            try:
-                pf = file(options.pidfile, 'r')
+                pf = open(options.pidfile, 'r')
                pid = int(pf.read().strip())
                pf.close()
            except (IOError, ValueError):
@@ -187,7 +187,7 @@ def main():
                # Write pid file
                pid = str(os.getpid())
                try:
-                    pf = file(options.pidfile, 'w+')
+                    pf = open(options.pidfile, 'w+')
                except IOError as e:
                    print("Failed to write PID file: %s" % (e), file=sys.stderr)
                    sys.exit(1)
@@ -271,7 +271,7 @@ def main():
                # Write pid file
                pid = str(os.getpid())
                try:
-                    pf = file(options.pidfile, 'w+')
+                    pf = open(options.pidfile, 'w+')
                except IOError as e:
                    log.error("Failed to write child PID file: %s" % (e))
                    sys.exit(1)

diff --git a/bin/diamond-setup b/bin/diamond-setup
@@ -78,7 +78,7 @@ def getCollectors(path):


 def typeToString(key):
-    if isinstance(obj.config[key], basestring):
+    if isinstance(obj.config[key], str):
        user_val = obj.config[key]
    elif isinstance(obj.config[key], bool):
        user_val = str(obj.config[key])
@@ -95,15 +95,15 @@ def typeToString(key):
 def stringToType(key, val):
    if type(obj.config[key]) is type(val):
        config_file[key] = val
-    elif isinstance(obj.config[key], basestring):
+    elif isinstance(obj.config[key], str):
        if val.lower() == 'false':
            config_file[key] = False
        elif val.lower() == 'true':
            config_file[key] = True
        else:
            config_file[key] = val
    elif isinstance(obj.config[key], bool):
-        if isinstance(val, basestring):
+        if isinstance(val, str):
            config_file[key] = str_to_bool(val)
        else:
            config_file[key] = bool(val)
@@ -117,7 +117,7 @@ def stringToType(key, val):


 def boolCheck(val):
-    if isinstance(val, basestring):
+    if isinstance(val, str):
        return str_to_bool(val)
    elif isinstance(val, bool):
        return val
@@ -137,7 +137,7 @@ def configureKey(key):
    print("\n")
    if key in default_conf_help:
        print(default_conf_help[key])
-    val = raw_input(key + ' [' + user_val + ']: ')
+    val = input(key + ' [' + user_val + ']: ')

    # Empty user input? Default to current value
    if len(val) == 0:
@@ -191,7 +191,7 @@ if __name__ == "__main__":
        print(config['server']['collectors_config_path'])
        print('Please type yes to continue')

-        val = raw_input('Are you sure? ')
+        val = input('Are you sure? ')
        if val != 'yes':
            sys.exit()

@@ -257,8 +257,8 @@ if __name__ == "__main__":

            config_file.write()

-        except IOError as (errno, strerror):
-            print("I/O error({}): {}".format(errno, strerror))
+        except IOError as err:
+            print("I/O error({}): {}".format(err.errno, err.strerror))
        except KeyboardInterrupt:
            print()
            sys.exit()

diff --git a/setup.py b/setup.py
@@ -5,6 +5,7 @@
 import os
 from glob import glob
 import platform
+import distro


 def running_under_virtualenv():
@@ -42,11 +43,11 @@ def running_under_virtualenv():
        ('share/diamond/user_scripts', []),
    ]

-    distro = platform.dist()[0]
-    distro_major_version = platform.dist()[1].split('.')[0]
-    if not distro:
+    distro_name = distro.name()
+    distro_major_version = int(distro.version().split('.')[0])
+    if not distro_name:
        if 'amzn' in platform.uname()[2]:
-            distro = 'centos'
+            distro_name = 'centos'

    if running_under_virtualenv():
        data_files.append(('etc/diamond',
@@ -65,20 +66,20 @@ def running_under_virtualenv():
        data_files.append(('/var/log/diamond',
                           ['.keep']))

-        if distro == 'Ubuntu':
+        if distro_name == 'Ubuntu':
            if distro_major_version >= 16:
                data_files.append(('/usr/lib/systemd/system',
                                   ['rpm/systemd/diamond.service']))
            else:
                data_files.append(('/etc/init',
                                   ['debian/diamond.upstart']))
-        if distro in ['centos', 'redhat', 'debian', 'fedora', 'oracle']:
+        if distro_name in ['centos', 'redhat', 'debian', 'fedora', 'oracle']:
            data_files.append(('/etc/init.d',
                               ['bin/init.d/diamond']))
-            if distro_major_version >= 7 and not distro == 'debian':
+            if distro_major_version >= 7 and not distro_name == 'debian':
                data_files.append(('/usr/lib/systemd/system',
                                   ['rpm/systemd/diamond.service']))
-            elif distro_major_version >= 6 and not distro == 'debian':
+            elif distro_major_version >= 6 and not distro_name == 'debian':
                data_files.append(('/etc/init',
                                   ['rpm/upstart/diamond.conf']))

@@ -88,11 +89,7 @@ def running_under_virtualenv():
    if running_under_virtualenv():
        install_requires = ['configobj', 'psutil', ]
    else:
-        if distro in ['debian', 'Ubuntu']:
-            install_requires = ['python-configobj', 'python-psutil', ]
-        # Default back to pip style requires
-        else:
-            install_requires = ['configobj', 'psutil', ]
+        install_requires = ['configobj', 'psutil', ]


 def get_version():
@@ -150,7 +147,7 @@ def pkgPath(root, path, rpath="/"):
    packages=['diamond', 'diamond.handler', 'diamond.utils'],
    scripts=['bin/diamond', 'bin/diamond-setup'],
    data_files=data_files,
-    python_requires='~=2.7',
+    python_requires='~=3.0',
    install_requires=install_requires,
    classifiers=[
        'Programming Language :: Python',

diff --git a/src/collectors/aurora/aurora.py b/src/collectors/aurora/aurora.py
@@ -1,6 +1,6 @@
 import diamond.collector

-import urllib2
+from urllib.request import urlopen


 class AuroraCollector(diamond.collector.Collector):
@@ -32,7 +32,7 @@ def collect(self):
                                   self.config['host'],
                                   self.config['port'])

-        response = urllib2.urlopen(url)
+        response = urlopen(url)

        for line in response.readlines():
            properties = line.split()

diff --git a/src/collectors/bind/bind.py b/src/collectors/bind/bind.py
@@ -11,7 +11,7 @@
 """

 import diamond.collector
-import urllib2
+from urllib.request import urlopen
 import xml.etree.cElementTree as ElementTree


@@ -69,7 +69,7 @@ def clean_counter(self, name, value):

    def collect(self):
        try:
-            req = urllib2.urlopen('http://%s:%d/' % (
+            req = urlopen('http://%s:%d/' % (
                self.config['host'], int(self.config['port'])))
        except Exception as e:
            self.log.error('Couldnt connect to bind: %s', e)

diff --git a/src/collectors/celerymon/celerymon.py b/src/collectors/celerymon/celerymon.py
@@ -18,7 +18,7 @@
 """

 import diamond.collector
-import urllib2
+from urllib.request import urlopen
 import time

 try:
@@ -68,7 +68,7 @@ def collect(self):

        celerymon_url = "http://%s:%s/api/task/?since=%i" % (
            host, port, self.LastCollectTime)
-        response = urllib2.urlopen(celerymon_url)
+        response = urlopen(celerymon_url)
        body = response.read()
        celery_data = json.loads(body)


diff --git a/src/collectors/conntrack/conntrack.py b/src/collectors/conntrack/conntrack.py
@@ -50,12 +50,12 @@ def collect(self):
        collected = {}
        files = []

-        if isinstance(self.config['dir'], basestring):
+        if isinstance(self.config['dir'], str):
            dirs = [d.strip() for d in self.config['dir'].split(',')]
        elif isinstance(self.config['dir'], list):
            dirs = self.config['dir']

-        if isinstance(self.config['files'], basestring):
+        if isinstance(self.config['files'], str):
            files = [f.strip() for f in self.config['files'].split(',')]
        elif isinstance(self.config['files'], list):
            files = self.config['files']

diff --git a/src/collectors/darner/darner.py b/src/collectors/darner/darner.py
@@ -117,7 +117,7 @@ def collect(self):
        hosts = self.config.get('hosts')

        # Convert a string config value to be an array
-        if isinstance(hosts, basestring):
+        if isinstance(hosts, str):
            hosts = [hosts]

        for host in hosts:

diff --git a/src/collectors/diskspace/diskspace.py b/src/collectors/diskspace/diskspace.py
@@ -78,7 +78,7 @@ def process_config(self):
        super(DiskSpaceCollector, self).process_config()
        # Precompile things
        self.exclude_filters = self.config['exclude_filters']
-        if isinstance(self.exclude_filters, basestring):
+        if isinstance(self.exclude_filters, str):
            self.exclude_filters = [self.exclude_filters]

        if not self.exclude_filters:
@@ -87,7 +87,7 @@ def process_config(self):
            self.exclude_reg = re.compile('|'.join(self.exclude_filters))

        self.filesystems = []
-        if isinstance(self.config['filesystems'], basestring):
+        if isinstance(self.config['filesystems'], str):
            for filesystem in self.config['filesystems'].split(','):
                self.filesystems.append(filesystem.strip())
        elif isinstance(self.config['filesystems'], list):

diff --git a/src/collectors/dropwizard/dropwizard.py b/src/collectors/dropwizard/dropwizard.py
@@ -5,7 +5,8 @@

 """

-import urllib2
+from urllib.error import HTTPError
+from urllib.request import urlopen

 try:
    import json
@@ -45,8 +46,8 @@ def collect(self):
        url = 'http://%s:%i/metrics' % (
            self.config['host'], int(self.config['port']))
        try:
-            response = urllib2.urlopen(url)
-        except urllib2.HTTPError as err:
+            response = urlopen(url)
+        except HTTPError as err:
            self.log.error("%s: %s", url, err)
            return


diff --git a/src/collectors/dseopscenter/dseopscenter.py b/src/collectors/dseopscenter/dseopscenter.py
@@ -4,7 +4,7 @@
 Collect the DataStax OpsCenter metrics
 """

-import urllib2
+from urllib.request import urlopen
 import datetime

 try:
@@ -119,7 +119,7 @@ def _get_schema(self):
                                             int(self.config['port']),
                                             self.config['cluster_id'])
        try:
-            response = urllib2.urlopen(url)
+            response = urlopen(url)
        except Exception as err:
            self.log.error('%s: %s', url, err)
            return False
@@ -157,7 +157,7 @@ def _get(self, start, end, step=60):
            self.config['default_tail_opts'])

        try:
-            response = urllib2.urlopen(url)
+            response = urlopen(url)
        except Exception as err:
            self.log.error('%s: %s', url, err)
            return False

diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py
@@ -13,7 +13,7 @@

 """

-import urllib2
+from urllib.request import Request, urlopen
 import base64
 import re
 from diamond.collector import str_to_bool
@@ -33,7 +33,7 @@ class ElasticSearchCollector(diamond.collector.Collector):
    def process_config(self):
        super(ElasticSearchCollector, self).process_config()
        instance_list = self.config['instances']
-        if isinstance(instance_list, basestring):
+        if isinstance(instance_list, str):
            instance_list = [instance_list]

        if len(instance_list) == 0:
@@ -111,12 +111,12 @@ def _get(self, scheme, host, port, path, assert_key=None):
        """
        url = '%s://%s:%i/%s' % (scheme, host, port, path)
        try:
-            request = urllib2.Request(url)
+            request = Request(url)
            if self.config['user'] and self.config['password']:
                base64string = base64.standard_b64encode(
                    '%s:%s' % (self.config['user'], self.config['password']))
                request.add_header("Authorization", "Basic %s" % base64string)
-            response = urllib2.urlopen(request)
+            response = urlopen(request)
        except Exception as err:
            self.log.error("%s: %s" % (url, err))
            return False

diff --git a/src/collectors/endecadgraph/endecadgraph.py b/src/collectors/endecadgraph/endecadgraph.py
@@ -11,8 +11,8 @@

 """

+from urllib.request import urlopen
 import diamond.collector
-import urllib2
 from StringIO import StringIO
 import re
 import xml.etree.cElementTree as ElementTree
@@ -111,7 +111,7 @@ def walkXML(context, elemList):
        url = 'http://%s:%d/admin?op=stats' % (self.config['host'],
                                               self.config['port'])
        try:
-            xml = urllib2.urlopen(url, timeout=self.config['timeout']).read()
+            xml = urlopen(url, timeout=self.config['timeout']).read()
        except Exception as e:
            self.log.error('Could not connect to endeca on %s: %s' % (url, e))
            return {}

diff --git a/src/collectors/etcdstat/etcdstat.py b/src/collectors/etcdstat/etcdstat.py
@@ -12,9 +12,10 @@
 ```
 """

+from urllib.error import HTTPError
+from urllib.request import urlopen
 import diamond.collector
 import json
-import urllib2

 METRICS_KEYS = ['sendPkgRate',
                'recvPkgRate',
@@ -100,8 +101,8 @@ def get_metrics(self, category):
            url = "%s://%s:%s/v2/stats/%s" % (protocol, self.config['host'],
                                              self.config['port'], category)

-            return json.load(urllib2.urlopen(url, **opts))
-        except (urllib2.HTTPError, ValueError) as err:
+            return json.load(urlopen(url, **opts))
+        except (HTTPError, ValueError) as err:
            self.log.error('Unable to read JSON response: %s' % err)
            return {}