From 21ccf40fb1030baa659e2265b8e14412eec0fe59 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 1 Mar 2015 19:44:01 -0600
Subject: [PATCH 001/263] Added deprecation warning to the old WebAPI class

---
 shodan/api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shodan/api.py b/shodan/api.py
index 907f411..635f82b 100644
--- a/shodan/api.py
+++ b/shodan/api.py
@@ -113,6 +113,7 @@ def __init__(self, key):
         key -- your API key
         
         """
+        print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()')
         self.api_key = key
         self.base_url = 'http://www.shodanhq.com/api/'
         self.base_exploits_url = 'https://exploits.shodan.io/'

From f550b8b48940ce42f4ef32fba26e1b144a49f2d0 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 1 Mar 2015 19:44:39 -0600
Subject: [PATCH 002/263] Split the code into separate files Added methods for
 interacting with the Shodan Alerts API (unstable)

---
 shodan/__init__.py  |   5 +-
 shodan/client.py    | 140 +++++++++++++++++---------------------------
 shodan/exception.py |   8 +++
 shodan/helpers.py   |  64 ++++++++++++++++++++
 shodan/stream.py    |  70 ++++++++++++++++++++++
 5 files changed, 198 insertions(+), 89 deletions(-)
 create mode 100644 shodan/exception.py
 create mode 100644 shodan/helpers.py
 create mode 100644 shodan/stream.py

diff --git a/shodan/__init__.py b/shodan/__init__.py
index 1c20901..7d4b04d 100644
--- a/shodan/__init__.py
+++ b/shodan/__init__.py
@@ -1,2 +1,3 @@
-from .api import WebAPI
-from .client import APIError, Shodan
+from shodan.api import WebAPI
+from shodan.client import Shodan
+from shodan.exception import APIError
diff --git a/shodan/client.py b/shodan/client.py
index 0eb8f65..4310a8a 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -6,21 +6,16 @@
 
 This module implements the Shodan API.
 
-:copyright: (c) 2014 by John Matherly
+:copyright: (c) 2014-2015 by John Matherly
 """
 
 import requests
 import simplejson
 import time
 
-
-class APIError(Exception):
-    """This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
-    def __init__(self, value):
-        self.value = value
-    
-    def __str__(self):
-        return self.value
+import shodan.exception as exception
+import shodan.helpers as helpers
+import shodan.stream as stream
 
 
 class Shodan:
@@ -66,7 +61,7 @@ def search(self, query, page=1, facets=None):
                 'page': page,
             }
             if facets:
-                facet_str = self.parent._create_facet_string(facets)
+                facet_str = helpers.create_facet_string(facets)
                 query_args['facets'] = facet_str
 
             return self.parent._request('/api/search', query_args, service='exploits')
@@ -86,65 +81,10 @@ def count(self, query, facets=None):
                 'query': query,
             }
             if facets:
-                facet_str = self.parent._create_facet_string(facets)
+                facet_str = helpers.create_facet_string(facets)
                 query_args['facets'] = facet_str
 
             return self.parent._request('/api/count', query_args, service='exploits')
-
-    class Stream:
-
-        base_url = 'https://stream.shodan.io'
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def _create_stream(self, name):
-            try:
-                req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True)
-            except:
-                raise APIError('Unable to contact the Shodan Streaming API')
-
-            if req.status_code != 200:
-                try:
-                    raise APIError(data.json()['error'])
-                except:
-                    pass
-                raise APIError('Invalid API key or you do not have access to the Streaming API')
-            return req
-
-        def banners(self):
-            """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
-            API subscription plans and for those it only returns a fraction of the data.
-            """
-            stream = self._create_stream('/shodan/banners')
-            for line in stream.iter_lines():
-                if line:
-                    banner = simplejson.loads(line)
-                    yield banner
-
-        def ports(self, ports):
-            """
-            A filtered version of the "banners" stream to only return banners that match the ports of interest.
-
-            :param ports: A list of ports to return banner data on.
-            :type ports: int[]
-            """
-            stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]))
-            for line in stream.iter_lines():
-                if line:
-                    banner = simplejson.loads(line)
-                    yield banner
-
-        def geo(self):
-            """
-            A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream
-            in case you only care about the geolocation information.
-            """
-            stream = self._create_stream('/shodan/geo')
-            for line in stream.iter_lines():
-                if line:
-                    banner = simplejson.loads(line)
-                    yield banner
     
     def __init__(self, key):
         """Initializes the API object.
@@ -157,7 +97,7 @@ def __init__(self, key):
         self.base_exploits_url = 'https://exploits.shodan.io'
         self.exploits = self.Exploits(self)
         self.tools = self.Tools(self)
-        self.stream = self.Stream(self)
+        self.stream = stream.Stream(key)
     
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.
@@ -186,41 +126,28 @@ def _request(self, function, params, service='shodan', method='get'):
             else:
                 data = requests.get(base_url + function, params=params)
         except:
-            raise APIError('Unable to connect to Shodan')
+            raise exception.APIError('Unable to connect to Shodan')
 
         # Check that the API key wasn't rejected
         if data.status_code == 401:
             try:
-                raise APIError(data.json()['error'])
+                raise exception.APIError(data.json()['error'])
             except:
                 pass
-            raise APIError('Invalid API key')
+            raise exception.APIError('Invalid API key')
         
         # Parse the text into JSON
         try:
             data = data.json()
         except:
-            raise APIError('Unable to parse JSON response')
+            raise exception.APIError('Unable to parse JSON response')
         
         # Raise an exception if an error occurred
         if type(data) == dict and data.get('error', None):
-            raise APIError(data['error'])
+            raise exception.APIError(data['error'])
         
         # Return the data
         return data
-
-    def _create_facet_string(self, facets):
-        """Converts a Python list of facets into a comma-separated string that can be understood by
-        the Shodan API.
-        """
-        facet_str = ''
-        for facet in facets:
-            if isinstance(facet, basestring):
-                facet_str += facet
-            else:
-                facet_str += '%s:%s'  % (facet[0], facet[1])
-            facet_str += ','
-        return facet_str[:-1]
     
     def count(self, query, facets=None):
         """Returns the total number of search results for the query.
@@ -236,7 +163,7 @@ def count(self, query, facets=None):
             'query': query,
         }
         if facets:
-            facet_str = self._create_facet_string(facets)
+            facet_str = helpers.create_facet_string(facets)
             query_args['facets'] = facet_str
         return self._request('/shodan/host/count', query_args)
     
@@ -306,7 +233,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
             args['page'] = page
 
         if facets:
-            facet_str = self._create_facet_string(facets)
+            facet_str = helpers.create_facet_string(facets)
             args['facets'] = facet_str
         
         return self._request('/shodan/host/search', args)
@@ -421,3 +348,42 @@ def queries_tags(self, size=10):
         }
         return self._request('/shodan/query/tags', args)
 
+    def create_alert(self, name, ip, expires=0):
+        """Search the directory of saved search queries in Shodan.
+
+        :param query: The number of tags to return
+        :type page: int
+
+        :returns: A list of tags.
+        """
+        data = {
+            'name': name,
+            'filters': {
+                'ip': ip,
+            },
+            'expires': expires,
+        }
+
+        response = helpers.api_request(self.api_key, '/shodan/alert', data=data, params={}, base_url='https://oldapi.shodan.io', method='post')
+
+        return response
+
+    def alerts(self, aid=None):
+        """List all of the active alerts that the user created."""
+        if aid:
+            func = '/shodan/alert/%s/info' % aid
+        else:
+            func = '/shodan/alert/info'
+
+        response = helpers.api_request(self.api_key, func, params={}, base_url='https://oldapi.shodan.io')
+
+        return response
+
+    def delete_alert(self, aid):
+        """Delete the alert with the given ID."""
+        func = '/shodan/alert/%s' % aid
+
+        response = helpers.api_request(self.api_key, func, params={}, method='delete', base_url='https://oldapi.shodan.io')
+
+        return response
+
diff --git a/shodan/exception.py b/shodan/exception.py
new file mode 100644
index 0000000..a34c8e1
--- /dev/null
+++ b/shodan/exception.py
@@ -0,0 +1,8 @@
+
+class APIError(Exception):
+    """This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
+    def __init__(self, value):
+        self.value = value
+    
+    def __str__(self):
+        return self.value
diff --git a/shodan/helpers.py b/shodan/helpers.py
new file mode 100644
index 0000000..bed0240
--- /dev/null
+++ b/shodan/helpers.py
@@ -0,0 +1,64 @@
+
+import requests
+import shodan.exception
+import simplejson
+
+def create_facet_string(facets):
+    """Converts a Python list of facets into a comma-separated string that can be understood by
+    the Shodan API.
+    """
+    facet_str = ''
+    for facet in facets:
+        if isinstance(facet, basestring):
+            facet_str += facet
+        else:
+            facet_str += '%s:%s'  % (facet[0], facet[1])
+        facet_str += ','
+    return facet_str[:-1]
+
+    
+def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get'):
+    """General-purpose function to create web requests to SHODAN.
+    
+    Arguments:
+        function  -- name of the function you want to execute
+        params    -- dictionary of parameters for the function
+    
+    Returns
+        A dictionary containing the function's results.
+    
+    """
+    # Add the API key parameter automatically
+    params['key'] = key
+
+    # Send the request
+    try:
+        if method.lower() == 'post':
+            data = requests.post(base_url + function, simplejson.dumps(data), params=params, headers={'content-type': 'application/json'})
+        elif method.lower() == 'delete':
+            data = requests.delete(base_url + function, params=params)
+        else:
+            data = requests.get(base_url + function, params=params)
+    except:
+        raise shodan.exception.APIError('Unable to connect to Shodan')
+
+    # Check that the API key wasn't rejected
+    if data.status_code == 401:
+        try:
+            raise shodan.exception.APIError(data.json()['error'])
+        except:
+            pass
+        raise shodan.exception.APIError('Invalid API key')
+    
+    # Parse the text into JSON
+    try:
+        data = data.json()
+    except:
+        raise shodan.exception.APIError('Unable to parse JSON response')
+    
+    # Raise an exception if an error occurred
+    if type(data) == dict and data.get('error', None):
+        raise shodan.exception.APIError(data['error'])
+    
+    # Return the data
+    return data
\ No newline at end of file
diff --git a/shodan/stream.py b/shodan/stream.py
new file mode 100644
index 0000000..df9dc84
--- /dev/null
+++ b/shodan/stream.py
@@ -0,0 +1,70 @@
+import requests
+import simplejson
+
+import shodan.exception
+
+class Stream:
+
+    base_url = 'https://stream.shodan.io'
+
+    def __init__(self, api_key):
+        self.api_key = api_key
+
+    def _create_stream(self, name):
+        try:
+            req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True)
+        except:
+            raise exception.APIError('Unable to contact the Shodan Streaming API')
+
+        if req.status_code != 200:
+            try:
+                raise exception.APIError(data.json()['error'])
+            except:
+                pass
+            raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
+        return req
+
+    def alert(self, aid=None):
+        if aid:
+            stream = self._create_stream('/shodan/alert/%s' % aid)
+        else:
+            stream = self._create_stream('/shodan/alert')
+
+        for line in stream.iter_lines():
+            if line:
+                banner = simplejson.loads(line)
+                yield banner
+
+    def banners(self):
+        """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
+        API subscription plans and for those it only returns a fraction of the data.
+        """
+        stream = self._create_stream('/shodan/banners')
+        for line in stream.iter_lines():
+            if line:
+                banner = simplejson.loads(line)
+                yield banner
+
+    def ports(self, ports):
+        """
+        A filtered version of the "banners" stream to only return banners that match the ports of interest.
+
+        :param ports: A list of ports to return banner data on.
+        :type ports: int[]
+        """
+        stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]))
+        for line in stream.iter_lines():
+            if line:
+                banner = simplejson.loads(line)
+                yield banner
+
+    def geo(self):
+        """
+        A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream
+        in case you only care about the geolocation information.
+        """
+        stream = self._create_stream('/shodan/geo')
+        for line in stream.iter_lines():
+            if line:
+                banner = simplejson.loads(line)
+                yield banner
\ No newline at end of file

From 708cf2e423134d512097b311f333127fcdeaa0fd Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 13 Mar 2015 14:26:30 -0500
Subject: [PATCH 003/263] Added "info" command to shodan cli Added "alert list"
 command to shodan cli Added "alert remove" command to shodan cli Improved
 "scan" command of shodan cli to wait for results using the new Alert API
 Bumped version Refactored Shodan() code into separate modules

---
 bin/shodan       | 136 +++++++++++++++++++++++++++++++++++++++++++++--
 setup.py         |   2 +-
 shodan/alert.py  |  10 ++++
 shodan/client.py |  20 +++++--
 shodan/stream.py |  23 ++++----
 5 files changed, 172 insertions(+), 19 deletions(-)
 create mode 100644 shodan/alert.py

diff --git a/bin/shodan b/bin/shodan
index d327699..4874933 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -18,6 +18,7 @@ The following commands are currently supported:
 """
 
 import click
+import collections
 import datetime
 import gzip
 import os
@@ -93,6 +94,59 @@ def init(key):
     os.chmod(keyfile, 0600)
 
 
+@main.group()
+def alert():
+    pass
+
+@alert.command(name='list')
+@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
+def alert_list(expired):
+    """Returns the number of results for a search"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        results = api.alerts(include_expired=expired)
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+
+    if len(results) > 0:
+        click.echo('# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
+        # click.echo('#' * 65)
+        for alert in results:
+            click.echo(
+                '{:16} {:<30} {:<35} '.format(
+                    click.style(alert['id'],  fg='yellow'),
+                    click.style(alert['name'], fg='cyan'),
+                    click.style(', '.join(alert['filters']['ip']), fg='white')
+                ),
+                nl=False
+            )
+
+            if 'expired' in alert and alert['expired']:
+                click.echo(click.style('expired', fg='red'))
+            else:
+                click.echo('')
+    else:
+        click.echo("You haven't created any alerts yet.")
+
+
+@alert.command(name='remove')
+@click.argument('alert_id', metavar='<alert ID>')
+def alert_remove(alert_id):
+    """Returns the number of results for a search"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        results = api.delete_alert(alert_id)
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+    click.echo("Alert deleted")
+
+
 @main.command()
 @click.argument('query', metavar='<search query>', nargs=-1)
 def count(query):
@@ -181,6 +235,21 @@ def download(limit, filename, query):
         click.echo(click.style('Saved %s results into file %s' % (count, filename), 'green'))
 
 
+@main.command()
+def info():
+    """Shows general information about your account"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        results = api.info()
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+
+    click.echo("""Query credits available: {0}
+Scan credits available: {1}
+    """.format(results['query_credits'], results['scan_credits']))
+
+
 @main.command()
 @click.option('--color/--no-color', default=True)
 @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data')
@@ -247,17 +316,76 @@ def myip():
 
 
 @main.command()
+@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=30, type=int)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan(netblocks):
+def scan(wait, netblocks):
     """Scan an IP/ netblock using Shodan."""
     key = get_api_key()
     api = shodan.Shodan(key)
 
     # Submit the IPs for scanning
     try:
-        results = api.scan(netblocks)
+        # Setup an alert to wait for responses
+        alert = api.create_alert('Scan', netblocks)
+
+        # Submit the scan
+        scan = api.scan(netblocks)
+        click.echo(click.style('Success (%s)! ', fg='green') % scan['id'] + '%s host(s) submitted (%s scan credits remaining)' % (scan['count'], scan['credits_left']))
+
+        # Wait for responses
+        if wait > 0:
+            click.echo('Waiting for results...')
+
+            # Now wait a few seconds for items to get returned
+            hosts = collections.defaultdict(list)
+            done = False
+            while not done:
+                try:
+                    for banner in api.stream.alert(timeout=wait):
+                        click.echo('Open: {0}:{1}'.format(banner['ip_str'], banner['port']))
+                        hosts[banner['ip_str']].append(banner)
+                except shodan.APIError, e:
+                    if done:
+                        break
 
-        click.echo(click.style('Success! ', fg='green') + '%s host(s) submitted (%s scan credits remaining)' % (results['count'], results['credits_left']))
+                    scan = api.scan_status(scan['id'])
+                    if scan['status'] == 'DONE':
+                        done = True
+                except Exception, e:
+                    raise click.ClickException(repr(e))
+
+            # Cleanup
+            click.echo('Cleaning up...')
+            api.delete_alert(alert['id'])
+
+            if hosts:
+                click.echo('')
+                click.echo('Summary')
+                click.echo('-------')
+                click.echo('')
+
+                for ip in hosts:
+                    click.echo(click.style(ip, fg='cyan'))
+
+                    host = hosts[ip][-1]
+                    if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
+                        click.echo('Country: {0}'.format(host['location']['country_name']))
+
+                        if 'city' in host['location'] and host['location']['city']:
+                            click.echo('City: {0}'.format(host['location']['city']))
+                    if 'org' in host and host['org']:
+                        click.echo('Organization: {0}'.format(host['org']))
+                    if 'os' in host and host['os']:
+                        click.echo('Operating system: {0}'.format(host['os']))
+                    click.echo('')
+
+                    # Print all the open ports:
+                    for banner in hosts[ip]:
+                        click.echo('Port: {0}'.format(click.style(str(banner['port']), fg='yellow')))
+                        click.echo(banner['data'].strip())
+                        click.echo('')
+            else:
+                click.echo('No open ports found or the host has been recently crawled and cant get scanned again so soon.')
     except shodan.APIError, e:
         raise click.ClickException(e.value)
 
@@ -360,7 +488,7 @@ def stats(limit, facets, query):
         print '# Top %s %s' % (limit, facet)
 
         for item in results['facets'][facet]:
-            print '  {:28s}'.format(item['value']), '{:12,d}'.format(item['count'])
+            print '  {:28s}'.format(item['value'].encode('ascii', errors='replace')), '{:12,d}'.format(item['count'])
 
         print ''
 
diff --git a/setup.py b/setup.py
index b42a7d4..9b5980d 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.2.6',
+    version = '1.3.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/alert.py b/shodan/alert.py
new file mode 100644
index 0000000..52a7a7c
--- /dev/null
+++ b/shodan/alert.py
@@ -0,0 +1,10 @@
+
+class Alert:
+	def __init__(self):
+		self.id = None
+		self.name = None
+		self.api_key = None
+		self.filters = None
+		self.credits = None
+		self.created = None
+		self.expires = None
diff --git a/shodan/client.py b/shodan/client.py
index 4310a8a..ee1b4bb 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -202,6 +202,16 @@ def scan(self, ips):
         }
 
         return self._request('/shodan/scan', params, method='post')
+
+    def scan_status(self, scan_id):
+        """Get the status information about a previously submitted scan.
+
+        :param id: The unique ID for the scan that was submitted
+        :type id: str
+
+        :returns: A dictionary with general information about the scan, including its status in getting processed.
+        """
+        return self._request('/shodan/scan/%s' % scan_id, {})
     
     def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
         """Search the SHODAN database.
@@ -364,18 +374,20 @@ def create_alert(self, name, ip, expires=0):
             'expires': expires,
         }
 
-        response = helpers.api_request(self.api_key, '/shodan/alert', data=data, params={}, base_url='https://oldapi.shodan.io', method='post')
+        response = helpers.api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post')
 
         return response
 
-    def alerts(self, aid=None):
+    def alerts(self, aid=None, include_expired=True):
         """List all of the active alerts that the user created."""
         if aid:
             func = '/shodan/alert/%s/info' % aid
         else:
             func = '/shodan/alert/info'
 
-        response = helpers.api_request(self.api_key, func, params={}, base_url='https://oldapi.shodan.io')
+        response = helpers.api_request(self.api_key, func, params={
+            'include_expired': include_expired,
+            })
 
         return response
 
@@ -383,7 +395,7 @@ def delete_alert(self, aid):
         """Delete the alert with the given ID."""
         func = '/shodan/alert/%s' % aid
 
-        response = helpers.api_request(self.api_key, func, params={}, method='delete', base_url='https://oldapi.shodan.io')
+        response = helpers.api_request(self.api_key, func, params={}, method='delete')
 
         return response
 
diff --git a/shodan/stream.py b/shodan/stream.py
index df9dc84..9ce0faa 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,7 +1,7 @@
 import requests
 import simplejson
 
-import shodan.exception
+import shodan.exception as exception
 
 class Stream:
 
@@ -10,9 +10,9 @@ class Stream:
     def __init__(self, api_key):
         self.api_key = api_key
 
-    def _create_stream(self, name):
+    def _create_stream(self, name, timeout=None):
         try:
-            req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True)
+            req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
         except:
             raise exception.APIError('Unable to contact the Shodan Streaming API')
 
@@ -24,16 +24,19 @@ def _create_stream(self, name):
             raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
         return req
 
-    def alert(self, aid=None):
+    def alert(self, aid=None, timeout=None):
         if aid:
-            stream = self._create_stream('/shodan/alert/%s' % aid)
+            stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
         else:
-            stream = self._create_stream('/shodan/alert')
+            stream = self._create_stream('/shodan/alert', timeout=timeout)
 
-        for line in stream.iter_lines():
-            if line:
-                banner = simplejson.loads(line)
-                yield banner
+        try:
+            for line in stream.iter_lines():
+                if line:
+                    banner = simplejson.loads(line)
+                    yield banner
+        except requests.exceptions.ConnectionError, e:
+            raise exception.APIError('Stream timed out')
 
     def banners(self):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to

From 5f9bd8e505216a48e237817dd781e46744a56724 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Tue, 17 Mar 2015 03:18:33 -0500
Subject: [PATCH 004/263] Added command to scan the Internet via Shodan "shodan
 scan internet <port> <protocol>" Bumped version

---
 bin/shodan       | 84 ++++++++++++++++++++++++++++++++++++++++++++++--
 setup.py         |  2 +-
 shodan/client.py | 24 ++++++++++++++
 3 files changed, 107 insertions(+), 3 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 4874933..f1e609d 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -315,10 +315,90 @@ def myip():
         raise click.ClickException(e.value)
 
 
-@main.command()
+@main.group()
+def scan():
+    pass
+
+
+@scan.command(name='internet')
+@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True)
+@click.argument('port', type=int)
+@click.argument('protocol', type=str)
+def scan_internet(quiet, port, protocol):
+    """Scan the Internet for a specific port and protocol using the Shodan infrastructure."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        # Submit the request to Shodan
+        click.echo('Submitting Internet scan to Shodan...', nl=False)
+        scan = api.scan_internet(port, protocol)
+        click.echo('Done')
+
+        # Create the output file
+        filename = '{0}-{1}.json.gz'.format(port, protocol)
+        counter = 0
+        with gzip.open(filename, 'w') as fout:
+            click.echo('Saving results to file: {0}'.format(filename))
+
+            # Start listening for results
+            done = False
+
+            # Keep listening for results until the scan is done
+            click.echo('Waiting for data, please stand by...')
+            while not done:
+                try:
+                    for banner in api.stream.ports([port]):
+                        counter += 1
+                        fout.write(simplejson.dumps(banner) + '\n')
+
+                        if not quiet:
+                            click.echo('{0:<40} {1:<20} {2}'.format(
+                                    click.style(banner['ip_str'], fg=COLORIZE_FIELDS['ip_str']),
+                                    click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
+                                    ';'.join(banner['hostnames'])
+                                )
+                            )
+
+                        # Check the scan status every few records
+                        if counter % 10000:
+                            scan = api.scan_status(scan['id'])
+                            if scan['status'] == 'DONE':
+                                done = True
+
+                except shodan.APIError, e:
+                    if done:
+                        break
+
+                    scan = api.scan_status(scan['id'])
+                    if scan['status'] == 'DONE':
+                        done = True
+                except Exception, e:
+                    raise click.ClickException(repr(e))
+        click.echo('Scan finished: {0} devices found'.format(counter))
+
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+
+
+@scan.command(name='protocols')
+def scan_protocols():
+    """List the protocols that you can scan with using Shodan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        protocols = api.protocols()
+
+        for name, description in protocols.iteritems():
+            click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description)
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+
+
+@scan.command(name='submit')
 @click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=30, type=int)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan(wait, netblocks):
+def scan_submit(wait, netblocks):
     """Scan an IP/ netblock using Shodan."""
     key = get_api_key()
     api = shodan.Shodan(key)
diff --git a/setup.py b/setup.py
index 9b5980d..aeeaf39 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.0',
+    version = '1.3.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/client.py b/shodan/client.py
index ee1b4bb..85bc1b6 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -186,6 +186,13 @@ def info(self):
         """
         return self._request('/api-info', {})
 
+    def protocols(self):
+        """Get a list of protocols that the Shodan on-demand scanning API supports.
+
+        :returns: A dictionary containing the protocol name and description.
+        """
+        return self._request('/shodan/protocols', {})
+
     def scan(self, ips):
         """Scan a network using Shodan
 
@@ -203,6 +210,23 @@ def scan(self, ips):
 
         return self._request('/shodan/scan', params, method='post')
 
+    def scan_internet(self, port, protocol):
+        """Scan a network using Shodan
+
+        :param port: The port that should get scanned.
+        :type port: int
+        :param port: The name of the protocol as returned by the protocols() method.
+        :type port: str
+
+        :returns: A dictionary with a unique ID to check on the scan progress.
+        """
+        params = {
+            'port': port,
+            'protocol': protocol,
+        }
+
+        return self._request('/shodan/scan/internet', params, method='post')
+
     def scan_status(self, scan_id):
         """Get the status information about a previously submitted scan.
 

From a3a99472750b3415c50a54f2f9997bed4f449864 Mon Sep 17 00:00:00 2001
From: Jim Sella <jsella@gmail.com>
Date: Tue, 24 Mar 2015 12:33:14 -0600
Subject: [PATCH 005/263] Allow access to raw streamed data to reduce CPU
 overhead of converting to JSON and back again to a string for streaming to
 disk.

---
 shodan/stream.py | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 9ce0faa..8eaf9af 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -24,7 +24,7 @@ def _create_stream(self, name, timeout=None):
             raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
         return req
 
-    def alert(self, aid=None, timeout=None):
+    def alert(self, aid=None, timeout=None, raw = False):
         if aid:
             stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
         else:
@@ -33,22 +33,28 @@ def alert(self, aid=None, timeout=None):
         try:
             for line in stream.iter_lines():
                 if line:
-                    banner = simplejson.loads(line)
-                    yield banner
+                    if raw:
+                        yield line
+                    else:
+                        banner = simplejson.loads(line)
+                        yield banner
         except requests.exceptions.ConnectionError, e:
             raise exception.APIError('Stream timed out')
 
-    def banners(self):
+    def banners(self, raw = False):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
         API subscription plans and for those it only returns a fraction of the data.
         """
         stream = self._create_stream('/shodan/banners')
         for line in stream.iter_lines():
             if line:
-                banner = simplejson.loads(line)
-                yield banner
+                if raw:
+                    yield line
+                else:
+                    banner = simplejson.loads(line)
+                    yield banner
 
-    def ports(self, ports):
+    def ports(self, ports, raw = False):
         """
         A filtered version of the "banners" stream to only return banners that match the ports of interest.
 
@@ -58,10 +64,13 @@ def ports(self, ports):
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]))
         for line in stream.iter_lines():
             if line:
-                banner = simplejson.loads(line)
-                yield banner
+                if raw:
+                    yield line
+                else:
+                    banner = simplejson.loads(line)
+                    yield banner
 
-    def geo(self):
+    def geo(self, raw = False):
         """
         A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream
         in case you only care about the geolocation information.
@@ -69,5 +78,8 @@ def geo(self):
         stream = self._create_stream('/shodan/geo')
         for line in stream.iter_lines():
             if line:
-                banner = simplejson.loads(line)
-                yield banner
\ No newline at end of file
+                if raw:
+                    yield line
+                else:
+                    banner = simplejson.loads(line)
+                    yield banner

From e7882c7b2507a122ab004f809b48c733e35af715 Mon Sep 17 00:00:00 2001
From: Jim Sella <jsella@gmail.com>
Date: Tue, 24 Mar 2015 12:44:56 -0600
Subject: [PATCH 006/263] Follow libraries existing coding style.

---
 shodan/stream.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 8eaf9af..552d880 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -24,7 +24,7 @@ def _create_stream(self, name, timeout=None):
             raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
         return req
 
-    def alert(self, aid=None, timeout=None, raw = False):
+    def alert(self, aid=None, timeout=None, raw=False):
         if aid:
             stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
         else:
@@ -41,7 +41,7 @@ def alert(self, aid=None, timeout=None, raw = False):
         except requests.exceptions.ConnectionError, e:
             raise exception.APIError('Stream timed out')
 
-    def banners(self, raw = False):
+    def banners(self, raw=False):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
         API subscription plans and for those it only returns a fraction of the data.
         """
@@ -54,7 +54,7 @@ def banners(self, raw = False):
                     banner = simplejson.loads(line)
                     yield banner
 
-    def ports(self, ports, raw = False):
+    def ports(self, ports, raw=False):
         """
         A filtered version of the "banners" stream to only return banners that match the ports of interest.
 
@@ -70,7 +70,7 @@ def ports(self, ports, raw = False):
                     banner = simplejson.loads(line)
                     yield banner
 
-    def geo(self, raw = False):
+    def geo(self, raw=False):
         """
         A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream
         in case you only care about the geolocation information.

From fc432be261339ffd70673bbe8d3d6390666c5568 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 14 Jun 2015 11:36:15 -0500
Subject: [PATCH 007/263] Added "host" command to the shodan CLI

---
 bin/shodan       | 79 +++++++++++++++++++++++++++++++++++++++++++++++-
 setup.py         |  2 +-
 shodan/stream.py |  1 +
 3 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index f1e609d..1e39518 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -36,6 +36,7 @@ COLORIZE_FIELDS = {
     'data': 'white',
     'hostnames': 'magenta',
     'org': 'cyan',
+    'vulns': 'red',
 }
 
 
@@ -235,6 +236,82 @@ def download(limit, filename, query):
         click.echo(click.style('Saved %s results into file %s' % (count, filename), 'green'))
 
 
+@main.command()
+@click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str)
+@click.argument('ip', metavar='<ip address>')
+def host(format, ip):
+    """Scan an IP/ netblock using Shodan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        host = api.host(ip)
+
+        # General info
+        click.echo(click.style(ip, fg='green'))
+        if len(host['hostnames']) > 0:
+            click.echo('{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
+
+        if 'city' in host and host['city']:
+            click.echo('{:25s}{}'.format('City:', host['city']))
+
+        if 'country_name' in host and host['country_name']:
+            click.echo('{:25s}{}'.format('Country:', host['country_name']))
+
+        if 'os' in host and host['os']:
+            click.echo('{:25s}{}'.format('Operating System:', host['os']))
+
+        if 'org' in host and host['org']:
+            click.echo('{:25s}{}'.format('Organization:', host['org']))
+
+        click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
+
+        # Output the vulnerabilities the host has
+        if 'vulns' in host and len(host['vulns']) > 0:
+            vulns = []
+            for vuln in host['vulns']:
+                if vuln.startswith('!'):
+                    continue
+                if vuln.upper() == 'CVE-2014-0160':
+                    vulns.append(click.style('Heartbleed', fg='red'))
+                else:
+                    vulns.append(click.style(vuln, fg='red'))
+
+            if len(vulns) > 0:
+                click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
+
+                for vuln in vulns:
+                    click.echo(vuln + '\t', nl=False)
+
+                click.echo('')
+
+        click.echo('')
+
+        click.echo('Ports:')
+        for banner in sorted(host['data'], key=lambda k: k['port']):
+            product = ''
+            version = ''
+            if 'product' in banner:
+                product = banner['product']
+            if 'version' in banner:
+                version = '({})'.format(banner['version'])
+
+            click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False)
+            click.echo('{} {}'.format(product, version))
+
+            # Show optional ssl info
+            if 'ssl' in banner:
+                if 'versions' in banner['ssl']:
+                    click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
+                if 'dhparams' in banner['ssl']:
+                    click.echo('\t|-- Diffie-Hellman Parameters:')
+                    click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
+                    if 'fingerprint' in banner['ssl']['dhparams']:
+                        click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+
+
 @main.command()
 def info():
     """Shows general information about your account"""
@@ -396,7 +473,7 @@ def scan_protocols():
 
 
 @scan.command(name='submit')
-@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=30, type=int)
+@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=0, type=int)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
 def scan_submit(wait, netblocks):
     """Scan an IP/ netblock using Shodan."""
diff --git a/setup.py b/setup.py
index aeeaf39..815d779 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.1',
+    version = '1.3.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/stream.py b/shodan/stream.py
index 9ce0faa..8a369e5 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -18,6 +18,7 @@ def _create_stream(self, name, timeout=None):
 
         if req.status_code != 200:
             try:
+                req.close()
                 raise exception.APIError(data.json()['error'])
             except:
                 pass

From f142a5be525acc70bc1b0b703ece4cb6fd84a356 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 14 Jun 2015 17:29:35 -0500
Subject: [PATCH 008/263] Improved output for "stats" command for the Shodan
 CLI

---
 bin/shodan | 7 ++++---
 setup.py   | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 1e39518..e6ac641 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -642,12 +642,13 @@ def stats(limit, facets, query):
 
     # Print the stats tables
     for facet in results['facets']:
-        print '# Top %s %s' % (limit, facet)
+        click.echo('Top {} Results for Facet: {}'.format(limit, facet))
 
         for item in results['facets'][facet]:
-            print '  {:28s}'.format(item['value'].encode('ascii', errors='replace')), '{:12,d}'.format(item['count'])
+            click.echo(click.style('{:28s}'.format(item['value'].encode('ascii', errors='replace')), fg='cyan'), nl=False)
+            click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
 
-        print ''
+        click.echo('')
 
 
 @main.command()
diff --git a/setup.py b/setup.py
index 815d779..a8f1b6f 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.2',
+    version = '1.3.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 2c5f593119aedaf360f55f2cf55700d84af3157f Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Tue, 30 Jun 2015 23:09:23 -0500
Subject: [PATCH 009/263] Removed waiting for scan results for now

---
 bin/shodan | 5 ++---
 setup.py   | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index e6ac641..c28b429 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -482,9 +482,6 @@ def scan_submit(wait, netblocks):
 
     # Submit the IPs for scanning
     try:
-        # Setup an alert to wait for responses
-        alert = api.create_alert('Scan', netblocks)
-
         # Submit the scan
         scan = api.scan(netblocks)
         click.echo(click.style('Success (%s)! ', fg='green') % scan['id'] + '%s host(s) submitted (%s scan credits remaining)' % (scan['count'], scan['credits_left']))
@@ -492,6 +489,8 @@ def scan_submit(wait, netblocks):
         # Wait for responses
         if wait > 0:
             click.echo('Waiting for results...')
+            # Setup an alert to wait for responses
+            alert = api.create_alert('Scan', netblocks)
 
             # Now wait a few seconds for items to get returned
             hosts = collections.defaultdict(list)
diff --git a/setup.py b/setup.py
index a8f1b6f..ad54d7b 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.3',
+    version = '1.3.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From eb09d7d0ff0ee5ceba4856833bb7e5a2cd00328b Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 3 Jul 2015 18:24:39 -0500
Subject: [PATCH 010/263] Remove SSL warning message that clutters up the
 output

---
 shodan/client.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/shodan/client.py b/shodan/client.py
index 85bc1b6..dbbafdb 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -18,6 +18,13 @@
 import shodan.stream as stream
 
 
+# Try to disable the SSL warnings in urllib3
+try:
+    requests.packages.urllib3.disable_warnings()
+except:
+    pass
+
+
 class Shodan:
     """Wrapper around the Shodan REST and Streaming APIs
 

From f05207abcab831b52a72cd754136f5480167fe8b Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 3 Jul 2015 18:25:05 -0500
Subject: [PATCH 011/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ad54d7b..481168f 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.4',
+    version = '1.3.5',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 7da5f7de9a9b01270c7b74785d21d8547ef1f5e8 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Thu, 23 Jul 2015 23:14:34 -0500
Subject: [PATCH 012/263] Improved the "scan" command to wait for results
 Pretty output for the "scan" command

---
 bin/shodan       | 200 +++++++++++++++++++++++++++++++++++++++--------
 shodan/client.py |   7 +-
 2 files changed, 172 insertions(+), 35 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index c28b429..383fbae 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -21,10 +21,14 @@ import click
 import collections
 import datetime
 import gzip
+import itertools
 import os
 import os.path
 import shodan
 import simplejson
+import socket
+import sys
+import threading
 import time
 
 # Constants
@@ -39,6 +43,8 @@ COLORIZE_FIELDS = {
     'vulns': 'red',
 }
 
+# Make "-h" work like "--help"
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 # Utility methods
 def get_api_key():
@@ -69,7 +75,7 @@ def open_file(directory, timestr):
         return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', 1)
 
 
-@click.group()
+@click.group(context_settings=CONTEXT_SETTINGS)
 def main():
     pass
 
@@ -99,10 +105,27 @@ def init(key):
 def alert():
     pass
 
+
+@alert.command(name='clear')
+def alert_clear():
+    """Remove all alerts"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        alerts = api.alerts()
+        for alert in alerts:
+            click.echo('Removing {} ({})'.format(alert['name'], alert['id']))
+            api.delete_alert(alert['id'])
+    except shodan.APIError, e:
+        raise click.ClickException(e.value)
+    click.echo("Alerts deleted")
+
 @alert.command(name='list')
 @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
 def alert_list(expired):
-    """Returns the number of results for a search"""
+    """List all the active alerts"""
     key = get_api_key()
 
     # Get the list
@@ -136,7 +159,7 @@ def alert_list(expired):
 @alert.command(name='remove')
 @click.argument('alert_id', metavar='<alert ID>')
 def alert_remove(alert_id):
-    """Returns the number of results for a search"""
+    """Remove the specified alert"""
     key = get_api_key()
 
     # Get the list
@@ -473,33 +496,72 @@ def scan_protocols():
 
 
 @scan.command(name='submit')
-@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=0, type=int)
+@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=10, type=int)
+@click.option('--filename', help='Save the results in the given file.', default='', type=str)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan_submit(wait, netblocks):
+def scan_submit(wait, filename, netblocks):
     """Scan an IP/ netblock using Shodan."""
     key = get_api_key()
     api = shodan.Shodan(key)
+    alert = None
 
     # Submit the IPs for scanning
     try:
         # Submit the scan
         scan = api.scan(netblocks)
-        click.echo(click.style('Success (%s)! ', fg='green') % scan['id'] + '%s host(s) submitted (%s scan credits remaining)' % (scan['count'], scan['credits_left']))
 
-        # Wait for responses
-        if wait > 0:
-            click.echo('Waiting for results...')
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
+
+        click.echo('')
+        click.echo('Starting Shodan scan at {} ({} scan credits left)'.format(now, scan['credits_left']))
+        # click.echo(click.style('Success (%s)! ', fg='green') % scan['id'] + '%s host(s) submitted (%s scan credits remaining)' % (scan['count'], scan['credits_left']))
+
+        # Return immediately
+        if wait <= 0:
+            click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
+        else:
             # Setup an alert to wait for responses
-            alert = api.create_alert('Scan', netblocks)
+            alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks)
+
+            # Create the output file if necessary
+            filename = filename.strip()
+            fout = None
+            if filename != '':
+                # Add the appropriate extension if it's not there atm
+                if not filename.endswith('.json.gz'):
+                    filename += '.json.gz'
+                fout = gzip.open(filename, 'w')
+
+            # Start a spinner
+            finished_event = threading.Event()
+            progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
+            progress_bar_thread.start()
 
             # Now wait a few seconds for items to get returned
-            hosts = collections.defaultdict(list)
+            hosts = collections.defaultdict(dict)
             done = False
+            scan_start = time.time()
+            cache = {}
             while not done:
                 try:
-                    for banner in api.stream.alert(timeout=wait):
-                        click.echo('Open: {0}:{1}'.format(banner['ip_str'], banner['port']))
-                        hosts[banner['ip_str']].append(banner)
+                    for banner in api.stream.alert(aid=alert['id'], timeout=wait):
+                        ip = banner.get('ip', banner.get('ipv6', None))
+                        if not ip:
+                            continue
+
+                        # Don't show duplicate banners
+                        cache_key = '{}:{}'.format(ip, banner['port'])
+                        if cache_key not in cache:
+                            hosts[banner['ip_str']][banner['port']] = banner
+                            cache[cache_key] = True
+
+                        # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
+                        if time.time() - scan_start >= 60:
+                            scan = api.scan_status(scan['id'])
+                            if scan['status'] == 'DONE':
+                                done = True
+                                break
+
                 except shodan.APIError, e:
                     if done:
                         break
@@ -507,43 +569,102 @@ def scan_submit(wait, netblocks):
                     scan = api.scan_status(scan['id'])
                     if scan['status'] == 'DONE':
                         done = True
+                except socket.timeout, e:
+                    done = True
                 except Exception, e:
                     raise click.ClickException(repr(e))
 
-            # Cleanup
-            click.echo('Cleaning up...')
-            api.delete_alert(alert['id'])
+            finished_event.set()
+            progress_bar_thread.join()
+
+            def print_field(name, value):
+                click.echo('  {:25s}{}'.format(name, value))
+
+            def print_banner(banner):
+                click.echo('    {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False)
+
+                if 'product' in banner:
+                    click.echo(banner['product'], nl=False)
+
+                    if 'version' in banner:
+                        click.echo(' ({})'.format(banner['version']), nl=False)
 
-            if hosts:
-                click.echo('')
-                click.echo('Summary')
-                click.echo('-------')
                 click.echo('')
 
-                for ip in hosts:
-                    click.echo(click.style(ip, fg='cyan'))
+                # Show optional ssl info
+                if 'ssl' in banner:
+                    if 'versions' in banner['ssl']:
+                        # Only print SSL versions if they were successfully tested
+                        versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
+                        if len(versions) > 0:
+                            click.echo('    |-- SSL Versions: {}'.format(', '.join(versions)))
+                    if 'dhparams' in banner['ssl']:
+                        click.echo('    |-- Diffie-Hellman Parameters:')
+                        click.echo('        {:15s}{}\n        {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
+                        if 'fingerprint' in banner['ssl']['dhparams']:
+                            click.echo('        {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+
+            if hosts:
+                # Remove the remaining spinner character
+                click.echo('\b ')
+
+                for ip in sorted(hosts):
+                    host = hosts[ip].items()[0]
+
+                    click.echo(click.style(ip, fg='cyan'), nl=False)
+                    if 'hostnames' in host and host['hostnames']:
+                        click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False)
+                    click.echo('')
 
-                    host = hosts[ip][-1]
                     if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
-                        click.echo('Country: {0}'.format(host['location']['country_name']))
+                        print_field('Country', host['location']['country_name'])
 
                         if 'city' in host['location'] and host['location']['city']:
-                            click.echo('City: {0}'.format(host['location']['city']))
+                            print_field('City', host['location']['city'])
                     if 'org' in host and host['org']:
-                        click.echo('Organization: {0}'.format(host['org']))
+                        print_field('Organization', host['org'])
                     if 'os' in host and host['os']:
-                        click.echo('Operating system: {0}'.format(host['os']))
+                        print_field('Operating System', host['os'])
                     click.echo('')
 
+                    # Output the vulnerabilities the host has
+                    if 'vulns' in host and len(host['vulns']) > 0:
+                        vulns = []
+                        for vuln in host['vulns']:
+                            if vuln.startswith('!'):
+                                continue
+                            if vuln.upper() == 'CVE-2014-0160':
+                                vulns.append(click.style('Heartbleed', fg='red'))
+                            else:
+                                vulns.append(click.style(vuln, fg='red'))
+
+                        if len(vulns) > 0:
+                            click.echo('  {:25s}'.format('Vulnerabilities:'), nl=False)
+
+                            for vuln in vulns:
+                                click.echo(vuln + '\t', nl=False)
+
+                            click.echo('')
+
                     # Print all the open ports:
-                    for banner in hosts[ip]:
-                        click.echo('Port: {0}'.format(click.style(str(banner['port']), fg='yellow')))
-                        click.echo(banner['data'].strip())
-                        click.echo('')
+                    click.echo('  Open Ports:')
+                    for port in sorted(hosts[ip]):
+                        print_banner(hosts[ip][port])
+
+                        # Save the banner in a file if necessary
+                        if fout:
+                            fout.write(simplejson.dumps(hosts[ip][port]) + '\n')
+
+                    click.echo('')
             else:
-                click.echo('No open ports found or the host has been recently crawled and cant get scanned again so soon.')
+                # Prepend a \b to remove the spinner
+                click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
     except shodan.APIError, e:
         raise click.ClickException(e.value)
+    finally:
+        # Remove any alert
+        if alert:
+            api.delete_alert(alert['id'])
 
 
 @main.command()
@@ -658,12 +779,16 @@ def stats(limit, facets, query):
 @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str)
 @click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
 @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
-def stream(color, fields, separator, limit, datadir, ports, quiet):
+@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
+def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
     api = shodan.Shodan(key)
 
+    # Temporarily change the baseurl
+    api.stream.base_url = streamer
+
     # Strip out any whitespace in the fields and turn them into an array
     fields = [item.strip() for item in fields.split(',')]
 
@@ -745,5 +870,12 @@ def stream(color, fields, separator, limit, datadir, ports, quiet):
             time.sleep(2)
 
 
+def async_spinner(finished):
+    spinner = itertools.cycle(['-', '/', '|', '\\'])
+    while not finished.is_set():
+        sys.stdout.write('\b{}'.format(spinner.next()))
+        sys.stdout.flush()
+        finished.wait(0.2)
+
 if __name__ == '__main__':
     main()
diff --git a/shodan/client.py b/shodan/client.py
index dbbafdb..0f6334e 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -18,7 +18,12 @@
 import shodan.stream as stream
 
 
-# Try to disable the SSL warnings in urllib3
+# Try to disable the SSL warnings in urllib3 since not everybody can install
+# C extensions. If you're able to install C extensions you can try to run:
+#
+# pip install requests[security]
+#
+# Which will download libraries that offer more full-featured SSL classes
 try:
     requests.packages.urllib3.disable_warnings()
 except:

From 899e9f6cfb7e5494a8bc348337dbff00f77f197f Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 24 Jul 2015 04:20:45 -0500
Subject: [PATCH 013/263] Made the streaming of results from a scan more
 reliable By default attempt 1 retry of an API call if it failed for
 connection reasons.

---
 bin/shodan          | 19 ++++++++++++++++---
 setup.py            |  2 +-
 shodan/exception.py |  4 ++++
 shodan/helpers.py   | 28 +++++++++++++++++++---------
 shodan/stream.py    | 11 +++++++----
 5 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 383fbae..911bda3 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -496,7 +496,7 @@ def scan_protocols():
 
 
 @scan.command(name='submit')
-@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=10, type=int)
+@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int)
 @click.option('--filename', help='Save the results in the given file.', default='', type=str)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
 def scan_submit(wait, filename, netblocks):
@@ -514,7 +514,6 @@ def scan_submit(wait, filename, netblocks):
 
         click.echo('')
         click.echo('Starting Shodan scan at {} ({} scan credits left)'.format(now, scan['credits_left']))
-        # click.echo(click.style('Success (%s)! ', fg='green') % scan['id'] + '%s host(s) submitted (%s scan credits remaining)' % (scan['count'], scan['credits_left']))
 
         # Return immediately
         if wait <= 0:
@@ -563,6 +562,14 @@ def scan_submit(wait, filename, netblocks):
                                 break
 
                 except shodan.APIError, e:
+                    # If the connection timed out before the timeout, that means the streaming server
+                    # that the user tried to reach is down. In that case, lets wait briefly and try
+                    # to connect again!
+                    if (time.time() - scan_start) < wait:
+                        time.sleep(0.5)
+                        continue
+
+                    # Exit if the scan was flagged as done somehow
                     if done:
                         break
 
@@ -570,6 +577,12 @@ def scan_submit(wait, filename, netblocks):
                     if scan['status'] == 'DONE':
                         done = True
                 except socket.timeout, e:
+                    # If the connection timed out before the timeout, that means the streaming server
+                    # that the user tried to reach is down. In that case, lets wait a second and try
+                    # to connect again!
+                    if (time.time() - scan_start) < wait:
+                        continue
+
                     done = True
                 except Exception, e:
                     raise click.ClickException(repr(e))
@@ -609,7 +622,7 @@ def scan_submit(wait, filename, netblocks):
                 click.echo('\b ')
 
                 for ip in sorted(hosts):
-                    host = hosts[ip].items()[0]
+                    host = hosts[ip].items()[0][1]
 
                     click.echo(click.style(ip, fg='cyan'), nl=False)
                     if 'hostnames' in host and host['hostnames']:
diff --git a/setup.py b/setup.py
index 481168f..556d669 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from distutils.core import setup
+from setuptools import setup
 
 setup(
     name = 'shodan',
diff --git a/shodan/exception.py b/shodan/exception.py
index a34c8e1..9bc0d03 100644
--- a/shodan/exception.py
+++ b/shodan/exception.py
@@ -6,3 +6,7 @@ def __init__(self, value):
     
     def __str__(self):
         return self.value
+
+class APITimeout(APIError):
+	pass
+
diff --git a/shodan/helpers.py b/shodan/helpers.py
index bed0240..6c4e289 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -17,7 +17,7 @@ def create_facet_string(facets):
     return facet_str[:-1]
 
     
-def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get'):
+def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get', retries=1):
     """General-purpose function to create web requests to SHODAN.
     
     Arguments:
@@ -32,14 +32,24 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     params['key'] = key
 
     # Send the request
-    try:
-        if method.lower() == 'post':
-            data = requests.post(base_url + function, simplejson.dumps(data), params=params, headers={'content-type': 'application/json'})
-        elif method.lower() == 'delete':
-            data = requests.delete(base_url + function, params=params)
-        else:
-            data = requests.get(base_url + function, params=params)
-    except:
+    tries = 0
+    error = False
+    while tries <= retries:
+        try:
+            if method.lower() == 'post':
+                data = requests.post(base_url + function, simplejson.dumps(data), params=params, headers={'content-type': 'application/json'})
+            elif method.lower() == 'delete':
+                data = requests.delete(base_url + function, params=params)
+            else:
+                data = requests.get(base_url + function, params=params)
+
+            # Exit out of the loop
+            break
+        except:
+            error = True
+            tries += 1
+
+    if error and tries >= retries:
         raise shodan.exception.APIError('Unable to connect to Shodan')
 
     # Check that the API key wasn't rejected
diff --git a/shodan/stream.py b/shodan/stream.py
index 8a369e5..bed0ebf 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,6 +1,7 @@
 import requests
 import simplejson
 
+import socket
 import shodan.exception as exception
 
 class Stream:
@@ -13,14 +14,16 @@ def __init__(self, api_key):
     def _create_stream(self, name, timeout=None):
         try:
             req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
-        except:
+        except Exception, e:
             raise exception.APIError('Unable to contact the Shodan Streaming API')
 
         if req.status_code != 200:
             try:
-                req.close()
-                raise exception.APIError(data.json()['error'])
-            except:
+                data = simplejson.loads(req.text)
+                raise exception.APIError(data['error'])
+            except exception.APIError, e:
+                raise
+            except Exception, e:
                 pass
             raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
         return req

From d4a07409d4c0b42333d76770cc69c7dd614d6e01 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Mon, 3 Aug 2015 22:08:29 -0500
Subject: [PATCH 014/263] Added API key validation step during "shodan init"
 Only wait for Internet-scan results if the port isn't normally scanned Added
 api.ports() method to get list of constantly scanned ports

---
 bin/shodan       | 91 ++++++++++++++++++++++++++----------------------
 setup.py         |  2 +-
 shodan/client.py |  7 ++++
 3 files changed, 58 insertions(+), 42 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 911bda3..5ee3ed9 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -92,6 +92,14 @@ def init(key):
         except OSError:
             raise click.ClickException('Unable to create directory to store the Shodan API key (%s)' % shodan_dir)
 
+    # Make sure it's a valid API key
+    key = key.strip()
+    try:
+        api = shodan.Shodan(key)
+        test = api.info()
+    except shodan.APIError, e:
+        raise click.ClickException('Invalid API key')
+
     # Store the API key in the user's directory
     keyfile = shodan_dir + '/api_key'
     with open(keyfile, 'w') as fout:
@@ -435,48 +443,49 @@ def scan_internet(quiet, port, protocol):
         scan = api.scan_internet(port, protocol)
         click.echo('Done')
 
-        # Create the output file
-        filename = '{0}-{1}.json.gz'.format(port, protocol)
-        counter = 0
-        with gzip.open(filename, 'w') as fout:
-            click.echo('Saving results to file: {0}'.format(filename))
-
-            # Start listening for results
-            done = False
-
-            # Keep listening for results until the scan is done
-            click.echo('Waiting for data, please stand by...')
-            while not done:
-                try:
-                    for banner in api.stream.ports([port]):
-                        counter += 1
-                        fout.write(simplejson.dumps(banner) + '\n')
-
-                        if not quiet:
-                            click.echo('{0:<40} {1:<20} {2}'.format(
-                                    click.style(banner['ip_str'], fg=COLORIZE_FIELDS['ip_str']),
-                                    click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
-                                    ';'.join(banner['hostnames'])
+        # If the requested port is part of the regular Shodan crawling, then
+        # we don't know when the scan is done so lets return immediately and
+        # let the user decide when to stop waiting for further results.
+        official_ports = api.ports()
+        if port in official_ports:
+            click.echo('The request port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
+        else:
+            # Create the output file
+            filename = '{0}-{1}.json.gz'.format(port, protocol)
+            counter = 0
+            with gzip.open(filename, 'w') as fout:
+                click.echo('Saving results to file: {0}'.format(filename))
+
+                # Start listening for results
+                done = False
+
+                # Keep listening for results until the scan is done
+                click.echo('Waiting for data, please stand by...')
+                while not done:
+                    try:
+                        for banner in api.stream.ports([port], timeout=30):
+                            counter += 1
+                            fout.write(simplejson.dumps(banner) + '\n')
+
+                            if not quiet:
+                                click.echo('{0:<40} {1:<20} {2}'.format(
+                                        click.style(banner['ip_str'], fg=COLORIZE_FIELDS['ip_str']),
+                                        click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
+                                        ';'.join(banner['hostnames'])
+                                    )
                                 )
-                            )
-
-                        # Check the scan status every few records
-                        if counter % 10000:
-                            scan = api.scan_status(scan['id'])
-                            if scan['status'] == 'DONE':
-                                done = True
-
-                except shodan.APIError, e:
-                    if done:
-                        break
-
-                    scan = api.scan_status(scan['id'])
-                    if scan['status'] == 'DONE':
-                        done = True
-                except Exception, e:
-                    raise click.ClickException(repr(e))
-        click.echo('Scan finished: {0} devices found'.format(counter))
-
+                    except shodan.APIError, e:
+                        # We stop waiting for results if the scan has been processed by the crawlers and
+                        # there haven't been new results in a while
+                        if done:
+                            break
+
+                        scan = api.scan_status(scan['id'])
+                        if scan['status'] == 'DONE':
+                            done = True
+                    except Exception, e:
+                        raise click.ClickException(repr(e))
+            click.echo('Scan finished: {0} devices found'.format(counter))
     except shodan.APIError, e:
         raise click.ClickException(e.value)
 
diff --git a/setup.py b/setup.py
index 556d669..9c060c3 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.5',
+    version = '1.3.6',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/client.py b/shodan/client.py
index 0f6334e..34e335b 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -198,6 +198,13 @@ def info(self):
         """
         return self._request('/api-info', {})
 
+    def ports(self):
+        """Get a list of ports that Shodan crawls
+
+        :returns: An array containing the ports that Shodan crawls for.
+        """
+        return self._request('/shodan/ports', {})
+
     def protocols(self):
         """Get a list of protocols that the Shodan on-demand scanning API supports.
 

From 8366884991d4dc482a4223bf3aeece698453b865 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Mon, 3 Aug 2015 22:42:55 -0500
Subject: [PATCH 015/263] Fixed typo

---
 bin/shodan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 5ee3ed9..79a4e79 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -448,7 +448,7 @@ def scan_internet(quiet, port, protocol):
         # let the user decide when to stop waiting for further results.
         official_ports = api.ports()
         if port in official_ports:
-            click.echo('The request port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
+            click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
         else:
             # Create the output file
             filename = '{0}-{1}.json.gz'.format(port, protocol)

From 0bed712307fb5738f12fe00933a4fe3ed3d6e7fc Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Mon, 3 Aug 2015 22:45:20 -0500
Subject: [PATCH 016/263] Added timeout option to real-time streams Removed
 deprecated /shodan/geo stream

---
 shodan/stream.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 33c7dfe..fe29a7f 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -45,11 +45,11 @@ def alert(self, aid=None, timeout=None, raw=False):
         except requests.exceptions.ConnectionError, e:
             raise exception.APIError('Stream timed out')
 
-    def banners(self, raw=False):
+    def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
         API subscription plans and for those it only returns a fraction of the data.
         """
-        stream = self._create_stream('/shodan/banners')
+        stream = self._create_stream('/shodan/banners', timeout=timeout)
         for line in stream.iter_lines():
             if line:
                 if raw:
@@ -58,28 +58,14 @@ def banners(self, raw=False):
                     banner = simplejson.loads(line)
                     yield banner
 
-    def ports(self, ports, raw=False):
+    def ports(self, ports, raw=False, timeout=None):
         """
         A filtered version of the "banners" stream to only return banners that match the ports of interest.
 
         :param ports: A list of ports to return banner data on.
         :type ports: int[]
         """
-        stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]))
-        for line in stream.iter_lines():
-            if line:
-                if raw:
-                    yield line
-                else:
-                    banner = simplejson.loads(line)
-                    yield banner
-
-    def geo(self, raw=False):
-        """
-        A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream
-        in case you only care about the geolocation information.
-        """
-        stream = self._create_stream('/shodan/geo')
+        stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
         for line in stream.iter_lines():
             if line:
                 if raw:

From 2b1b060351ceeab8a17d2299e6eb71b101b9c8e1 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Wed, 2 Sep 2015 06:13:09 -0500
Subject: [PATCH 017/263] Bumped version Added the ability to filter results
 using "parse" and saving the filtered results in a new file

---
 bin/shodan | 114 ++++++++++++++++++++++++++++++++++++++++++++---------
 setup.py   |   2 +-
 2 files changed, 96 insertions(+), 20 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 79a4e79..18a9500 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -7,8 +7,10 @@ Note: Always run "shodan init <api key>" before trying to execute any other comm
 A simple interface to search Shodan, download data and parse compressed JSON files.
 The following commands are currently supported:
 
+    alert
     count
     download
+    host
     init
     myip
     parse
@@ -74,6 +76,57 @@ def timestr():
 def open_file(directory, timestr):
         return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', 1)
 
+def iterate_files(files):
+    for filename in files:
+        # Create a file handle depending on the filetype
+        if filename.endswith('.gz'):
+            fin = gzip.open(filename, 'r')
+        else:
+            fin = open(filename, 'r')
+
+        for line in fin:
+            # Convert the JSON into a native Python object
+            banner = simplejson.loads(line)
+            yield banner
+
+def get_banner_field(banner, flat_field):
+    # The provided field is a collapsed form of the actual field
+    fields = flat_field.split('.')
+
+    try:
+        current_obj = banner
+        for field in fields:
+            current_obj = current_obj[field]
+        return current_obj
+    except:
+        pass
+
+    return None
+
+def match_filters(banner, filters):
+    for args in filters:
+        flat_field, check = args.split(':')
+        value = get_banner_field(banner, flat_field)
+
+        # It must match all filters to be allowed
+        field_type = type(value)
+
+        # For lists of strings we see whether the desired value is contained in the field
+        if field_type == list or isinstance(value, basestring):
+            if check not in value:
+                return False
+        elif field_type == int:
+            if int(check) != value:
+                return False
+        elif field_type == float:
+            if float(check) != value:
+                return False
+        else:
+            # Ignore unknown types
+            pass
+
+    return True
+
 
 @click.group(context_settings=CONTEXT_SETTINGS)
 def main():
@@ -361,44 +414,58 @@ Scan credits available: {1}
 @main.command()
 @click.option('--color/--no-color', default=True)
 @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data')
+@click.option('--filters', '-f', help='Filter the results for specific values using key:value pairs.', multiple=True)
+@click.option('--filename', '-O', help='Save the filtered results in the given file (append if file exists).')
 @click.option('--separator', help='The separator between the properties of the search results.', default='\t')
-@click.argument('filename', metavar='<filename>', type=click.Path(exists=True))
-def parse(color, fields, separator, filename):
+@click.argument('filenames', metavar='<filenames>', type=click.Path(exists=True), nargs=-1)
+def parse(color, fields, filters, filename, separator, filenames):
     """Extract information out of compressed JSON files."""
-    # Make sure it's some sort of json file
-    if not filename.endswith('.json.gz') and not filename.endswith('.json'):
-        raise click.ClickException('Invalid file, please make sure it is a valid Shodan JSON file')
-
     # Strip out any whitespace in the fields and turn them into an array
     fields = [item.strip() for item in fields.split(',')]
 
     if len(fields) == 0:
         raise click.ClickException('Please define at least one property to show')
 
-    # Create a file handle depending on the filetype
-    if filename.endswith('.gz'):
-        fin = gzip.open(filename, 'r')
-    else:
-        fin = open(filename, 'r')
+    has_filters = len(filters) > 0
+
+
+    # Setup the output file handle
+    fout = None
+    if filename:
+        # If no filters were provided raise an error since it doesn't make much sense w/out them
+        if not has_filters:
+            raise click.ClickException('Output file specified without any filters. Need to use filters with this option.')
 
-    for line in fin:
-        # Convert the JSON into a native Python object
-        banner = simplejson.loads(line)
+        # Add the appropriate extension if it's not there atm
+        if not filename.endswith('.json.gz'):
+            filename += '.json.gz'
+        fout = gzip.open(filename, 'a')
+
+    for banner in iterate_files(filenames):
         row = ''
 
+        # Validate the banner against any provided filters
+        if has_filters and not match_filters(banner, filters):
+            continue
+
+        # Append the data
+        if fout:
+            fout.write(simplejson.dumps(banner) + '\n')
+
         # Loop over all the fields and print the banner as a row
         for field in fields:
             tmp = ''
-            if field in banner and banner[field]:
-                field_type = type(banner[field])
+            value = get_banner_field(banner, field)
+            if value:
+                field_type = type(value)
 
                 # If the field is an array then merge it together
                 if field_type == list:
-                    tmp = ';'.join(banner[field])
+                    tmp = ';'.join(value)
                 elif field_type in [int, float]:
-                    tmp = str(banner[field])
+                    tmp = str(value)
                 else:
-                    tmp = escape_data(banner[field])
+                    tmp = escape_data(value)
 
                 # Colorize certain fields if the user wants it
                 if color:
@@ -480,6 +547,15 @@ def scan_internet(quiet, port, protocol):
                         if done:
                             break
 
+                        scan = api.scan_status(scan['id'])
+                        if scan['status'] == 'DONE':
+                            done = True
+                    except socket.timeout, e:
+                        # We stop waiting for results if the scan has been processed by the crawlers and
+                        # there haven't been new results in a while
+                        if done:
+                            break
+
                         scan = api.scan_status(scan['id'])
                         if scan['status'] == 'DONE':
                             done = True
diff --git a/setup.py b/setup.py
index 9c060c3..3e3237e 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.6',
+    version = '1.3.7',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 7a2bbe2b65ef5e5e5e842227215503dbcbf6b028 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Thu, 3 Sep 2015 04:21:28 -0500
Subject: [PATCH 018/263] Added "convert" subcommand to "shodan" CLI Added KML
 output for "convert" subcommand

---
 bin/shodan | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)

diff --git a/bin/shodan b/bin/shodan
index 18a9500..79ae1cb 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -133,6 +133,152 @@ def main():
     pass
 
 
+@main.command()
+@click.argument('input', metavar='<input file>')
+@click.argument('format', metavar='<output format>', type=click.Choice(['kml']))
+def convert(input, format):
+    """Convert the given input data file into a different format.
+
+    Example: shodan convert data.json.gz kml
+    """
+    #---------------------------------------
+    # KML Support
+    #---------------------------------------
+    KML_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+  <Document>"""
+    KML_FOOTER = """</Document></kml>"""
+    def kml_writer(fout, host):
+        try:
+            ip = host.get('ip_str', host.get('ipv6', None))
+            lat, lon = host['location']['latitude'], host['location']['longitude']
+
+            placemark = '<Placemark><name><![CDATA[<h1 style="margin-bottom:0;padding-bottom:0;font-size:1.5em">{}</h1>]]></name>'.format(ip)
+            placemark += '<description><![CDATA['
+
+            if 'hostnames' in host and host['hostnames']:
+                placemark += '<div><a style="color: #999;margin-top:-10px;padding-top:0;" href="http://{0}" target="_blank">{0}</a></div>'.format(host['hostnames'][0])
+
+            test = """
+    <table>
+        <tbody>
+          <tr>
+            <td>City</td>
+            <th>Albuquerque</th>
+          </tr>
+          <tr>
+            <td>Country</td>
+            <th>United States</th>
+          </tr>
+          <tr>
+            <td>Organization</td>
+            <th>Nexcess.net L.L.C.</th>
+          </tr>
+        </tbody>
+    </table>
+    <h2>Ports</h2>
+    <ul>
+            """
+
+            placemark += '<h2>Ports</h2><ul>'
+
+            for port in host['ports']:
+                placemark += """
+                    <li style="background-color: #1CA8DD;
+                    color: #FFF;
+                    float: left;
+                    font-family: Arial;
+                    font-size: 12px;
+                    font-weight: bold;
+                    height: 48px;
+                    margin: 5px;
+                    position: relative;
+                    text-shadow: none;
+                    width: 48px;"><span style="color: #FFF;
+                    height: 34px;
+                    position: absolute;
+                    text-align: center;
+                    top: 30%;
+                    width: 48px;">{}</span>
+                        </li>
+                """.format(port)
+
+            placemark += '</ul><div style="clear:both"></div>'
+
+            placemark += """
+                <div style="text-align:center"><a href="https://www.shodan.io/host/{0}" style="display: inline-block;
+                    padding: 4px 10px;
+                    margin-bottom: 0px;
+                    font-size: 13px;
+                    line-height: 18px;
+                    color: #333;
+                    text-align: center;
+                    text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.75);
+                    vertical-align: middle;
+                    cursor: pointer;
+                    background-color: #F5F5F5;
+                    background-image: -moz-linear-gradient(center top , #FFF, #E6E6E6);
+                    background-repeat: repeat-x;
+                    border-width: 1px;
+                    border-style: solid;
+                    border-color: #CCC #CCC #B3B3B3;
+                    -moz-border-top-colors: none;
+                    -moz-border-right-colors: none;
+                    -moz-border-bottom-colors: none;
+                    -moz-border-left-colors: none;
+                    border-image: none;
+                    border-radius: 4px;
+                    box-shadow: 0px 1px 0px rgba(255, 255, 255, 0.2) inset, 0px 1px 2px rgba(0, 0, 0, 0.05);" target="_blank">View Details</a></div>
+                    <div>powered by <a href="https://www.shodan.io" target="_blank">Shodan</a></div>
+            """.format(ip)
+
+            placemark += ']]></description>'
+            placemark += '<Point><coordinates>{},{}</coordinates></Point>'.format(lon, lat)
+            placemark += '</Placemark>'
+
+            fout.write(placemark)
+        except Exception, e:
+            pass
+
+    # Get the basename for the input file
+    basename = input.replace('.json.gz', '').replace('.json', '')
+
+    # Add the new file extension based on the format
+    filename = '{}.{}'.format(basename, format)
+
+    # Open the output file
+    fout = open(filename, 'w')
+    
+    # Start a spinner
+    finished_event = threading.Event()
+    progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
+    progress_bar_thread.start()
+
+    fout.write(KML_HEADER)
+
+    hosts = {}
+    for banner in iterate_files([input]):
+        ip = banner.get('ip_str', banner.get('ipv6', None))
+        if not ip:
+            continue
+
+        if ip not in hosts:
+            hosts[ip] = banner
+            hosts[ip]['ports'] = []
+
+        hosts[ip]['ports'].append(banner['port'])
+
+    for ip, host in hosts.iteritems():
+        kml_writer(fout, host)
+
+    fout.write(KML_FOOTER)
+
+    finished_event.set()
+    progress_bar_thread.join()
+
+    click.echo(click.style('Successfully created new file: {}'.format(filename), fg='green'))
+
+
 @main.command()
 @click.argument('key', metavar='<api key>')
 def init(key):

From 7ac5326e4186f1c18c75dc3c790a3bc9601ff4b2 Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 16:27:30 +0000
Subject: [PATCH 019/263] fixed gross tabs in alert.py

---
 shodan/alert.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/shodan/alert.py b/shodan/alert.py
index 52a7a7c..7a89e90 100644
--- a/shodan/alert.py
+++ b/shodan/alert.py
@@ -1,10 +1,9 @@
-
 class Alert:
-	def __init__(self):
-		self.id = None
-		self.name = None
-		self.api_key = None
-		self.filters = None
-		self.credits = None
-		self.created = None
-		self.expires = None
+    def __init__(self):
+        self.id = None
+        self.name = None
+        self.api_key = None
+        self.filters = None
+        self.credits = None
+        self.created = None
+        self.expires = None

From 210c2b473a5f095f2c4a9abf761777611a769b02 Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 16:51:50 +0000
Subject: [PATCH 020/263] fixed imports

---
 shodan/api.py       | 10 +++-------
 shodan/client.py    | 18 +++++++++---------
 shodan/exception.py |  7 +++++++
 shodan/helpers.py   | 17 +++++++++--------
 shodan/stream.py    | 15 ++++++++-------
 shodan/threatnet.py |  4 +++-
 shodan/wps.py       |  6 ++----
 7 files changed, 41 insertions(+), 36 deletions(-)

diff --git a/shodan/api.py b/shodan/api.py
index 635f82b..99cc901 100644
--- a/shodan/api.py
+++ b/shodan/api.py
@@ -10,14 +10,10 @@
     from urllib.request     import urlopen
     from urllib.parse       import urlencode
 
-__all__ = ['WebAPI']
+from .exception import WebAPIError
 
-class WebAPIError(Exception):
-    def __init__(self, value):
-        self.value = value
-    
-    def __str__(self):
-        return self.value
+
+__all__ = ['WebAPI']
 
 
 class WebAPI:
diff --git a/shodan/client.py b/shodan/client.py
index 34e335b..050643a 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -13,9 +13,9 @@
 import simplejson
 import time
 
-import shodan.exception as exception
-import shodan.helpers as helpers
-import shodan.stream as stream
+from .exception import APIError
+from .helpers import *
+from .stream import Stream
 
 
 # Try to disable the SSL warnings in urllib3 since not everybody can install
@@ -109,7 +109,7 @@ def __init__(self, key):
         self.base_exploits_url = 'https://exploits.shodan.io'
         self.exploits = self.Exploits(self)
         self.tools = self.Tools(self)
-        self.stream = stream.Stream(key)
+        self.stream = Stream(key)
     
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.
@@ -138,25 +138,25 @@ def _request(self, function, params, service='shodan', method='get'):
             else:
                 data = requests.get(base_url + function, params=params)
         except:
-            raise exception.APIError('Unable to connect to Shodan')
+            raise APIError('Unable to connect to Shodan')
 
         # Check that the API key wasn't rejected
         if data.status_code == 401:
             try:
-                raise exception.APIError(data.json()['error'])
+                raise APIError(data.json()['error'])
             except:
                 pass
-            raise exception.APIError('Invalid API key')
+            raise APIError('Invalid API key')
         
         # Parse the text into JSON
         try:
             data = data.json()
         except:
-            raise exception.APIError('Unable to parse JSON response')
+            raise APIError('Unable to parse JSON response')
         
         # Raise an exception if an error occurred
         if type(data) == dict and data.get('error', None):
-            raise exception.APIError(data['error'])
+            raise APIError(data['error'])
         
         # Return the data
         return data
diff --git a/shodan/exception.py b/shodan/exception.py
index 9bc0d03..97e9e25 100644
--- a/shodan/exception.py
+++ b/shodan/exception.py
@@ -1,3 +1,10 @@
+class WebAPIError(Exception):
+    def __init__(self, value):
+        self.value = value
+    
+    def __str__(self):
+        return self.value
+
 
 class APIError(Exception):
     """This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 6c4e289..5331734 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -1,8 +1,9 @@
-
 import requests
-import shodan.exception
 import simplejson
 
+from .exception import APIError
+
+
 def create_facet_string(facets):
     """Converts a Python list of facets into a comma-separated string that can be understood by
     the Shodan API.
@@ -50,25 +51,25 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
             tries += 1
 
     if error and tries >= retries:
-        raise shodan.exception.APIError('Unable to connect to Shodan')
+        raise APIError('Unable to connect to Shodan')
 
     # Check that the API key wasn't rejected
     if data.status_code == 401:
         try:
-            raise shodan.exception.APIError(data.json()['error'])
+            raise APIError(data.json()['error'])
         except:
             pass
-        raise shodan.exception.APIError('Invalid API key')
+        raise APIError('Invalid API key')
     
     # Parse the text into JSON
     try:
         data = data.json()
     except:
-        raise shodan.exception.APIError('Unable to parse JSON response')
+        raise APIError('Unable to parse JSON response')
     
     # Raise an exception if an error occurred
     if type(data) == dict and data.get('error', None):
-        raise shodan.exception.APIError(data['error'])
+        raise APIError(data['error'])
     
     # Return the data
-    return data
\ No newline at end of file
+    return data
diff --git a/shodan/stream.py b/shodan/stream.py
index fe29a7f..56a303a 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,8 +1,9 @@
+import socket
+
 import requests
 import simplejson
 
-import socket
-import shodan.exception as exception
+from .exception import APIError
 
 class Stream:
 
@@ -15,17 +16,17 @@ def _create_stream(self, name, timeout=None):
         try:
             req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
         except Exception, e:
-            raise exception.APIError('Unable to contact the Shodan Streaming API')
+            raise APIError('Unable to contact the Shodan Streaming API')
 
         if req.status_code != 200:
             try:
                 data = simplejson.loads(req.text)
-                raise exception.APIError(data['error'])
-            except exception.APIError, e:
+                raise APIError(data['error'])
+            except APIError, e:
                 raise
             except Exception, e:
                 pass
-            raise exception.APIError('Invalid API key or you do not have access to the Streaming API')
+            raise APIError('Invalid API key or you do not have access to the Streaming API')
         return req
 
     def alert(self, aid=None, timeout=None, raw=False):
@@ -43,7 +44,7 @@ def alert(self, aid=None, timeout=None, raw=False):
                         banner = simplejson.loads(line)
                         yield banner
         except requests.exceptions.ConnectionError, e:
-            raise exception.APIError('Stream timed out')
+            raise APIError('Stream timed out')
 
     def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
diff --git a/shodan/threatnet.py b/shodan/threatnet.py
index eca547f..a252da9 100644
--- a/shodan/threatnet.py
+++ b/shodan/threatnet.py
@@ -1,6 +1,8 @@
 import requests
 import simplejson
-from .client import APIError
+
+from .exception import APIError
+
 
 class Threatnet:
     """Wrapper around the Threatnet REST and Streaming APIs
diff --git a/shodan/wps.py b/shodan/wps.py
index af9249e..30dd215 100644
--- a/shodan/wps.py
+++ b/shodan/wps.py
@@ -4,10 +4,7 @@
 Wrappers around the SkyHook and Google Locations APIs to resolve
 wireless routers' MAC addresses (BSSID) to physical locations.
 """
-try:
-    from json       import dumps, loads
-except:
-    from simplejson import dumps, loads
+from simplejson import dumps, loads
 
 try:
     from urllib2    import Request, urlopen
@@ -16,6 +13,7 @@
     from urllib.request     import Request, urlopen
     from urllib.parse       import urlencode
 
+
 class Skyhook:
     
     """Not yet ready for production, use the GoogleLocation class instead."""

From 955ab2e21f688e69328ac6f6130aa53d2e96cd1f Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 16:55:36 +0000
Subject: [PATCH 021/263] now completed fix for imports. woops, forgot to save

---
 shodan/api.py    | 6 +++---
 shodan/client.py | 2 +-
 shodan/stream.py | 1 +
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/shodan/api.py b/shodan/api.py
index 99cc901..635127a 100644
--- a/shodan/api.py
+++ b/shodan/api.py
@@ -1,6 +1,3 @@
-# The simplejson library has better JSON-parsing than the standard library and is more often updated
-from simplejson import dumps, loads
-
 try:
     # Python 2
     from urllib2    import urlopen
@@ -10,6 +7,9 @@
     from urllib.request     import urlopen
     from urllib.parse       import urlencode
 
+# The simplejson library has better JSON-parsing than the standard library and is more often updated
+from simplejson import dumps, loads
+
 from .exception import WebAPIError
 
 
diff --git a/shodan/client.py b/shodan/client.py
index 050643a..e041184 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -8,10 +8,10 @@
 
 :copyright: (c) 2014-2015 by John Matherly
 """
+import time
 
 import requests
 import simplejson
-import time
 
 from .exception import APIError
 from .helpers import *
diff --git a/shodan/stream.py b/shodan/stream.py
index 56a303a..d9e1312 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -5,6 +5,7 @@
 
 from .exception import APIError
 
+
 class Stream:
 
     base_url = 'https://stream.shodan.io'

From 900cfefee6541383aaf34e057180fad524f1c0d9 Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 17:00:18 +0000
Subject: [PATCH 022/263] fixed try/except to be py 2 and 3 compliant

---
 shodan/stream.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index d9e1312..18286c3 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -16,16 +16,16 @@ def __init__(self, api_key):
     def _create_stream(self, name, timeout=None):
         try:
             req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
-        except Exception, e:
+        except Exception as e:
             raise APIError('Unable to contact the Shodan Streaming API')
 
         if req.status_code != 200:
             try:
                 data = simplejson.loads(req.text)
                 raise APIError(data['error'])
-            except APIError, e:
+            except APIError as e:
                 raise
-            except Exception, e:
+            except Exception as e:
                 pass
             raise APIError('Invalid API key or you do not have access to the Streaming API')
         return req
@@ -44,7 +44,7 @@ def alert(self, aid=None, timeout=None, raw=False):
                     else:
                         banner = simplejson.loads(line)
                         yield banner
-        except requests.exceptions.ConnectionError, e:
+        except requests.exceptions.ConnectionError as e:
             raise APIError('Stream timed out')
 
     def banners(self, raw=False, timeout=None):

From ac283e2fa78b3628afec054d4bdc144fc9d56ffc Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 17:50:49 +0000
Subject: [PATCH 023/263] minor fixes to api.py

---
 shodan/api.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/shodan/api.py b/shodan/api.py
index 635127a..3bafce4 100644
--- a/shodan/api.py
+++ b/shodan/api.py
@@ -40,7 +40,7 @@ def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None):
             
             """
             if sources:
-                query += ' source:' + ','.join(sources)
+                query += ' source:%s' % (','.join(sources))
             if cve:
                 query += ' cve:%s' % (str(cve).strip())
             if osvdb:
@@ -76,6 +76,7 @@ def search(self, query, **kwargs):
             A dictionary with 2 main items: matches (list) and total (int).
             """
             return self.parent.search(query, sources=['exploitdb'])
+
     
     class Msf:
         
@@ -143,7 +144,7 @@ def _request(self, function, params, service='shodan'):
         except:
             raise WebAPIError('Unable to connect to Shodan')
         
-        # Parse the text into JSON
+        # Parse the text from JSON to a dict
         data = loads(data)
         
         # Raise an exception if an error occurred
@@ -216,7 +217,7 @@ def search(self, query, page=1, limit=None, offset=None):
         }
         if limit:
             args['l'] = limit
-            if offset:
-                args['o'] = offset
+        if offset:
+            args['o'] = offset
         
         return self._request('search', args)

From 28269fa598e11cb94956287e0062ced0cb98515e Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 17:58:44 +0000
Subject: [PATCH 024/263] minor fixes to client.py

---
 shodan/client.py | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index e041184..5a6785e 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 """
 shodan.client
 ~~~~~~~~~~~~~
@@ -14,7 +13,7 @@
 import simplejson
 
 from .exception import APIError
-from .helpers import *
+from .helpers import api_request, create_facet_string
 from .stream import Stream
 
 
@@ -73,8 +72,7 @@ def search(self, query, page=1, facets=None):
                 'page': page,
             }
             if facets:
-                facet_str = helpers.create_facet_string(facets)
-                query_args['facets'] = facet_str
+                query_args['facets'] = create_facet_string(facets)
 
             return self.parent._request('/api/search', query_args, service='exploits')
             
@@ -93,8 +91,7 @@ def count(self, query, facets=None):
                 'query': query,
             }
             if facets:
-                facet_str = helpers.create_facet_string(facets)
-                query_args['facets'] = facet_str
+                query_args['facets'] = create_facet_string(facets)
 
             return self.parent._request('/api/count', query_args, service='exploits')
     
@@ -175,8 +172,7 @@ def count(self, query, facets=None):
             'query': query,
         }
         if facets:
-            facet_str = helpers.create_facet_string(facets)
-            query_args['facets'] = facet_str
+            query_args['facets'] = create_facet_string(facets)
         return self._request('/shodan/host/count', query_args)
     
     def host(self, ip, history=False):
@@ -286,8 +282,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
             args['page'] = page
 
         if facets:
-            facet_str = helpers.create_facet_string(facets)
-            args['facets'] = facet_str
+            args['facets'] = create_facet_string(facets)
         
         return self._request('/shodan/host/search', args)
     
@@ -417,7 +412,7 @@ def create_alert(self, name, ip, expires=0):
             'expires': expires,
         }
 
-        response = helpers.api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post')
+        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post')
 
         return response
 
@@ -428,7 +423,7 @@ def alerts(self, aid=None, include_expired=True):
         else:
             func = '/shodan/alert/info'
 
-        response = helpers.api_request(self.api_key, func, params={
+        response = api_request(self.api_key, func, params={
             'include_expired': include_expired,
             })
 
@@ -438,7 +433,7 @@ def delete_alert(self, aid):
         """Delete the alert with the given ID."""
         func = '/shodan/alert/%s' % aid
 
-        response = helpers.api_request(self.api_key, func, params={}, method='delete')
+        response = api_request(self.api_key, func, params={}, method='delete')
 
         return response
 

From def350181a8db63d743eb0948365c14b71d75626 Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 17:59:07 +0000
Subject: [PATCH 025/263] super minor fix to exceptions.py

---
 shodan/exception.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shodan/exception.py b/shodan/exception.py
index 97e9e25..11d89d3 100644
--- a/shodan/exception.py
+++ b/shodan/exception.py
@@ -14,6 +14,7 @@ def __init__(self, value):
     def __str__(self):
         return self.value
 
+
 class APITimeout(APIError):
 	pass
 

From fd3f0c7baac3479c08255a915ab80890b6d972bb Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 18:12:47 +0000
Subject: [PATCH 026/263] fixes to stream.py. abstracted common functionality
 to hidden method

---
 shodan/stream.py | 43 +++++++++++++++----------------------------
 1 file changed, 15 insertions(+), 28 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 18286c3..0801bbb 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,5 +1,3 @@
-import socket
-
 import requests
 import simplejson
 
@@ -21,7 +19,7 @@ def _create_stream(self, name, timeout=None):
 
         if req.status_code != 200:
             try:
-                data = simplejson.loads(req.text)
+                data = req.json()
                 raise APIError(data['error'])
             except APIError as e:
                 raise
@@ -30,35 +28,29 @@ def _create_stream(self, name, timeout=None):
             raise APIError('Invalid API key or you do not have access to the Streaming API')
         return req
 
+    def _iter_stream(self, stream, raw):
+        for line in stream.iter_lines():
+            if line:
+                if raw:
+                    yield line
+                else:
+                    yield simplejson.loads(line)
+
     def alert(self, aid=None, timeout=None, raw=False):
         if aid:
             stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
         else:
             stream = self._create_stream('/shodan/alert', timeout=timeout)
-
-        try:
-            for line in stream.iter_lines():
-                if line:
-                    if raw:
-                        yield line
-                    else:
-                        banner = simplejson.loads(line)
-                        yield banner
-        except requests.exceptions.ConnectionError as e:
-            raise APIError('Stream timed out')
+        for line in self._iter_stream(stream, raw):
+            yield line
 
     def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
         API subscription plans and for those it only returns a fraction of the data.
         """
         stream = self._create_stream('/shodan/banners', timeout=timeout)
-        for line in stream.iter_lines():
-            if line:
-                if raw:
-                    yield line
-                else:
-                    banner = simplejson.loads(line)
-                    yield banner
+        for line in self._iter_stream(stream, raw):
+            yield line
 
     def ports(self, ports, raw=False, timeout=None):
         """
@@ -68,10 +60,5 @@ def ports(self, ports, raw=False, timeout=None):
         :type ports: int[]
         """
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
-        for line in stream.iter_lines():
-            if line:
-                if raw:
-                    yield line
-                else:
-                    banner = simplejson.loads(line)
-                    yield banner
+        for line in self._iter_stream(stream, raw):
+            yield line

From 5cea814d097aeca1b4a0a23304a4ebe11a716503 Mon Sep 17 00:00:00 2001
From: Sean Beck <seanmckaybeck@gmail.com>
Date: Tue, 29 Sep 2015 18:29:27 +0000
Subject: [PATCH 027/263] fixed basestring. all tests passing python3

Fixes #6
---
 shodan/helpers.py    |  5 +++++
 tests/test_shodan.py | 18 ++++++++++++++----
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 5331734..3f8dc10 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -3,6 +3,11 @@
 
 from .exception import APIError
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 
 def create_facet_string(facets):
     """Converts a Python list of facets into a comma-separated string that can be understood by
diff --git a/tests/test_shodan.py b/tests/test_shodan.py
index e1b39c8..396a059 100644
--- a/tests/test_shodan.py
+++ b/tests/test_shodan.py
@@ -1,6 +1,12 @@
 import unittest
 import shodan
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
+
 class ShodanTests(unittest.TestCase):
 
 	api = None
@@ -115,7 +121,7 @@ def test_invalid_key(self):
 		raised = False
 		try:
 			api.search('something')
-		except shodan.APIError, e:
+		except shodan.APIError as e:
 			raised = True
 
 		self.assertTrue(raised)
@@ -124,7 +130,7 @@ def test_invalid_host_ip(self):
 		raised = False
 		try:
 			host = self.api.host('test')
-		except shodan.APIError, e:
+		except shodan.APIError as e:
 			raised = True
 
 		self.assertTrue(raised)
@@ -133,7 +139,7 @@ def test_search_empty_query(self):
 		raised = False
 		try:
 			self.api.search('')
-		except shodan.APIError, e:
+		except shodan.APIError as e:
 			raised = True
 		self.assertTrue(raised)
 
@@ -142,6 +148,10 @@ def test_search_advanced_query(self):
 		raised = False
 		try:
 			self.api.search(self.QUERIES['advanced'])
-		except shodan.APIError, e:
+		except shodan.APIError as e:
 			raised = True
 		self.assertTrue(raised)
+
+
+if __name__ == '__main__':
+    unittest.main()

From 2c61087d287d7eae3b27ca24335f3a0cbe891f89 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 25 Oct 2015 04:16:01 -0500
Subject: [PATCH 028/263] Added connection timeout to streams Fixed bug in
 "stats" command where numeric facets would error out

---
 bin/shodan | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 79ae1cb..e454d04 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -105,7 +105,7 @@ def get_banner_field(banner, flat_field):
 
 def match_filters(banner, filters):
     for args in filters:
-        flat_field, check = args.split(':')
+        flat_field, check = args.split(':', 1)
         value = get_banner_field(banner, flat_field)
 
         # It must match all filters to be allowed
@@ -1009,7 +1009,7 @@ def stats(limit, facets, query):
         click.echo('Top {} Results for Facet: {}'.format(limit, facet))
 
         for item in results['facets'][facet]:
-            click.echo(click.style('{:28s}'.format(item['value'].encode('ascii', errors='replace')), fg='cyan'), nl=False)
+            click.echo(click.style('{:28s}'.format(str(item['value']).encode('ascii', errors='replace')), fg='cyan'), nl=False)
             click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
@@ -1048,9 +1048,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
 
     # Decide which stream to subscribe to based on whether or not ports were selected
     if ports:
-        stream = api.stream.ports(ports)
+        stream = api.stream.ports(ports, timeout=5)
     else:
-        stream = api.stream.banners()
+        stream = api.stream.banners(timeout=5)
 
     counter = 0
     quit = False

From 728b464ba0f34e78450c3b292e2de0c83f056ee7 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 25 Oct 2015 19:03:51 -0500
Subject: [PATCH 029/263] Improved shodan CLI help documentation

---
 bin/shodan | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index e454d04..c2dcf9a 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -470,7 +470,7 @@ def download(limit, filename, query):
 @click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str)
 @click.argument('ip', metavar='<ip address>')
 def host(format, ip):
-    """Scan an IP/ netblock using Shodan."""
+    """View all available information for an IP address"""
     key = get_api_key()
     api = shodan.Shodan(key)
 
@@ -638,6 +638,7 @@ def myip():
 
 @main.group()
 def scan():
+    """Scan an IP/ netblock using Shodan."""
     pass
 
 
@@ -983,6 +984,7 @@ def search(color, fields, limit, separator, query):
 @click.option('--facets', help='List of facets to get statistics for.', default='country,org')
 @click.argument('query', metavar='<search query>', nargs=-1)
 def stats(limit, facets, query):
+    """Provide summary information about a search query"""
     # Setup Shodan
     key = get_api_key()
     api = shodan.Shodan(key)

From 933a0f5532ea5e29548a865b70cb4f8599b989eb Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Sun, 25 Oct 2015 20:01:39 -0500
Subject: [PATCH 030/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3e3237e..b3889f5 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.7',
+    version = '1.3.8',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From d5332e0eeb52a228de9c4c96ece519e80a121283 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Wed, 28 Oct 2015 22:33:39 -0500
Subject: [PATCH 031/263] Bugfixes Improved documentation Improved IPv6 support

---
 bin/shodan | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index c2dcf9a..21df4d3 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -66,6 +66,11 @@ def get_api_key():
 
     raise click.ClickException('Please run "shodan init <api key>" before using this command')
 
+def get_ip(banner):
+    if 'ipv6' in banner:
+        return banner['ipv6']
+    return banner['ip_str']
+
 
 def escape_data(args):
     return args.encode('ascii', 'replace').replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
@@ -683,7 +688,7 @@ def scan_internet(quiet, port, protocol):
 
                             if not quiet:
                                 click.echo('{0:<40} {1:<20} {2}'.format(
-                                        click.style(banner['ip_str'], fg=COLORIZE_FIELDS['ip_str']),
+                                        click.style(get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
                                         click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
                                         ';'.join(banner['hostnames'])
                                     )
@@ -783,7 +788,7 @@ def scan_submit(wait, filename, netblocks):
                         # Don't show duplicate banners
                         cache_key = '{}:{}'.format(ip, banner['port'])
                         if cache_key not in cache:
-                            hosts[banner['ip_str']][banner['port']] = banner
+                            hosts[get_ip(banner)][banner['port']] = banner
                             cache[cache_key] = True
 
                         # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
@@ -817,6 +822,8 @@ def scan_submit(wait, filename, netblocks):
 
                     done = True
                 except Exception, e:
+                    finished_event.set()
+                    progress_bar_thread.join()
                     raise click.ClickException(repr(e))
 
             finished_event.set()

From c5dd3957af37df88f3a0bf7021505fc16b78d3e6 Mon Sep 17 00:00:00 2001
From: Aaron Kaplan <aaron@lo-res.org>
Date: Wed, 11 Nov 2015 13:21:27 +0100
Subject: [PATCH 032/263] make timeout configurable; terminate on timeout so
 that a while ; do shodan --stream --datadir ...; done script can resume on
 timeouts

---
 bin/shodan | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 21df4d3..6d904e3 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1032,8 +1032,9 @@ def stats(limit, facets, query):
 @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str)
 @click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
 @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
+@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=5, type=int)
 @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -1057,9 +1058,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
 
     # Decide which stream to subscribe to based on whether or not ports were selected
     if ports:
-        stream = api.stream.ports(ports, timeout=5)
+        stream = api.stream.ports(ports, timeout=timeout)
     else:
-        stream = api.stream.banners(timeout=5)
+        stream = api.stream.banners(timeout=timeout)
 
     counter = 0
     quit = False
@@ -1116,10 +1117,14 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
                         row += separator
 
                     click.echo(row)
+        except requests.exceptions.Timeout:
+            raise click.ClickException('Connection timed out')
+            # let some exteran while $TRUE script handle the reconnects XXX FIXME : should be improved
         except KeyboardInterrupt:
             quit = True
         except:
             # For other errors lets just wait a few seconds and try to reconnect again
+            # XXX FIXME: this does not seem to reconnect since it's in the while not quit loop!
             time.sleep(2)
 
 

From bf1df5a14ef3c9a06e2849e0d2ec6cb67d9a67c6 Mon Sep 17 00:00:00 2001
From: Aaron Kaplan <aaron@lo-res.org>
Date: Wed, 11 Nov 2015 13:21:27 +0100
Subject: [PATCH 033/263] forgot to import requests

---
 bin/shodan | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 21df4d3..6d904e3 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1032,8 +1032,9 @@ def stats(limit, facets, query):
 @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str)
 @click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
 @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
+@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=5, type=int)
 @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -1057,9 +1058,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
 
     # Decide which stream to subscribe to based on whether or not ports were selected
     if ports:
-        stream = api.stream.ports(ports, timeout=5)
+        stream = api.stream.ports(ports, timeout=timeout)
     else:
-        stream = api.stream.banners(timeout=5)
+        stream = api.stream.banners(timeout=timeout)
 
     counter = 0
     quit = False
@@ -1116,10 +1117,14 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, streamer):
                         row += separator
 
                     click.echo(row)
+        except requests.exceptions.Timeout:
+            raise click.ClickException('Connection timed out')
+            # let some exteran while $TRUE script handle the reconnects XXX FIXME : should be improved
         except KeyboardInterrupt:
             quit = True
         except:
             # For other errors lets just wait a few seconds and try to reconnect again
+            # XXX FIXME: this does not seem to reconnect since it's in the while not quit loop!
             time.sleep(2)
 
 

From 06c7549648af8502173a6c59d8d2f1c74ba938a4 Mon Sep 17 00:00:00 2001
From: Aaron Kaplan <aaron@lo-res.org>
Date: Wed, 11 Nov 2015 13:29:25 +0100
Subject: [PATCH 034/263] forgot to import requests

---
 bin/shodan | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/shodan b/bin/shodan
index 6d904e3..25a84c4 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -31,6 +31,7 @@ import simplejson
 import socket
 import sys
 import threading
+import requests
 import time
 
 # Constants

From a8c218343e48e7cc35d6ac3356c15714b720f09f Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Thu, 19 Nov 2015 19:31:32 -0600
Subject: [PATCH 035/263] Use simplejson for parsing JSON due to unicode
 parsing issues in older versions of Python

---
 shodan/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 0801bbb..0fcea9b 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -19,7 +19,7 @@ def _create_stream(self, name, timeout=None):
 
         if req.status_code != 200:
             try:
-                data = req.json()
+                data = simplejson.loads(req.text)
                 raise APIError(data['error'])
             except APIError as e:
                 raise

From 34bf269d21e2207435387fe11ed0b898b718f287 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 20 Nov 2015 14:29:48 -0600
Subject: [PATCH 036/263] Bumped version after merging pull requests

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b3889f5..14ba87f 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.8',
+    version = '1.3.9',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From cd369cd22014dd05b0d9ad5d5afd19c25b9355e8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 4 Dec 2015 11:57:37 -0600
Subject: [PATCH 037/263] Raise an API Error message when an alert stream times
 out

---
 shodan/stream.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 0fcea9b..efb9024 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -41,8 +41,12 @@ def alert(self, aid=None, timeout=None, raw=False):
             stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
         else:
             stream = self._create_stream('/shodan/alert', timeout=timeout)
-        for line in self._iter_stream(stream, raw):
-            yield line
+
+        try:
+            for line in self._iter_stream(stream, raw):
+                yield line
+        except requests.exceptions.ConnectionError as e:
+            raise APIError('Stream timed out')
 
     def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to

From aab96480f802d3b0db355769796598942c200849 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 4 Dec 2015 16:30:48 -0600
Subject: [PATCH 038/263] Improved Python3 support

---
 bin/shodan       | 80 ++++++++++++++++++++++++++++--------------------
 shodan/client.py |  6 ++++
 2 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 25a84c4..2e3c9db 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -49,6 +49,12 @@ COLORIZE_FIELDS = {
 # Make "-h" work like "--help"
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
+# Define a basestring type if necessary for Python3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
 # Utility methods
 def get_api_key():
     shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
@@ -60,7 +66,7 @@ def get_api_key():
         raise click.ClickException('Please run "shodan init <api key>" before using this command')
 
     # Make sure it is a read-only file
-    os.chmod(keyfile, 0600)
+    os.chmod(keyfile, 0o600)
 
     with open(keyfile, 'r') as fin:
         return fin.read().strip()
@@ -74,7 +80,10 @@ def get_ip(banner):
 
 
 def escape_data(args):
-    return args.encode('ascii', 'replace').replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+    # Ensure the provided string isn't unicode data
+    if not isinstance(args, str):
+        args = args.encode('ascii', 'replace')
+    return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
 
 def timestr():
         return datetime.datetime.utcnow().strftime('%Y-%m-%d')
@@ -133,6 +142,10 @@ def match_filters(banner, filters):
 
     return True
 
+def write_banner(fout, banner):
+    line = simplejson.dumps(banner) + '\n'
+    fout.write(line.encode('utf-8'))
+
 
 @click.group(context_settings=CONTEXT_SETTINGS)
 def main():
@@ -242,8 +255,8 @@ def convert(input, format):
             placemark += '<Point><coordinates>{},{}</coordinates></Point>'.format(lon, lat)
             placemark += '</Placemark>'
 
-            fout.write(placemark)
-        except Exception, e:
+            fout.write(placemark.encode('utf-8'))
+        except Exception as e:
             pass
 
     # Get the basename for the input file
@@ -274,7 +287,7 @@ def convert(input, format):
 
         hosts[ip]['ports'].append(banner['port'])
 
-    for ip, host in hosts.iteritems():
+    for ip, host in iter(hosts.items()):
         kml_writer(fout, host)
 
     fout.write(KML_FOOTER)
@@ -302,7 +315,7 @@ def init(key):
     try:
         api = shodan.Shodan(key)
         test = api.info()
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException('Invalid API key')
 
     # Store the API key in the user's directory
@@ -311,7 +324,7 @@ def init(key):
         fout.write(key.strip())
         click.echo(click.style('Successfully initialized', fg='green'))
 
-    os.chmod(keyfile, 0600)
+    os.chmod(keyfile, 0o600)
 
 
 @main.group()
@@ -331,7 +344,7 @@ def alert_clear():
         for alert in alerts:
             click.echo('Removing {} ({})'.format(alert['name'], alert['id']))
             api.delete_alert(alert['id'])
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
     click.echo("Alerts deleted")
 
@@ -345,7 +358,7 @@ def alert_list(expired):
     api = shodan.Shodan(key)
     try:
         results = api.alerts(include_expired=expired)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
     if len(results) > 0:
@@ -379,7 +392,7 @@ def alert_remove(alert_id):
     api = shodan.Shodan(key)
     try:
         results = api.delete_alert(alert_id)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
     click.echo("Alert deleted")
 
@@ -401,7 +414,7 @@ def count(query):
     api = shodan.Shodan(key)
     try:
         results = api.count(query)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
     click.echo(results['total'])
@@ -458,7 +471,7 @@ def download(limit, filename, query):
             cursor = api.search_cursor(query)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
-                    fout.write(simplejson.dumps(banner) + '\n')
+                    write_banner(fout, banner)
                     count += 1
 
                     if count >= limit:
@@ -544,7 +557,7 @@ def host(format, ip):
                     click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
                     if 'fingerprint' in banner['ssl']['dhparams']:
                         click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
 
@@ -555,7 +568,7 @@ def info():
     api = shodan.Shodan(key)
     try:
         results = api.info()
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
     click.echo("""Query credits available: {0}
@@ -602,7 +615,7 @@ def parse(color, fields, filters, filename, separator, filenames):
 
         # Append the data
         if fout:
-            fout.write(simplejson.dumps(banner) + '\n')
+            write_banner(fout, banner)
 
         # Loop over all the fields and print the banner as a row
         for field in fields:
@@ -638,7 +651,7 @@ def myip():
     api = shodan.Shodan(key)
     try:
         click.echo(api.tools.myip())
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
 
@@ -685,7 +698,7 @@ def scan_internet(quiet, port, protocol):
                     try:
                         for banner in api.stream.ports([port], timeout=30):
                             counter += 1
-                            fout.write(simplejson.dumps(banner) + '\n')
+                            write_banner(fout, banner)
 
                             if not quiet:
                                 click.echo('{0:<40} {1:<20} {2}'.format(
@@ -694,7 +707,7 @@ def scan_internet(quiet, port, protocol):
                                         ';'.join(banner['hostnames'])
                                     )
                                 )
-                    except shodan.APIError, e:
+                    except shodan.APIError as e:
                         # We stop waiting for results if the scan has been processed by the crawlers and
                         # there haven't been new results in a while
                         if done:
@@ -703,7 +716,7 @@ def scan_internet(quiet, port, protocol):
                         scan = api.scan_status(scan['id'])
                         if scan['status'] == 'DONE':
                             done = True
-                    except socket.timeout, e:
+                    except socket.timeout as e:
                         # We stop waiting for results if the scan has been processed by the crawlers and
                         # there haven't been new results in a while
                         if done:
@@ -712,10 +725,10 @@ def scan_internet(quiet, port, protocol):
                         scan = api.scan_status(scan['id'])
                         if scan['status'] == 'DONE':
                             done = True
-                    except Exception, e:
+                    except Exception as e:
                         raise click.ClickException(repr(e))
             click.echo('Scan finished: {0} devices found'.format(counter))
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
 
@@ -727,9 +740,9 @@ def scan_protocols():
     try:
         protocols = api.protocols()
 
-        for name, description in protocols.iteritems():
+        for name, description in iter(protocols.items()):
             click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
 
@@ -799,7 +812,7 @@ def scan_submit(wait, filename, netblocks):
                                 done = True
                                 break
 
-                except shodan.APIError, e:
+                except shodan.APIError as e:
                     # If the connection timed out before the timeout, that means the streaming server
                     # that the user tried to reach is down. In that case, lets wait briefly and try
                     # to connect again!
@@ -814,7 +827,7 @@ def scan_submit(wait, filename, netblocks):
                     scan = api.scan_status(scan['id'])
                     if scan['status'] == 'DONE':
                         done = True
-                except socket.timeout, e:
+                except socket.timeout as e:
                     # If the connection timed out before the timeout, that means the streaming server
                     # that the user tried to reach is down. In that case, lets wait a second and try
                     # to connect again!
@@ -822,7 +835,8 @@ def scan_submit(wait, filename, netblocks):
                         continue
 
                     done = True
-                except Exception, e:
+                except Exception as e:
+                    raise
                     finished_event.set()
                     progress_bar_thread.join()
                     raise click.ClickException(repr(e))
@@ -862,7 +876,7 @@ def scan_submit(wait, filename, netblocks):
                 click.echo('\b ')
 
                 for ip in sorted(hosts):
-                    host = hosts[ip].items()[0][1]
+                    host = next(iter(hosts[ip].items()))[1]
 
                     click.echo(click.style(ip, fg='cyan'), nl=False)
                     if 'hostnames' in host and host['hostnames']:
@@ -906,13 +920,13 @@ def scan_submit(wait, filename, netblocks):
 
                         # Save the banner in a file if necessary
                         if fout:
-                            fout.write(simplejson.dumps(hosts[ip][port]) + '\n')
+                            write_banner(fout, hosts[ip][port])
 
                     click.echo('')
             else:
                 # Prepend a \b to remove the spinner
                 click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
     finally:
         # Remove any alert
@@ -951,7 +965,7 @@ def search(color, fields, limit, separator, query):
     api = shodan.Shodan(key)
     try:
         results = api.search(query, limit=limit)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
     # We buffer the entire output so we can use click's pager functionality
@@ -1011,7 +1025,7 @@ def stats(limit, facets, query):
     api = shodan.Shodan(key)
     try:
         results = api.count(query, facets=facets)
-    except shodan.APIError, e:
+    except shodan.APIError as e:
         raise click.ClickException(e.value)
 
     # Print the stats tables
@@ -1089,7 +1103,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
                             last_time = cur_time
                             fout.close()
                             fout = open_file(datadir, last_time)
-                    fout.write(simplejson.dumps(banner) + '\n')
+                    write_banner(fout, banner)
 
                 # Print the banner information to stdout
                 if not quiet:
@@ -1132,7 +1146,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
 def async_spinner(finished):
     spinner = itertools.cycle(['-', '/', '|', '\\'])
     while not finished.is_set():
-        sys.stdout.write('\b{}'.format(spinner.next()))
+        sys.stdout.write('\b{}'.format(next(spinner)))
         sys.stdout.flush()
         finished.wait(0.2)
 
diff --git a/shodan/client.py b/shodan/client.py
index 5a6785e..c44eb38 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -28,6 +28,12 @@
 except:
     pass
 
+# Define a basestring type if necessary for Python3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
 
 class Shodan:
     """Wrapper around the Shodan REST and Streaming APIs

From 8eae5b41f05f5375f36987baae5d8795b3b880f7 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 4 Dec 2015 16:49:06 -0600
Subject: [PATCH 039/263] Bumped version up a minor due to significant Python3
 improvements

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 14ba87f..87f1773 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.3.9',
+    version = '1.4.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 26b01d2ae03d8b0484e4ad69c00aa90e6897b912 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 4 Dec 2015 17:08:42 -0600
Subject: [PATCH 040/263] Treat a limit of 0 the same as -1 when downloading
 data (#23)

---
 bin/shodan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 2e3c9db..92fa910 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -462,7 +462,7 @@ def download(limit, filename, query):
         limit = total
 
     # A limit of -1 means that we should download all the data
-    if limit == -1:
+    if limit <= 0:
         limit = total
 
     with gzip.open(filename, 'w') as fout:

From 1d8aad75c4be33c54f84b98fd60518bdca1e785e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 4 Dec 2015 17:12:11 -0600
Subject: [PATCH 041/263] Define a minimum version for the requests library
 (#25) Bumped version

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 87f1773..a7da6ae 100755
--- a/setup.py
+++ b/setup.py
@@ -4,14 +4,14 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.0',
+    version = '1.4.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan'],
     scripts = ['bin/shodan'],
-    install_requires=["simplejson", "requests", "click", "colorama"],
+    install_requires=["simplejson", "requests>=2.2.1", "click", "colorama"],
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',

From aeb1ae7b9d7bf6d9c8dbbed1e72872779fb05f3d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 6 Dec 2015 06:27:43 -0600
Subject: [PATCH 042/263] Improved exception handling and added advanced scan
 mode

---
 .gitignore       | 10 ++++++++++
 bin/shodan       |  1 -
 shodan/client.py | 20 +++++++++++++++++---
 shodan/stream.py |  3 +++
 4 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7c7a28f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,10 @@
+build/*
+*.tar.gz
+*.json.gz
+*.kml
+*.egg
+*.pyc
+shodan.egg-info/*
+tmp/*
+shodan/cli/*
+MANIFEST
\ No newline at end of file
diff --git a/bin/shodan b/bin/shodan
index 92fa910..6e03df1 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -836,7 +836,6 @@ def scan_submit(wait, filename, netblocks):
 
                     done = True
                 except Exception as e:
-                    raise
                     finished_event.set()
                     progress_bar_thread.join()
                     raise click.ClickException(repr(e))
diff --git a/shodan/client.py b/shodan/client.py
index c44eb38..880fe52 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -217,16 +217,30 @@ def protocols(self):
     def scan(self, ips):
         """Scan a network using Shodan
 
-        :param ips: A list of IPs or netblocks in CIDR notation
-        :type ips: str
+        :param ips: A list of IPs or netblocks in CIDR notation or an object structured like:
+                    {
+                        "9.9.9.9": [
+                            (443, "https"),
+                            (8080, "http")
+                        ],
+                        "1.1.1.0/24": [
+                            (503, "modbus")
+                        ]
+                    }
+        :type ips: str or dict
 
         :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left.
         """
         if isinstance(ips, basestring):
             ips = [ips]
+        
+        if isinstance(ips, dict):
+            networks = simplejson.dumps(ips)
+        else:
+            networks = ','.join(ips)
 
         params = {
-            'ips': ','.join(ips),
+            'ips': networks,
         }
 
         return self._request('/shodan/scan', params, method='post')
diff --git a/shodan/stream.py b/shodan/stream.py
index efb9024..a6a607f 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,5 +1,6 @@
 import requests
 import simplejson
+import ssl
 
 from .exception import APIError
 
@@ -47,6 +48,8 @@ def alert(self, aid=None, timeout=None, raw=False):
                 yield line
         except requests.exceptions.ConnectionError as e:
             raise APIError('Stream timed out')
+        except ssl.SSLError as e:
+            raise APIError('Stream timed out')
 
     def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to

From 01fe013ee3885045bd126114bad4c3fed3c028c1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 8 Jan 2016 18:34:14 -0600
Subject: [PATCH 043/263] host() method now supports looking up multiple IPs at
 once

---
 shodan/client.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 880fe52..cb1bba0 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -189,10 +189,13 @@ def host(self, ip, history=False):
         :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise.
         :type history: bool
         """
+        if isinstance(ips, basestring):
+            ips = [ips]
+        
         params = {}
         if history:
             params['history'] = history
-        return self._request('/shodan/host/%s' % ip, params)
+        return self._request('/shodan/host/%s' % ','.join(ips), params)
     
     def info(self):
         """Returns information about the current API key, such as a list of add-ons

From 780e6aaf30844b4cbfc084155253ac6ee77540e5 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 8 Jan 2016 18:36:22 -0600
Subject: [PATCH 044/263] Bumped version and fixed typo

---
 setup.py         | 2 +-
 shodan/client.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index a7da6ae..b8da7d2 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.1',
+    version = '1.4.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/client.py b/shodan/client.py
index cb1bba0..11e3594 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -181,7 +181,7 @@ def count(self, query, facets=None):
             query_args['facets'] = create_facet_string(facets)
         return self._request('/shodan/host/count', query_args)
     
-    def host(self, ip, history=False):
+    def host(self, ips, history=False):
         """Get all available information on an IP.
 
         :param ip: IP of the computer

From 0d57a324d0832ce220e1806ef1b2b67ab2a7a8ab Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 8 Jan 2016 22:03:31 -0600
Subject: [PATCH 045/263] Added utility method "iterate_files()" to loop over
 json.gz files generated by Shodan

---
 shodan/helpers.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 3f8dc10..d876ff3 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -1,3 +1,4 @@
+import gzip
 import requests
 import simplejson
 
@@ -78,3 +79,21 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     
     # Return the data
     return data
+
+
+def iterate_files(files):
+    """Loop over all the records of the provided Shodan output file(s)."""
+    if isinstance(files, basestring):
+        files = [files]
+    
+    for filename in files:
+        # Create a file handle depending on the filetype
+        if filename.endswith('.gz'):
+            fin = gzip.open(filename, 'r')
+        else:
+            fin = open(filename, 'r')
+
+        for line in fin:
+            # Convert the JSON into a native Python object
+            banner = simplejson.loads(line)
+            yield banner
\ No newline at end of file

From d0b90f5b4d236a3a958b1d7fb5b485f5831a9610 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Jan 2016 13:46:03 -0600
Subject: [PATCH 046/263] Improved timeout handling for streams

---
 bin/shodan       | 11 ++++++++---
 shodan/stream.py |  4 ++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 6e03df1..71d7ccd 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1046,7 +1046,7 @@ def stats(limit, facets, query):
 @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str)
 @click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
 @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
-@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=5, type=int)
+@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=0, type=int)
 @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
 def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer):
     """Stream data in real-time."""
@@ -1137,9 +1137,14 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         except KeyboardInterrupt:
             quit = True
         except:
-            # For other errors lets just wait a few seconds and try to reconnect again
+            # For other errors lets just wait a bit and try to reconnect again
             # XXX FIXME: this does not seem to reconnect since it's in the while not quit loop!
-            time.sleep(2)
+            # Decide which stream to subscribe to based on whether or not ports were selected
+            if ports:
+                stream = api.stream.ports(ports, timeout=timeout)
+            else:
+                stream = api.stream.banners(timeout=timeout)
+            time.sleep(1)
 
 
 def async_spinner(finished):
diff --git a/shodan/stream.py b/shodan/stream.py
index a6a607f..80d0dc4 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -13,6 +13,10 @@ def __init__(self, api_key):
         self.api_key = api_key
 
     def _create_stream(self, name, timeout=None):
+        # The user doesn't want to use a timeout
+        if timeout <= 0:
+            timeout = None
+        
         try:
             req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
         except Exception as e:

From 732f36d04fa83f9b332f704d460a55457698acfe Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Jan 2016 20:33:29 -0600
Subject: [PATCH 047/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b8da7d2..7922655 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.2',
+    version = '1.4.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 9887909b50b39337538b95339329c04d011236f7 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Jan 2016 20:50:51 -0600
Subject: [PATCH 048/263] Updated documentation with best practices Update
 "shodan" CLI to use new helpers

---
 bin/shodan                     | 18 +++---------------
 docs/examples/basic-search.rst |  2 +-
 docs/examples/cert-stream.rst  |  9 +++++----
 3 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 71d7ccd..79549c6 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -27,6 +27,7 @@ import itertools
 import os
 import os.path
 import shodan
+import shodan.helpers as helpers
 import simplejson
 import socket
 import sys
@@ -91,19 +92,6 @@ def timestr():
 def open_file(directory, timestr):
         return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', 1)
 
-def iterate_files(files):
-    for filename in files:
-        # Create a file handle depending on the filetype
-        if filename.endswith('.gz'):
-            fin = gzip.open(filename, 'r')
-        else:
-            fin = open(filename, 'r')
-
-        for line in fin:
-            # Convert the JSON into a native Python object
-            banner = simplejson.loads(line)
-            yield banner
-
 def get_banner_field(banner, flat_field):
     # The provided field is a collapsed form of the actual field
     fields = flat_field.split('.')
@@ -276,7 +264,7 @@ def convert(input, format):
     fout.write(KML_HEADER)
 
     hosts = {}
-    for banner in iterate_files([input]):
+    for banner in helpers.iterate_files([input]):
         ip = banner.get('ip_str', banner.get('ipv6', None))
         if not ip:
             continue
@@ -606,7 +594,7 @@ def parse(color, fields, filters, filename, separator, filenames):
             filename += '.json.gz'
         fout = gzip.open(filename, 'a')
 
-    for banner in iterate_files(filenames):
+    for banner in helpers.iterate_files(filenames):
         row = ''
 
         # Validate the banner against any provided filters
diff --git a/docs/examples/basic-search.rst b/docs/examples/basic-search.rst
index 58605ad..e61e38d 100644
--- a/docs/examples/basic-search.rst
+++ b/docs/examples/basic-search.rst
@@ -32,6 +32,6 @@ Basic Shodan Search
 		# Loop through the matches and print each IP
 		for service in result['matches']:
 			print service['ip_str']
-	except Exception, e:
+	except Exception as e:
 		print 'Error: %s' % e
 		sys.exit(1)
\ No newline at end of file
diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst
index fce8a65..afa5e25 100644
--- a/docs/examples/cert-stream.rst
+++ b/docs/examples/cert-stream.rst
@@ -20,7 +20,7 @@ information.
 	#
 	# WARNING: This script only works with people that have a subscription API plan!
 	# And by default the Streaming API only returns 1% of the data that Shodan gathers.
-	# If you wish to have more access please contact us at support@shodan.io for pricing
+	# If you wish to have more access please contact us at sales@shodan.io for pricing
 	# information.
 	#
 	# Author: achillean
@@ -37,10 +37,11 @@ information.
 
 	    print 'Listening for certs...'
 	    for banner in api.stream.ports([443, 8443]):
-	        if 'opts' in banner and 'pem' in banner['opts']:
-	            print banner['opts']['pem']
+			if 'ssl' in banner:
+				# Print out all the SSL information that Shodan has collected
+				print banner['ssl']
 	    
-	except Exception, e:
+	except Exception as e:
 	    print 'Error: %s' % e
 	    sys.exit(1)
 

From 6cc2bce8fba7946da3270f528c084a0871e51a0b Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Jan 2016 21:32:19 -0600
Subject: [PATCH 049/263] Add gifcreator.py

---
 docs/examples/gifcreator.rst | 112 +++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 docs/examples/gifcreator.rst

diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst
new file mode 100644
index 0000000..ad9efc8
--- /dev/null
+++ b/docs/examples/gifcreator.rst
@@ -0,0 +1,112 @@
+GIF Creator
+-----------
+
+Shodan keeps a full history of all the information that has been gathered on an IP address. With the API,
+you're able to retrieve that history and we're going to use that to create a tool that outputs GIFs made of
+the screenshots that the Shodan crawlers gather.
+
+The below code requires the following Python packages:
+
+	- arrow
+	- shodan
+
+The **arrow** package is used to parse the *timestamp* field of the banner into a Python `datetime` object.
+
+In addition to the above Python packages, you also need to have the **ImageMagick** software installed. If you're
+working on Ubuntu or another distro using **apt** you can run the following command:
+
+.. code-block:: bash
+	
+	sudo apt-get install imagemagick
+
+This will provide us with the **convert** command which is needed to merge several images into an animated GIF.
+
+There are a few key Shodan methods/ parameters that make the script work:
+
+1. :py:func:`shodan.helpers.iterate_files()` to loop through the Shodan data file
+2. **history** flag on the :py:func:`shodan.Shodan.host` method to get all the banners for an IP that Shodan has collected over the years
+
+
+
+.. code-block:: python
+
+	#!/usr/bin/env python
+	# gifcreator.py
+	#
+	# Dependencies:
+	# - arrow
+	# - shodan
+	#
+	# Installation:
+	# sudo easy_install arrow shodan
+	# sudo apt-get install imagemagick
+	#
+	# Usage:
+	# 1. Download a json.gz file using the website or the Shodan command-line tool (https://cli.shodan.io).
+	#    For example:
+	#        shodan download screenshots.json.gz has_screenshot:true
+	# 2. Run the tool on the file:
+	#        python gifcreator.py screenshots.json.gz
+	
+	import arrow
+	import os
+	import shodan
+	import shodan.helpers as helpers
+	import sys
+	
+	
+	# Settings
+	API_KEY = ''
+	MIN_SCREENS = 5	# Number of screenshots that Shodan needs to have in order to make a GIF
+	MAX_SCREENS = 24
+	
+	if len(sys.argv) != 2:
+		print('Usage: {} <shodan-data.json.gz>'.format(sys.argv[0]))
+		sys.exit(1)
+	
+	# GIFs are stored in the local "data" directory
+	os.mkdir('data')
+	
+	# We need to connect to the API to lookup the historical host information
+	api = shodan.Shodan(API_KEY)
+	
+	# Use the shodan.helpers.iterate_files() method to loop over the Shodan data file
+	for result in helpers.iterate_files(sys.argv[1]):
+		# Get the historic info
+		host = api.host(result['ip_str'], history=True)
+		
+		# Count how many screenshots this host has
+		screenshots = []
+		for banner in host['data']:
+			# Extract the image from the banner data
+			if 'opts' in banner and 'screenshot' in banner['opts']:
+				# Sort the images by the time they were collected so the GIF will loop
+				# based on the local time regardless of which day the banner was taken.
+				timestamp = arrow.get(banner['timestamp']).time()
+				sort_key = timestamp.hour
+				screenshots.append((
+					sort_key,
+					banner['opts']['screenshot']['data']
+				))
+				
+				# Ignore any further screenshots if we already have MAX_SCREENS number of images
+				if len(screenshots) >= MAX_SCREENS:
+					break
+		
+		# Extract the screenshots and turn them into a GIF if we've got the necessary
+		# amount of images.
+		if len(screenshots) >= MIN_SCREENS:
+			for (i, screenshot) in enumerate(sorted(screenshots, key=lambda x: x[0], reverse=True)):
+				open('/tmp/gif-image-{}.jpg'.format(i), 'w').write(screenshot[1].decode('base64'))
+			
+			# Create the actual GIF using the  ImageMagick "convert" command
+			os.system('convert -layers OptimizePlus -delay 5x10 /tmp/gif-image-*.jpg -loop 0 +dither -colors 256 -depth 8 data/{}.gif'.format(result['ip_str']))
+	
+			# Clean up the temporary files
+			os.system('rm -f /tmp/gif-image-*.jpg')
+	
+			# Show a progress indicator
+			print result['ip_str']
+
+
+The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101
\ No newline at end of file

From 8ac1e5a250ad149e9e9d55337af1c54a45397a1d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Jan 2016 21:32:55 -0600
Subject: [PATCH 050/263] Add gifcreator to index page

---
 docs/index.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/index.rst b/docs/index.rst
index dd52b20..db16146 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -26,6 +26,7 @@ Examples
     examples/basic-search
     examples/query-summary
     examples/cert-stream
+    examples/gifcreator
 
 API Reference
 ~~~~~~~~~~~~~

From ae1ceeaeb9a9a7fccecd50df77104746ebd55207 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 17 Jan 2016 16:02:15 -0600
Subject: [PATCH 051/263] Added helper method: get_screenshot Bumped version

---
 setup.py          | 2 +-
 shodan/helpers.py | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 7922655..206548e 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.3',
+    version = '1.4.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/helpers.py b/shodan/helpers.py
index d876ff3..d35d848 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -96,4 +96,9 @@ def iterate_files(files):
         for line in fin:
             # Convert the JSON into a native Python object
             banner = simplejson.loads(line)
-            yield banner
\ No newline at end of file
+            yield banner
+
+def get_screenshot(banner):
+    if 'opts' in banner and 'screenshot' in banner['opts']:
+        return banner['opts']['screenshot']
+    return None

From 921d1ce3087f25212f2e7666598c7e9826cc3d05 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 17 Jan 2016 19:09:44 -0600
Subject: [PATCH 052/263] Added the "countries" and "asn" streams

---
 shodan/stream.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/shodan/stream.py b/shodan/stream.py
index 80d0dc4..4bafc9a 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -55,6 +55,17 @@ def alert(self, aid=None, timeout=None, raw=False):
         except ssl.SSLError as e:
             raise APIError('Stream timed out')
 
+    def asn(self, asn, raw=False, timeout=None):
+        """
+        A filtered version of the "banners" stream to only return banners that match the ASNs of interest.
+
+        :param asn: A list of ASN to return banner data on.
+        :type asn: string[]
+        """
+        stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout)
+        for line in self._iter_stream(stream, raw):
+            yield line
+
     def banners(self, raw=False, timeout=None):
         """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to
         API subscription plans and for those it only returns a fraction of the data.
@@ -63,6 +74,17 @@ def banners(self, raw=False, timeout=None):
         for line in self._iter_stream(stream, raw):
             yield line
 
+    def countries(self, countries, raw=False, timeout=None):
+        """
+        A filtered version of the "banners" stream to only return banners that match the countries of interest.
+
+        :param countries: A list of countries to return banner data on.
+        :type countries: string[]
+        """
+        stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout)
+        for line in self._iter_stream(stream, raw):
+            yield line
+
     def ports(self, ports, raw=False, timeout=None):
         """
         A filtered version of the "banners" stream to only return banners that match the ports of interest.
@@ -73,3 +95,4 @@ def ports(self, ports, raw=False, timeout=None):
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
+            

From 2333ef24e69f9cec5c2a4599b177f87d77a6a517 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 17 Jan 2016 21:14:47 -0600
Subject: [PATCH 053/263] Added --countries and --asn to shodan CLI stream
 command

---
 bin/shodan | 59 ++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 11 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 79549c6..44a1660 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1036,7 +1036,9 @@ def stats(limit, facets, query):
 @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
 @click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=0, type=int)
 @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer):
+@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
+@click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -1050,19 +1052,56 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
 
     if len(fields) == 0:
         raise click.ClickException('Please define at least one property to show')
+    
+    # The user must choose "ports", "countries", "asn" or nothing - can't select multiple
+    # filtered streams at once.
+    stream_type = []
+    if ports:
+        stream_type.append('ports')
+    if countries:
+        stream_type.append('countries')
+    if asn:
+        stream_type.append('asn')
+    
+    if len(stream_type) > 1:
+        raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
 
+    stream_args = None
+    
     # Turn the list of ports into integers
     if ports:
         try:
-            ports = [int(item.strip()) for item in ports.split(',')]
+            stream_args = [int(item.strip()) for item in ports.split(',')]
         except:
             raise click.ClickException('Invalid list of ports')
+    
+    if asn:
+        stream_args = asn.split(',')
+    
+    if countries:
+        stream_args = countries.split(',')
+    
+    # Flatten the list of stream types
+    # Possible values are:
+    # - all
+    # - asn
+    # - countries
+    # - ports
+    if len(stream_type) == 1:
+        stream_type = stream_type[0]
+    else:
+        stream_type = 'all'
 
     # Decide which stream to subscribe to based on whether or not ports were selected
-    if ports:
-        stream = api.stream.ports(ports, timeout=timeout)
-    else:
-        stream = api.stream.banners(timeout=timeout)
+    def _create_stream(name, args, timeout):
+        return {
+            'all': api.stream.banners(timeout=timeout),
+            'asn': api.stream.asn(args, timeout=timeout),
+            'countries': api.stream.countries(args, timeout=timeout),
+            'ports': api.stream.ports(args, timeout=timeout),
+        }.get(name, 'all')
+    
+    stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
     counter = 0
     quit = False
@@ -1127,12 +1166,10 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         except:
             # For other errors lets just wait a bit and try to reconnect again
             # XXX FIXME: this does not seem to reconnect since it's in the while not quit loop!
-            # Decide which stream to subscribe to based on whether or not ports were selected
-            if ports:
-                stream = api.stream.ports(ports, timeout=timeout)
-            else:
-                stream = api.stream.banners(timeout=timeout)
             time.sleep(1)
+            
+            # Create a new stream object to subscribe to
+            stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
 
 def async_spinner(finished):

From 32fecce040796da6901d4d0227ebc6e26ccec1b5 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 17 Jan 2016 21:48:57 -0600
Subject: [PATCH 054/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 206548e..652c908 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.4',
+    version = '1.4.5',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From b382cad624bc707e08f120ad100728282ea2e0d1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 22 Jan 2016 20:34:28 -0600
Subject: [PATCH 055/263] Disable minification when downloading data

---
 bin/shodan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 44a1660..7eedd90 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -456,7 +456,7 @@ def download(limit, filename, query):
     with gzip.open(filename, 'w') as fout:
         count = 0
         try:
-            cursor = api.search_cursor(query)
+            cursor = api.search_cursor(query, minify=False)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
                     write_banner(fout, banner)

From 7c8de8450130a87138652a8a5d0aef42e22bea56 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 18 Feb 2016 00:41:23 -0600
Subject: [PATCH 056/263] Added "csv" format to "shodan convert"

---
 .gitignore                       |   3 +-
 bin/shodan                       | 131 +++----------------------------
 setup.py                         |   2 +-
 shodan/cli/__init__.py           |   0
 shodan/cli/converter/__init__.py |   2 +
 shodan/cli/converter/base.py     |   8 ++
 shodan/cli/converter/csvc.py     |  87 ++++++++++++++++++++
 shodan/cli/converter/kml.py      | 127 ++++++++++++++++++++++++++++++
 8 files changed, 238 insertions(+), 122 deletions(-)
 create mode 100644 shodan/cli/__init__.py
 create mode 100644 shodan/cli/converter/__init__.py
 create mode 100644 shodan/cli/converter/base.py
 create mode 100644 shodan/cli/converter/csvc.py
 create mode 100644 shodan/cli/converter/kml.py

diff --git a/.gitignore b/.gitignore
index 7c7a28f..430a392 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,5 +6,4 @@ build/*
 *.pyc
 shodan.egg-info/*
 tmp/*
-shodan/cli/*
-MANIFEST
\ No newline at end of file
+MANIFEST
diff --git a/bin/shodan b/bin/shodan
index 7eedd90..9557c8a 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -35,6 +35,9 @@ import threading
 import requests
 import time
 
+# The file converters that are used to go from .json.gz to various other formats
+from shodan.cli.converter import CsvConverter, KmlConverter
+
 # Constants
 SHODAN_CONFIG_DIR = '~/.shodan/'
 ARRAY_SEPARATOR = ';'
@@ -142,111 +145,12 @@ def main():
 
 @main.command()
 @click.argument('input', metavar='<input file>')
-@click.argument('format', metavar='<output format>', type=click.Choice(['kml']))
+@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv']))
 def convert(input, format):
     """Convert the given input data file into a different format.
 
     Example: shodan convert data.json.gz kml
     """
-    #---------------------------------------
-    # KML Support
-    #---------------------------------------
-    KML_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
-<kml xmlns="http://www.opengis.net/kml/2.2">
-  <Document>"""
-    KML_FOOTER = """</Document></kml>"""
-    def kml_writer(fout, host):
-        try:
-            ip = host.get('ip_str', host.get('ipv6', None))
-            lat, lon = host['location']['latitude'], host['location']['longitude']
-
-            placemark = '<Placemark><name><![CDATA[<h1 style="margin-bottom:0;padding-bottom:0;font-size:1.5em">{}</h1>]]></name>'.format(ip)
-            placemark += '<description><![CDATA['
-
-            if 'hostnames' in host and host['hostnames']:
-                placemark += '<div><a style="color: #999;margin-top:-10px;padding-top:0;" href="http://{0}" target="_blank">{0}</a></div>'.format(host['hostnames'][0])
-
-            test = """
-    <table>
-        <tbody>
-          <tr>
-            <td>City</td>
-            <th>Albuquerque</th>
-          </tr>
-          <tr>
-            <td>Country</td>
-            <th>United States</th>
-          </tr>
-          <tr>
-            <td>Organization</td>
-            <th>Nexcess.net L.L.C.</th>
-          </tr>
-        </tbody>
-    </table>
-    <h2>Ports</h2>
-    <ul>
-            """
-
-            placemark += '<h2>Ports</h2><ul>'
-
-            for port in host['ports']:
-                placemark += """
-                    <li style="background-color: #1CA8DD;
-                    color: #FFF;
-                    float: left;
-                    font-family: Arial;
-                    font-size: 12px;
-                    font-weight: bold;
-                    height: 48px;
-                    margin: 5px;
-                    position: relative;
-                    text-shadow: none;
-                    width: 48px;"><span style="color: #FFF;
-                    height: 34px;
-                    position: absolute;
-                    text-align: center;
-                    top: 30%;
-                    width: 48px;">{}</span>
-                        </li>
-                """.format(port)
-
-            placemark += '</ul><div style="clear:both"></div>'
-
-            placemark += """
-                <div style="text-align:center"><a href="https://www.shodan.io/host/{0}" style="display: inline-block;
-                    padding: 4px 10px;
-                    margin-bottom: 0px;
-                    font-size: 13px;
-                    line-height: 18px;
-                    color: #333;
-                    text-align: center;
-                    text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.75);
-                    vertical-align: middle;
-                    cursor: pointer;
-                    background-color: #F5F5F5;
-                    background-image: -moz-linear-gradient(center top , #FFF, #E6E6E6);
-                    background-repeat: repeat-x;
-                    border-width: 1px;
-                    border-style: solid;
-                    border-color: #CCC #CCC #B3B3B3;
-                    -moz-border-top-colors: none;
-                    -moz-border-right-colors: none;
-                    -moz-border-bottom-colors: none;
-                    -moz-border-left-colors: none;
-                    border-image: none;
-                    border-radius: 4px;
-                    box-shadow: 0px 1px 0px rgba(255, 255, 255, 0.2) inset, 0px 1px 2px rgba(0, 0, 0, 0.05);" target="_blank">View Details</a></div>
-                    <div>powered by <a href="https://www.shodan.io" target="_blank">Shodan</a></div>
-            """.format(ip)
-
-            placemark += ']]></description>'
-            placemark += '<Point><coordinates>{},{}</coordinates></Point>'.format(lon, lat)
-            placemark += '</Placemark>'
-
-            fout.write(placemark.encode('utf-8'))
-        except Exception as e:
-            pass
-
     # Get the basename for the input file
     basename = input.replace('.json.gz', '').replace('.json', '')
 
@@ -261,29 +165,18 @@ def convert(input, format):
     progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
     progress_bar_thread.start()
 
-    fout.write(KML_HEADER)
-
-    hosts = {}
-    for banner in helpers.iterate_files([input]):
-        ip = banner.get('ip_str', banner.get('ipv6', None))
-        if not ip:
-            continue
-
-        if ip not in hosts:
-            hosts[ip] = banner
-            hosts[ip]['ports'] = []
-
-        hosts[ip]['ports'].append(banner['port'])
-
-    for ip, host in iter(hosts.items()):
-        kml_writer(fout, host)
-
-    fout.write(KML_FOOTER)
+    # Initialize the file converter
+    converter = {
+        'kml': KmlConverter,
+        'csv': CsvConverter,
+    }.get(format)(fout)
+    
+    converter.process([input])
 
     finished_event.set()
     progress_bar_thread.join()
 
-    click.echo(click.style('Successfully created new file: {}'.format(filename), fg='green'))
+    click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green'))
 
 
 @main.command()
diff --git a/setup.py b/setup.py
index 652c908..737586f 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.5',
+    version = '1.4.6',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/__init__.py b/shodan/cli/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py
new file mode 100644
index 0000000..f99203a
--- /dev/null
+++ b/shodan/cli/converter/__init__.py
@@ -0,0 +1,2 @@
+from .csvc import *
+from .kml import *
diff --git a/shodan/cli/converter/base.py b/shodan/cli/converter/base.py
new file mode 100644
index 0000000..9dc83c2
--- /dev/null
+++ b/shodan/cli/converter/base.py
@@ -0,0 +1,8 @@
+
+class Converter:
+
+    def __init__(self, fout):
+        self.fout = fout
+    
+    def process(self, fout):
+        pass
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
new file mode 100644
index 0000000..6e22b62
--- /dev/null
+++ b/shodan/cli/converter/csvc.py
@@ -0,0 +1,87 @@
+
+from .base import Converter
+from ...helpers import iterate_files
+
+from collections import MutableMapping
+from csv import writer as csv_writer, excel
+
+
+class CsvConverter(Converter):
+
+    fields = [
+        'data',
+        'hostnames',
+        'ip',
+        'ip_str',
+        'ipv6',
+        'org',
+        'isp',
+        'location.country_code',
+        'location.city',
+        'location.country_name',
+        'location.latitude',
+        'location.longitude',
+        'os',
+        'asn',
+        'port',
+        'transport',
+        'product',
+        'version',
+        
+        'ssl.cipher.version',
+        'ssl.cipher.bits',
+        'ssl.cipher.name',
+        'ssl.alpn',
+        'ssl.versions',
+        'ssl.cert.serial',
+        'ssl.cert.fingerprint.sha1',
+        'ssl.cert.fingerprint.sha256',
+        
+        'html',
+        'title',
+    ]
+    
+    def process(self, files):
+        writer = csv_writer(self.fout, dialect=excel)
+        
+        # Write the header
+        writer.writerow(self.fields)
+        
+        for banner in iterate_files(files):
+            try:
+                row = []
+                for field in self.fields:
+                    value = self.banner_field(banner, field)
+                    row.append(value)
+                writer.writerow(row)
+            except:
+                pass
+    
+    def banner_field(self, banner, flat_field):
+        # The provided field is a collapsed form of the actual field
+        fields = flat_field.split('.')
+    
+        try:
+            current_obj = banner
+            for field in fields:
+                current_obj = current_obj[field]
+            
+            # Convert a list into a concatenated string
+            if isinstance(current_obj, list):
+                current_obj = ','.join([str(i) for i in current_obj])
+            
+            return current_obj
+        except:
+            pass
+    
+        return ''
+    
+    def flatten(self, d, parent_key='', sep='.'):
+        items = []
+        for k, v in d.items():
+            new_key = parent_key + sep + k if parent_key else k
+            if isinstance(v, MutableMapping):
+                items.extend(flatten(v, new_key, sep=sep).items())
+            else:
+                items.append((new_key, v))
+        return dict(items)
\ No newline at end of file
diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py
new file mode 100644
index 0000000..7bcac11
--- /dev/null
+++ b/shodan/cli/converter/kml.py
@@ -0,0 +1,127 @@
+
+from .base import Converter
+from ...helpers import iterate_files
+
+class KmlConverter(Converter):
+    
+    def header(self):
+        self.fout.write("""<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+  <Document>""")
+    
+    def footer(self):
+        self.fout.write("""</Document></kml>""")
+    
+    def process(self, files):
+        # Write the header
+        self.header()
+        
+        hosts = {}
+        for banner in iterate_files(files):
+            ip = banner.get('ip_str', banner.get('ipv6', None))
+            if not ip:
+                continue
+    
+            if ip not in hosts:
+                hosts[ip] = banner
+                hosts[ip]['ports'] = []
+    
+            hosts[ip]['ports'].append(banner['port'])
+    
+        for ip, host in iter(hosts.items()):
+            self.write(host)
+        
+        self.footer()
+            
+    
+    def write(self, host):
+        try:
+            ip = host.get('ip_str', host.get('ipv6', None))
+            lat, lon = host['location']['latitude'], host['location']['longitude']
+
+            placemark = '<Placemark><name><![CDATA[<h1 style="margin-bottom:0;padding-bottom:0;font-size:1.5em">{}</h1>]]></name>'.format(ip)
+            placemark += '<description><![CDATA['
+
+            if 'hostnames' in host and host['hostnames']:
+                placemark += '<div><a style="color: #999;margin-top:-10px;padding-top:0;" href="http://{0}" target="_blank">{0}</a></div>'.format(host['hostnames'][0])
+
+            test = """
+    <table>
+        <tbody>
+          <tr>
+            <td>City</td>
+            <th>Albuquerque</th>
+          </tr>
+          <tr>
+            <td>Country</td>
+            <th>United States</th>
+          </tr>
+          <tr>
+            <td>Organization</td>
+            <th>Nexcess.net L.L.C.</th>
+          </tr>
+        </tbody>
+    </table>
+    <h2>Ports</h2>
+    <ul>
+            """
+
+            placemark += '<h2>Ports</h2><ul>'
+
+            for port in host['ports']:
+                placemark += """
+                    <li style="background-color: #1CA8DD;
+                    color: #FFF;
+                    float: left;
+                    font-family: Arial;
+                    font-size: 12px;
+                    font-weight: bold;
+                    height: 48px;
+                    margin: 5px;
+                    position: relative;
+                    text-shadow: none;
+                    width: 48px;"><span style="color: #FFF;
+                    height: 34px;
+                    position: absolute;
+                    text-align: center;
+                    top: 30%;
+                    width: 48px;">{}</span>
+                        </li>
+                """.format(port)
+
+            placemark += '</ul><div style="clear:both"></div>'
+
+            placemark += """
+                <div style="text-align:center"><a href="https://www.shodan.io/host/{0}" style="display: inline-block;
+                    padding: 4px 10px;
+                    margin-bottom: 0px;
+                    font-size: 13px;
+                    line-height: 18px;
+                    color: #333;
+                    text-align: center;
+                    text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.75);
+                    vertical-align: middle;
+                    cursor: pointer;
+                    background-color: #F5F5F5;
+                    background-image: -moz-linear-gradient(center top , #FFF, #E6E6E6);
+                    background-repeat: repeat-x;
+                    border-width: 1px;
+                    border-style: solid;
+                    border-color: #CCC #CCC #B3B3B3;
+                    -moz-border-top-colors: none;
+                    -moz-border-right-colors: none;
+                    -moz-border-bottom-colors: none;
+                    -moz-border-left-colors: none;
+                    border-image: none;
+                    border-radius: 4px;
+                    box-shadow: 0px 1px 0px rgba(255, 255, 255, 0.2) inset, 0px 1px 2px rgba(0, 0, 0, 0.05);" target="_blank">View Details</a></div>
+                    <div>powered by <a href="https://www.shodan.io" target="_blank">Shodan</a></div>
+            """.format(ip)
+
+            placemark += ']]></description>'
+            placemark += '<Point><coordinates>{},{}</coordinates></Point>'.format(lon, lat)
+            placemark += '</Placemark>'
+
+            self.fout.write(placemark.encode('utf-8'))
+        except Exception as e:
+            pass

From a8aa9424fee5e85c5f448dab904031d807704324 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 20 Feb 2016 13:45:00 -0600
Subject: [PATCH 057/263] Fixed a bug where sub-packages weren't distributed on
 installation

---
 MANIFEST.in            |   3 +-
 setup.py               |   4 +-
 shodan/cli/worldmap.py | 268 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 272 insertions(+), 3 deletions(-)
 create mode 100644 shodan/cli/worldmap.py

diff --git a/MANIFEST.in b/MANIFEST.in
index 2088d21..645827f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,4 @@
 include AUTHORS
 include LICENSE
-include MANIFEST.in
+include requirements.txt
+recursive-include shodan *.py
diff --git a/setup.py b/setup.py
index 737586f..642a673 100755
--- a/setup.py
+++ b/setup.py
@@ -4,12 +4,12 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.6',
+    version = '1.4.8',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
-    packages = ['shodan'],
+    packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     scripts = ['bin/shodan'],
     install_requires=["simplejson", "requests>=2.2.1", "click", "colorama"],
     classifiers = [
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
new file mode 100644
index 0000000..9ce8095
--- /dev/null
+++ b/shodan/cli/worldmap.py
@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+'''
+F-Secure Virus World Map console edition
+
+See README.md for more details
+
+Copyright 2012-2013 Jyrki Muukkonen
+
+Released under the MIT license.
+See LICENSE.txt or http://www.opensource.org/licenses/mit-license.php
+
+ASCII map in map-world-01.txt is copyright:
+ "Map 1998 Matthew Thomas. Freely usable as long as this line is included"
+
+'''
+import curses
+import json
+import locale
+import os
+import random
+import sys
+import time
+import urllib2
+
+
+STREAMS = {
+    'filetest': 'wm3stream.json',
+    'wm3': 'http://worldmap3.f-secure.com/api/stream/',
+}
+
+MAPS = {
+    'world': {
+        # offset (as (y, x) for curses...)
+        'corners': (1, 4, 23, 73),
+        # lat top, lon left, lat bottom, lon right
+        'coords': [90.0, -180.0, -90.0, 180.0],
+        'file': 'map-world-01.txt',
+    }
+}
+
+
+class AsciiMap(object):
+    """
+    Helper class for handling map drawing and coordinate calculations
+    """
+    def __init__(self, map_name='world', map_conf=None, window=None, encoding=None):
+        if map_conf is None:
+            map_conf = MAPS[map_name]
+        with open(map_conf['file'], 'rb') as mapf:
+            self.map = mapf.read()
+        self.coords = map_conf['coords']
+        self.corners = map_conf['corners']
+        if window is None:
+            window = curses.newwin(0, 0)
+        self.window = window
+
+        self.data = []
+        self.data_timestamp = None
+
+        # JSON contents _should_ be UTF8 (so, python internal unicode here...)
+        if encoding is None:
+            encoding = locale.getpreferredencoding()
+        self.encoding = encoding
+
+        # check if we can use transparent background or not
+        if curses.can_change_color():
+            curses.use_default_colors()
+            background = -1
+        else:
+            background = curses.COLOR_BLACK
+
+        tmp_colors = [
+            ('red', curses.COLOR_RED, background),
+            ('blue', curses.COLOR_BLUE, background),
+            ('pink', curses.COLOR_MAGENTA, background)
+        ]
+
+        self.colors = {}
+        if curses.has_colors():
+            for i, (name, fgcolor, bgcolor) in enumerate(tmp_colors, 1):
+                curses.init_pair(i, fgcolor, bgcolor)
+                self.colors[name] = i
+
+    def latlon_to_coords(self, lat, lon):
+        """
+        Convert lat/lon coordinates to character positions.
+        Very naive version, assumes that we are drawing the whole world
+        TODO: filter out stuff that doesn't fit
+        TODO: make it possible to use "zoomed" maps
+        """
+        width = (self.corners[3]-self.corners[1])
+        height = (self.corners[2]-self.corners[0])
+
+        # change to 0-180, 0-360
+        abs_lat = -lat+90
+        abs_lon = lon+180
+        x = (abs_lon/360.0)*width + self.corners[1]
+        y = (abs_lat/180.0)*height + self.corners[0]
+        return int(x), int(y)
+
+    def set_data(self, data):
+        """
+        Set / convert internal data.
+        For now it just selects a random set to show (good enough for demo purposes)
+        TODO: could use deque to show all entries
+        """
+        entries = []
+        formats = [
+            u"{name} / {country} {city}",
+            u"{name} / {country}",
+            u"{name}",
+            u"{type}",
+        ]
+        dets = data.get('detections', [])
+        for det in random.sample(dets, min(len(dets), 5)):
+            #"city": "Montoire-sur-le-loir",
+            #"country": "FR",
+            #"lat": "47.7500",
+            #"long": "0.8667",
+            #"name": "Trojan.Generic.7555308",
+            #"type": "Trojan"
+            desc = "Detection"
+            # keeping it unicode here, encode() for curses later on
+            for fmt in formats:
+                try:
+                    desc = fmt.format(**det)
+                    break
+                except StandardError:
+                    pass
+            entry = (
+                float(det['lat']),
+                float(det['long']),
+                '*',
+                desc,
+                curses.A_BOLD,
+                'red',
+            )
+            entries.append(entry)
+        self.data = entries
+        # for debugging... maybe it could be shown again now that we have the live stream support
+        #self.data_timestamp =  data.get('response_generated')
+
+    def draw(self, target):
+        """ Draw internal data to curses window """
+        self.window.clear()
+        self.window.addstr(0, 0, self.map)
+        debugdata = [
+            (60.16, 24.94, '*', self.data_timestamp, curses.A_BOLD, 'blue'), # Helsinki
+            #(90, -180, '1', 'top left', curses.A_BOLD, 'blue'),
+            #(-90, -180, '2', 'bottom left', curses.A_BOLD, 'blue'),
+            #(90, 180, '3', 'top right', curses.A_BOLD, 'blue'),
+            #(-90, 180, '4', 'bottom right', curses.A_BOLD, 'blue'),
+        ]
+        # FIXME: position to be defined in map config?
+        row = self.corners[2]-6
+        items_to_show = 5
+        for lat, lon, char, desc, attrs, color in debugdata + self.data:
+            # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html
+            if desc:
+                desc = desc.encode(self.encoding, 'ignore')
+            if items_to_show <= 0:
+                break
+            char_x, char_y = self.latlon_to_coords(lat, lon)
+            if self.colors and color:
+                attrs |= curses.color_pair(self.colors[color])
+            self.window.addstr(char_y, char_x, char, attrs)
+            if desc:
+                det_show = "%s %s" % (char, desc)
+            else:
+                det_show = None
+
+            if det_show is not None:
+                try:
+                    self.window.addstr(row, 1, det_show, attrs)
+                    row += 1
+                    items_to_show -= 1
+                except StandardError:
+                    # FIXME: check window size before addstr()
+                    break
+        self.window.overwrite(target)
+        self.window.leaveok(1)
+
+
+class MapApp(object):
+    """ Virus World Map ncurses application """
+    def __init__(self, api_key):
+        self.api_key = api_key
+        self.data = None
+        self.last_fetch = 0
+        self.sleep = 10  # tenths of seconds, for curses.halfdelay()
+
+    def fetch_data(self, epoch_now, force_refresh=False):
+        """ (Re)fetch data from JSON stream """
+        refresh = False
+        if force_refresh or self.data is None:
+            refresh = True
+        else:
+            # json data usually has: "polling_interval": 120
+            try:
+                poll_interval = int(self.data['polling_interval'])
+            except (ValueError, KeyError):
+                poll_interval = 60
+            if self.last_fetch + poll_interval <= epoch_now:
+                refresh = True
+
+        if refresh:
+            try:
+                self.data = json.load(urllib2.urlopen(self.stream_url))
+                self.last_fetch = epoch_now
+            except StandardError:
+                pass
+        return refresh
+
+    def run(self, scr):
+        """ Initialize and run the application """
+        m = AsciiMap()
+        curses.halfdelay(self.sleep)
+        while True:
+            now = int(time.time())
+            refresh = self.fetch_data(now)
+            m.set_data(self.data)
+            m.draw(scr)
+            scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD)
+            scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD)
+
+            event = scr.getch()
+            if event == ord("q"):
+                break
+
+            # if in replay mode?
+            #elif event == ord('-'):
+            #    self.sleep = min(self.sleep+10, 100)
+            #    curses.halfdelay(self.sleep)
+            #elif event == ord('+'):
+            #    self.sleep = max(self.sleep-10, 10)
+            #    curses.halfdelay(self.sleep)
+
+            elif event == ord('r'):
+                # force refresh
+                refresh = True
+            elif event == ord('c'):
+                # enter config mode
+                pass
+            elif event == ord('h'):
+                # show help screen
+                pass
+            elif event == ord('m'):
+                # cycle maps
+                pass
+
+            # redraw window (to fix encoding/rendering bugs and to hide other messages to same tty)
+            # user pressed 'r' or new data was fetched
+            if refresh:
+                m.window.redrawwin()
+
+
+def main(argv=None):
+    """ Main function / entry point """
+    if argv is None:
+        argv = sys.argv[1:]
+    conf = {}
+    if len(argv):
+        conf['stream_url'] = argv[0]
+    app = MapApp(conf)
+    return curses.wrapper(app.run_curses_app)
+
+if __name__ == '__main__':
+    sys.exit(main())

From 02e9c472921d5fc1096b4ed684395559e2fa5725 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 10 Mar 2016 22:30:27 -0600
Subject: [PATCH 058/263] Added experimental methods to API client

---
 shodan/client.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/shodan/client.py b/shodan/client.py
index 11e3594..62560d3 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -101,6 +101,21 @@ def count(self, query, facets=None):
 
             return self.parent._request('/api/count', query_args, service='exploits')
     
+    class Labs:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def honeyscore(self, ip):
+            """Calculate the probability of an IP being an ICS honeypot.
+            
+            :param ip: IP address of the device
+            :type ip: str
+
+            :returns: int -- honeyscore ranging from 0.0 to 1.0
+            """
+            return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
+    
     def __init__(self, key):
         """Initializes the API object.
         
@@ -111,6 +126,7 @@ def __init__(self, key):
         self.base_url = 'https://api.shodan.io'
         self.base_exploits_url = 'https://exploits.shodan.io'
         self.exploits = self.Exploits(self)
+        self.labs = self.Labs(self)
         self.tools = self.Tools(self)
         self.stream = Stream(key)
     

From 21f0cd7a8ad9ee6140e02fb3bca5411f5391ef56 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 10 Mar 2016 22:30:43 -0600
Subject: [PATCH 059/263] Bump version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 642a673..b856d9a 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.4.8',
+    version = '1.5.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From be90e5ef3138136dfd6d7203fcfeadf7a360d9e6 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 11 Mar 2016 00:25:34 -0600
Subject: [PATCH 060/263] Added "honeyscore" command to the CLI

---
 bin/shodan | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/bin/shodan b/bin/shodan
index 9557c8a..3f7e264 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1065,6 +1065,27 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
             stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
 
+@main.command()
+@click.argument('ip', metavar='<IP address>')
+def honeyscore(ip):
+    """Check whether the IP is a honeypot or not."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        score = api.labs.honeyscore(ip)
+        
+        if score == 1.0:
+            click.echo(click.style('Honeypot detected', fg='red'))
+        elif score > 0.5:
+            click.echo(click.style('Probably a honeypot', fg='yellow'))
+        else:
+            click.echo(click.style('Not a honeypot', fg='green'))
+        
+        click.echo('Score: {}'.format(score))
+    except:
+        click.ClickException('Unable to calculate honeyscore')
+
 def async_spinner(finished):
     spinner = itertools.cycle(['-', '/', '|', '\\'])
     while not finished.is_set():

From c0b1ee1ca020803ed0c41778a31cf7a078636964 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 20 Mar 2016 19:30:07 -0500
Subject: [PATCH 061/263] Fixed error propagation bug

---
 shodan/client.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index 62560d3..c386150 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -160,12 +160,14 @@ def _request(self, function, params, service='shodan', method='get'):
             raise APIError('Unable to connect to Shodan')
 
         # Check that the API key wasn't rejected
-        if data.status_code == 401:
+        if data.status_code >= 401 and data.status_code < 500:
             try:
-                raise APIError(data.json()['error'])
-            except:
-                pass
-            raise APIError('Invalid API key')
+                # Return the actual error message if the API returned valid JSON
+                error = data.json()['error']
+            except Exception as e:
+                error = 'Invalid API key'
+            
+            raise APIError(error)
         
         # Parse the text into JSON
         try:
@@ -174,7 +176,7 @@ def _request(self, function, params, service='shodan', method='get'):
             raise APIError('Unable to parse JSON response')
         
         # Raise an exception if an error occurred
-        if type(data) == dict and data.get('error', None):
+        if type(data) == dict and 'error' in data:
             raise APIError(data['error'])
         
         # Return the data

From bdb5af082201fcddbd593d0ca970729c82f9df98 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 20 Mar 2016 20:52:59 -0500
Subject: [PATCH 062/263] Bump version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b856d9a..6a6bc4a 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.0',
+    version = '1.5.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From c10d875b95707463fd0ece3390a0e526dafd0f0a Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 25 Mar 2016 04:25:38 -0500
Subject: [PATCH 063/263] Added the ability to create network alerts and
 subscribe to them using the CLI Added the --history flag to the "shodan host"
 command to show the full list of ports/ banners that are currently stored
 Bumped version

---
 bin/shodan | 46 ++++++++++++++++++++++++++++++++++++++++------
 setup.py   |  2 +-
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 3f7e264..65dbe58 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -210,6 +210,7 @@ def init(key):
 
 @main.group()
 def alert():
+    """Manage the network alerts for your account"""
     pass
 
 
@@ -229,6 +230,23 @@ def alert_clear():
         raise click.ClickException(e.value)
     click.echo("Alerts deleted")
 
+@alert.command(name='create')
+@click.argument('name', metavar='<name>')
+@click.argument('netblock', metavar='<netblock>')
+def alert_create(name, netblock):
+    """Create a network alert to monitor an external network"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        alert = api.create_alert(name, netblock)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    
+    click.echo(click.style('Successfully created network alert!', fg='green'))
+    click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan'))
+
 @alert.command(name='list')
 @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
 def alert_list(expired):
@@ -368,14 +386,15 @@ def download(limit, filename, query):
 
 @main.command()
 @click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str)
+@click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True)
 @click.argument('ip', metavar='<ip address>')
-def host(format, ip):
+def host(format, history, ip):
     """View all available information for an IP address"""
     key = get_api_key()
     api = shodan.Shodan(key)
 
     try:
-        host = api.host(ip)
+        host = api.host(ip, history=history)
 
         # General info
         click.echo(click.style(ip, fg='green'))
@@ -427,7 +446,13 @@ def host(format, ip):
                 version = '({})'.format(banner['version'])
 
             click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False)
-            click.echo('{} {}'.format(product, version))
+            click.echo('{} {}'.format(product, version), nl=False)
+            
+            if history:
+                # Format the timestamp to only show the year-month-day
+                date = banner['timestamp'][:10]
+                click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
+            click.echo('')
 
             # Show optional ssl info
             if 'ssl' in banner:
@@ -931,7 +956,8 @@ def stats(limit, facets, query):
 @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
 @click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
 @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn):
+@click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn, alert):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -955,6 +981,8 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         stream_type.append('countries')
     if asn:
         stream_type.append('asn')
+    if alert:
+        stream_type.append('alert')
     
     if len(stream_type) > 1:
         raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
@@ -968,6 +996,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         except:
             raise click.ClickException('Invalid list of ports')
     
+    if alert:
+        alert = alert.strip()
+        if alert.lower() != 'all':
+            stream_args = alert
+    
     if asn:
         stream_args = asn.split(',')
     
@@ -989,6 +1022,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
     def _create_stream(name, args, timeout):
         return {
             'all': api.stream.banners(timeout=timeout),
+            'alert': api.stream.alert(args, timeout=timeout),
             'asn': api.stream.asn(args, timeout=timeout),
             'countries': api.stream.countries(args, timeout=timeout),
             'ports': api.stream.ports(args, timeout=timeout),
@@ -1053,12 +1087,12 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
                     click.echo(row)
         except requests.exceptions.Timeout:
             raise click.ClickException('Connection timed out')
-            # let some exteran while $TRUE script handle the reconnects XXX FIXME : should be improved
         except KeyboardInterrupt:
             quit = True
+        except shodan.APIError as e:
+            raise click.ClickException(e.value)
         except:
             # For other errors lets just wait a bit and try to reconnect again
-            # XXX FIXME: this does not seem to reconnect since it's in the while not quit loop!
             time.sleep(1)
             
             # Create a new stream object to subscribe to
diff --git a/setup.py b/setup.py
index 6a6bc4a..213eaad 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.1',
+    version = '1.5.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 88fc133a0e6b68e00d70cc82999ae35e49a8dd35 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 25 Apr 2016 03:07:55 -0500
Subject: [PATCH 064/263] Added "scan status" command Added "--verbose" option
 to "scan submit"

---
 bin/shodan | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 65dbe58..1d8e058 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -655,8 +655,9 @@ def scan_protocols():
 @scan.command(name='submit')
 @click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int)
 @click.option('--filename', help='Save the results in the given file.', default='', type=str)
+@click.option('--verbose', default=False, is_flag=True)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan_submit(wait, filename, netblocks):
+def scan_submit(wait, filename, verbose, netblocks):
     """Scan an IP/ netblock using Shodan."""
     key = get_api_key()
     api = shodan.Shodan(key)
@@ -670,7 +671,10 @@ def scan_submit(wait, filename, netblocks):
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
 
         click.echo('')
-        click.echo('Starting Shodan scan at {} ({} scan credits left)'.format(now, scan['credits_left']))
+        click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
+        
+        if verbose:
+            click.echo('# Scan ID: {}'.format(scan['id']))
 
         # Return immediately
         if wait <= 0:
@@ -714,6 +718,10 @@ def scan_submit(wait, filename, netblocks):
                         # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
                         if time.time() - scan_start >= 60:
                             scan = api.scan_status(scan['id'])
+                            
+                            if verbose:
+                                click.echo('# Scan status: {}'.format(scan['status']))
+                                
                             if scan['status'] == 'DONE':
                                 done = True
                                 break
@@ -733,6 +741,9 @@ def scan_submit(wait, filename, netblocks):
                     scan = api.scan_status(scan['id'])
                     if scan['status'] == 'DONE':
                         done = True
+                    
+                    if verbose:
+                        click.echo('# Scan status: {}'.format(scan['status']))
                 except socket.timeout as e:
                     # If the connection timed out before the timeout, that means the streaming server
                     # that the user tried to reach is down. In that case, lets wait a second and try
@@ -839,6 +850,19 @@ def scan_submit(wait, filename, netblocks):
             api.delete_alert(alert['id'])
 
 
+@scan.command(name='status')
+@click.argument('scan_id', type=str)
+def scan_status(scan_id):
+    """Check the status of an on-demand scan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        scan = api.scan_status(scan_id)
+        click.echo(scan['status'])
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+
 @main.command()
 @click.option('--color/--no-color', default=True)
 @click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data')

From 72e74aabe54bd826e951cf1f951b91a56f8d1067 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 26 Apr 2016 05:02:08 -0500
Subject: [PATCH 065/263] Bump version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 213eaad..84279fa 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.2',
+    version = '1.5.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 5c85d30112b8467cdb1e67734449b9998ac58096 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 26 Apr 2016 17:10:44 -0500
Subject: [PATCH 066/263] Added the ability to force a scan of an IP

---
 bin/shodan       | 5 +++--
 shodan/client.py | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 1d8e058..b90da55 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -655,9 +655,10 @@ def scan_protocols():
 @scan.command(name='submit')
 @click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int)
 @click.option('--filename', help='Save the results in the given file.', default='', type=str)
+@click.option('--force', default=False, is_flag=True)
 @click.option('--verbose', default=False, is_flag=True)
 @click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan_submit(wait, filename, verbose, netblocks):
+def scan_submit(wait, filename, force, verbose, netblocks):
     """Scan an IP/ netblock using Shodan."""
     key = get_api_key()
     api = shodan.Shodan(key)
@@ -666,7 +667,7 @@ def scan_submit(wait, filename, verbose, netblocks):
     # Submit the IPs for scanning
     try:
         # Submit the scan
-        scan = api.scan(netblocks)
+        scan = api.scan(netblocks, force=force)
 
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
 
diff --git a/shodan/client.py b/shodan/client.py
index c386150..1ba73d9 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -235,7 +235,7 @@ def protocols(self):
         """
         return self._request('/shodan/protocols', {})
 
-    def scan(self, ips):
+    def scan(self, ips, force=False):
         """Scan a network using Shodan
 
         :param ips: A list of IPs or netblocks in CIDR notation or an object structured like:
@@ -249,6 +249,8 @@ def scan(self, ips):
                         ]
                     }
         :type ips: str or dict
+        :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users.
+        :type force: bool
 
         :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left.
         """
@@ -262,6 +264,7 @@ def scan(self, ips):
 
         params = {
             'ips': networks,
+            'force': force,
         }
 
         return self._request('/shodan/scan', params, method='post')

From ab495e40ec037e516daae5357b24166708e80b08 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 24 Jun 2016 07:39:17 -0500
Subject: [PATCH 067/263] Added saving to CSV to the "stats" command

---
 bin/shodan | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index b90da55..2b2877b 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -21,6 +21,7 @@ The following commands are currently supported:
 
 import click
 import collections
+import csv
 import datetime
 import gzip
 import itertools
@@ -934,8 +935,9 @@ def search(color, fields, limit, separator, query):
 @main.command()
 @click.option('--limit', help='The number of results to return.', default=10, type=int)
 @click.option('--facets', help='List of facets to get statistics for.', default='country,org')
+@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None)
 @click.argument('query', metavar='<search query>', nargs=-1)
-def stats(limit, facets, query):
+def stats(limit, facets, filename, query):
     """Provide summary information about a search query"""
     # Setup Shodan
     key = get_api_key()
@@ -960,13 +962,60 @@ def stats(limit, facets, query):
 
     # Print the stats tables
     for facet in results['facets']:
-        click.echo('Top {} Results for Facet: {}'.format(limit, facet))
+        click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet))
 
         for item in results['facets'][facet]:
             click.echo(click.style('{:28s}'.format(str(item['value']).encode('ascii', errors='replace')), fg='cyan'), nl=False)
             click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
+        
+    # Create the output file if requested
+    fout = None
+    if filename:
+        if not filename.endswith('.csv'):
+            filename += '.csv'
+        fout = open(filename, 'w')
+        writer = csv.writer(fout, dialect=csv.excel)
+        
+        # Write the header
+        writer.writerow(['Query', query])
+        
+        # Add an empty line to separate rows
+        writer.writerow([])
+        
+        # Write the header that contains the facets
+        row = []
+        for facet in results['facets']:
+            row.append(facet)
+            row.append('')
+        writer.writerow(row)
+        
+        # Every facet has 2 columns (key, value)
+        counter = 0
+        has_items = True
+        while has_items:
+            row = ['' for i in range(len(results['facets']) * 2)]
+            
+            pos = 0
+            has_items = False
+            for facet in results['facets']:
+                values = results['facets'][facet]
+                
+                # Add the values for the facet into the current row
+                if len(values) > counter:
+                    has_items = True
+                    row[pos] = values[counter]['value']
+                    row[pos+1] = values[counter]['count']
+                
+                pos += 2
+            
+            # Write out the row
+            if has_items:
+                writer.writerow(row)
+            
+            # Move to the next row of values
+            counter += 1
 
 
 @main.command()

From e476792d24c1befe911aea958d0293c611e66333 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 24 Jun 2016 07:39:34 -0500
Subject: [PATCH 068/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 84279fa..b1282ff 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.3',
+    version = '1.5.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 230b3d15ccf35f300195e6cee45044dce8d3deaa Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 28 Jun 2016 12:28:49 -0500
Subject: [PATCH 069/263] Improved parse in CLI

---
 bin/shodan | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/shodan b/bin/shodan
index 2b2877b..9f1a199 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -114,6 +114,10 @@ def match_filters(banner, filters):
     for args in filters:
         flat_field, check = args.split(':', 1)
         value = get_banner_field(banner, flat_field)
+        
+        # If the field doesn't exist on the banner then ignore the record
+        if not value:
+            return False
 
         # It must match all filters to be allowed
         field_type = type(value)

From 778a53099f40d5a795e18922551b0b08656962e1 Mon Sep 17 00:00:00 2001
From: Beda Kosata <bedrich.kosata@nic.cz>
Date: Sun, 17 Jul 2016 15:08:00 +0200
Subject: [PATCH 070/263] use requests.Session for connection reuse

---
 shodan/client.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index 1ba73d9..3706e4c 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -129,7 +129,8 @@ def __init__(self, key):
         self.labs = self.Labs(self)
         self.tools = self.Tools(self)
         self.stream = Stream(key)
-    
+        self._session = requests.Session()
+            
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.
         
@@ -153,9 +154,9 @@ def _request(self, function, params, service='shodan', method='get'):
         # Send the request
         try:
             if method.lower() == 'post':
-                data = requests.post(base_url + function, params)
+                data = self._session.post(base_url + function, params)
             else:
-                data = requests.get(base_url + function, params=params)
+                data = self._session.get(base_url + function, params=params)
         except:
             raise APIError('Unable to connect to Shodan')
 

From 019a1b0692246f5806803c1cbb8b788290ac7403 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 17 Jul 2016 12:55:46 -0500
Subject: [PATCH 071/263] Bumped version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b1282ff..9c58099 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.4',
+    version = '1.5.5',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 2fd6c4318a6f8910b2ce65b1c6d9e414e1bafe5d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 22 Aug 2016 17:53:58 -0500
Subject: [PATCH 072/263] Updated the timeout for Internet scans to 90 seconds

---
 bin/shodan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 9f1a199..505aa0f 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -607,7 +607,7 @@ def scan_internet(quiet, port, protocol):
                 click.echo('Waiting for data, please stand by...')
                 while not done:
                     try:
-                        for banner in api.stream.ports([port], timeout=30):
+                        for banner in api.stream.ports([port], timeout=90):
                             counter += 1
                             write_banner(fout, banner)
 

From 34c90619a400d494e20826c499f999e134422867 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 31 Aug 2016 20:55:25 -0500
Subject: [PATCH 073/263] Changed default compression level to the highest
 level and exposed it as an option in the "stream" command

---
 bin/shodan | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 505aa0f..63233d2 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -93,8 +93,8 @@ def escape_data(args):
 def timestr():
         return datetime.datetime.utcnow().strftime('%Y-%m-%d')
 
-def open_file(directory, timestr):
-        return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', 1)
+def open_file(directory, timestr, compresslevel=9):
+        return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
 
 def get_banner_field(banner, flat_field):
     # The provided field is a collapsed form of the actual field
@@ -1035,7 +1035,8 @@ def stats(limit, facets, filename, query):
 @click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
 @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
 @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn, alert):
+@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn, alert, compresslevel):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -1114,7 +1115,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
     fout = None
 
     if datadir:
-        fout = open_file(datadir, last_time)
+        fout = open_file(datadir, last_time, compresslevel)
 
     while not quit:
         try:

From 3fe77b0860e9c09cd28fa5df58d69fc479ac3a1c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 6 Sep 2016 16:38:23 -0500
Subject: [PATCH 074/263] Fixed a bug that prevented "stats" from working in
 Python3

---
 bin/shodan | 8 +++++++-
 setup.py   | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 63233d2..64e31e4 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -969,7 +969,13 @@ def stats(limit, facets, filename, query):
         click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet))
 
         for item in results['facets'][facet]:
-            click.echo(click.style('{:28s}'.format(str(item['value']).encode('ascii', errors='replace')), fg='cyan'), nl=False)
+            value = item['value']
+            if isinstance(value, basestring):
+                value = value.encode('ascii', errors='replace').decode('ascii')
+            else:
+                value = str(value)
+            
+            click.echo(click.style('{:28s}'.format(value), fg='cyan'), nl=False)
             click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
diff --git a/setup.py b/setup.py
index 9c58099..f54d8ce 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.5',
+    version = '1.5.6',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From fbcebef8eb21b9f06c09874f079eb4aa1fe95878 Mon Sep 17 00:00:00 2001
From: Vaclav Bartos <bartos@cesnet.cz>
Date: Thu, 6 Oct 2016 16:17:18 +0200
Subject: [PATCH 075/263] Added support for "minify" parameter of "host" method

The minify parameter is already a part of API, but it wasn't included in python client.
---
 shodan/client.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 3706e4c..ecf7fa3 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -200,13 +200,15 @@ def count(self, query, facets=None):
             query_args['facets'] = create_facet_string(facets)
         return self._request('/shodan/host/count', query_args)
     
-    def host(self, ips, history=False):
+    def host(self, ips, history=False, minify=False):
         """Get all available information on an IP.
 
         :param ip: IP of the computer
         :type ip: str
         :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise.
         :type history: bool
+        :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise.
+        :type minify: bool
         """
         if isinstance(ips, basestring):
             ips = [ips]
@@ -214,6 +216,8 @@ def host(self, ips, history=False):
         params = {}
         if history:
             params['history'] = history
+        if minify:
+            params['minify'] = minify
         return self._request('/shodan/host/%s' % ','.join(ips), params)
     
     def info(self):

From 479ed0a4df06cfa76c52fdd7390eeb127ec415b9 Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Wed, 7 Dec 2016 02:06:26 -0600
Subject: [PATCH 076/263] Allow the development of plugins for the CLI

---
 bin/shodan             | 40 +++++++++++-----------------------------
 setup.py               |  4 ++--
 shodan/cli/helpers.py  | 25 +++++++++++++++++++++++++
 shodan/cli/settings.py | 10 ++++++++++
 4 files changed, 48 insertions(+), 31 deletions(-)
 create mode 100644 shodan/cli/helpers.py
 create mode 100644 shodan/cli/settings.py

diff --git a/bin/shodan b/bin/shodan
index 64e31e4..11bf731 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -40,16 +40,14 @@ import time
 from shodan.cli.converter import CsvConverter, KmlConverter
 
 # Constants
-SHODAN_CONFIG_DIR = '~/.shodan/'
-ARRAY_SEPARATOR = ';'
-COLORIZE_FIELDS = {
-    'ip_str': 'green',
-    'port': 'yellow',
-    'data': 'white',
-    'hostnames': 'magenta',
-    'org': 'cyan',
-    'vulns': 'red',
-}
+from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS
+
+# Helper methods
+from shodan.cli.helpers import get_api_key
+
+# Allow 3rd-parties to develop custom commands
+from click_plugins import with_plugins
+from pkg_resources import iter_entry_points
 
 # Make "-h" work like "--help"
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -61,23 +59,6 @@ except NameError:
     basestring = str
 
 # Utility methods
-def get_api_key():
-    shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
-    keyfile = shodan_dir + '/api_key'
-
-    # If the file doesn't yet exist let the user know that they need to
-    # initialize the shodan cli
-    if not os.path.exists(keyfile):
-        raise click.ClickException('Please run "shodan init <api key>" before using this command')
-
-    # Make sure it is a read-only file
-    os.chmod(keyfile, 0o600)
-
-    with open(keyfile, 'r') as fin:
-        return fin.read().strip()
-
-    raise click.ClickException('Please run "shodan init <api key>" before using this command')
-
 def get_ip(banner):
     if 'ipv6' in banner:
         return banner['ipv6']
@@ -91,10 +72,10 @@ def escape_data(args):
     return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
 
 def timestr():
-        return datetime.datetime.utcnow().strftime('%Y-%m-%d')
+    return datetime.datetime.utcnow().strftime('%Y-%m-%d')
 
 def open_file(directory, timestr, compresslevel=9):
-        return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
+    return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
 
 def get_banner_field(banner, flat_field):
     # The provided field is a collapsed form of the actual field
@@ -143,6 +124,7 @@ def write_banner(fout, banner):
     fout.write(line.encode('utf-8'))
 
 
+@with_plugins(iter_entry_points('shodan.cli.plugins'))
 @click.group(context_settings=CONTEXT_SETTINGS)
 def main():
     pass
diff --git a/setup.py b/setup.py
index f54d8ce..d90154d 100755
--- a/setup.py
+++ b/setup.py
@@ -4,14 +4,14 @@
 
 setup(
     name = 'shodan',
-    version = '1.5.6',
+    version = '1.6.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     scripts = ['bin/shodan'],
-    install_requires=["simplejson", "requests>=2.2.1", "click", "colorama"],
+    install_requires=["simplejson", "requests>=2.2.1", "click", "click-plugins", "colorama"],
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
new file mode 100644
index 0000000..8fafe33
--- /dev/null
+++ b/shodan/cli/helpers.py
@@ -0,0 +1,25 @@
+'''
+Helper methods to create your own CLI commands.
+'''
+import click
+import os
+
+from .settings import SHODAN_CONFIG_DIR
+
+def get_api_key():
+    '''Returns the API key of the current logged-in user.'''
+    shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
+    keyfile = shodan_dir + '/api_key'
+
+    # If the file doesn't yet exist let the user know that they need to
+    # initialize the shodan cli
+    if not os.path.exists(keyfile):
+        raise click.ClickException('Please run "shodan init <api key>" before using this command')
+
+    # Make sure it is a read-only file
+    os.chmod(keyfile, 0o600)
+
+    with open(keyfile, 'r') as fin:
+        return fin.read().strip()
+
+    raise click.ClickException('Please run "shodan init <api key>" before using this command')
diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
new file mode 100644
index 0000000..6f27e5d
--- /dev/null
+++ b/shodan/cli/settings.py
@@ -0,0 +1,10 @@
+
+SHODAN_CONFIG_DIR = '~/.shodan/'
+COLORIZE_FIELDS = {
+    'ip_str': 'green',
+    'port': 'yellow',
+    'data': 'white',
+    'hostnames': 'magenta',
+    'org': 'cyan',
+    'vulns': 'red',
+}

From d754b50c37920dd979642cc8308b1285317304fb Mon Sep 17 00:00:00 2001
From: achillean <jmath@shodan.io>
Date: Fri, 16 Dec 2016 14:54:13 -0600
Subject: [PATCH 077/263] Adjust copyright years

---
 shodan/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 3706e4c..e384a34 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -5,7 +5,7 @@
 
 This module implements the Shodan API.
 
-:copyright: (c) 2014-2015 by John Matherly
+:copyright: (c) 2014- by John Matherly
 """
 import time
 

From 3d408c3ecdbec522cfca4f180e04b0fb1dd76321 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 18 Dec 2016 21:30:33 -0600
Subject: [PATCH 078/263] Added GeoJSON converter Added helper method to get an
 IP from the banner Bump version

---
 bin/shodan                       | 15 +++------
 setup.py                         |  2 +-
 shodan/cli/converter/__init__.py |  1 +
 shodan/cli/converter/geojson.py  | 57 ++++++++++++++++++++++++++++++++
 shodan/helpers.py                |  6 ++++
 5 files changed, 70 insertions(+), 11 deletions(-)
 create mode 100644 shodan/cli/converter/geojson.py

diff --git a/bin/shodan b/bin/shodan
index 11bf731..cc26ac7 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -37,7 +37,7 @@ import requests
 import time
 
 # The file converters that are used to go from .json.gz to various other formats
-from shodan.cli.converter import CsvConverter, KmlConverter
+from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter
 
 # Constants
 from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS
@@ -58,12 +58,6 @@ try:
 except NameError:
     basestring = str
 
-# Utility methods
-def get_ip(banner):
-    if 'ipv6' in banner:
-        return banner['ipv6']
-    return banner['ip_str']
-
 
 def escape_data(args):
     # Ensure the provided string isn't unicode data
@@ -132,7 +126,7 @@ def main():
 
 @main.command()
 @click.argument('input', metavar='<input file>')
-@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv']))
+@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json']))
 def convert(input, format):
     """Convert the given input data file into a different format.
 
@@ -156,6 +150,7 @@ def convert(input, format):
     converter = {
         'kml': KmlConverter,
         'csv': CsvConverter,
+        'geo.json': GeoJsonConverter,
     }.get(format)(fout)
     
     converter.process([input])
@@ -595,7 +590,7 @@ def scan_internet(quiet, port, protocol):
 
                             if not quiet:
                                 click.echo('{0:<40} {1:<20} {2}'.format(
-                                        click.style(get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
+                                        click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
                                         click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
                                         ';'.join(banner['hostnames'])
                                     )
@@ -700,7 +695,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
                         # Don't show duplicate banners
                         cache_key = '{}:{}'.format(ip, banner['port'])
                         if cache_key not in cache:
-                            hosts[get_ip(banner)][banner['port']] = banner
+                            hosts[helpers.get_ip(banner)][banner['port']] = banner
                             cache[cache_key] = True
 
                         # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
diff --git a/setup.py b/setup.py
index d90154d..9452b3c 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.0',
+    version = '1.6.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py
index f99203a..b880502 100644
--- a/shodan/cli/converter/__init__.py
+++ b/shodan/cli/converter/__init__.py
@@ -1,2 +1,3 @@
 from .csvc import *
 from .kml import *
+from .geojson import *
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
new file mode 100644
index 0000000..8d6d7d0
--- /dev/null
+++ b/shodan/cli/converter/geojson.py
@@ -0,0 +1,57 @@
+
+from .base import Converter
+from ...helpers import get_ip, iterate_files
+
+class GeoJsonConverter(Converter):
+    
+    def header(self):
+        self.fout.write("""{
+            "type": "FeatureCollection",
+            "features": [
+        """)
+    
+    def footer(self):
+        self.fout.write("""{ }]}""")
+    
+    def process(self, files):
+        # Write the header
+        self.header()
+        
+        hosts = {}
+        for banner in iterate_files(files):
+            ip = get_ip(banner)
+            if not ip:
+                continue
+    
+            if ip not in hosts:
+                hosts[ip] = banner
+                hosts[ip]['ports'] = []
+    
+            hosts[ip]['ports'].append(banner['port'])
+    
+        for ip, host in iter(hosts.items()):
+            self.write(host)
+        
+        self.footer()
+            
+    
+    def write(self, host):
+        try:
+            ip = get_ip(host)
+            lat, lon = host['location']['latitude'], host['location']['longitude']
+
+            feature = """{
+                "type": "Feature",
+                "id": "{}",
+                "properties": {
+                    "name": "{}"
+                 },
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [{}, {}]
+                }
+            }""".format(ip, ip, lat, lon)
+
+            self.fout.write(feature)
+        except Exception as e:
+            pass
diff --git a/shodan/helpers.py b/shodan/helpers.py
index d35d848..f8aa1ec 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -102,3 +102,9 @@ def get_screenshot(banner):
     if 'opts' in banner and 'screenshot' in banner['opts']:
         return banner['opts']['screenshot']
     return None
+
+
+def get_ip(banner):
+    if 'ipv6' in banner:
+        return banner['ipv6']
+    return banner['ip_str']

From 2b5a6012642f57781691d46750089d931b0e75e4 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 28 Dec 2016 19:57:06 -0600
Subject: [PATCH 079/263] Fixed bug when API error didn't contain an "error"
 property

---
 setup.py         | 2 +-
 shodan/client.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 9452b3c..aafcf26 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.1',
+    version = '1.6.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/client.py b/shodan/client.py
index 6010b8a..f491990 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -161,7 +161,7 @@ def _request(self, function, params, service='shodan', method='get'):
             raise APIError('Unable to connect to Shodan')
 
         # Check that the API key wasn't rejected
-        if data.status_code >= 401 and data.status_code < 500:
+        if data.status_code == 401:
             try:
                 # Return the actual error message if the API returned valid JSON
                 error = data.json()['error']

From 4e25fa657f7b990b331fe155c70594866b799423 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 29 Dec 2016 17:57:23 -0600
Subject: [PATCH 080/263] Move open_file and write_banner into helpers

---
 bin/shodan        | 28 ++++++++++++----------------
 setup.py          |  2 +-
 shodan/helpers.py |  9 +++++++++
 3 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index cc26ac7..e6d4804 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -68,7 +68,7 @@ def escape_data(args):
 def timestr():
     return datetime.datetime.utcnow().strftime('%Y-%m-%d')
 
-def open_file(directory, timestr, compresslevel=9):
+def open_streaming_file(directory, timestr, compresslevel=9):
     return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
 
 def get_banner_field(banner, flat_field):
@@ -113,10 +113,6 @@ def match_filters(banner, filters):
 
     return True
 
-def write_banner(fout, banner):
-    line = simplejson.dumps(banner) + '\n'
-    fout.write(line.encode('utf-8'))
-
 
 @with_plugins(iter_entry_points('shodan.cli.plugins'))
 @click.group(context_settings=CONTEXT_SETTINGS)
@@ -346,13 +342,13 @@ def download(limit, filename, query):
     if limit <= 0:
         limit = total
 
-    with gzip.open(filename, 'w') as fout:
+    with helpers.open_file(filename, 'w') as fout:
         count = 0
         try:
             cursor = api.search_cursor(query, minify=False)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
-                    write_banner(fout, banner)
+                    helpers.write_banner(fout, banner)
                     count += 1
 
                     if count >= limit:
@@ -492,7 +488,7 @@ def parse(color, fields, filters, filename, separator, filenames):
         # Add the appropriate extension if it's not there atm
         if not filename.endswith('.json.gz'):
             filename += '.json.gz'
-        fout = gzip.open(filename, 'a')
+        fout = helpers.open_file(filename)
 
     for banner in helpers.iterate_files(filenames):
         row = ''
@@ -503,7 +499,7 @@ def parse(color, fields, filters, filename, separator, filenames):
 
         # Append the data
         if fout:
-            write_banner(fout, banner)
+            helpers.write_banner(fout, banner)
 
         # Loop over all the fields and print the banner as a row
         for field in fields:
@@ -574,7 +570,7 @@ def scan_internet(quiet, port, protocol):
             # Create the output file
             filename = '{0}-{1}.json.gz'.format(port, protocol)
             counter = 0
-            with gzip.open(filename, 'w') as fout:
+            with helpers.open_file(filename, 'w') as fout:
                 click.echo('Saving results to file: {0}'.format(filename))
 
                 # Start listening for results
@@ -586,7 +582,7 @@ def scan_internet(quiet, port, protocol):
                     try:
                         for banner in api.stream.ports([port], timeout=90):
                             counter += 1
-                            write_banner(fout, banner)
+                            helpers.write_banner(fout, banner)
 
                             if not quiet:
                                 click.echo('{0:<40} {1:<20} {2}'.format(
@@ -673,7 +669,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
                 # Add the appropriate extension if it's not there atm
                 if not filename.endswith('.json.gz'):
                     filename += '.json.gz'
-                fout = gzip.open(filename, 'w')
+                fout = helpers.open_file(filename, 'w')
 
             # Start a spinner
             finished_event = threading.Event()
@@ -819,7 +815,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
 
                         # Save the banner in a file if necessary
                         if fout:
-                            write_banner(fout, hosts[ip][port])
+                            helpers.write_banner(fout, hosts[ip][port])
 
                     click.echo('')
             else:
@@ -1098,7 +1094,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
     fout = None
 
     if datadir:
-        fout = open_file(datadir, last_time, compresslevel)
+        fout = open_streaming_file(datadir, last_time, compresslevel)
 
     while not quit:
         try:
@@ -1117,8 +1113,8 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
                     if cur_time != last_time:
                             last_time = cur_time
                             fout.close()
-                            fout = open_file(datadir, last_time)
-                    write_banner(fout, banner)
+                            fout = open_streaming_file(datadir, last_time)
+                    helpers.write_banner(fout, banner)
 
                 # Print the banner information to stdout
                 if not quiet:
diff --git a/setup.py b/setup.py
index aafcf26..81f252e 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.2',
+    version = '1.6.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/helpers.py b/shodan/helpers.py
index f8aa1ec..73a5a72 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -108,3 +108,12 @@ def get_ip(banner):
     if 'ipv6' in banner:
         return banner['ipv6']
     return banner['ip_str']
+
+
+def open_file(filename, mode='a', compresslevel=9):
+    return gzip.open(filename, mode, compresslevel)
+
+
+def write_banner(fout, banner):
+    line = simplejson.dumps(banner) + '\n'
+    fout.write(line.encode('utf-8'))

From 9119242cee11769f3fafa35642a53240c914ce52 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 9 Jan 2017 15:56:50 -0600
Subject: [PATCH 081/263] Improved properties validation for host command

---
 bin/shodan | 8 ++++----
 setup.py   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index e6d4804..350fe23 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -418,9 +418,9 @@ def host(format, history, ip):
         for banner in sorted(host['data'], key=lambda k: k['port']):
             product = ''
             version = ''
-            if 'product' in banner:
+            if 'product' in banner and banner['product']:
                 product = banner['product']
-            if 'version' in banner:
+            if 'version' in banner and banner['version']:
                 version = '({})'.format(banner['version'])
 
             click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False)
@@ -434,9 +434,9 @@ def host(format, history, ip):
 
             # Show optional ssl info
             if 'ssl' in banner:
-                if 'versions' in banner['ssl']:
+                if 'versions' in banner['ssl'] and banner['ssl']['versions']:
                     click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
-                if 'dhparams' in banner['ssl']:
+                if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
                     click.echo('\t|-- Diffie-Hellman Parameters:')
                     click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
                     if 'fingerprint' in banner['ssl']['dhparams']:
diff --git a/setup.py b/setup.py
index 81f252e..ce726ba 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.3',
+    version = '1.6.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From d897f9a64d8949ad63360d2094c443d28ef05f11 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 7 Apr 2017 17:16:29 -0500
Subject: [PATCH 082/263] Fixed bug where the "stream" command didn't print
 nested properties

---
 bin/shodan | 11 ++++++-----
 setup.py   |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 350fe23..298bb4c 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1123,16 +1123,17 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
                     # Loop over all the fields and print the banner as a row
                     for field in fields:
                         tmp = ''
-                        if field in banner and banner[field]:
-                            field_type = type(banner[field])
+                        value = get_banner_field(banner, field)
+                        if value:
+                            field_type = type(value)
 
                             # If the field is an array then merge it together
                             if field_type == list:
-                                tmp = ';'.join(banner[field])
+                                tmp = ';'.join(value)
                             elif field_type in [int, float]:
-                                tmp = str(banner[field])
+                                tmp = str(value)
                             else:
-                                tmp = escape_data(banner[field])
+                                tmp = escape_data(value)
 
                             # Colorize certain fields if the user wants it
                             if color:
diff --git a/setup.py b/setup.py
index ce726ba..ab84368 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.4',
+    version = '1.6.5',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 29531e394775469a758de906984782de69a5c2dd Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 18 May 2017 17:27:48 -0500
Subject: [PATCH 083/263] Migrate to the stdlib json module

---
 bin/shodan          |  1 -
 requirements.txt    |  2 +-
 setup.py            |  4 +--
 shodan/api.py       |  3 +--
 shodan/client.py    |  4 +--
 shodan/helpers.py   |  8 +++---
 shodan/stream.py    |  6 ++---
 shodan/threatnet.py |  8 +++---
 shodan/wps.py       | 64 ---------------------------------------------
 9 files changed, 17 insertions(+), 83 deletions(-)
 delete mode 100644 shodan/wps.py

diff --git a/bin/shodan b/bin/shodan
index 298bb4c..1a8d609 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -29,7 +29,6 @@ import os
 import os.path
 import shodan
 import shodan.helpers as helpers
-import simplejson
 import socket
 import sys
 import threading
diff --git a/requirements.txt b/requirements.txt
index 05b1ace..bbf7b6a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 click
+click-plugins
 colorama
 requests
-simplejson
diff --git a/setup.py b/setup.py
index ab84368..6038efd 100755
--- a/setup.py
+++ b/setup.py
@@ -4,14 +4,14 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.5',
+    version = '1.6.6',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     scripts = ['bin/shodan'],
-    install_requires=["simplejson", "requests>=2.2.1", "click", "click-plugins", "colorama"],
+    install_requires=["requests>=2.2.1", "click", "click-plugins", "colorama"],
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
diff --git a/shodan/api.py b/shodan/api.py
index 3bafce4..2f54f08 100644
--- a/shodan/api.py
+++ b/shodan/api.py
@@ -7,8 +7,7 @@
     from urllib.request     import urlopen
     from urllib.parse       import urlencode
 
-# The simplejson library has better JSON-parsing than the standard library and is more often updated
-from simplejson import dumps, loads
+from json import dumps, loads
 
 from .exception import WebAPIError
 
diff --git a/shodan/client.py b/shodan/client.py
index f491990..4e77099 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -10,7 +10,7 @@
 import time
 
 import requests
-import simplejson
+import json
 
 from .exception import APIError
 from .helpers import api_request, create_facet_string
@@ -263,7 +263,7 @@ def scan(self, ips, force=False):
             ips = [ips]
         
         if isinstance(ips, dict):
-            networks = simplejson.dumps(ips)
+            networks = json.dumps(ips)
         else:
             networks = ','.join(ips)
 
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 73a5a72..6f01420 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -1,6 +1,6 @@
 import gzip
 import requests
-import simplejson
+import json
 
 from .exception import APIError
 
@@ -44,7 +44,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     while tries <= retries:
         try:
             if method.lower() == 'post':
-                data = requests.post(base_url + function, simplejson.dumps(data), params=params, headers={'content-type': 'application/json'})
+                data = requests.post(base_url + function, json.dumps(data), params=params, headers={'content-type': 'application/json'})
             elif method.lower() == 'delete':
                 data = requests.delete(base_url + function, params=params)
             else:
@@ -95,7 +95,7 @@ def iterate_files(files):
 
         for line in fin:
             # Convert the JSON into a native Python object
-            banner = simplejson.loads(line)
+            banner = json.loads(line)
             yield banner
 
 def get_screenshot(banner):
@@ -115,5 +115,5 @@ def open_file(filename, mode='a', compresslevel=9):
 
 
 def write_banner(fout, banner):
-    line = simplejson.dumps(banner) + '\n'
+    line = json.dumps(banner) + '\n'
     fout.write(line.encode('utf-8'))
diff --git a/shodan/stream.py b/shodan/stream.py
index 4bafc9a..5151a3e 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -1,5 +1,5 @@
 import requests
-import simplejson
+import json
 import ssl
 
 from .exception import APIError
@@ -24,7 +24,7 @@ def _create_stream(self, name, timeout=None):
 
         if req.status_code != 200:
             try:
-                data = simplejson.loads(req.text)
+                data = json.loads(req.text)
                 raise APIError(data['error'])
             except APIError as e:
                 raise
@@ -39,7 +39,7 @@ def _iter_stream(self, stream, raw):
                 if raw:
                     yield line
                 else:
-                    yield simplejson.loads(line)
+                    yield json.loads(line)
 
     def alert(self, aid=None, timeout=None, raw=False):
         if aid:
diff --git a/shodan/threatnet.py b/shodan/threatnet.py
index a252da9..792a924 100644
--- a/shodan/threatnet.py
+++ b/shodan/threatnet.py
@@ -1,5 +1,5 @@
 import requests
-import simplejson
+import json
 
 from .exception import APIError
 
@@ -37,21 +37,21 @@ def events(self):
             stream = self._create_stream('/threatnet/events')
             for line in stream.iter_lines():
                 if line:
-                    banner = simplejson.loads(line)
+                    banner = json.loads(line)
                     yield banner
 
         def backscatter(self):
             stream = self._create_stream('/threatnet/backscatter')
             for line in stream.iter_lines():
                 if line:
-                    banner = simplejson.loads(line)
+                    banner = json.loads(line)
                     yield banner
 
         def activity(self):
             stream = self._create_stream('/threatnet/ssh')
             for line in stream.iter_lines():
                 if line:
-                    banner = simplejson.loads(line)
+                    banner = json.loads(line)
                     yield banner
     
     def __init__(self, key):
diff --git a/shodan/wps.py b/shodan/wps.py
deleted file mode 100644
index 30dd215..0000000
--- a/shodan/wps.py
+++ /dev/null
@@ -1,64 +0,0 @@
-"""
-WiFi Positioning System
-
-Wrappers around the SkyHook and Google Locations APIs to resolve
-wireless routers' MAC addresses (BSSID) to physical locations.
-"""
-from simplejson import dumps, loads
-
-try:
-    from urllib2    import Request, urlopen
-    from urllib     import urlencode
-except:
-    from urllib.request     import Request, urlopen
-    from urllib.parse       import urlencode
-
-
-class Skyhook:
-    
-    """Not yet ready for production, use the GoogleLocation class instead."""
-    
-    def __init__(self, username='api', realm='shodan'):
-        self.username = username
-        self.realm = realm
-        self.url = 'https://api.skyhookwireless.com/wps2/location'
-    
-    def locate(self, mac):
-        # Remove the ':'
-        mac = mac.replace(':', '')
-        data = """<?xml version='1.0'?>  
-        <LocationRQ xmlns='http://skyhookwireless.com/wps/2005' version='2.6' street-address-lookup='full'>  
-          <authentication version='2.0'>  
-            <simple>  
-              <username>%s</username>  
-              <realm>%s</realm>  
-            </simple>  
-          </authentication>  
-          <access-point>  
-            <mac>%s</mac>  
-            <signal-strength>-50</signal-strength>  
-          </access-point>  
-        </LocationRQ>""" % (self.username, self.realm, mac)
-        request = Request(url=self.url, data=data, headers={'Content-type': 'text/xml'})
-        response = urlopen(request)
-        result = response.read()
-        return result
-
-class GoogleLocation:
-    
-    def __init__(self):
-        self.url = 'http://www.google.com/loc/json'
-    
-    def locate(self, mac):
-        data = {
-            'version': '1.1.0',
-            'request_address': True,
-            'wifi_towers': [{
-                'mac_address': mac,
-                'ssid': 'g',
-                'signal_strength': -72
-            }]
-        }
-        response = urlopen(self.url, dumps(data))
-        data = response.read()
-        return loads(data)

From ae3f862b67afa3717a218437d4952aed658aca9a Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 21 May 2017 14:01:23 -0500
Subject: [PATCH 084/263] Take advantage of "ujson" if it's installed

---
 shodan/helpers.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 6f01420..cea83d9 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -1,6 +1,14 @@
 import gzip
 import requests
-import json
+
+# Try to use ujson for parsing JSON if it's available
+# It's significantly faster at encoding/ decoding JSON but it doesn't support as
+# many options as the standard library. As such, we're mostly interested in using it for
+# decoding since reading/ parsing files will use up the most time.
+try:
+    import ujson as json
+except:
+    import json
 
 from .exception import APIError
 

From 47c3ae2d151a074cc9f9d75be01c3e0ea9f931e5 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 21 May 2017 15:36:21 -0500
Subject: [PATCH 085/263] Only use ujson module if "fast" option is specified
 in iterate_files() since ujson doesnt parse large banners (ex. HTTP) by
 default

---
 setup.py          |  2 +-
 shodan/helpers.py | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/setup.py b/setup.py
index 6038efd..c6d6b3a 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.6',
+    version = '1.6.7',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/helpers.py b/shodan/helpers.py
index cea83d9..8eda7a6 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -1,14 +1,6 @@
 import gzip
 import requests
-
-# Try to use ujson for parsing JSON if it's available
-# It's significantly faster at encoding/ decoding JSON but it doesn't support as
-# many options as the standard library. As such, we're mostly interested in using it for
-# decoding since reading/ parsing files will use up the most time.
-try:
-    import ujson as json
-except:
-    import json
+import json
 
 from .exception import APIError
 
@@ -89,8 +81,18 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     return data
 
 
-def iterate_files(files):
+def iterate_files(files, fast=False):
     """Loop over all the records of the provided Shodan output file(s)."""
+    if fast:
+        # Try to use ujson for parsing JSON if it's available and the user requested faster throughput
+        # It's significantly faster at encoding/ decoding JSON but it doesn't support as
+        # many options as the standard library. As such, we're mostly interested in using it for
+        # decoding since reading/ parsing files will use up the most time.
+        try:
+            from ujson import loads
+        except:
+            from json import loads
+    
     if isinstance(files, basestring):
         files = [files]
     

From 832c2af3f514a4dc0aa758bc782c350ed51ae5c1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 21 May 2017 15:37:31 -0500
Subject: [PATCH 086/263] Fix typo

---
 shodan/helpers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 8eda7a6..9f9b7b5 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -83,6 +83,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
 
 def iterate_files(files, fast=False):
     """Loop over all the records of the provided Shodan output file(s)."""
+    from json import loads
     if fast:
         # Try to use ujson for parsing JSON if it's available and the user requested faster throughput
         # It's significantly faster at encoding/ decoding JSON but it doesn't support as
@@ -91,7 +92,7 @@ def iterate_files(files, fast=False):
         try:
             from ujson import loads
         except:
-            from json import loads
+            pass
     
     if isinstance(files, basestring):
         files = [files]
@@ -105,7 +106,7 @@ def iterate_files(files, fast=False):
 
         for line in fin:
             # Convert the JSON into a native Python object
-            banner = json.loads(line)
+            banner = loads(line)
             yield banner
 
 def get_screenshot(banner):

From d97441b7301f87c0fa6a48a6acc9fc08dcd00b70 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 31 May 2017 10:33:40 -0500
Subject: [PATCH 087/263] Added helper method humanize_bytes

---
 shodan/helpers.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 9f9b7b5..eb25c74 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -128,3 +128,39 @@ def open_file(filename, mode='a', compresslevel=9):
 def write_banner(fout, banner):
     line = json.dumps(banner) + '\n'
     fout.write(line.encode('utf-8'))
+
+
+def humanize_bytes(bytes, precision=1):
+    """Return a humanized string representation of a number of bytes.
+
+    >>> humanize_bytes(1)
+    '1 byte'
+    >>> humanize_bytes(1024)
+    '1.0 kB'
+    >>> humanize_bytes(1024*123)
+    '123.0 kB'
+    >>> humanize_bytes(1024*12342)
+    '12.1 MB'
+    >>> humanize_bytes(1024*12342,2)
+    '12.05 MB'
+    >>> humanize_bytes(1024*1234,2)
+    '1.21 MB'
+    >>> humanize_bytes(1024*1234*1111,2)
+    '1.31 GB'
+    >>> humanize_bytes(1024*1234*1111,1)
+    '1.3 GB'
+    """
+    abbrevs = (
+        (1<<50L, 'PB'),
+        (1<<40L, 'TB'),
+        (1<<30L, 'GB'),
+        (1<<20L, 'MB'),
+        (1<<10L, 'kB'),
+        (1, 'bytes')
+    )
+    if bytes == 1:
+        return '1 byte'
+    for factor, suffix in abbrevs:
+        if bytes >= factor:
+            break
+    return '%.*f %s' % (precision, bytes / factor, suffix)

From 53840be840127a44b9ccc70bc75c1481b8d87c44 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 31 May 2017 10:35:36 -0500
Subject: [PATCH 088/263] Bump version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c6d6b3a..52765a1 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.7',
+    version = '1.6.8',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 2fb7f7f7daa5ffb019e0aea7154366d376fcd172 Mon Sep 17 00:00:00 2001
From: zopar <evidio@hotmail.it>
Date: Fri, 2 Jun 2017 17:13:25 +0100
Subject: [PATCH 089/263] Update helpers.py

Works on python 3 and solved problem about rounding in python 2
---
 shodan/helpers.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index eb25c74..b9ddc13 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -132,7 +132,6 @@ def write_banner(fout, banner):
 
 def humanize_bytes(bytes, precision=1):
     """Return a humanized string representation of a number of bytes.
-
     >>> humanize_bytes(1)
     '1 byte'
     >>> humanize_bytes(1024)
@@ -150,17 +149,15 @@ def humanize_bytes(bytes, precision=1):
     >>> humanize_bytes(1024*1234*1111,1)
     '1.3 GB'
     """
-    abbrevs = (
-        (1<<50L, 'PB'),
-        (1<<40L, 'TB'),
-        (1<<30L, 'GB'),
-        (1<<20L, 'MB'),
-        (1<<10L, 'kB'),
-        (1, 'bytes')
-    )
+
     if bytes == 1:
         return '1 byte'
-    for factor, suffix in abbrevs:
-        if bytes >= factor:
-            break
-    return '%.*f %s' % (precision, bytes / factor, suffix)
+    if bytes < 1024:
+        return '%.*f %s' % (precision, bytes, "bytes")
+    
+    suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
+    multiple = 1024.0    #.0 force float on python 2
+    for suffix in suffixes:
+        bytes /= multiple
+        if bytes < multiple:
+            return '%.*f %s' % (precision, bytes, suffix)

From 8dd9b293fc4db1e0cacc987eb1eabb96974de473 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 9 Jun 2017 14:29:13 -0500
Subject: [PATCH 090/263] New release to apply fix from #40

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 52765a1..3e1b719 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.8',
+    version = '1.6.9',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 93b2f61e7081bf8672e0c1a021837550efe85cdc Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 9 Jun 2017 14:34:17 -0500
Subject: [PATCH 091/263] Fixed a bug where humanize_bytes didn't return past
 1024 PB

---
 shodan/helpers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index b9ddc13..f95631b 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -161,3 +161,4 @@ def humanize_bytes(bytes, precision=1):
         bytes /= multiple
         if bytes < multiple:
             return '%.*f %s' % (precision, bytes, suffix)
+    return '%.*f %s' % (precision, bytes, suffix)

From 4ef43ace6b9197b67be79a790c3bff897d72de5d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 10 Jun 2017 01:12:11 -0500
Subject: [PATCH 092/263] Force UTF-8 encoding for Python output (potential fix
 for #34)

---
 bin/shodan | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/shodan b/bin/shodan
index 1a8d609..bacaef1 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 Shodan CLI
 

From 0f522fb33985fefc07f85e952e3236fecb877f0d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 16 Jun 2017 19:26:49 -0500
Subject: [PATCH 093/263] Added Excel file converter Fixed Python3 bug when
 iterating files Use the requirements.txt information for setup.py

---
 .gitignore                       |   1 +
 bin/shodan                       |   5 +-
 requirements.txt                 |   3 +-
 setup.py                         |   4 +-
 shodan/cli/converter/__init__.py |   7 +-
 shodan/cli/converter/excel.py    | 130 +++++++++++++++++++++++++++++++
 shodan/helpers.py                |   3 +
 7 files changed, 146 insertions(+), 7 deletions(-)
 create mode 100644 shodan/cli/converter/excel.py

diff --git a/.gitignore b/.gitignore
index 430a392..afb5610 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ build/*
 shodan.egg-info/*
 tmp/*
 MANIFEST
+.vscode/
diff --git a/bin/shodan b/bin/shodan
index bacaef1..9089e76 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -37,7 +37,7 @@ import requests
 import time
 
 # The file converters that are used to go from .json.gz to various other formats
-from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter
+from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter
 
 # Constants
 from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS
@@ -122,7 +122,7 @@ def main():
 
 @main.command()
 @click.argument('input', metavar='<input file>')
-@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json']))
+@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json', 'xlsx']))
 def convert(input, format):
     """Convert the given input data file into a different format.
 
@@ -147,6 +147,7 @@ def convert(input, format):
         'kml': KmlConverter,
         'csv': CsvConverter,
         'geo.json': GeoJsonConverter,
+        'xlsx': ExcelConverter,
     }.get(format)(fout)
     
     converter.process([input])
diff --git a/requirements.txt b/requirements.txt
index bbf7b6a..4fa2ed6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 click
 click-plugins
 colorama
-requests
+requests>=2.2.1
+XlsxWriter
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 3e1b719..dfc1ebc 100755
--- a/setup.py
+++ b/setup.py
@@ -2,6 +2,8 @@
 
 from setuptools import setup
 
+dependencies = open('requirements.txt', 'r').read().split('\n')
+
 setup(
     name = 'shodan',
     version = '1.6.9',
@@ -11,7 +13,7 @@
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     scripts = ['bin/shodan'],
-    install_requires=["requests>=2.2.1", "click", "click-plugins", "colorama"],
+    install_requires = dependencies,
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py
index b880502..1a697db 100644
--- a/shodan/cli/converter/__init__.py
+++ b/shodan/cli/converter/__init__.py
@@ -1,3 +1,4 @@
-from .csvc import *
-from .kml import *
-from .geojson import *
+from .csvc import CsvConverter
+from .kml import KmlConverter
+from .geojson import GeoJsonConverter
+from .excel import ExcelConverter
\ No newline at end of file
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
new file mode 100644
index 0000000..f5d2bb7
--- /dev/null
+++ b/shodan/cli/converter/excel.py
@@ -0,0 +1,130 @@
+
+from .base import Converter
+from ...helpers import iterate_files, get_ip
+
+from collections import defaultdict, MutableMapping
+from xlsxwriter import Workbook
+
+
+class ExcelConverter(Converter):
+
+    fields = [
+        'port',
+        'timestamp',
+        'data',
+        'hostnames',
+        'org',
+        'isp',
+        'location.country_name',
+        'location.country_code',
+        'location.city',
+        'os',
+        'asn',
+        'transport',
+        'product',
+        'version',
+        
+        'http.server',
+        'http.title',
+    ]
+
+    field_names = {
+        'org': 'Organization',
+        'isp': 'ISP',
+        'location.country_code': 'Country ISO Code',
+        'location.country_name': 'Country',
+        'location.city': 'City',
+        'os': 'OS',
+        'asn': 'ASN',
+
+        'http.server': 'Web Server',
+        'http.title': 'Website Title',
+    }
+    
+    def process(self, files):
+        # Get the filename from the already-open file handle
+        filename = self.fout.name
+
+        # Close the existing file as the XlsxWriter library handles that for us
+        self.fout.close()
+
+        # Create the new workbook
+        workbook = Workbook(filename)
+
+        # Define some common styles/ formats
+        bold = workbook.add_format({
+            'bold': 1,
+        })
+        
+        # Create the main worksheet where all the raw data is shown
+        main_sheet = workbook.add_worksheet('Raw Data')
+
+        # Write the header
+        main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently
+        main_sheet.set_column(0, 0, 20)
+        
+        row = 0
+        col = 1
+        for field in self.fields:
+            name = self.field_names.get(field, field.capitalize())
+            main_sheet.write(row, col, name, bold)
+            col += 1
+        row += 1
+
+        total = 0
+        ports = defaultdict(int)
+        for banner in iterate_files(files):
+            try:
+                # Build the list that contains all the relevant values
+                data = []
+                for field in self.fields:
+                    value = self.banner_field(banner, field)
+                    data.append(value)
+                
+                # Write those values to the main workbook
+                # Starting off w/ the special "IP" property
+                main_sheet.write_string(row, 0, get_ip(banner))
+                col = 1
+
+                for value in data:
+                    main_sheet.write(row, col, value)
+                    col += 1
+                row += 1
+            except:
+                pass
+            
+            # Aggregate summary information
+            total += 1
+            ports[banner['port']] += 1
+        
+        summary_sheet = workbook.add_worksheet('Summary')
+        summary_sheet.write(0, 0, 'Total', bold)
+        summary_sheet.write(0, 1, total)
+
+        # Ports Distribution
+        summary_sheet.write(0, 3, 'Ports Distribution', bold)
+        row = 1
+        col = 3
+        for key, value in sorted(ports.iteritems(), reverse=True, key=lambda (k, v): (v, k)):
+            summary_sheet.write(row, col, key)
+            summary_sheet.write(row, col + 1, value)
+            row += 1
+    
+    def banner_field(self, banner, flat_field):
+        # The provided field is a collapsed form of the actual field
+        fields = flat_field.split('.')
+    
+        try:
+            current_obj = banner
+            for field in fields:
+                current_obj = current_obj[field]
+            
+            # Convert a list into a concatenated string
+            if isinstance(current_obj, list):
+                current_obj = ','.join([str(i) for i in current_obj])
+            
+            return current_obj
+        except:
+            pass
+    
+        return ''
\ No newline at end of file
diff --git a/shodan/helpers.py b/shodan/helpers.py
index f95631b..dcf0afe 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -105,6 +105,9 @@ def iterate_files(files, fast=False):
             fin = open(filename, 'r')
 
         for line in fin:
+            # Ensure the line has been decoded into a string to prevent errors w/ Python3
+            line = line.decode('utf-8')
+
             # Convert the JSON into a native Python object
             banner = loads(line)
             yield banner

From e94e9140e72fbfe57cade5e0c43b1b3fdf726186 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 16 Jun 2017 19:27:25 -0500
Subject: [PATCH 094/263] Bump version for new release

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index dfc1ebc..5ae4cee 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.9',
+    version = '1.6.10',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 02cb5ff31a5de70396db431e3a03bacf61e503c1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 16 Jun 2017 19:32:25 -0500
Subject: [PATCH 095/263] Fixed lambda usage

---
 setup.py                      | 2 +-
 shodan/cli/converter/excel.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 5ae4cee..e2357bd 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.10',
+    version = '1.6.11',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index f5d2bb7..a5b8942 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -105,7 +105,7 @@ def process(self, files):
         summary_sheet.write(0, 3, 'Ports Distribution', bold)
         row = 1
         col = 3
-        for key, value in sorted(ports.iteritems(), reverse=True, key=lambda (k, v): (v, k)):
+        for key, value in sorted(ports.iteritems(), reverse=True, key=lambda kv: (kv[1], kv[0])):
             summary_sheet.write(row, col, key)
             summary_sheet.write(row, col + 1, value)
             row += 1

From f885d0f12d6e8e1f3cb9de478995f37467c19c99 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 16 Jun 2017 19:34:49 -0500
Subject: [PATCH 096/263] Fixed Python3 error due to iteritems()

---
 setup.py                      | 2 +-
 shodan/cli/converter/excel.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index e2357bd..43370ed 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.11',
+    version = '1.6.12',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index a5b8942..c22da90 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -105,7 +105,7 @@ def process(self, files):
         summary_sheet.write(0, 3, 'Ports Distribution', bold)
         row = 1
         col = 3
-        for key, value in sorted(ports.iteritems(), reverse=True, key=lambda kv: (kv[1], kv[0])):
+        for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])):
             summary_sheet.write(row, col, key)
             summary_sheet.write(row, col + 1, value)
             row += 1

From 220d679adfd9bcfaadb4a676e523f340a33b4ebd Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 20 Jun 2017 17:36:17 -0500
Subject: [PATCH 097/263] Added "images" convert output format to let users
 extract images from Shodan data files (#42)

---
 bin/shodan                       | 10 +++++--
 setup.py                         |  2 +-
 shodan/cli/converter/__init__.py |  5 ++--
 shodan/cli/converter/images.py   | 51 ++++++++++++++++++++++++++++++++
 4 files changed, 62 insertions(+), 6 deletions(-)
 create mode 100644 shodan/cli/converter/images.py

diff --git a/bin/shodan b/bin/shodan
index 9089e76..5f590ef 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -37,7 +37,7 @@ import requests
 import time
 
 # The file converters that are used to go from .json.gz to various other formats
-from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter
+from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter
 
 # Constants
 from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS
@@ -122,7 +122,7 @@ def main():
 
 @main.command()
 @click.argument('input', metavar='<input file>')
-@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json', 'xlsx']))
+@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json', 'images', 'xlsx']))
 def convert(input, format):
     """Convert the given input data file into a different format.
 
@@ -147,6 +147,7 @@ def convert(input, format):
         'kml': KmlConverter,
         'csv': CsvConverter,
         'geo.json': GeoJsonConverter,
+        'images': ImagesConverter,
         'xlsx': ExcelConverter,
     }.get(format)(fout)
     
@@ -155,7 +156,10 @@ def convert(input, format):
     finished_event.set()
     progress_bar_thread.join()
 
-    click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green'))
+    if format == 'images':
+        click.echo(click.style('\rSuccessfully extracted images to directory: {}'.format(converter.dirname), fg='green'))
+    else:
+        click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green'))
 
 
 @main.command()
diff --git a/setup.py b/setup.py
index 43370ed..fab0384 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.6.12',
+    version = '1.7.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py
index 1a697db..507ca0b 100644
--- a/shodan/cli/converter/__init__.py
+++ b/shodan/cli/converter/__init__.py
@@ -1,4 +1,5 @@
 from .csvc import CsvConverter
-from .kml import KmlConverter
+from .excel import ExcelConverter
 from .geojson import GeoJsonConverter
-from .excel import ExcelConverter
\ No newline at end of file
+from .images import ImagesConverter
+from .kml import KmlConverter
\ No newline at end of file
diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py
new file mode 100644
index 0000000..65c5a11
--- /dev/null
+++ b/shodan/cli/converter/images.py
@@ -0,0 +1,51 @@
+
+from .base import Converter
+from ...helpers import iterate_files, get_ip, get_screenshot
+
+from collections import defaultdict, MutableMapping
+from xlsxwriter import Workbook
+
+import os
+
+
+class ImagesConverter(Converter):
+
+    # The Images converter is special in that it creates a directory and there's
+    # special code in the Shodan CLI that relies on the "dirname" property to let
+    # the user know where the images have been stored.
+    dirname = None
+    
+    def process(self, files):
+        # Get the filename from the already-open file handle and use it as
+        # the directory name to store the images.
+        self.dirname = self.fout.name[:-7] + '-images'
+
+        # Remove the original file that was created
+        self.fout.close()
+        os.unlink(self.fout.name)
+
+        # Create the directory if it doesn't yet exist
+        if not os.path.exists(self.dirname):
+            os.mkdir(self.dirname)
+
+        # Close the existing file as the XlsxWriter library handles that for us
+        self.fout.close()
+
+        # Loop through all the banners in the data file
+        for banner in iterate_files(files):
+            screenshot = get_screenshot(banner)
+            if screenshot:
+                filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port'])
+
+                # If a file with the name already exists then count up until we
+                # create a new, unique filename
+                counter = 0
+                tmpname = filename
+                while os.path.exists(tmpname + '.jpg'):
+                    tmpname = '{}-{}'.format(filename, counter)
+                    counter += 1
+                filename = tmpname + '.jpg'
+
+                fout = open(filename, 'w')
+                fout.write(screenshot['data'].decode('base64'))
+                fout.close()

From 70abb8c1772fa5e54378a0110d634675009d8502 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 20 Jun 2017 17:48:58 -0500
Subject: [PATCH 098/263] Add the ability to save results from host lookups via
 the CLI (#43)

---
 bin/shodan | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 5f590ef..ccab705 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -370,8 +370,10 @@ def download(limit, filename, query):
 @main.command()
 @click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str)
 @click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True)
+@click.option('--filename', '-O', help='Save the host information in the given file (append if file exists).', default=None)
+@click.option('--save', '-S', help='Save the host information in the a file named after the IP (append if file exists).', default=False, is_flag=True)
 @click.argument('ip', metavar='<ip address>')
-def host(format, history, ip):
+def host(format, history, filename, save, ip):
     """View all available information for an IP address"""
     key = get_api_key()
     api = shodan.Shodan(key)
@@ -446,8 +448,24 @@ def host(format, history, ip):
                     click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
                     if 'fingerprint' in banner['ssl']['dhparams']:
                         click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+    
+        # Store the results
+        if filename or save:
+            if save:
+                filename = '{}.json.gz'.format(ip)
+            
+            # Add the appropriate extension if it's not there atm
+            if not filename.endswith('.json.gz'):
+                filename += '.json.gz'
+            
+            # Create/ append to the file
+            fout = helpers.open_file(filename)
+
+            for banner in sorted(host['data'], key=lambda k: k['port']):
+                helpers.write_banner(fout, banner)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
+        
 
 
 @main.command()

From a5fcbe3cd0310ec5621f8334b3cf2f1a5b3df6f6 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 20 Jun 2017 18:05:37 -0500
Subject: [PATCH 099/263] Python3 fixes for outputting images (#42)

---
 setup.py                       | 2 +-
 shodan/cli/converter/images.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index fab0384..df3d29f 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.0',
+    version = '1.7.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py
index 65c5a11..b239b4d 100644
--- a/shodan/cli/converter/images.py
+++ b/shodan/cli/converter/images.py
@@ -2,8 +2,8 @@
 from .base import Converter
 from ...helpers import iterate_files, get_ip, get_screenshot
 
-from collections import defaultdict, MutableMapping
-from xlsxwriter import Workbook
+# Needed for decoding base64-strings in Python3
+from codecs import decode
 
 import os
 
@@ -46,6 +46,6 @@ def process(self, files):
                     counter += 1
                 filename = tmpname + '.jpg'
 
-                fout = open(filename, 'w')
-                fout.write(screenshot['data'].decode('base64'))
+                fout = open(filename, 'wb')
+                fout.write(decode(screenshot['data'].encode(), 'base64'))
                 fout.close()

From 50b0449468d93690d00fd6ae17e377f28670d74c Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 4 Jul 2017 16:15:10 +0200
Subject: [PATCH 100/263] stream: handle timeout=None

None (default) can't be compared with integers
---
 shodan/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 5151a3e..d6d20ce 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -14,7 +14,7 @@ def __init__(self, api_key):
 
     def _create_stream(self, name, timeout=None):
         # The user doesn't want to use a timeout
-        if timeout <= 0:
+        if timeout and timeout <= 0:
             timeout = None
         
         try:

From 16a2af8f4c79c1d9835f217859808afb316e80d2 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 4 Jul 2017 16:18:04 +0200
Subject: [PATCH 101/263] stream: automatically decode to unicode

fixes streaming on python3
---
 shodan/stream.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 5151a3e..44ed60f 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -31,10 +31,12 @@ def _create_stream(self, name, timeout=None):
             except Exception as e:
                 pass
             raise APIError('Invalid API key or you do not have access to the Streaming API')
+        if req.encoding is None:
+            req.encoding = 'utf-8'
         return req
 
     def _iter_stream(self, stream, raw):
-        for line in stream.iter_lines():
+        for line in stream.iter_lines(decode_unicode=True):
             if line:
                 if raw:
                     yield line

From 85d0498bde5730b4a999736e115fcb09b8d03273 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 4 Jul 2017 17:22:34 +0200
Subject: [PATCH 102/263] Include docs in packages

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 645827f..5fdf9c3 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,5 @@
 include AUTHORS
 include LICENSE
 include requirements.txt
+graft docs
 recursive-include shodan *.py

From 47cdaf5192270aa7b93f51e42044b46b88a425e7 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 5 Jul 2017 16:20:25 -0500
Subject: [PATCH 103/263] Bump version

---
 .gitignore | 1 +
 setup.py   | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index afb5610..eca7a0a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ shodan.egg-info/*
 tmp/*
 MANIFEST
 .vscode/
+PKG-INFO
\ No newline at end of file
diff --git a/setup.py b/setup.py
index df3d29f..ed7127f 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.1',
+    version = '1.7.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
@@ -14,6 +14,7 @@
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     scripts = ['bin/shodan'],
     install_requires = dependencies,
+    keywords = ['security', 'network'],
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',

From 690bb0daffd5ee5487f19dfc8b49e5deb672c7b2 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 5 Jul 2017 16:21:06 -0500
Subject: [PATCH 104/263] Remove PKG-INFO as its built during PyPi sdist

---
 PKG-INFO | 15 ---------------
 1 file changed, 15 deletions(-)
 delete mode 100644 PKG-INFO

diff --git a/PKG-INFO b/PKG-INFO
deleted file mode 100644
index 1fa284c..0000000
--- a/PKG-INFO
+++ /dev/null
@@ -1,15 +0,0 @@
-Metadata-Version: 1.1
-Name: shodan
-Version: 0.7.0
-Summary: Python library for working with SHODAN
-Home-page: http://github.com/achillean/shodan-python/
-Author: John Matherly
-Author-email: jmath@shodanhq.com
-License: UNKNOWN
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
\ No newline at end of file

From 53437b9fdd1d97985d1f08f131e1c8756b411242 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 5 Jul 2017 16:43:52 -0500
Subject: [PATCH 105/263] Fixed the bug #47 which was caused by the CLI using a
 timeout value of "0" which resulted in the "requests" library failing to
 connect

---
 setup.py         | 2 +-
 shodan/stream.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index ed7127f..703cc3b 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.2',
+    version = '1.7.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/stream.py b/shodan/stream.py
index 8a02851..9db92db 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -14,7 +14,8 @@ def __init__(self, api_key):
 
     def _create_stream(self, name, timeout=None):
         # The user doesn't want to use a timeout
-        if timeout and timeout <= 0:
+        # If the timeout is specified as 0 then we also don't want to have a timeout
+        if ( timeout and timeout <= 0 ) or ( timeout == 0 ):
             timeout = None
         
         try:

From 744ca965525ebbf04bd8db624d3f93e93d4b880f Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 17 Jul 2017 00:58:01 -0500
Subject: [PATCH 106/263] Added "shodan radar" command

---
 bin/shodan             |  10 +++
 setup.py               |   2 +-
 shodan/cli/worldmap.py | 154 +++++++++++++++++++----------------------
 3 files changed, 82 insertions(+), 84 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index ccab705..a1302ba 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1202,6 +1202,16 @@ def honeyscore(ip):
     except:
         click.ClickException('Unable to calculate honeyscore')
 
+
+@main.command()
+def radar():
+    """Check whether the IP is a honeypot or not."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    from shodan.cli.worldmap import launch_map
+    launch_map(api)
+
 def async_spinner(finished):
     spinner = itertools.cycle(['-', '/', '|', '\\'])
     while not finished.is_set():
diff --git a/setup.py b/setup.py
index 703cc3b..cf2cecb 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.3',
+    version = '1.7.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 9ce8095..bda0b6a 100644
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -14,13 +14,11 @@
 
 '''
 import curses
-import json
 import locale
-import os
 import random
-import sys
 import time
-import urllib2
+
+from shodan.helpers import get_ip
 
 
 STREAMS = {
@@ -34,7 +32,31 @@
         'corners': (1, 4, 23, 73),
         # lat top, lon left, lat bottom, lon right
         'coords': [90.0, -180.0, -90.0, 180.0],
-        'file': 'map-world-01.txt',
+        'data': '''
+               . _..::__:  ,-"-"._       |7       ,     _,.__             
+       _.___ _ _<_>`!(._`.`-.    /        _._     `_ ,_/  '  '-._.---.-.__
+     .{     " " `-==,',._\{  \  / {)     / _ ">_,-' `                mt-2_
+      \_.:--.       `._ )`^-. "'      , [_/(                       __,/-' 
+     '"'     \         "    _L       oD_,--'                )     /. (|   
+              |           ,'         _)_.\\._<> 6              _,' /  '   
+              `.         /          [_/_'` `"(                <'}  )      
+               \\    .-. )          /   `-'"..' `:._          _)  '       
+        `        \  (  `(          /         `:\  > \  ,-^.  /' '         
+                  `._,   ""        |           \`'   \|   ?_)  {\         
+                     `=.---.       `._._       ,'     "`  |' ,- '.        
+                       |    `-._        |     /          `:`<_|h--._      
+                       (        >       .     | ,          `=.__.`-'\     
+                        `.     /        |     |{|              ,-.,\     .
+                         |   ,'          \   / `'            ,"     \     
+                         |  /             |_'                |  __  /     
+                         | |                                 '-'  `-'   \.
+                         |/                                        "    / 
+                         \.                                            '  
+                                                                          
+                          ,/           ______._.--._ _..---.---------._   
+         ,-----"-..?----_/ )      _,-'"             "                  (  
+    Map 1998 Matthew Thomas. Freely usable as long as this line is included
+'''
     }
 }
 
@@ -46,8 +68,7 @@ class AsciiMap(object):
     def __init__(self, map_name='world', map_conf=None, window=None, encoding=None):
         if map_conf is None:
             map_conf = MAPS[map_name]
-        with open(map_conf['file'], 'rb') as mapf:
-            self.map = mapf.read()
+        self.map = map_conf['data']
         self.coords = map_conf['coords']
         self.corners = map_conf['corners']
         if window is None:
@@ -101,35 +122,22 @@ def latlon_to_coords(self, lat, lon):
     def set_data(self, data):
         """
         Set / convert internal data.
-        For now it just selects a random set to show (good enough for demo purposes)
-        TODO: could use deque to show all entries
+        For now it just selects a random set to show.
         """
         entries = []
-        formats = [
-            u"{name} / {country} {city}",
-            u"{name} / {country}",
-            u"{name}",
-            u"{type}",
-        ]
-        dets = data.get('detections', [])
-        for det in random.sample(dets, min(len(dets), 5)):
-            #"city": "Montoire-sur-le-loir",
-            #"country": "FR",
-            #"lat": "47.7500",
-            #"long": "0.8667",
-            #"name": "Trojan.Generic.7555308",
-            #"type": "Trojan"
-            desc = "Detection"
-            # keeping it unicode here, encode() for curses later on
-            for fmt in formats:
-                try:
-                    desc = fmt.format(**det)
-                    break
-                except StandardError:
-                    pass
+
+        # Grab 5 random banners to display
+        for banner in random.sample(data, min(len(data), 5)):
+            desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code'])
+            if banner['location']['city']:
+                desc += ' {}'.format(banner['location']['city'])
+            
+            if 'tags' in banner and banner['tags']:
+                desc += ' / {}'.format(','.join(banner['tags']))
+            
             entry = (
-                float(det['lat']),
-                float(det['long']),
+                float(banner['location']['latitude']),
+                float(banner['location']['longitude']),
                 '*',
                 desc,
                 curses.A_BOLD,
@@ -137,24 +145,16 @@ def set_data(self, data):
             )
             entries.append(entry)
         self.data = entries
-        # for debugging... maybe it could be shown again now that we have the live stream support
-        #self.data_timestamp =  data.get('response_generated')
 
     def draw(self, target):
         """ Draw internal data to curses window """
         self.window.clear()
         self.window.addstr(0, 0, self.map)
-        debugdata = [
-            (60.16, 24.94, '*', self.data_timestamp, curses.A_BOLD, 'blue'), # Helsinki
-            #(90, -180, '1', 'top left', curses.A_BOLD, 'blue'),
-            #(-90, -180, '2', 'bottom left', curses.A_BOLD, 'blue'),
-            #(90, 180, '3', 'top right', curses.A_BOLD, 'blue'),
-            #(-90, 180, '4', 'bottom right', curses.A_BOLD, 'blue'),
-        ]
+
         # FIXME: position to be defined in map config?
         row = self.corners[2]-6
         items_to_show = 5
-        for lat, lon, char, desc, attrs, color in debugdata + self.data:
+        for lat, lon, char, desc, attrs, color in self.data:
             # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html
             if desc:
                 desc = desc.encode(self.encoding, 'ignore')
@@ -183,11 +183,12 @@ def draw(self, target):
 
 class MapApp(object):
     """ Virus World Map ncurses application """
-    def __init__(self, api_key):
-        self.api_key = api_key
+    def __init__(self, api):
+        self.api = api
         self.data = None
         self.last_fetch = 0
         self.sleep = 10  # tenths of seconds, for curses.halfdelay()
+        self.polling_interval = 60
 
     def fetch_data(self, epoch_now, force_refresh=False):
         """ (Re)fetch data from JSON stream """
@@ -195,20 +196,22 @@ def fetch_data(self, epoch_now, force_refresh=False):
         if force_refresh or self.data is None:
             refresh = True
         else:
-            # json data usually has: "polling_interval": 120
-            try:
-                poll_interval = int(self.data['polling_interval'])
-            except (ValueError, KeyError):
-                poll_interval = 60
-            if self.last_fetch + poll_interval <= epoch_now:
+            if self.last_fetch + self.polling_interval <= epoch_now:
                 refresh = True
-
+        
         if refresh:
             try:
-                self.data = json.load(urllib2.urlopen(self.stream_url))
+                # Grab 20 banners from the main stream
+                banners = []
+                for banner in self.api.stream.banners():
+                    if 'location' in banner and banner['location']['latitude']:
+                        banners.append(banner)
+                    if len(banners) >= 20:
+                        break
+                self.data = banners
                 self.last_fetch = epoch_now
             except StandardError:
-                pass
+                raise
         return refresh
 
     def run(self, scr):
@@ -223,46 +226,31 @@ def run(self, scr):
             scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD)
             scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD)
 
+            # Key Input
+            # q     - Quit
             event = scr.getch()
-            if event == ord("q"):
+            if event == ord('q'):
                 break
 
-            # if in replay mode?
-            #elif event == ord('-'):
-            #    self.sleep = min(self.sleep+10, 100)
-            #    curses.halfdelay(self.sleep)
-            #elif event == ord('+'):
-            #    self.sleep = max(self.sleep-10, 10)
-            #    curses.halfdelay(self.sleep)
-
-            elif event == ord('r'):
-                # force refresh
-                refresh = True
-            elif event == ord('c'):
-                # enter config mode
-                pass
-            elif event == ord('h'):
-                # show help screen
-                pass
-            elif event == ord('m'):
-                # cycle maps
-                pass
-
             # redraw window (to fix encoding/rendering bugs and to hide other messages to same tty)
             # user pressed 'r' or new data was fetched
             if refresh:
                 m.window.redrawwin()
 
 
+def launch_map(api):
+    app = MapApp(api)
+    return curses.wrapper(app.run)
+
+
 def main(argv=None):
     """ Main function / entry point """
-    if argv is None:
-        argv = sys.argv[1:]
-    conf = {}
-    if len(argv):
-        conf['stream_url'] = argv[0]
-    app = MapApp(conf)
-    return curses.wrapper(app.run_curses_app)
+    from shodan import Shodan
+    from shodan.cli.helpers import get_api_key
+
+    api = Shodan(get_api_key())
+    return launch_map(api)
 
 if __name__ == '__main__':
+    import sys
     sys.exit(main())

From 61023005f232ce88b4ed710c0769c24ffe71cb30 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 17 Jul 2017 00:59:56 -0500
Subject: [PATCH 107/263] Code cleanup

---
 shodan/cli/worldmap.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index bda0b6a..ad7b46a 100644
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -21,11 +21,6 @@
 from shodan.helpers import get_ip
 
 
-STREAMS = {
-    'filetest': 'wm3stream.json',
-    'wm3': 'http://worldmap3.f-secure.com/api/stream/',
-}
-
 MAPS = {
     'world': {
         # offset (as (y, x) for curses...)

From 2d2e3f5cfda5d22d9e3c1371085bc8ee89ae86ba Mon Sep 17 00:00:00 2001
From: francozappa <antonioli.daniele@gmail.com>
Date: Wed, 19 Jul 2017 15:25:51 +0800
Subject: [PATCH 108/263] Facet default is 5 items

---
 docs/examples/query-summary.rst | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst
index d54a8ba..7a60716 100644
--- a/docs/examples/query-summary.rst
+++ b/docs/examples/query-summary.rst
@@ -32,18 +32,18 @@ and country.
 	    'port',
 	    'asn',
 
-	    # We only care about the top 5 countries, this is how we let Shodan know to return 5 instead of the
-	    # default 10 for a facet. If you want to see more than 10, you could do ('country', 1000) for example
+	    # We only care about the top 3 countries, this is how we let Shodan know to return 3 instead of the
+	    # default 5 for a facet. If you want to see more than 5, you could do ('country', 1000) for example
 	    # to see the top 1,000 countries for a search query.
-	    ('country', 5),
+	    ('country', 3),
 	]
 
 	FACET_TITLES = {
-	    'org': 'Top 10 Organizations',
-	    'domain': 'Top 10 Domains',
-	    'port': 'Top 10 Ports',
-	    'asn': 'Top 10 Autonomous Systems',
-	    'country': 'Top 5 Countries',
+	    'org': 'Top 5 Organizations',
+	    'domain': 'Top 5 Domains',
+	    'port': 'Top 5 Ports',
+	    'asn': 'Top 5 Autonomous Systems',
+	    'country': 'Top 3 Countries',
 	}
 
 	# Input validation
@@ -89,38 +89,36 @@ and country.
 	Query: apache
 	Total Results: 34612043
 
-	Top 10 Organizations
+	Top 5 Organizations
 	Amazon.com: 808061
 	Ecommerce Corporation: 788704
 	Verio Web Hosting: 760112
 	Unified Layer: 627827
 	GoDaddy.com, LLC: 567004
 
-	Top 10 Domains
+	Top 5 Domains
 	secureserver.net: 562047
 	unifiedlayer.com: 494399
 	t-ipconnect.de: 385792
 	netart.pl: 194817
 	wanadoo.fr: 151925
 
-	Top 10 Ports
+	Top 5 Ports
 	80: 24118703
 	443: 8330932
 	8080: 1479050
 	81: 359025
 	8443: 231441
 
-	Top 10 Autonomous Systems
+	Top 5 Autonomous Systems
 	as32392: 580002
 	as2914: 465786
 	as26496: 414998
 	as48030: 332000
 	as8560: 255774
 
-	Top 5 Countries
+	Top 3 Countries
 	US: 13227366
 	DE: 2900530
 	JP: 2014506
-	CN: 1722048
-	GB: 1209938
 	"""

From 2042e5adf69d2eaf38f00cb068db8c83171c3a91 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Mon, 24 Jul 2017 11:12:51 +0200
Subject: [PATCH 109/263] mark worldmap as executable

---
 shodan/cli/worldmap.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 shodan/cli/worldmap.py

diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
old mode 100644
new mode 100755

From 794a95c2bd51131f2ba5369fae0f45a9a3d5ce09 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 30 Aug 2017 11:39:17 -0500
Subject: [PATCH 110/263] Handle Cloudflare timeouts

---
 setup.py         |  2 +-
 shodan/stream.py | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index cf2cecb..01606d5 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.4',
+    version = '1.7.5',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/stream.py b/shodan/stream.py
index 9db92db..fa7c67c 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -19,7 +19,17 @@ def _create_stream(self, name, timeout=None):
             timeout = None
         
         try:
-            req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
+            while True:
+                req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
+
+                # Status code 524 is special to Cloudflare
+                # It means that no data was sent from the streaming servers which caused Cloudflare
+                # to terminate the connection.
+                #
+                # We only want to exit if there was a timeout specified or the HTTP status code is
+                # not specific to Cloudflare.
+                if req.status_code != 524 or timeout >= 0:
+                    break
         except Exception as e:
             raise APIError('Unable to contact the Shodan Streaming API')
 

From eba2f7b43d5aa6397ed4466dcde4fab74139dc8d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 4 Dec 2017 23:55:34 -0600
Subject: [PATCH 111/263] Add basic support for the Bulk Data API

---
 bin/shodan       | 31 +++++++++++++++++++++++++++++++
 setup.py         |  2 +-
 shodan/client.py | 20 ++++++++++++++++++++
 shodan/stream.py | 14 +++++++++++++-
 4 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index a1302ba..497dbd8 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -302,6 +302,37 @@ def count(query):
     click.echo(results['total'])
 
 
+@main.group()
+def data():
+    """Bulk data access to Shodan"""
+    pass
+
+
+@data.command(name='list')
+@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str)
+def data_list(dataset):
+    """List available datasets or the files within those datasets."""
+    # Setup the API connection
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    if dataset:
+        # Show the files within this dataset
+        files = api.data.list_files(dataset)
+
+        for file in files:
+            click.echo(click.style('{:20s}'.format(file['name']), fg='cyan'), nl=False)
+            click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False)
+            click.echo('{}'.format(file['url']))
+    else:
+        # If no dataset was provided then show a list of all datasets
+        datasets = api.data.list_datasets()
+
+        for ds in datasets:
+            click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False)
+            click.echo('{}'.format(ds['description']))
+
+
 @main.command()
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
 @click.argument('filename', metavar='<filename>')
diff --git a/setup.py b/setup.py
index 01606d5..d890fc7 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.5',
+    version = '1.7.6',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/client.py b/shodan/client.py
index 4e77099..de2cd78 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -44,6 +44,25 @@ class Shodan:
     :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API.
     """
     
+    class Data:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def list_datasets(self):
+            """Returns a list of datasets that the user has permission to download.
+
+            :returns: A list of objects where every object describes a dataset
+            """
+            return self.parent._request('/shodan/data', {})
+
+        def list_files(self, dataset):
+            """Returns a list of files that belong to the given dataset.
+
+            :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url'
+            """
+            return self.parent._request('/shodan/data/{}'.format(dataset), {})
+    
     class Tools:
 
         def __init__(self, parent):
@@ -125,6 +144,7 @@ def __init__(self, key):
         self.api_key = key
         self.base_url = 'https://api.shodan.io'
         self.base_exploits_url = 'https://exploits.shodan.io'
+        self.data = self.Data(self)
         self.exploits = self.Exploits(self)
         self.labs = self.Labs(self)
         self.tools = self.Tools(self)
diff --git a/shodan/stream.py b/shodan/stream.py
index fa7c67c..864b1c5 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -46,13 +46,25 @@ def _create_stream(self, name, timeout=None):
             req.encoding = 'utf-8'
         return req
 
-    def _iter_stream(self, stream, raw):
+    def _iter_stream(self, stream, raw, timeout=None):
         for line in stream.iter_lines(decode_unicode=True):
+            # The Streaming API sends out heartbeat messages that are newlines
+            # We want to ignore those messages since they don't contain any data
             if line:
                 if raw:
                     yield line
                 else:
                     yield json.loads(line)
+            else:
+                # If the user specified a timeout then we want to keep track of how long we've
+                # been getting heartbeat messages and exit the loop if it's been too long since
+                # we've seen any activity.
+                if timeout:
+                    # TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke
+                    # the ability to use inactivity timeouts (the connection timeout still works). The timeout is
+                    # mostly needed when doing on-demand scans and wanting to temporarily consume data from a
+                    # network alert.
+                    pass
 
     def alert(self, aid=None, timeout=None, raw=False):
         if aid:

From 73a0c8b46d10f2411b755938c0659d051bc9e25f Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 5 Dec 2017 12:09:29 +0100
Subject: [PATCH 112/263] Add changelog for last releases

---
 CHANGES | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 CHANGES

diff --git a/CHANGES b/CHANGES
new file mode 100644
index 0000000..147a3e7
--- /dev/null
+++ b/CHANGES
@@ -0,0 +1,37 @@
+CHANGES
+=======
+
+in development
+--------------
+* added CHANGES file
+
+1.7.6
+-----
+* Add basic support for the Bulk Data API
+
+1.7.5
+-----
+ * Handle Cloudflare timeouts
+
+1.7.4
+-----
+ * Added "shodan radar" command
+
+1.7.3
+-----
+ *  Fixed the bug #47 which was caused by the CLI using a timeout value of "0" which resulted in the "requests" library failing to connect
+
+1.7.2
+-----
+ * stream: automatically decode to unicode, fixes streaming on python3 (#45)
+ * Include docs in packages (#46)
+ * stream: handle timeout=None, None (default) can't be compared with integers (#44)
+
+1.7.1
+-----
+ * Python3 fixes for outputting images (#42)
+ * Add the ability to save results from host lookups via the CLI (#43)
+
+1.7.0
+-----
+ * Added "images" convert output format to let users extract images from Shodan data files (#42)

From 569aa82bf1f6b81a565741f0a787ad3e128f793e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 7 Dec 2017 20:29:07 -0600
Subject: [PATCH 113/263] Added "shodan data download" command Updated list of
 available top-level commands in the comments

---
 CHANGES    |  4 ++++
 bin/shodan | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 setup.py   |  2 +-
 3 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 147a3e7..1863461 100644
--- a/CHANGES
+++ b/CHANGES
@@ -5,6 +5,10 @@ in development
 --------------
 * added CHANGES file
 
+1.7.7
+-----
+* Added "shodan data download" command to help download bulk data files
+
 1.7.6
 -----
 * Add basic support for the Bulk Data API
diff --git a/bin/shodan b/bin/shodan
index 497dbd8..20423ef 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -9,14 +9,21 @@ A simple interface to search Shodan, download data and parse compressed JSON fil
 The following commands are currently supported:
 
     alert
+    convert
     count
+    data
     download
+    honeyscore
     host
+    info
     init
     myip
     parse
+    radar
     scan
     search
+    stats
+    stream
 
 """
 
@@ -333,6 +340,59 @@ def data_list(dataset):
             click.echo('{}'.format(ds['description']))
 
 
+@data.command(name='download')
+@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int)
+@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.')
+@click.argument('dataset', metavar='<dataset>')
+@click.argument('name', metavar='<file>')
+def data_download(chunksize, filename, dataset, name):
+    # Setup the API connection
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    # Get the file object that the user requested which will contain the URL and total file size
+    file = None
+    try:
+        files = api.data.list_files(dataset)
+        for tmp in files:
+            if tmp['name'] == name:
+                file = tmp
+                break
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    
+    # The file isn't available
+    if not file:
+        raise click.ClickException('File not found')
+    
+    # Start downloading the file
+    response = requests.get(file['url'], stream=True)
+
+    # Figure out the size of the file based on the headers
+    filesize = response.headers.get('content-length', None)
+    if not filesize:
+        # Fall back to using the filesize provided by the API
+        filesize = file['size']
+    else:
+        filesize = int(filesize)
+    
+    chunk_size = 1024
+    limit = filesize / chunk_size
+
+    # Create a default filename based on the dataset and the filename within that dataset
+    if not filename:
+        filename = '{}-{}'.format(dataset, name)
+
+    # Open the output file and start writing to it in chunks
+    with open(filename, 'wb') as fout:
+        with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar:
+            for chunk in bar:
+                if chunk:
+                    fout.write(chunk)
+    
+    click.echo(click.style('Download completed: {}'.format(filename), 'green'))
+
+
 @main.command()
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
 @click.argument('filename', metavar='<filename>')
diff --git a/setup.py b/setup.py
index d890fc7..1e4a214 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.6',
+    version = '1.7.7',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From 37d9fd9b52264b25242b1b86b4df2c1536dea4f4 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Mon, 15 Jan 2018 17:44:35 +0100
Subject: [PATCH 114/263] Fix small issues found by lgtm

---
 bin/shodan            | 81 +++++++++++++++++++++----------------------
 shodan/cli/helpers.py |  2 --
 2 files changed, 40 insertions(+), 43 deletions(-)

diff --git a/bin/shodan b/bin/shodan
index 20423ef..4790fb2 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -96,7 +96,7 @@ def match_filters(banner, filters):
     for args in filters:
         flat_field, check = args.split(':', 1)
         value = get_banner_field(banner, flat_field)
-        
+
         # If the field doesn't exist on the banner then ignore the record
         if not value:
             return False
@@ -143,7 +143,7 @@ def convert(input, format):
 
     # Open the output file
     fout = open(filename, 'w')
-    
+
     # Start a spinner
     finished_event = threading.Event()
     progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
@@ -157,7 +157,7 @@ def convert(input, format):
         'images': ImagesConverter,
         'xlsx': ExcelConverter,
     }.get(format)(fout)
-    
+
     converter.process([input])
 
     finished_event.set()
@@ -233,7 +233,7 @@ def alert_create(name, netblock):
         alert = api.create_alert(name, netblock)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-    
+
     click.echo(click.style('Successfully created network alert!', fg='green'))
     click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan'))
 
@@ -360,11 +360,11 @@ def data_download(chunksize, filename, dataset, name):
                 break
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-    
+
     # The file isn't available
     if not file:
         raise click.ClickException('File not found')
-    
+
     # Start downloading the file
     response = requests.get(file['url'], stream=True)
 
@@ -375,7 +375,7 @@ def data_download(chunksize, filename, dataset, name):
         filesize = file['size']
     else:
         filesize = int(filesize)
-    
+
     chunk_size = 1024
     limit = filesize / chunk_size
 
@@ -389,7 +389,7 @@ def data_download(chunksize, filename, dataset, name):
             for chunk in bar:
                 if chunk:
                     fout.write(chunk)
-    
+
     click.echo(click.style('Download completed: {}'.format(filename), 'green'))
 
 
@@ -523,7 +523,7 @@ def host(format, history, filename, save, ip):
 
             click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False)
             click.echo('{} {}'.format(product, version), nl=False)
-            
+
             if history:
                 # Format the timestamp to only show the year-month-day
                 date = banner['timestamp'][:10]
@@ -539,16 +539,16 @@ def host(format, history, filename, save, ip):
                     click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
                     if 'fingerprint' in banner['ssl']['dhparams']:
                         click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
-    
+
         # Store the results
         if filename or save:
             if save:
                 filename = '{}.json.gz'.format(ip)
-            
+
             # Add the appropriate extension if it's not there atm
             if not filename.endswith('.json.gz'):
                 filename += '.json.gz'
-            
+
             # Create/ append to the file
             fout = helpers.open_file(filename)
 
@@ -556,7 +556,7 @@ def host(format, history, filename, save, ip):
                 helpers.write_banner(fout, banner)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-        
+
 
 
 @main.command()
@@ -765,7 +765,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
 
         click.echo('')
         click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
-        
+
         if verbose:
             click.echo('# Scan ID: {}'.format(scan['id']))
 
@@ -811,10 +811,10 @@ def scan_submit(wait, filename, force, verbose, netblocks):
                         # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
                         if time.time() - scan_start >= 60:
                             scan = api.scan_status(scan['id'])
-                            
+
                             if verbose:
                                 click.echo('# Scan status: {}'.format(scan['status']))
-                                
+
                             if scan['status'] == 'DONE':
                                 done = True
                                 break
@@ -834,7 +834,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
                     scan = api.scan_status(scan['id'])
                     if scan['status'] == 'DONE':
                         done = True
-                    
+
                     if verbose:
                         click.echo('# Scan status: {}'.format(scan['status']))
                 except socket.timeout as e:
@@ -1045,7 +1045,6 @@ def stats(limit, facets, filename, query):
     facets = [(facet, limit) for facet in facets]
 
     # Perform the search
-    api = shodan.Shodan(key)
     try:
         results = api.count(query, facets=facets)
     except shodan.APIError as e:
@@ -1061,12 +1060,12 @@ def stats(limit, facets, filename, query):
                 value = value.encode('ascii', errors='replace').decode('ascii')
             else:
                 value = str(value)
-            
+
             click.echo(click.style('{:28s}'.format(value), fg='cyan'), nl=False)
             click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
-        
+
     # Create the output file if requested
     fout = None
     if filename:
@@ -1074,43 +1073,43 @@ def stats(limit, facets, filename, query):
             filename += '.csv'
         fout = open(filename, 'w')
         writer = csv.writer(fout, dialect=csv.excel)
-        
+
         # Write the header
         writer.writerow(['Query', query])
-        
+
         # Add an empty line to separate rows
         writer.writerow([])
-        
+
         # Write the header that contains the facets
         row = []
         for facet in results['facets']:
             row.append(facet)
             row.append('')
         writer.writerow(row)
-        
+
         # Every facet has 2 columns (key, value)
         counter = 0
         has_items = True
         while has_items:
             row = ['' for i in range(len(results['facets']) * 2)]
-            
+
             pos = 0
             has_items = False
             for facet in results['facets']:
                 values = results['facets'][facet]
-                
+
                 # Add the values for the facet into the current row
                 if len(values) > counter:
                     has_items = True
                     row[pos] = values[counter]['value']
                     row[pos+1] = values[counter]['count']
-                
+
                 pos += 2
-            
+
             # Write out the row
             if has_items:
                 writer.writerow(row)
-            
+
             # Move to the next row of values
             counter += 1
 
@@ -1143,7 +1142,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
 
     if len(fields) == 0:
         raise click.ClickException('Please define at least one property to show')
-    
+
     # The user must choose "ports", "countries", "asn" or nothing - can't select multiple
     # filtered streams at once.
     stream_type = []
@@ -1155,30 +1154,30 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         stream_type.append('asn')
     if alert:
         stream_type.append('alert')
-    
+
     if len(stream_type) > 1:
         raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
 
     stream_args = None
-    
+
     # Turn the list of ports into integers
     if ports:
         try:
             stream_args = [int(item.strip()) for item in ports.split(',')]
         except:
             raise click.ClickException('Invalid list of ports')
-    
+
     if alert:
         alert = alert.strip()
         if alert.lower() != 'all':
             stream_args = alert
-    
+
     if asn:
         stream_args = asn.split(',')
-    
+
     if countries:
         stream_args = countries.split(',')
-    
+
     # Flatten the list of stream types
     # Possible values are:
     # - all
@@ -1199,7 +1198,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
             'countries': api.stream.countries(args, timeout=timeout),
             'ports': api.stream.ports(args, timeout=timeout),
         }.get(name, 'all')
-    
+
     stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
     counter = 0
@@ -1267,7 +1266,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         except:
             # For other errors lets just wait a bit and try to reconnect again
             time.sleep(1)
-            
+
             # Create a new stream object to subscribe to
             stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
@@ -1281,17 +1280,17 @@ def honeyscore(ip):
 
     try:
         score = api.labs.honeyscore(ip)
-        
+
         if score == 1.0:
             click.echo(click.style('Honeypot detected', fg='red'))
         elif score > 0.5:
             click.echo(click.style('Probably a honeypot', fg='yellow'))
         else:
             click.echo(click.style('Not a honeypot', fg='green'))
-        
+
         click.echo('Score: {}'.format(score))
     except:
-        click.ClickException('Unable to calculate honeyscore')
+        raise click.ClickException('Unable to calculate honeyscore')
 
 
 @main.command()
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 8fafe33..24f7b83 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -21,5 +21,3 @@ def get_api_key():
 
     with open(keyfile, 'r') as fin:
         return fin.read().strip()
-
-    raise click.ClickException('Please run "shodan init <api key>" before using this command')

From c37a1719687adace2f3d3f4d8cfe11c460b392f9 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 24 Feb 2018 13:32:09 -0600
Subject: [PATCH 115/263] Updated comment for "shodan radar"

---
 bin/shodan | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 4790fb2..108acb6 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -1295,7 +1295,7 @@ def honeyscore(ip):
 
 @main.command()
 def radar():
-    """Check whether the IP is a honeypot or not."""
+    """Real-Time Map of some results as Shodan finds them."""
     key = get_api_key()
     api = shodan.Shodan(key)
 

From e4aadf2aa009da45a2516ff038fc9c76d705fb4f Mon Sep 17 00:00:00 2001
From: BlackVirusScript <BlackVirusScript@Tutanota.com>
Date: Sat, 17 Mar 2018 14:42:55 -0300
Subject: [PATCH 116/263] Create `console_script` `entry_point` for `shodan`
 script. (Like this the console script can be executed on any platform.)

---
 setup.py                         | 2 +-
 bin/shodan => shodan/__main__.py | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename bin/shodan => shodan/__main__.py (100%)
 mode change 100755 => 100644

diff --git a/setup.py b/setup.py
index 1e4a214..8dfa7a4 100755
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
-    scripts = ['bin/shodan'],
+    entry_points = {'console_scripts': ['shodan = shodan.__main__:main']},
     install_requires = dependencies,
     keywords = ['security', 'network'],
     classifiers = [
diff --git a/bin/shodan b/shodan/__main__.py
old mode 100755
new mode 100644
similarity index 100%
rename from bin/shodan
rename to shodan/__main__.py

From 5977ad054901efd8ae91d96750965582240138eb Mon Sep 17 00:00:00 2001
From: BlackVirusScript <BlackVirusScript@Tutanota.com>
Date: Fri, 23 Mar 2018 23:15:14 -0300
Subject: [PATCH 117/263] Update __main__.py

---
 shodan/__main__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 108acb6..f629803 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 """
 Shodan CLI
 

From 1a1c22af63968e7b49693a0f713d8d47a91645d5 Mon Sep 17 00:00:00 2001
From: Jeremy Bae <jswbae@gmail.com>
Date: Wed, 23 May 2018 14:15:49 +0900
Subject: [PATCH 118/263] Add last_update time

---
 bin/shodan | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/shodan b/bin/shodan
index 108acb6..da47cab 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -489,6 +489,9 @@ def host(format, history, filename, save, ip):
         if 'org' in host and host['org']:
             click.echo('{:25s}{}'.format('Organization:', host['org']))
 
+        if 'last_update' in host and host['last_update']:
+            click.echo('{:25s}{}'.format('Updated:', host['last_update']))
+
         click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
 
         # Output the vulnerabilities the host has

From 75f1ef0016f38bdf3b2e59ef9fa7650454f033c2 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 28 May 2018 06:52:57 -0500
Subject: [PATCH 119/263] Updated documentation to make it Python3 compatible
 and point to the new help center Prevent the worldmap from erroring out on
 city names that can't be printed to the terminal

---
 README.rst                    |  5 +++--
 docs/examples/cert-stream.rst |  5 ++---
 docs/index.rst                |  6 +++++-
 docs/tutorial.rst             | 28 ++++++++++++++--------------
 shodan/cli/worldmap.py        |  6 +++++-
 5 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/README.rst b/README.rst
index b53284f..0faf74e 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,8 @@ Features
 --------
 
 - Search Shodan
-- Streaming API support for real-time consumption of Shodan data
+- Fast IP lookups
+- Streaming API support for real-time consumption of Shodan firehose
 - Exploit search API fully implemented
 
 
@@ -32,4 +33,4 @@ Or if you don't have pip installed (which you should seriously install):
 Documentation
 -------------
 
-Documentation is available at http://shodan.readthedocs.org/.
+Documentation is available at https://shodan.readthedocs.org/ and https://help.shodan.io
diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst
index afa5e25..e3e72c1 100644
--- a/docs/examples/cert-stream.rst
+++ b/docs/examples/cert-stream.rst
@@ -39,9 +39,8 @@ information.
 	    for banner in api.stream.ports([443, 8443]):
 			if 'ssl' in banner:
 				# Print out all the SSL information that Shodan has collected
-				print banner['ssl']
+				print(banner['ssl'])
 	    
 	except Exception as e:
-	    print 'Error: %s' % e
+	    print('Error: {}'.format(e))
 	    sys.exit(1)
-
diff --git a/docs/index.rst b/docs/index.rst
index db16146..956e335 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -11,6 +11,10 @@ Streaming API. And as a bonus it also lets you search for exploits using the Sho
 If you're not sure where to start simply go through the "Getting Started" section of the documentation and work your
 way down through the examples.
 
+For more information about Shodan and how to use the API please visit our official help center at:
+
+    https://help.shodan.io
+
 Introduction
 ~~~~~~~~~~~~
 .. toctree::
@@ -33,4 +37,4 @@ API Reference
 .. toctree::
     :maxdepth: 2
 
-    api
\ No newline at end of file
+    api
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index e8efcb2..00c4d34 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -51,13 +51,13 @@ Now that we have our API object all good to go, we're ready to perform a search:
 		results = api.search('apache')
 		
 		# Show the results
-		print 'Results found: %s' % results['total']
+		print('Results found: {}'.format(results['total']))
 		for result in results['matches']:
-			print 'IP: %s' % result['ip_str']
-			print result['data']
-			print ''
+			print('IP: {}'.format(result['ip_str']))
+			print(result['data'])
+			print('')
 	except shodan.APIError, e:
-		print 'Error: %s' % e
+		print('Error: {}'.format(e))
 
 Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which
 returns a dictionary of result information. We then print how many results were found in total,
@@ -101,16 +101,16 @@ To see what Shodan has available on a specific IP we can use the :py:func:`Shoda
 	host = api.host('217.140.75.46')
 	
 	# Print general info
-	print """
-		IP: %s
-		Organization: %s
-		Operating System: %s
-	""" % (host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a'))
+	print("""
+		IP: {}
+		Organization: {}
+		Operating System: {}
+	""".format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')))
 	
 	# Print all banners
 	for item in host['data']:
-		print """
-			Port: %s
-			Banner: %s
+		print("""
+			Port: {}
+			Banner: {}
 			
-		""" % (item['port'], item['data'])
\ No newline at end of file
+		""".format(item['port'], item['data']))
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index ad7b46a..9804aa2 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -125,7 +125,11 @@ def set_data(self, data):
         for banner in random.sample(data, min(len(data), 5)):
             desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code'])
             if banner['location']['city']:
-                desc += ' {}'.format(banner['location']['city'])
+                # Not all cities can be encoded in ASCII so ignore any errors
+                try:
+                    desc += ' {}'.format(banner['location']['city'])
+                except:
+                    pass
             
             if 'tags' in banner and banner['tags']:
                 desc += ' / {}'.format(','.join(banner['tags']))

From f1fe2f2d4720062d1ef881584601957b7c09d753 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 28 May 2018 07:52:01 -0500
Subject: [PATCH 120/263] Add tcp/ udp information to ports when doing a host
 lookup (#64)

---
 bin/shodan | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index da47cab..7104eca 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -524,7 +524,9 @@ def host(format, history, filename, save, ip):
             if 'version' in banner and banner['version']:
                 version = '({})'.format(banner['version'])
 
-            click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False)
+            click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
+            click.echo('/', nl=False)
+            click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
             click.echo('{} {}'.format(product, version), nl=False)
 
             if history:

From c46bb245acc522c8eec62e547f337cfbcb280ce4 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 28 May 2018 08:08:08 -0500
Subject: [PATCH 121/263] Show an error message instead of an empty screen if a
 search doesn't return results (#62)

---
 bin/shodan | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/shodan b/bin/shodan
index 7104eca..994aaa6 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -994,6 +994,10 @@ def search(color, fields, limit, separator, query):
         results = api.search(query, limit=limit)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
+    
+    # Error out if no results were found
+    if results['total'] == 0:
+        raise click.ClickException('No search results found')
 
     # We buffer the entire output so we can use click's pager functionality
     output = ''

From ca08e3f25b677259b465cc813600b446ef808c6d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 28 May 2018 23:38:59 -0500
Subject: [PATCH 122/263] Show open ports even if the API key doesn't have
 access to the data on those ports (#63)

---
 bin/shodan | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/bin/shodan b/bin/shodan
index 994aaa6..c06e0bd 100755
--- a/bin/shodan
+++ b/bin/shodan
@@ -515,6 +515,26 @@ def host(format, history, filename, save, ip):
 
         click.echo('')
 
+        # If the user doesn't have access to SSL/ Telnet results then we need
+        # to pad the host['data'] property with empty banners so they still see
+        # the port listed as open. (#63)
+        if len(host['ports']) != len(host['data']):
+            # Find the ports the user can't see the data for
+            ports = host['ports']
+            for banner in host['data']:
+                if banner['port'] in ports:
+                    ports.remove(banner['port'])
+            
+            # Add the placeholder banners
+            for port in ports:
+                banner = {
+                    'port': port,
+                    'transport': 'tcp', # All the filtered services use TCP
+                    'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
+                    'placeholder': True, # Don't store this banner when the file is saved
+                }
+                host['data'].append(banner)
+
         click.echo('Ports:')
         for banner in sorted(host['data'], key=lambda k: k['port']):
             product = ''
@@ -558,7 +578,8 @@ def host(format, history, filename, save, ip):
             fout = helpers.open_file(filename)
 
             for banner in sorted(host['data'], key=lambda k: k['port']):
-                helpers.write_banner(fout, banner)
+                if 'placeholder' not in banner:
+                    helpers.write_banner(fout, banner)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 

From 4ccbe7152db100c3197da3fc597fb92becd880f5 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 29 May 2018 05:26:34 -0500
Subject: [PATCH 123/263] Release 1.8.0 Updated changelog with notable fixes/
 improvements

---
 CHANGES => CHANGELOG.md | 14 +++++++++-----
 setup.py                |  2 +-
 2 files changed, 10 insertions(+), 6 deletions(-)
 rename CHANGES => CHANGELOG.md (71%)

diff --git a/CHANGES b/CHANGELOG.md
similarity index 71%
rename from CHANGES
rename to CHANGELOG.md
index 1863461..34a1bb2 100644
--- a/CHANGES
+++ b/CHANGELOG.md
@@ -1,9 +1,13 @@
-CHANGES
-=======
+CHANGELOG
+=========
 
-in development
---------------
-* added CHANGES file
+1.8.0
+-----
+* Shodan CLI now installs properly on Windows (#66)
+* Improved output of "shodan host" (#64, #67)
+* Fixed bug that prevented an open port from being shown in "shodan host" (#63)
+* No longer show an empty page if "shodan search" didn't return results (#62)
+* Updated docs to make them Python3 compatible
 
 1.7.7
 -----
diff --git a/setup.py b/setup.py
index 8dfa7a4..022b5a8 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.7.7',
+    version = '1.8.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',

From f46e3836fe055330ec3e73ae74f1699b6230493d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 31 May 2018 21:33:27 -0500
Subject: [PATCH 124/263] Minor housekeeping tasks (updated docs, metadata,
 license years)

---
 LICENSE    |  2 +-
 README.rst | 33 ++++++++++++++++++++++++++++++++-
 setup.py   |  9 +++++++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/LICENSE b/LICENSE
index e344a97..0af97af 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2014 John Matherly <jmath@shodanhq.com>
+Copyright (c) 2014- John Matherly <jmath@shodan.io>
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation
diff --git a/README.rst b/README.rst
index 0faf74e..2f4d675 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,12 @@
 shodan: The official Python library for accessing Shodan
 ========================================================
 
+.. image:: https://img.shields.io/pypi/v/shodan.svg
+    :target: https://pypi.org/project/shodan/
+
+.. image:: https://img.shields.io/github/contributors/achillean/shodan-python.svg
+    :target: https://github.com/achillean/shodan-python/graphs/contributors
+
 Shodan is a search engine for Internet-connected devices. Google lets you search for websites,
 Shodan lets you search for devices. This library provides developers easy access to all of the
 data stored in Shodan in order to automate tasks and integrate into existing tools.
@@ -9,10 +15,35 @@ Features
 --------
 
 - Search Shodan
-- Fast IP lookups
+- `Fast/ bulk IP lookups <https://help.shodan.io/developer-fundamentals/looking-up-ip-info>`_
 - Streaming API support for real-time consumption of Shodan firehose
+- `Network alerts (aka private firehose) <https://help.shodan.io/guides/how-to-monitor-network>`_
 - Exploit search API fully implemented
+- Bulk data downloads
+
+
+Quick Start
+-----------
+
+```python
+from shodan import Shodan
+
+api = Shodan('MY API KEY')
+
+# Lookup an IP
+ipinfo = api.host('8.8.8.8')
+print(ipinfo)
+
+# Search for websites that have been "hacked"
+for banner in api.search_cursor('http.title:"hacked by"'):
+    print(banner)
+
+# Get the total number of industrial control systems services on the Internet
+ics_services = api.count('tag:ics')
+print('Industrial Control Systems: {}'.format(ics_services['total']))
+```
 
+Grab your API key from https://account.shodan.io
 
 Installation
 ------------
diff --git a/setup.py b/setup.py
index 022b5a8..fc6cbd6 100755
--- a/setup.py
+++ b/setup.py
@@ -16,10 +16,19 @@
     install_requires = dependencies,
     keywords = ['security', 'network'],
     classifiers = [
+        'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.6',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
 )

From 04fa27c9abf6257f75c0821187fa0f04bd902df5 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 31 May 2018 21:34:31 -0500
Subject: [PATCH 125/263] Fix README to use rst

---
 README.rst | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/README.rst b/README.rst
index 2f4d675..308428a 100644
--- a/README.rst
+++ b/README.rst
@@ -25,23 +25,22 @@ Features
 Quick Start
 -----------
 
-```python
-from shodan import Shodan
+.. code-block:: python
+    from shodan import Shodan
 
-api = Shodan('MY API KEY')
+    api = Shodan('MY API KEY')
 
-# Lookup an IP
-ipinfo = api.host('8.8.8.8')
-print(ipinfo)
+    # Lookup an IP
+    ipinfo = api.host('8.8.8.8')
+    print(ipinfo)
 
-# Search for websites that have been "hacked"
-for banner in api.search_cursor('http.title:"hacked by"'):
-    print(banner)
+    # Search for websites that have been "hacked"
+    for banner in api.search_cursor('http.title:"hacked by"'):
+        print(banner)
 
-# Get the total number of industrial control systems services on the Internet
-ics_services = api.count('tag:ics')
-print('Industrial Control Systems: {}'.format(ics_services['total']))
-```
+    # Get the total number of industrial control systems services on the Internet
+    ics_services = api.count('tag:ics')
+    print('Industrial Control Systems: {}'.format(ics_services['total']))
 
 Grab your API key from https://account.shodan.io
 

From 335c3bf822c24d77b88f3308a2eeebc2adf21a0e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 31 May 2018 21:35:23 -0500
Subject: [PATCH 126/263] Another fix for README

---
 README.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.rst b/README.rst
index 308428a..d0a40ae 100644
--- a/README.rst
+++ b/README.rst
@@ -26,6 +26,7 @@ Quick Start
 -----------
 
 .. code-block:: python
+
     from shodan import Shodan
 
     api = Shodan('MY API KEY')

From 88ec367e852eda0739611a59c95a15503b6ed951 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 2 Jun 2018 01:57:04 -0500
Subject: [PATCH 127/263] Disable the heartbeat messages for the Streaming API
 if the user requested a timeout (#70)

---
 shodan/stream.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 864b1c5..9864854 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -13,14 +13,25 @@ def __init__(self, api_key):
         self.api_key = api_key
 
     def _create_stream(self, name, timeout=None):
+        params = {
+            'key': self.api_key,
+        }
+        stream_url = self.base_url + name
+        
         # The user doesn't want to use a timeout
         # If the timeout is specified as 0 then we also don't want to have a timeout
         if ( timeout and timeout <= 0 ) or ( timeout == 0 ):
             timeout = None
         
+        # If the user requested a timeout then we need to disable heartbeat messages
+        # which are intended to keep stream connections alive even if there isn't any data
+        # flowing through.
+        if timeout:
+            params['heartbeat'] = False
+        
         try:
             while True:
-                req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout)
+                req = requests.get(stream_url, params=params, stream=True, timeout=timeout)
 
                 # Status code 524 is special to Cloudflare
                 # It means that no data was sent from the streaming servers which caused Cloudflare
@@ -46,7 +57,7 @@ def _create_stream(self, name, timeout=None):
             req.encoding = 'utf-8'
         return req
 
-    def _iter_stream(self, stream, raw, timeout=None):
+    def _iter_stream(self, stream, raw):
         for line in stream.iter_lines(decode_unicode=True):
             # The Streaming API sends out heartbeat messages that are newlines
             # We want to ignore those messages since they don't contain any data
@@ -55,16 +66,6 @@ def _iter_stream(self, stream, raw, timeout=None):
                     yield line
                 else:
                     yield json.loads(line)
-            else:
-                # If the user specified a timeout then we want to keep track of how long we've
-                # been getting heartbeat messages and exit the loop if it's been too long since
-                # we've seen any activity.
-                if timeout:
-                    # TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke
-                    # the ability to use inactivity timeouts (the connection timeout still works). The timeout is
-                    # mostly needed when doing on-demand scans and wanting to temporarily consume data from a
-                    # network alert.
-                    pass
 
     def alert(self, aid=None, timeout=None, raw=False):
         if aid:

From 915a5e50d846890ebd09bd68ff8af67fc7422e4b Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 2 Jun 2018 02:14:35 -0500
Subject: [PATCH 128/263] Bugfix release 1.8.1 Minor fix to check that dhparams
 is not empty before using it

---
 CHANGELOG.md       | 4 ++++
 setup.py           | 2 +-
 shodan/__main__.py | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 34a1bb2..f97dd26 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.8.1
+-----
+* Fixed bug that prevented **shodan scan submit** from finishing (#70)
+
 1.8.0
 -----
 * Shodan CLI now installs properly on Windows (#66)
diff --git a/setup.py b/setup.py
index fc6cbd6..6520d3a 100755
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.8.0',
+    version = '1.8.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 256f199..4cb1ca6 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -898,7 +898,7 @@ def print_banner(banner):
                         versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
                         if len(versions) > 0:
                             click.echo('    |-- SSL Versions: {}'.format(', '.join(versions)))
-                    if 'dhparams' in banner['ssl']:
+                    if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
                         click.echo('    |-- Diffie-Hellman Parameters:')
                         click.echo('        {:15s}{}\n        {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
                         if 'fingerprint' in banner['ssl']['dhparams']:

From 9cc9988609434eb52043d38fa531611a4195cd8a Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 2 Jun 2018 02:29:54 -0500
Subject: [PATCH 129/263] Add description which will be shown on PyPi

---
 setup.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 6520d3a..59acf83 100755
--- a/setup.py
+++ b/setup.py
@@ -2,18 +2,21 @@
 
 from setuptools import setup
 
-dependencies = open('requirements.txt', 'r').read().split('\n')
+DEPENDENCIES = open('requirements.txt', 'r').read().split('\n')
+README = open('README.md', 'r').read()
 
 setup(
     name = 'shodan',
     version = '1.8.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
+    long_description = README,
+    long_description_content_type = 'text/markdown',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',
     packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
     entry_points = {'console_scripts': ['shodan = shodan.__main__:main']},
-    install_requires = dependencies,
+    install_requires = DEPENDENCIES,
     keywords = ['security', 'network'],
     classifiers = [
         'Development Status :: 5 - Production/Stable',

From 6d755dd3135e74b9f1ad04361b8db2a62d471e67 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 2 Jun 2018 02:32:25 -0500
Subject: [PATCH 130/263] Update description to load the README.rst

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 59acf83..9aa775c 100755
--- a/setup.py
+++ b/setup.py
@@ -3,14 +3,14 @@
 from setuptools import setup
 
 DEPENDENCIES = open('requirements.txt', 'r').read().split('\n')
-README = open('README.md', 'r').read()
+README = open('README.rst', 'r').read()
 
 setup(
     name = 'shodan',
     version = '1.8.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
-    long_description_content_type = 'text/markdown',
+    long_description_content_type = 'text/x-rst',
     author = 'John Matherly',
     author_email = 'jmath@shodan.io',
     url = 'http://github.com/achillean/shodan-python/tree/master',

From bd1fb8927f74a873b55fe3913bff048f4743ccdd Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 10 Jul 2018 11:24:40 +0200
Subject: [PATCH 131/263] Add proxy parameter

fixes achillean/shodan-python#72
---
 CHANGELOG.md        |  4 +++
 shodan/__main__.py  |  4 +--
 shodan/client.py    | 87 ++++++++++++++++++++++++---------------------
 shodan/helpers.py   | 32 +++++++++--------
 shodan/stream.py    | 14 ++++----
 shodan/threatnet.py | 12 ++++---
 6 files changed, 86 insertions(+), 67 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f97dd26..7e43fa4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+unreleased
+----------
+* New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72)
+
 1.8.1
 -----
 * Fixed bug that prevented **shodan scan submit** from finishing (#70)
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 4cb1ca6..3a2dded 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -522,7 +522,7 @@ def host(format, history, filename, save, ip):
             for banner in host['data']:
                 if banner['port'] in ports:
                     ports.remove(banner['port'])
-            
+
             # Add the placeholder banners
             for port in ports:
                 banner = {
@@ -1013,7 +1013,7 @@ def search(color, fields, limit, separator, query):
         results = api.search(query, limit=limit)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-    
+
     # Error out if no results were found
     if results['total'] == 0:
         raise click.ClickException('No search results found')
diff --git a/shodan/client.py b/shodan/client.py
index de2cd78..6c19e48 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -43,7 +43,7 @@ class Shodan:
     :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API.
     :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API.
     """
-    
+
     class Data:
 
         def __init__(self, parent):
@@ -62,7 +62,7 @@ def list_files(self, dataset):
             :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url'
             """
             return self.parent._request('/shodan/data/{}'.format(dataset), {})
-    
+
     class Tools:
 
         def __init__(self, parent):
@@ -74,16 +74,16 @@ def myip(self):
             :returns: str -- your IP address
             """
             return self.parent._request('/tools/myip', {})
-    
+
     class Exploits:
 
         def __init__(self, parent):
             self.parent = parent
-            
+
         def search(self, query, page=1, facets=None):
             """Search the entire Shodan Exploits archive using the same query syntax
             as the website.
-            
+
             :param query: The exploit search query; same syntax as website.
             :type query: str
             :param facets: A list of strings or tuples to get summary information on.
@@ -100,17 +100,17 @@ def search(self, query, page=1, facets=None):
                 query_args['facets'] = create_facet_string(facets)
 
             return self.parent._request('/api/search', query_args, service='exploits')
-            
+
         def count(self, query, facets=None):
             """Search the entire Shodan Exploits archive but only return the total # of results,
             not the actual exploits.
-            
+
             :param query: The exploit search query; same syntax as website.
             :type query: str
             :param facets: A list of strings or tuples to get summary information on.
             :type facets: str
             :returns: dict -- a dictionary containing the results of the search.
-            
+
             """
             query_args = {
                 'query': query,
@@ -119,7 +119,7 @@ def count(self, query, facets=None):
                 query_args['facets'] = create_facet_string(facets)
 
             return self.parent._request('/api/count', query_args, service='exploits')
-    
+
     class Labs:
 
         def __init__(self, parent):
@@ -127,19 +127,21 @@ def __init__(self, parent):
 
         def honeyscore(self, ip):
             """Calculate the probability of an IP being an ICS honeypot.
-            
+
             :param ip: IP address of the device
             :type ip: str
 
             :returns: int -- honeyscore ranging from 0.0 to 1.0
             """
             return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
-    
-    def __init__(self, key):
+
+    def __init__(self, key, proxies=None):
         """Initializes the API object.
-        
+
         :param key: The Shodan API key.
         :type key: str
+        :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'}
+        :type key: dict
         """
         self.api_key = key
         self.base_url = 'https://api.shodan.io'
@@ -148,23 +150,25 @@ def __init__(self, key):
         self.exploits = self.Exploits(self)
         self.labs = self.Labs(self)
         self.tools = self.Tools(self)
-        self.stream = Stream(key)
+        self.stream = Stream(key, proxies=proxies)
         self._session = requests.Session()
-            
+        if proxies:
+            self._session.proxies.update(proxies)
+
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.
-        
+
         Arguments:
             function  -- name of the function you want to execute
             params    -- dictionary of parameters for the function
-        
+
         Returns
             A dictionary containing the function's results.
-        
+
         """
         # Add the API key parameter automatically
         params['key'] = self.api_key
-        
+
         # Determine the base_url based on which service we're interacting with
         base_url = {
             'shodan': self.base_url,
@@ -187,22 +191,22 @@ def _request(self, function, params, service='shodan', method='get'):
                 error = data.json()['error']
             except Exception as e:
                 error = 'Invalid API key'
-            
+
             raise APIError(error)
-        
+
         # Parse the text into JSON
         try:
             data = data.json()
         except:
             raise APIError('Unable to parse JSON response')
-        
+
         # Raise an exception if an error occurred
         if type(data) == dict and 'error' in data:
             raise APIError(data['error'])
-        
+
         # Return the data
         return data
-    
+
     def count(self, query, facets=None):
         """Returns the total number of search results for the query.
 
@@ -210,7 +214,7 @@ def count(self, query, facets=None):
         :type query: str
         :param facets: (optional) A list of properties to get summary information on
         :type facets: str
-        
+
         :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
         """
         query_args = {
@@ -219,7 +223,7 @@ def count(self, query, facets=None):
         if facets:
             query_args['facets'] = create_facet_string(facets)
         return self._request('/shodan/host/count', query_args)
-    
+
     def host(self, ips, history=False, minify=False):
         """Get all available information on an IP.
 
@@ -232,14 +236,14 @@ def host(self, ips, history=False, minify=False):
         """
         if isinstance(ips, basestring):
             ips = [ips]
-        
+
         params = {}
         if history:
             params['history'] = history
         if minify:
             params['minify'] = minify
         return self._request('/shodan/host/%s' % ','.join(ips), params)
-    
+
     def info(self):
         """Returns information about the current API key, such as a list of add-ons
         and other features that are enabled for the current user's API plan.
@@ -281,7 +285,7 @@ def scan(self, ips, force=False):
         """
         if isinstance(ips, basestring):
             ips = [ips]
-        
+
         if isinstance(ips, dict):
             networks = json.dumps(ips)
         else:
@@ -320,7 +324,7 @@ def scan_status(self, scan_id):
         :returns: A dictionary with general information about the scan, including its status in getting processed.
         """
         return self._request('/shodan/scan/%s' % scan_id, {})
-    
+
     def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
         """Search the SHODAN database.
 
@@ -336,8 +340,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
         :type facets: str
         :param minify: (optional) Whether to minify the banner and only return the important data
         :type minify: bool
-        
-        :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.        
+
+        :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
         """
         args = {
             'query': query,
@@ -352,9 +356,9 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         if facets:
             args['facets'] = create_facet_string(facets)
-        
+
         return self._request('/shodan/host/search', args)
-    
+
     def search_cursor(self, query, minify=True, retries=5):
         """Search the SHODAN database.
 
@@ -369,7 +373,7 @@ def search_cursor(self, query, minify=True, retries=5):
         :type minify: bool
         :param retries: (optional) How often to retry the search in case it times out
         :type minify: int
-        
+
         :returns: A search cursor that can be used as an iterator/ generator.
         """
         args = {
@@ -396,13 +400,13 @@ def search_cursor(self, query, minify=True, retries=5):
 
                 tries += 1
                 time.sleep(1.0) # wait 1 second if the search errored out for some reason
-    
+
     def search_tokens(self, query):
         """Returns information about the search query itself (filters used etc.)
 
         :param query: Search query; identical syntax to the website
         :type query: str
-        
+
         :returns: A dictionary with 4 main properties: filters, errors, attributes and string.
         """
         query_args = {
@@ -481,7 +485,8 @@ def create_alert(self, name, ip, expires=0):
             'expires': expires,
         }
 
-        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post')
+        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post',
+                               proxies=self._session.proxies)
 
         return response
 
@@ -494,7 +499,8 @@ def alerts(self, aid=None, include_expired=True):
 
         response = api_request(self.api_key, func, params={
             'include_expired': include_expired,
-            })
+            },
+            proxies=self._session.proxies)
 
         return response
 
@@ -502,7 +508,8 @@ def delete_alert(self, aid):
         """Delete the alert with the given ID."""
         func = '/shodan/alert/%s' % aid
 
-        response = api_request(self.api_key, func, params={}, method='delete')
+        response = api_request(self.api_key, func, params={}, method='delete',
+                               proxies=self._session.proxies)
 
         return response
 
diff --git a/shodan/helpers.py b/shodan/helpers.py
index dcf0afe..95df590 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -23,17 +23,19 @@ def create_facet_string(facets):
         facet_str += ','
     return facet_str[:-1]
 
-    
-def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get', retries=1):
+
+def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io',
+                method='get', retries=1, proxies=None):
     """General-purpose function to create web requests to SHODAN.
-    
+
     Arguments:
         function  -- name of the function you want to execute
         params    -- dictionary of parameters for the function
-    
+        proxies   -- a proxies array for the requests library
+
     Returns
         A dictionary containing the function's results.
-    
+
     """
     # Add the API key parameter automatically
     params['key'] = key
@@ -44,11 +46,13 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     while tries <= retries:
         try:
             if method.lower() == 'post':
-                data = requests.post(base_url + function, json.dumps(data), params=params, headers={'content-type': 'application/json'})
+                data = requests.post(base_url + function, json.dumps(data), params=params,
+                                     headers={'content-type': 'application/json'},
+                                     proxies=proxies)
             elif method.lower() == 'delete':
-                data = requests.delete(base_url + function, params=params)
+                data = requests.delete(base_url + function, params=params, proxies=proxies)
             else:
-                data = requests.get(base_url + function, params=params)
+                data = requests.get(base_url + function, params=params, proxies=proxies)
 
             # Exit out of the loop
             break
@@ -66,17 +70,17 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
         except:
             pass
         raise APIError('Invalid API key')
-    
+
     # Parse the text into JSON
     try:
         data = data.json()
     except:
         raise APIError('Unable to parse JSON response')
-    
+
     # Raise an exception if an error occurred
     if type(data) == dict and data.get('error', None):
         raise APIError(data['error'])
-    
+
     # Return the data
     return data
 
@@ -93,10 +97,10 @@ def iterate_files(files, fast=False):
             from ujson import loads
         except:
             pass
-    
+
     if isinstance(files, basestring):
         files = [files]
-    
+
     for filename in files:
         # Create a file handle depending on the filetype
         if filename.endswith('.gz'):
@@ -157,7 +161,7 @@ def humanize_bytes(bytes, precision=1):
         return '1 byte'
     if bytes < 1024:
         return '%.*f %s' % (precision, bytes, "bytes")
-    
+
     suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
     multiple = 1024.0    #.0 force float on python 2
     for suffix in suffixes:
diff --git a/shodan/stream.py b/shodan/stream.py
index 9864854..49ab633 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -9,29 +9,31 @@ class Stream:
 
     base_url = 'https://stream.shodan.io'
 
-    def __init__(self, api_key):
+    def __init__(self, api_key, proxies=None):
         self.api_key = api_key
+        self.proxies = proxies
 
     def _create_stream(self, name, timeout=None):
         params = {
             'key': self.api_key,
         }
         stream_url = self.base_url + name
-        
+
         # The user doesn't want to use a timeout
         # If the timeout is specified as 0 then we also don't want to have a timeout
         if ( timeout and timeout <= 0 ) or ( timeout == 0 ):
             timeout = None
-        
+
         # If the user requested a timeout then we need to disable heartbeat messages
         # which are intended to keep stream connections alive even if there isn't any data
         # flowing through.
         if timeout:
             params['heartbeat'] = False
-        
+
         try:
             while True:
-                req = requests.get(stream_url, params=params, stream=True, timeout=timeout)
+                req = requests.get(stream_url, params=params, stream=True, timeout=timeout,
+                                   proxies=self.proxies)
 
                 # Status code 524 is special to Cloudflare
                 # It means that no data was sent from the streaming servers which caused Cloudflare
@@ -121,4 +123,4 @@ def ports(self, ports, raw=False, timeout=None):
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
-            
+
diff --git a/shodan/threatnet.py b/shodan/threatnet.py
index 792a924..8b609f0 100644
--- a/shodan/threatnet.py
+++ b/shodan/threatnet.py
@@ -11,17 +11,19 @@ class Threatnet:
     :type key: str
     :ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API.
     """
-    
+
     class Stream:
 
         base_url = 'https://stream.shodan.io'
 
-        def __init__(self, parent):
+        def __init__(self, parent, proxies=None):
             self.parent = parent
+            self.proxies = proxies
 
         def _create_stream(self, name):
             try:
-                req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True)
+                req = requests.get(self.base_url + name, params={'key': self.parent.api_key},
+                                   stream=True, proxies=self.proxies)
             except:
                 raise APIError('Unable to contact the Shodan Streaming API')
 
@@ -53,10 +55,10 @@ def activity(self):
                 if line:
                     banner = json.loads(line)
                     yield banner
-    
+
     def __init__(self, key):
         """Initializes the API object.
-        
+
         :param key: The Shodan API key.
         :type key: str
         """

From c161708a3cdcec818360f5030a79c770520bcafc Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 1 Aug 2018 01:22:00 -0500
Subject: [PATCH 132/263] Release 1.9.0

---
 CHANGELOG.md | 2 +-
 setup.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e43fa4..840f1ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,7 @@
 CHANGELOG
 =========
 
-unreleased
+1.9.0
 ----------
 * New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72)
 
diff --git a/setup.py b/setup.py
index 9aa775c..c52e5e0 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.8.1',
+    version = '1.9.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',

From d8b124e64fd1ea6871b4617dcfcc7e5ae16508f4 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Wed, 1 Aug 2018 10:23:24 +0200
Subject: [PATCH 133/263] DOC: add changelog to manifest

---
 CHANGELOG.md | 6 +++++-
 MANIFEST.in  | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 840f1ab..5bdc40e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,12 @@
 CHANGELOG
 =========
 
-1.9.0
+unreleased
 ----------
+* The CHANGELOG is now part of the packages.
+
+1.9.0
+-----
 * New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72)
 
 1.8.1
diff --git a/MANIFEST.in b/MANIFEST.in
index 5fdf9c3..4e76fba 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,6 @@
 include AUTHORS
 include LICENSE
 include requirements.txt
+inlcude CHANGELOG.md
 graft docs
 recursive-include shodan *.py

From 48047b4973ddbde48bae1c2e21848f500d2ec7a1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@surtri.com>
Date: Wed, 1 Aug 2018 17:59:40 -0500
Subject: [PATCH 134/263] Fixed a typo

---
 MANIFEST.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 4e76fba..4ba799c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,6 @@
 include AUTHORS
 include LICENSE
 include requirements.txt
-inlcude CHANGELOG.md
+include CHANGELOG.md
 graft docs
 recursive-include shodan *.py

From e6137b5b63515607bb6fc8e5a8e8ca7c7a9e7068 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 14:14:15 -0500
Subject: [PATCH 135/263] Ensure strings that can contain non-ascii characters
 are treated as unicode for Python2 (#78)

---
 shodan/__main__.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 3a2dded..1c2dbe4 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -212,7 +212,7 @@ def alert_clear():
     try:
         alerts = api.alerts()
         for alert in alerts:
-            click.echo('Removing {} ({})'.format(alert['name'], alert['id']))
+            click.echo(u'Removing {} ({})'.format(alert['name'], alert['id']))
             api.delete_alert(alert['id'])
     except shodan.APIError as e:
         raise click.ClickException(e.value)
@@ -249,11 +249,11 @@ def alert_list(expired):
         raise click.ClickException(e.value)
 
     if len(results) > 0:
-        click.echo('# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
+        click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
         # click.echo('#' * 65)
         for alert in results:
             click.echo(
-                '{:16} {:<30} {:<35} '.format(
+                u'{:16} {:<30} {:<35} '.format(
                     click.style(alert['id'],  fg='yellow'),
                     click.style(alert['name'], fg='cyan'),
                     click.style(', '.join(alert['filters']['ip']), fg='white')
@@ -326,7 +326,7 @@ def data_list(dataset):
         files = api.data.list_files(dataset)
 
         for file in files:
-            click.echo(click.style('{:20s}'.format(file['name']), fg='cyan'), nl=False)
+            click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False)
             click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False)
             click.echo('{}'.format(file['url']))
     else:
@@ -473,19 +473,19 @@ def host(format, history, filename, save, ip):
         # General info
         click.echo(click.style(ip, fg='green'))
         if len(host['hostnames']) > 0:
-            click.echo('{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
+            click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
 
         if 'city' in host and host['city']:
-            click.echo('{:25s}{}'.format('City:', host['city']))
+            click.echo(u'{:25s}{}'.format('City:', host['city']))
 
         if 'country_name' in host and host['country_name']:
-            click.echo('{:25s}{}'.format('Country:', host['country_name']))
+            click.echo(u'{:25s}{}'.format('Country:', host['country_name']))
 
         if 'os' in host and host['os']:
-            click.echo('{:25s}{}'.format('Operating System:', host['os']))
+            click.echo(u'{:25s}{}'.format('Operating System:', host['os']))
 
         if 'org' in host and host['org']:
-            click.echo('{:25s}{}'.format('Organization:', host['org']))
+            click.echo(u'{:25s}{}'.format('Organization:', host['org']))
 
         if 'last_update' in host and host['last_update']:
             click.echo('{:25s}{}'.format('Updated:', host['last_update']))

From 743b445cf7babe9e3742bdff4406bf0b1760b7a8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 14:15:28 -0500
Subject: [PATCH 136/263] Add CHANGELOG entry for #78

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bdc40e..64817d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ CHANGELOG
 unreleased
 ----------
 * The CHANGELOG is now part of the packages.
+* Improved unicode handling in Python2 (#78)
 
 1.9.0
 -----

From a3e5855779456ba7a6bcba05c8a4ebf2b21a9743 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 15:52:58 -0500
Subject: [PATCH 137/263] Add tsv output format for shodan host (#65)

---
 CHANGELOG.md       |   1 +
 shodan/__main__.py |  97 ++----------------------------------
 shodan/cli/host.py | 121 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 126 insertions(+), 93 deletions(-)
 create mode 100644 shodan/cli/host.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64817d4..dae37d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ unreleased
 ----------
 * The CHANGELOG is now part of the packages.
 * Improved unicode handling in Python2 (#78)
+* Add `tsv` output format for **shodan host** (#65)
 
 1.9.0
 -----
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 1c2dbe4..f247432 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -49,6 +49,7 @@
 
 # Helper methods
 from shodan.cli.helpers import get_api_key
+from shodan.cli.host import HOST_PRINT
 
 # Allow 3rd-parties to develop custom commands
 from click_plugins import with_plugins
@@ -457,7 +458,7 @@ def download(limit, filename, query):
 
 
 @main.command()
-@click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str)
+@click.option('--format', help='The output format for the host information. Possible values are: pretty, tsv.', default='pretty', type=click.Choice(['pretty', 'tsv']))
 @click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True)
 @click.option('--filename', '-O', help='Save the host information in the given file (append if file exists).', default=None)
 @click.option('--save', '-S', help='Save the host information in the a file named after the IP (append if file exists).', default=False, is_flag=True)
@@ -470,98 +471,8 @@ def host(format, history, filename, save, ip):
     try:
         host = api.host(ip, history=history)
 
-        # General info
-        click.echo(click.style(ip, fg='green'))
-        if len(host['hostnames']) > 0:
-            click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
-
-        if 'city' in host and host['city']:
-            click.echo(u'{:25s}{}'.format('City:', host['city']))
-
-        if 'country_name' in host and host['country_name']:
-            click.echo(u'{:25s}{}'.format('Country:', host['country_name']))
-
-        if 'os' in host and host['os']:
-            click.echo(u'{:25s}{}'.format('Operating System:', host['os']))
-
-        if 'org' in host and host['org']:
-            click.echo(u'{:25s}{}'.format('Organization:', host['org']))
-
-        if 'last_update' in host and host['last_update']:
-            click.echo('{:25s}{}'.format('Updated:', host['last_update']))
-
-        click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
-
-        # Output the vulnerabilities the host has
-        if 'vulns' in host and len(host['vulns']) > 0:
-            vulns = []
-            for vuln in host['vulns']:
-                if vuln.startswith('!'):
-                    continue
-                if vuln.upper() == 'CVE-2014-0160':
-                    vulns.append(click.style('Heartbleed', fg='red'))
-                else:
-                    vulns.append(click.style(vuln, fg='red'))
-
-            if len(vulns) > 0:
-                click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
-
-                for vuln in vulns:
-                    click.echo(vuln + '\t', nl=False)
-
-                click.echo('')
-
-        click.echo('')
-
-        # If the user doesn't have access to SSL/ Telnet results then we need
-        # to pad the host['data'] property with empty banners so they still see
-        # the port listed as open. (#63)
-        if len(host['ports']) != len(host['data']):
-            # Find the ports the user can't see the data for
-            ports = host['ports']
-            for banner in host['data']:
-                if banner['port'] in ports:
-                    ports.remove(banner['port'])
-
-            # Add the placeholder banners
-            for port in ports:
-                banner = {
-                    'port': port,
-                    'transport': 'tcp', # All the filtered services use TCP
-                    'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
-                    'placeholder': True, # Don't store this banner when the file is saved
-                }
-                host['data'].append(banner)
-
-        click.echo('Ports:')
-        for banner in sorted(host['data'], key=lambda k: k['port']):
-            product = ''
-            version = ''
-            if 'product' in banner and banner['product']:
-                product = banner['product']
-            if 'version' in banner and banner['version']:
-                version = '({})'.format(banner['version'])
-
-            click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
-            click.echo('/', nl=False)
-            click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
-            click.echo('{} {}'.format(product, version), nl=False)
-
-            if history:
-                # Format the timestamp to only show the year-month-day
-                date = banner['timestamp'][:10]
-                click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
-            click.echo('')
-
-            # Show optional ssl info
-            if 'ssl' in banner:
-                if 'versions' in banner['ssl'] and banner['ssl']['versions']:
-                    click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
-                if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
-                    click.echo('\t|-- Diffie-Hellman Parameters:')
-                    click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
-                    if 'fingerprint' in banner['ssl']['dhparams']:
-                        click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+        # Print the host information to the terminal using the user-specified format
+        HOST_PRINT[format](host, history=history)
 
         # Store the results
         if filename or save:
diff --git a/shodan/cli/host.py b/shodan/cli/host.py
new file mode 100644
index 0000000..fcd440f
--- /dev/null
+++ b/shodan/cli/host.py
@@ -0,0 +1,121 @@
+# Helper methods for printing `host` information to the terminal.
+import click
+
+from shodan.helpers import get_ip
+
+
+def host_print_pretty(host, history=False):
+    """Show the host information in a user-friendly way and try to include
+    as much relevant information as possible."""
+    # General info
+    click.echo(click.style(get_ip(host), fg='green'))
+    if len(host['hostnames']) > 0:
+        click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames'])))
+
+    if 'city' in host and host['city']:
+        click.echo(u'{:25s}{}'.format('City:', host['city']))
+
+    if 'country_name' in host and host['country_name']:
+        click.echo(u'{:25s}{}'.format('Country:', host['country_name']))
+
+    if 'os' in host and host['os']:
+        click.echo(u'{:25s}{}'.format('Operating System:', host['os']))
+
+    if 'org' in host and host['org']:
+        click.echo(u'{:25s}{}'.format('Organization:', host['org']))
+
+    if 'last_update' in host and host['last_update']:
+        click.echo('{:25s}{}'.format('Updated:', host['last_update']))
+
+    click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports'])))
+
+    # Output the vulnerabilities the host has
+    if 'vulns' in host and len(host['vulns']) > 0:
+        vulns = []
+        for vuln in host['vulns']:
+            if vuln.startswith('!'):
+                continue
+            if vuln.upper() == 'CVE-2014-0160':
+                vulns.append(click.style('Heartbleed', fg='red'))
+            else:
+                vulns.append(click.style(vuln, fg='red'))
+
+        if len(vulns) > 0:
+            click.echo('{:25s}'.format('Vulnerabilities:'), nl=False)
+
+            for vuln in vulns:
+                click.echo(vuln + '\t', nl=False)
+
+            click.echo('')
+
+    click.echo('')
+
+    # If the user doesn't have access to SSL/ Telnet results then we need
+    # to pad the host['data'] property with empty banners so they still see
+    # the port listed as open. (#63)
+    if len(host['ports']) != len(host['data']):
+        # Find the ports the user can't see the data for
+        ports = host['ports']
+        for banner in host['data']:
+            if banner['port'] in ports:
+                ports.remove(banner['port'])
+
+        # Add the placeholder banners
+        for port in ports:
+            banner = {
+                'port': port,
+                'transport': 'tcp', # All the filtered services use TCP
+                'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
+                'placeholder': True, # Don't store this banner when the file is saved
+            }
+            host['data'].append(banner)
+
+    click.echo('Ports:')
+    for banner in sorted(host['data'], key=lambda k: k['port']):
+        product = ''
+        version = ''
+        if 'product' in banner and banner['product']:
+            product = banner['product']
+        if 'version' in banner and banner['version']:
+            version = '({})'.format(banner['version'])
+
+        click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
+        click.echo('/', nl=False)
+        click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
+        click.echo('{} {}'.format(product, version), nl=False)
+
+        if history:
+            # Format the timestamp to only show the year-month-day
+            date = banner['timestamp'][:10]
+            click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
+        click.echo('')
+
+        # Show optional ssl info
+        if 'ssl' in banner:
+            if 'versions' in banner['ssl'] and banner['ssl']['versions']:
+                click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
+            if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
+                click.echo('\t|-- Diffie-Hellman Parameters:')
+                click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
+                if 'fingerprint' in banner['ssl']['dhparams']:
+                    click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+
+
+def host_print_tsv(host, history=False):
+    """Show the host information in a succinct, grep-friendly manner."""
+    for banner in sorted(host['data'], key=lambda k: k['port']):
+        click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
+        click.echo('\t', nl=False)
+        click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
+
+        if history:
+            # Format the timestamp to only show the year-month-day
+            date = banner['timestamp'][:10]
+            click.echo(click.style('\t({})'.format(date), fg='white', dim=True), nl=False)
+        click.echo('')
+
+
+HOST_PRINT = {
+    'pretty': host_print_pretty,
+    'tsv': host_print_tsv,
+}
\ No newline at end of file

From 9767c0ace95563a6857fd2d1be80e232a419aca2 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 16:13:26 -0500
Subject: [PATCH 138/263] Add information/ links for the CLI

---
 README.rst | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index d0a40ae..c78af63 100644
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
-shodan: The official Python library for accessing Shodan
-========================================================
+shodan: The official Python library and CLI for accessing Shodan
+================================================================
 
 .. image:: https://img.shields.io/pypi/v/shodan.svg
     :target: https://pypi.org/project/shodan/
@@ -20,6 +20,12 @@ Features
 - `Network alerts (aka private firehose) <https://help.shodan.io/guides/how-to-monitor-network>`_
 - Exploit search API fully implemented
 - Bulk data downloads
+- `Command-line interface <https://cli.shodan.io>`_
+
+.. image:: https://asciinema.org/a/40357.png
+    :target: https://asciinema.org/~Shodan
+    :width: 400px
+    :align: center
 
 
 Quick Start

From 04852200f2536e08f470d36bb968605198db2e85 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 16:19:30 -0500
Subject: [PATCH 139/263] Shorten title so it fits without wrapping and link to
 custom-scaled image as Github doesn't seem to honor the ":width:" modifier

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index c78af63..0316ea0 100644
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
-shodan: The official Python library and CLI for accessing Shodan
-================================================================
+shodan: The official Python library and CLI for Shodan
+======================================================
 
 .. image:: https://img.shields.io/pypi/v/shodan.svg
     :target: https://pypi.org/project/shodan/

From 6e81234088661f77c9fd312f3d4fd814ea4b3a4b Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 16:20:02 -0500
Subject: [PATCH 140/263] Update the actual link to the new image

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 0316ea0..d0b1f10 100644
--- a/README.rst
+++ b/README.rst
@@ -22,7 +22,7 @@ Features
 - Bulk data downloads
 - `Command-line interface <https://cli.shodan.io>`_
 
-.. image:: https://asciinema.org/a/40357.png
+.. image:: https://cli.shodan.io/img/shodan-cli-preview.png
     :target: https://asciinema.org/~Shodan
     :width: 400px
     :align: center

From 5960167be753aa04e428cf8d0116e81c707555ee Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 17:49:53 -0500
Subject: [PATCH 141/263] Improved error handling around "shodan radar" (#74)

---
 CHANGELOG.md           | 1 +
 shodan/__main__.py     | 8 +++++++-
 shodan/cli/worldmap.py | 8 ++++++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dae37d5..73250be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ unreleased
 * The CHANGELOG is now part of the packages.
 * Improved unicode handling in Python2 (#78)
 * Add `tsv` output format for **shodan host** (#65)
+* Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74)
 
 1.9.0
 -----
diff --git a/shodan/__main__.py b/shodan/__main__.py
index f247432..3165c9a 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -1239,7 +1239,13 @@ def radar():
     api = shodan.Shodan(key)
 
     from shodan.cli.worldmap import launch_map
-    launch_map(api)
+
+    try:
+        launch_map(api)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    except Exception as e:
+        raise click.ClickException(u'{}'.format(e))
 
 def async_spinner(finished):
     spinner = itertools.cycle(['-', '/', '|', '\\'])
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 9804aa2..16f06d4 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -18,6 +18,7 @@
 import random
 import time
 
+from shodan.exception import APIError
 from shodan.helpers import get_ip
 
 
@@ -209,7 +210,7 @@ def fetch_data(self, epoch_now, force_refresh=False):
                         break
                 self.data = banners
                 self.last_fetch = epoch_now
-            except StandardError:
+            except APIError:
                 raise
         return refresh
 
@@ -221,7 +222,10 @@ def run(self, scr):
             now = int(time.time())
             refresh = self.fetch_data(now)
             m.set_data(self.data)
-            m.draw(scr)
+            try:
+                m.draw(scr)
+            except curses.error:
+                raise Exception('Terminal window too small')
             scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD)
             scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD)
 

From f3fb662988568b00d660f54dfe820a1c2beaae89 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 18:47:26 -0500
Subject: [PATCH 142/263] Improved error messages for exceptions raised during
 API requests (#77)

---
 CHANGELOG.md       |  1 +
 shodan/__main__.py |  2 +-
 shodan/client.py   | 10 +++++++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73250be..39f33d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ unreleased
 * Improved unicode handling in Python2 (#78)
 * Add `tsv` output format for **shodan host** (#65)
 * Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74)
+* Improved exception handling to improve debugging **shodan init** (#77)
 
 1.9.0
 -----
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 3165c9a..bc26f1d 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -186,7 +186,7 @@ def init(key):
         api = shodan.Shodan(key)
         test = api.info()
     except shodan.APIError as e:
-        raise click.ClickException('Invalid API key')
+        raise click.ClickException(e.value)
 
     # Store the API key in the user's directory
     keyfile = shodan_dir + '/api_key'
diff --git a/shodan/client.py b/shodan/client.py
index 6c19e48..56a89e8 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -190,9 +190,17 @@ def _request(self, function, params, service='shodan', method='get'):
                 # Return the actual error message if the API returned valid JSON
                 error = data.json()['error']
             except Exception as e:
-                error = 'Invalid API key'
+                # If the response looks like HTML then it's probably the 401 page that nginx returns
+                # for 401 responses by default
+                if data.text.startswith('<'):
+                    error = 'Invalid API key'
+                else:
+                    # Otherwise lets raise the error message
+                    error = u'{}'.format(e)
 
             raise APIError(error)
+        else if data.status_code == 403:
+            raise APIError('Access denied (403 Forbidden)')
 
         # Parse the text into JSON
         try:

From d768a0924d9355e4b250ca72808c74a3c00ec855 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 20:00:03 -0500
Subject: [PATCH 143/263] Fix typo

---
 shodan/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 56a89e8..0efb628 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -199,7 +199,7 @@ def _request(self, function, params, service='shodan', method='get'):
                     error = u'{}'.format(e)
 
             raise APIError(error)
-        else if data.status_code == 403:
+        elif data.status_code == 403:
             raise APIError('Access denied (403 Forbidden)')
 
         # Parse the text into JSON

From 868ac51be1ce70cf410a37ab2916278f17962cae Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 20:19:43 -0500
Subject: [PATCH 144/263] Release 1.9.1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c52e5e0..88b8511 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.9.0',
+    version = '1.9.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',

From 2775c5c646335da9548248083134ed41fbfcc01c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 20:22:34 -0500
Subject: [PATCH 145/263] Update CHANGELOG to match the release 1.9.1

---
 CHANGELOG.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39f33d5..99a5dd8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,8 @@
 CHANGELOG
 =========
 
-unreleased
-----------
+1.9.1
+-----
 * The CHANGELOG is now part of the packages.
 * Improved unicode handling in Python2 (#78)
 * Add `tsv` output format for **shodan host** (#65)

From 89e4d34c009c98cea14f93a8617bb419328b14ec Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 20:27:24 -0500
Subject: [PATCH 146/263] Remove API wrapper for the deprecated shodanhq.com
 website

---
 shodan/api.py | 222 --------------------------------------------------
 1 file changed, 222 deletions(-)
 delete mode 100644 shodan/api.py

diff --git a/shodan/api.py b/shodan/api.py
deleted file mode 100644
index 2f54f08..0000000
--- a/shodan/api.py
+++ /dev/null
@@ -1,222 +0,0 @@
-try:
-    # Python 2
-    from urllib2    import urlopen
-    from urllib     import urlencode
-except:
-    # Python 3
-    from urllib.request     import urlopen
-    from urllib.parse       import urlencode
-
-from json import dumps, loads
-
-from .exception import WebAPIError
-
-
-__all__ = ['WebAPI']
-
-
-class WebAPI:
-    """Wrapper around the SHODAN webservices API"""
-    
-    class Exploits:
-        
-        def __init__(self, parent):
-            self.parent = parent
-            
-        def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None):
-            """Search the entire Shodan Exploits archive using the same query syntax
-            as the website.
-            
-            Arguments:
-            query    -- exploit search query; same syntax as website
-            
-            Optional arguments:
-            sources  -- metasploit, cve, osvdb, exploitdb
-            cve      -- CVE identifier (ex. 2010-0432)
-            osvdb    -- OSVDB identifier (ex. 11666)
-            msb      -- Microsoft Security Bulletin ID (ex. MS05-030)
-            bid      -- Bugtraq identifier (ex. 13951)
-            
-            """
-            if sources:
-                query += ' source:%s' % (','.join(sources))
-            if cve:
-                query += ' cve:%s' % (str(cve).strip())
-            if osvdb:
-                query += ' osvdb:%s' % (str(osvdb).strip())
-            if msb:
-                query += ' msb:%s' % (str(msb).strip())
-            if bid:
-                query += ' bid:%s' % (str(bid).strip())
-            return self.parent._request('api', {'q': query}, service='exploits')
-    
-    class ExploitDb:
-        
-        def __init__(self, parent):
-            self.parent = parent
-        
-        def download(self, id):
-            """DEPRECATED
-            Download the exploit code from the ExploitDB archive.
-    
-            Arguments:
-            id    -- ID of the ExploitDB entry
-            """
-            query = '_id:%s' % id
-            return self.parent.search(query, sources=['exploitdb'])
-        
-        def search(self, query, **kwargs):
-            """Search the ExploitDB archive.
-    
-            Arguments:
-            query     -- Search terms
-    
-            Returns:
-            A dictionary with 2 main items: matches (list) and total (int).
-            """
-            return self.parent.search(query, sources=['exploitdb'])
-
-    
-    class Msf:
-        
-        def __init__(self, parent):
-            self.parent = parent
-            
-        def download(self, id):
-            """Download a metasploit module given the fullname (id) of it.
-            
-            Arguments:
-            id        -- fullname of the module (ex. auxiliary/admin/backupexec/dump)
-            
-            Returns:
-            A dictionary with the following fields:
-            filename        -- Name of the file
-            content-type    -- Mimetype
-            data            -- File content
-            """
-            query = '_id:%s' % id
-            return self.parent.search(query, sources=['metasploit'])
-        
-        def search(self, query, **kwargs):
-            """Search for a Metasploit module.
-            """
-            return self.parent.search(query, sources=['metasploit'])
-    
-    def __init__(self, key):
-        """Initializes the API object.
-        
-        Arguments:
-        key -- your API key
-        
-        """
-        print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()')
-        self.api_key = key
-        self.base_url = 'http://www.shodanhq.com/api/'
-        self.base_exploits_url = 'https://exploits.shodan.io/'
-        self.exploits = self.Exploits(self)
-        self.exploitdb = self.ExploitDb(self.exploits)
-        self.msf = self.Msf(self.exploits)
-    
-    def _request(self, function, params, service='shodan'):
-        """General-purpose function to create web requests to SHODAN.
-        
-        Arguments:
-        function  -- name of the function you want to execute
-        params      -- dictionary of parameters for the function
-        
-        Returns
-        A JSON string containing the function's results.
-        
-        """
-        # Add the API key parameter automatically
-        params['key'] = self.api_key
-        
-        # Determine the base_url based on which service we're interacting with
-        base_url = {
-            'shodan': self.base_url,
-            'exploits': self.base_exploits_url,
-        }.get(service, 'shodan')
-
-        # Send the request
-        try:
-            data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8')
-        except:
-            raise WebAPIError('Unable to connect to Shodan')
-        
-        # Parse the text from JSON to a dict
-        data = loads(data)
-        
-        # Raise an exception if an error occurred
-        if data.get('error', None):
-            raise WebAPIError(data['error'])
-        
-        # Return the data
-        return data
-    
-    def count(self, query):
-        """Returns the total number of search results for the query.
-        """
-        return self._request('count', {'q': query})
-    
-    def locations(self, query):
-        """Return a break-down of all the countries and cities that the results for
-        the given search are located in.
-        """
-        return self._request('locations', {'q': query})
-    
-    def fingerprint(self, banner):
-        """Determine the software based on the banner.
-        
-        Arguments:
-        banner  - HTTP banner
-        
-        Returns:
-        A list of software that matched the given banner.
-        """
-        return self._request('fingerprint', {'banner': banner})
-    
-    def host(self, ip):
-        """Get all available information on an IP.
-
-        Arguments:
-        ip    -- IP of the computer
-
-        Returns:
-        All available information SHODAN has on the given IP,
-        subject to API key restrictions.
-
-        """
-        return self._request('host', {'ip': ip})
-    
-    def info(self):
-        """Returns information about the current API key, such as a list of add-ons
-        and other features that are enabled for the current user's API plan.
-        """
-        return self._request('info', {})
-    
-    def search(self, query, page=1, limit=None, offset=None):
-        """Search the SHODAN database.
-        
-        Arguments:
-        query    -- search query; identical syntax to the website
-        
-        Optional arguments:
-        page     -- page number of the search results 
-        limit    -- number of results to return
-        offset   -- search offset to begin getting results from
-        
-        Returns:
-        A dictionary with 3 main items: matches, countries and total.
-        Visit the website for more detailed information.
-        
-        """
-        args = {
-            'q': query,
-            'p': page,
-        }
-        if limit:
-            args['l'] = limit
-        if offset:
-            args['o'] = offset
-        
-        return self._request('search', args)

From 2f8d21d65e6afe02fef0183b49e5bd023e8c371c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 17 Aug 2018 20:28:57 -0500
Subject: [PATCH 147/263] More code cleanup to remove deprecated classes

---
 shodan/__init__.py  | 1 -
 shodan/exception.py | 8 --------
 2 files changed, 9 deletions(-)

diff --git a/shodan/__init__.py b/shodan/__init__.py
index 7d4b04d..bdfecaa 100644
--- a/shodan/__init__.py
+++ b/shodan/__init__.py
@@ -1,3 +1,2 @@
-from shodan.api import WebAPI
 from shodan.client import Shodan
 from shodan.exception import APIError
diff --git a/shodan/exception.py b/shodan/exception.py
index 11d89d3..c4878b1 100644
--- a/shodan/exception.py
+++ b/shodan/exception.py
@@ -1,11 +1,3 @@
-class WebAPIError(Exception):
-    def __init__(self, value):
-        self.value = value
-    
-    def __str__(self):
-        return self.value
-
-
 class APIError(Exception):
     """This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
     def __init__(self, value):

From 64e9b6ba2c6550b68c28c9ede38dd88663789f51 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 30 Aug 2018 20:48:33 -0500
Subject: [PATCH 148/263] Major code cleanup using pylint Migrated large
 subcommands for the CLI into their own Python modules

---
 shodan/__main__.py              | 582 ++------------------------------
 shodan/cli/alert.py             |  91 +++++
 shodan/cli/converter/csvc.py    |   1 +
 shodan/cli/converter/geojson.py |   2 +-
 shodan/cli/converter/kml.py     |  23 +-
 shodan/cli/data.py              |  90 +++++
 shodan/cli/helpers.py           |  73 +++-
 shodan/cli/scan.py              | 314 +++++++++++++++++
 shodan/cli/worldmap.py          |   3 +
 shodan/helpers.py               |   1 +
 shodan/threatnet.py             |   2 +-
 11 files changed, 603 insertions(+), 579 deletions(-)
 create mode 100644 shodan/cli/alert.py
 create mode 100644 shodan/cli/data.py
 create mode 100644 shodan/cli/scan.py

diff --git a/shodan/__main__.py b/shodan/__main__.py
index bc26f1d..874e2d0 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -48,7 +48,7 @@
 from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS
 
 # Helper methods
-from shodan.cli.helpers import get_api_key
+from shodan.cli.helpers import async_spinner, get_api_key, escape_data, timestr, open_streaming_file, get_banner_field, match_filters
 from shodan.cli.host import HOST_PRINT
 
 # Allow 3rd-parties to develop custom commands
@@ -65,67 +65,23 @@
     basestring = str
 
 
-def escape_data(args):
-    # Ensure the provided string isn't unicode data
-    if not isinstance(args, str):
-        args = args.encode('ascii', 'replace')
-    return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
-
-def timestr():
-    return datetime.datetime.utcnow().strftime('%Y-%m-%d')
-
-def open_streaming_file(directory, timestr, compresslevel=9):
-    return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
-
-def get_banner_field(banner, flat_field):
-    # The provided field is a collapsed form of the actual field
-    fields = flat_field.split('.')
-
-    try:
-        current_obj = banner
-        for field in fields:
-            current_obj = current_obj[field]
-        return current_obj
-    except:
-        pass
-
-    return None
-
-def match_filters(banner, filters):
-    for args in filters:
-        flat_field, check = args.split(':', 1)
-        value = get_banner_field(banner, flat_field)
-
-        # If the field doesn't exist on the banner then ignore the record
-        if not value:
-            return False
-
-        # It must match all filters to be allowed
-        field_type = type(value)
-
-        # For lists of strings we see whether the desired value is contained in the field
-        if field_type == list or isinstance(value, basestring):
-            if check not in value:
-                return False
-        elif field_type == int:
-            if int(check) != value:
-                return False
-        elif field_type == float:
-            if float(check) != value:
-                return False
-        else:
-            # Ignore unknown types
-            pass
-
-    return True
-
-
+# Define the main entry point for all of our commands
+# and expose a way for 3rd-party plugins to tie into the Shodan CLI.
 @with_plugins(iter_entry_points('shodan.cli.plugins'))
 @click.group(context_settings=CONTEXT_SETTINGS)
 def main():
     pass
 
 
+# Large subcommands are stored in separate modules
+from shodan.cli.alert import alert
+from shodan.cli.data import data
+from shodan.cli.scan import scan
+main.add_command(alert)
+main.add_command(data)
+main.add_command(scan)
+
+
 @main.command()
 @click.argument('input', metavar='<input file>')
 @click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json', 'images', 'xlsx']))
@@ -184,7 +140,7 @@ def init(key):
     key = key.strip()
     try:
         api = shodan.Shodan(key)
-        test = api.info()
+        api.info()
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
@@ -196,95 +152,6 @@ def init(key):
 
     os.chmod(keyfile, 0o600)
 
-
-@main.group()
-def alert():
-    """Manage the network alerts for your account"""
-    pass
-
-
-@alert.command(name='clear')
-def alert_clear():
-    """Remove all alerts"""
-    key = get_api_key()
-
-    # Get the list
-    api = shodan.Shodan(key)
-    try:
-        alerts = api.alerts()
-        for alert in alerts:
-            click.echo(u'Removing {} ({})'.format(alert['name'], alert['id']))
-            api.delete_alert(alert['id'])
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-    click.echo("Alerts deleted")
-
-@alert.command(name='create')
-@click.argument('name', metavar='<name>')
-@click.argument('netblock', metavar='<netblock>')
-def alert_create(name, netblock):
-    """Create a network alert to monitor an external network"""
-    key = get_api_key()
-
-    # Get the list
-    api = shodan.Shodan(key)
-    try:
-        alert = api.create_alert(name, netblock)
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-    click.echo(click.style('Successfully created network alert!', fg='green'))
-    click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan'))
-
-@alert.command(name='list')
-@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
-def alert_list(expired):
-    """List all the active alerts"""
-    key = get_api_key()
-
-    # Get the list
-    api = shodan.Shodan(key)
-    try:
-        results = api.alerts(include_expired=expired)
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-    if len(results) > 0:
-        click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
-        # click.echo('#' * 65)
-        for alert in results:
-            click.echo(
-                u'{:16} {:<30} {:<35} '.format(
-                    click.style(alert['id'],  fg='yellow'),
-                    click.style(alert['name'], fg='cyan'),
-                    click.style(', '.join(alert['filters']['ip']), fg='white')
-                ),
-                nl=False
-            )
-
-            if 'expired' in alert and alert['expired']:
-                click.echo(click.style('expired', fg='red'))
-            else:
-                click.echo('')
-    else:
-        click.echo("You haven't created any alerts yet.")
-
-
-@alert.command(name='remove')
-@click.argument('alert_id', metavar='<alert ID>')
-def alert_remove(alert_id):
-    """Remove the specified alert"""
-    key = get_api_key()
-
-    # Get the list
-    api = shodan.Shodan(key)
-    try:
-        results = api.delete_alert(alert_id)
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-    click.echo("Alert deleted")
-
-
 @main.command()
 @click.argument('query', metavar='<search query>', nargs=-1)
 def count(query):
@@ -308,90 +175,6 @@ def count(query):
     click.echo(results['total'])
 
 
-@main.group()
-def data():
-    """Bulk data access to Shodan"""
-    pass
-
-
-@data.command(name='list')
-@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str)
-def data_list(dataset):
-    """List available datasets or the files within those datasets."""
-    # Setup the API connection
-    key = get_api_key()
-    api = shodan.Shodan(key)
-
-    if dataset:
-        # Show the files within this dataset
-        files = api.data.list_files(dataset)
-
-        for file in files:
-            click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False)
-            click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False)
-            click.echo('{}'.format(file['url']))
-    else:
-        # If no dataset was provided then show a list of all datasets
-        datasets = api.data.list_datasets()
-
-        for ds in datasets:
-            click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False)
-            click.echo('{}'.format(ds['description']))
-
-
-@data.command(name='download')
-@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int)
-@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.')
-@click.argument('dataset', metavar='<dataset>')
-@click.argument('name', metavar='<file>')
-def data_download(chunksize, filename, dataset, name):
-    # Setup the API connection
-    key = get_api_key()
-    api = shodan.Shodan(key)
-
-    # Get the file object that the user requested which will contain the URL and total file size
-    file = None
-    try:
-        files = api.data.list_files(dataset)
-        for tmp in files:
-            if tmp['name'] == name:
-                file = tmp
-                break
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-    # The file isn't available
-    if not file:
-        raise click.ClickException('File not found')
-
-    # Start downloading the file
-    response = requests.get(file['url'], stream=True)
-
-    # Figure out the size of the file based on the headers
-    filesize = response.headers.get('content-length', None)
-    if not filesize:
-        # Fall back to using the filesize provided by the API
-        filesize = file['size']
-    else:
-        filesize = int(filesize)
-
-    chunk_size = 1024
-    limit = filesize / chunk_size
-
-    # Create a default filename based on the dataset and the filename within that dataset
-    if not filename:
-        filename = '{}-{}'.format(dataset, name)
-
-    # Open the output file and start writing to it in chunks
-    with open(filename, 'wb') as fout:
-        with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar:
-            for chunk in bar:
-                if chunk:
-                    fout.write(chunk)
-
-    click.echo(click.style('Download completed: {}'.format(filename), 'green'))
-
-
 @main.command()
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
 @click.argument('filename', metavar='<filename>')
@@ -514,7 +297,7 @@ def info():
 @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data')
 @click.option('--filters', '-f', help='Filter the results for specific values using key:value pairs.', multiple=True)
 @click.option('--filename', '-O', help='Save the filtered results in the given file (append if file exists).')
-@click.option('--separator', help='The separator between the properties of the search results.', default='\t')
+@click.option('--separator', help='The separator between the properties of the search results.', default=u'\t')
 @click.argument('filenames', metavar='<filenames>', type=click.Path(exists=True), nargs=-1)
 def parse(color, fields, filters, filename, separator, filenames):
     """Extract information out of compressed JSON files."""
@@ -540,7 +323,7 @@ def parse(color, fields, filters, filename, separator, filenames):
         fout = helpers.open_file(filename)
 
     for banner in helpers.iterate_files(filenames):
-        row = ''
+        row = u''
 
         # Validate the banner against any provided filters
         if has_filters and not match_filters(banner, filters):
@@ -552,16 +335,16 @@ def parse(color, fields, filters, filename, separator, filenames):
 
         # Loop over all the fields and print the banner as a row
         for field in fields:
-            tmp = ''
+            tmp = u''
             value = get_banner_field(banner, field)
             if value:
                 field_type = type(value)
 
                 # If the field is an array then merge it together
                 if field_type == list:
-                    tmp = ';'.join(value)
+                    tmp = u';'.join(value)
                 elif field_type in [int, float]:
-                    tmp = str(value)
+                    tmp = u'{}'.format(value)
                 else:
                     tmp = escape_data(value)
 
@@ -588,309 +371,6 @@ def myip():
         raise click.ClickException(e.value)
 
 
-@main.group()
-def scan():
-    """Scan an IP/ netblock using Shodan."""
-    pass
-
-
-@scan.command(name='internet')
-@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True)
-@click.argument('port', type=int)
-@click.argument('protocol', type=str)
-def scan_internet(quiet, port, protocol):
-    """Scan the Internet for a specific port and protocol using the Shodan infrastructure."""
-    key = get_api_key()
-    api = shodan.Shodan(key)
-
-    try:
-        # Submit the request to Shodan
-        click.echo('Submitting Internet scan to Shodan...', nl=False)
-        scan = api.scan_internet(port, protocol)
-        click.echo('Done')
-
-        # If the requested port is part of the regular Shodan crawling, then
-        # we don't know when the scan is done so lets return immediately and
-        # let the user decide when to stop waiting for further results.
-        official_ports = api.ports()
-        if port in official_ports:
-            click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
-        else:
-            # Create the output file
-            filename = '{0}-{1}.json.gz'.format(port, protocol)
-            counter = 0
-            with helpers.open_file(filename, 'w') as fout:
-                click.echo('Saving results to file: {0}'.format(filename))
-
-                # Start listening for results
-                done = False
-
-                # Keep listening for results until the scan is done
-                click.echo('Waiting for data, please stand by...')
-                while not done:
-                    try:
-                        for banner in api.stream.ports([port], timeout=90):
-                            counter += 1
-                            helpers.write_banner(fout, banner)
-
-                            if not quiet:
-                                click.echo('{0:<40} {1:<20} {2}'.format(
-                                        click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
-                                        click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
-                                        ';'.join(banner['hostnames'])
-                                    )
-                                )
-                    except shodan.APIError as e:
-                        # We stop waiting for results if the scan has been processed by the crawlers and
-                        # there haven't been new results in a while
-                        if done:
-                            break
-
-                        scan = api.scan_status(scan['id'])
-                        if scan['status'] == 'DONE':
-                            done = True
-                    except socket.timeout as e:
-                        # We stop waiting for results if the scan has been processed by the crawlers and
-                        # there haven't been new results in a while
-                        if done:
-                            break
-
-                        scan = api.scan_status(scan['id'])
-                        if scan['status'] == 'DONE':
-                            done = True
-                    except Exception as e:
-                        raise click.ClickException(repr(e))
-            click.echo('Scan finished: {0} devices found'.format(counter))
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-
-@scan.command(name='protocols')
-def scan_protocols():
-    """List the protocols that you can scan with using Shodan."""
-    key = get_api_key()
-    api = shodan.Shodan(key)
-    try:
-        protocols = api.protocols()
-
-        for name, description in iter(protocols.items()):
-            click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description)
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-
-@scan.command(name='submit')
-@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int)
-@click.option('--filename', help='Save the results in the given file.', default='', type=str)
-@click.option('--force', default=False, is_flag=True)
-@click.option('--verbose', default=False, is_flag=True)
-@click.argument('netblocks', metavar='<ip address>', nargs=-1)
-def scan_submit(wait, filename, force, verbose, netblocks):
-    """Scan an IP/ netblock using Shodan."""
-    key = get_api_key()
-    api = shodan.Shodan(key)
-    alert = None
-
-    # Submit the IPs for scanning
-    try:
-        # Submit the scan
-        scan = api.scan(netblocks, force=force)
-
-        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
-
-        click.echo('')
-        click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
-
-        if verbose:
-            click.echo('# Scan ID: {}'.format(scan['id']))
-
-        # Return immediately
-        if wait <= 0:
-            click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
-        else:
-            # Setup an alert to wait for responses
-            alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks)
-
-            # Create the output file if necessary
-            filename = filename.strip()
-            fout = None
-            if filename != '':
-                # Add the appropriate extension if it's not there atm
-                if not filename.endswith('.json.gz'):
-                    filename += '.json.gz'
-                fout = helpers.open_file(filename, 'w')
-
-            # Start a spinner
-            finished_event = threading.Event()
-            progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
-            progress_bar_thread.start()
-
-            # Now wait a few seconds for items to get returned
-            hosts = collections.defaultdict(dict)
-            done = False
-            scan_start = time.time()
-            cache = {}
-            while not done:
-                try:
-                    for banner in api.stream.alert(aid=alert['id'], timeout=wait):
-                        ip = banner.get('ip', banner.get('ipv6', None))
-                        if not ip:
-                            continue
-
-                        # Don't show duplicate banners
-                        cache_key = '{}:{}'.format(ip, banner['port'])
-                        if cache_key not in cache:
-                            hosts[helpers.get_ip(banner)][banner['port']] = banner
-                            cache[cache_key] = True
-
-                        # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
-                        if time.time() - scan_start >= 60:
-                            scan = api.scan_status(scan['id'])
-
-                            if verbose:
-                                click.echo('# Scan status: {}'.format(scan['status']))
-
-                            if scan['status'] == 'DONE':
-                                done = True
-                                break
-
-                except shodan.APIError as e:
-                    # If the connection timed out before the timeout, that means the streaming server
-                    # that the user tried to reach is down. In that case, lets wait briefly and try
-                    # to connect again!
-                    if (time.time() - scan_start) < wait:
-                        time.sleep(0.5)
-                        continue
-
-                    # Exit if the scan was flagged as done somehow
-                    if done:
-                        break
-
-                    scan = api.scan_status(scan['id'])
-                    if scan['status'] == 'DONE':
-                        done = True
-
-                    if verbose:
-                        click.echo('# Scan status: {}'.format(scan['status']))
-                except socket.timeout as e:
-                    # If the connection timed out before the timeout, that means the streaming server
-                    # that the user tried to reach is down. In that case, lets wait a second and try
-                    # to connect again!
-                    if (time.time() - scan_start) < wait:
-                        continue
-
-                    done = True
-                except Exception as e:
-                    finished_event.set()
-                    progress_bar_thread.join()
-                    raise click.ClickException(repr(e))
-
-            finished_event.set()
-            progress_bar_thread.join()
-
-            def print_field(name, value):
-                click.echo('  {:25s}{}'.format(name, value))
-
-            def print_banner(banner):
-                click.echo('    {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False)
-
-                if 'product' in banner:
-                    click.echo(banner['product'], nl=False)
-
-                    if 'version' in banner:
-                        click.echo(' ({})'.format(banner['version']), nl=False)
-
-                click.echo('')
-
-                # Show optional ssl info
-                if 'ssl' in banner:
-                    if 'versions' in banner['ssl']:
-                        # Only print SSL versions if they were successfully tested
-                        versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
-                        if len(versions) > 0:
-                            click.echo('    |-- SSL Versions: {}'.format(', '.join(versions)))
-                    if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
-                        click.echo('    |-- Diffie-Hellman Parameters:')
-                        click.echo('        {:15s}{}\n        {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
-                        if 'fingerprint' in banner['ssl']['dhparams']:
-                            click.echo('        {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
-
-            if hosts:
-                # Remove the remaining spinner character
-                click.echo('\b ')
-
-                for ip in sorted(hosts):
-                    host = next(iter(hosts[ip].items()))[1]
-
-                    click.echo(click.style(ip, fg='cyan'), nl=False)
-                    if 'hostnames' in host and host['hostnames']:
-                        click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False)
-                    click.echo('')
-
-                    if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
-                        print_field('Country', host['location']['country_name'])
-
-                        if 'city' in host['location'] and host['location']['city']:
-                            print_field('City', host['location']['city'])
-                    if 'org' in host and host['org']:
-                        print_field('Organization', host['org'])
-                    if 'os' in host and host['os']:
-                        print_field('Operating System', host['os'])
-                    click.echo('')
-
-                    # Output the vulnerabilities the host has
-                    if 'vulns' in host and len(host['vulns']) > 0:
-                        vulns = []
-                        for vuln in host['vulns']:
-                            if vuln.startswith('!'):
-                                continue
-                            if vuln.upper() == 'CVE-2014-0160':
-                                vulns.append(click.style('Heartbleed', fg='red'))
-                            else:
-                                vulns.append(click.style(vuln, fg='red'))
-
-                        if len(vulns) > 0:
-                            click.echo('  {:25s}'.format('Vulnerabilities:'), nl=False)
-
-                            for vuln in vulns:
-                                click.echo(vuln + '\t', nl=False)
-
-                            click.echo('')
-
-                    # Print all the open ports:
-                    click.echo('  Open Ports:')
-                    for port in sorted(hosts[ip]):
-                        print_banner(hosts[ip][port])
-
-                        # Save the banner in a file if necessary
-                        if fout:
-                            helpers.write_banner(fout, hosts[ip][port])
-
-                    click.echo('')
-            else:
-                # Prepend a \b to remove the spinner
-                click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-    finally:
-        # Remove any alert
-        if alert:
-            api.delete_alert(alert['id'])
-
-
-@scan.command(name='status')
-@click.argument('scan_id', type=str)
-def scan_status(scan_id):
-    """Check the status of an on-demand scan."""
-    key = get_api_key()
-    api = shodan.Shodan(key)
-    try:
-        scan = api.scan_status(scan_id)
-        click.echo(scan['status'])
-    except shodan.APIError as e:
-        raise click.ClickException(e.value)
-
-
 @main.command()
 @click.option('--color/--no-color', default=True)
 @click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data')
@@ -930,9 +410,9 @@ def search(color, fields, limit, separator, query):
         raise click.ClickException('No search results found')
 
     # We buffer the entire output so we can use click's pager functionality
-    output = ''
+    output = u''
     for banner in results['matches']:
-        row = ''
+        row = u''
 
         # Loop over all the fields and print the banner as a row
         for field in fields:
@@ -942,9 +422,9 @@ def search(color, fields, limit, separator, query):
 
                 # If the field is an array then merge it together
                 if field_type == list:
-                    tmp = ';'.join(banner[field])
+                    tmp = u';'.join(banner[field])
                 elif field_type in [int, float]:
-                    tmp = str(banner[field])
+                    tmp = u'{}'.format(banner[field])
                 else:
                     tmp = escape_data(banner[field])
 
@@ -957,7 +437,7 @@ def search(color, fields, limit, separator, query):
             row += separator
 
             # click.echo(out + separator, nl=False)
-        output += row + '\n'
+        output += row + u'\n'
         # click.echo('')
     click.echo_via_pager(output)
 
@@ -1030,6 +510,7 @@ def stats(limit, facets, filename, query):
         counter = 0
         has_items = True
         while has_items:
+            # pylint: disable=W0612
             row = ['' for i in range(len(results['facets']) * 2)]
 
             pos = 0
@@ -1170,20 +651,20 @@ def _create_stream(name, args, timeout):
 
                 # Print the banner information to stdout
                 if not quiet:
-                    row = ''
+                    row = u''
 
                     # Loop over all the fields and print the banner as a row
                     for field in fields:
-                        tmp = ''
+                        tmp = u''
                         value = get_banner_field(banner, field)
                         if value:
                             field_type = type(value)
 
                             # If the field is an array then merge it together
                             if field_type == list:
-                                tmp = ';'.join(value)
+                                tmp = u';'.join(value)
                             elif field_type in [int, float]:
-                                tmp = str(value)
+                                tmp = u'{}'.format(value)
                             else:
                                 tmp = escape_data(value)
 
@@ -1247,12 +728,5 @@ def radar():
     except Exception as e:
         raise click.ClickException(u'{}'.format(e))
 
-def async_spinner(finished):
-    spinner = itertools.cycle(['-', '/', '|', '\\'])
-    while not finished.is_set():
-        sys.stdout.write('\b{}'.format(next(spinner)))
-        sys.stdout.flush()
-        finished.wait(0.2)
-
 if __name__ == '__main__':
     main()
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
new file mode 100644
index 0000000..779c44c
--- /dev/null
+++ b/shodan/cli/alert.py
@@ -0,0 +1,91 @@
+import click
+import shodan
+
+from shodan.cli.helpers import get_api_key
+
+@click.group()
+def alert():
+    """Manage the network alerts for your account"""
+    pass
+
+
+@alert.command(name='clear')
+def alert_clear():
+    """Remove all alerts"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        alerts = api.alerts()
+        for alert in alerts:
+            click.echo(u'Removing {} ({})'.format(alert['name'], alert['id']))
+            api.delete_alert(alert['id'])
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    click.echo("Alerts deleted")
+
+@alert.command(name='create')
+@click.argument('name', metavar='<name>')
+@click.argument('netblock', metavar='<netblock>')
+def alert_create(name, netblock):
+    """Create a network alert to monitor an external network"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        alert = api.create_alert(name, netblock)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.echo(click.style('Successfully created network alert!', fg='green'))
+    click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan'))
+
+@alert.command(name='list')
+@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
+def alert_list(expired):
+    """List all the active alerts"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        results = api.alerts(include_expired=expired)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    if len(results) > 0:
+        click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
+        # click.echo('#' * 65)
+        for alert in results:
+            click.echo(
+                u'{:16} {:<30} {:<35} '.format(
+                    click.style(alert['id'],  fg='yellow'),
+                    click.style(alert['name'], fg='cyan'),
+                    click.style(', '.join(alert['filters']['ip']), fg='white')
+                ),
+                nl=False
+            )
+
+            if 'expired' in alert and alert['expired']:
+                click.echo(click.style('expired', fg='red'))
+            else:
+                click.echo('')
+    else:
+        click.echo("You haven't created any alerts yet.")
+
+
+@alert.command(name='remove')
+@click.argument('alert_id', metavar='<alert ID>')
+def alert_remove(alert_id):
+    """Remove the specified alert"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        api.delete_alert(alert_id)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    click.echo("Alert deleted")
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 6e22b62..4e08f63 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -81,6 +81,7 @@ def flatten(self, d, parent_key='', sep='.'):
         for k, v in d.items():
             new_key = parent_key + sep + k if parent_key else k
             if isinstance(v, MutableMapping):
+                # pylint: disable=E0602
                 items.extend(flatten(v, new_key, sep=sep).items())
             else:
                 items.append((new_key, v))
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index 8d6d7d0..2cc55e4 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -53,5 +53,5 @@ def write(self, host):
             }""".format(ip, ip, lat, lon)
 
             self.fout.write(feature)
-        except Exception as e:
+        except:
             pass
diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py
index 7bcac11..c8cb2cc 100644
--- a/shodan/cli/converter/kml.py
+++ b/shodan/cli/converter/kml.py
@@ -45,27 +45,6 @@ def write(self, host):
             if 'hostnames' in host and host['hostnames']:
                 placemark += '<div><a style="color: #999;margin-top:-10px;padding-top:0;" href="http://{0}" target="_blank">{0}</a></div>'.format(host['hostnames'][0])
 
-            test = """
-    <table>
-        <tbody>
-          <tr>
-            <td>City</td>
-            <th>Albuquerque</th>
-          </tr>
-          <tr>
-            <td>Country</td>
-            <th>United States</th>
-          </tr>
-          <tr>
-            <td>Organization</td>
-            <th>Nexcess.net L.L.C.</th>
-          </tr>
-        </tbody>
-    </table>
-    <h2>Ports</h2>
-    <ul>
-            """
-
             placemark += '<h2>Ports</h2><ul>'
 
             for port in host['ports']:
@@ -123,5 +102,5 @@ def write(self, host):
             placemark += '</Placemark>'
 
             self.fout.write(placemark.encode('utf-8'))
-        except Exception as e:
+        except:
             pass
diff --git a/shodan/cli/data.py b/shodan/cli/data.py
new file mode 100644
index 0000000..7cd7228
--- /dev/null
+++ b/shodan/cli/data.py
@@ -0,0 +1,90 @@
+import click
+import requests
+import shodan
+import shodan.helpers as helpers
+
+from shodan.cli.helpers import get_api_key
+
+
+@click.group()
+def data():
+    """Bulk data access to Shodan"""
+    pass
+
+
+@data.command(name='list')
+@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str)
+def data_list(dataset):
+    """List available datasets or the files within those datasets."""
+    # Setup the API connection
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    if dataset:
+        # Show the files within this dataset
+        files = api.data.list_files(dataset)
+
+        for file in files:
+            click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False)
+            click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False)
+            click.echo('{}'.format(file['url']))
+    else:
+        # If no dataset was provided then show a list of all datasets
+        datasets = api.data.list_datasets()
+
+        for ds in datasets:
+            click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False)
+            click.echo('{}'.format(ds['description']))
+
+
+@data.command(name='download')
+@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int)
+@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.')
+@click.argument('dataset', metavar='<dataset>')
+@click.argument('name', metavar='<file>')
+def data_download(chunksize, filename, dataset, name):
+    # Setup the API connection
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    # Get the file object that the user requested which will contain the URL and total file size
+    file = None
+    try:
+        files = api.data.list_files(dataset)
+        for tmp in files:
+            if tmp['name'] == name:
+                file = tmp
+                break
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    # The file isn't available
+    if not file:
+        raise click.ClickException('File not found')
+
+    # Start downloading the file
+    response = requests.get(file['url'], stream=True)
+
+    # Figure out the size of the file based on the headers
+    filesize = response.headers.get('content-length', None)
+    if not filesize:
+        # Fall back to using the filesize provided by the API
+        filesize = file['size']
+    else:
+        filesize = int(filesize)
+
+    chunk_size = 1024
+    limit = filesize / chunk_size
+
+    # Create a default filename based on the dataset and the filename within that dataset
+    if not filename:
+        filename = '{}-{}'.format(dataset, name)
+
+    # Open the output file and start writing to it in chunks
+    with open(filename, 'wb') as fout:
+        with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar:
+            for chunk in bar:
+                if chunk:
+                    fout.write(chunk)
+
+    click.echo(click.style('Download completed: {}'.format(filename), 'green'))
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 24f7b83..711fe4c 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -1,8 +1,12 @@
 '''
-Helper methods to create your own CLI commands.
+Helper methods used across the CLI commands.
 '''
 import click
+import datetime
+import gzip
+import itertools
 import os
+import sys
 
 from .settings import SHODAN_CONFIG_DIR
 
@@ -21,3 +25,70 @@ def get_api_key():
 
     with open(keyfile, 'r') as fin:
         return fin.read().strip()
+
+
+def escape_data(args):
+    # Ensure the provided string isn't unicode data
+    if not isinstance(args, str):
+        args = args.encode('ascii', 'replace')
+    return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+
+
+def timestr():
+    return datetime.datetime.utcnow().strftime('%Y-%m-%d')
+
+
+def open_streaming_file(directory, timestr, compresslevel=9):
+    return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
+
+
+def get_banner_field(banner, flat_field):
+    # The provided field is a collapsed form of the actual field
+    fields = flat_field.split('.')
+
+    try:
+        current_obj = banner
+        for field in fields:
+            current_obj = current_obj[field]
+        return current_obj
+    except:
+        pass
+
+    return None
+
+
+def match_filters(banner, filters):
+    for args in filters:
+        flat_field, check = args.split(':', 1)
+        value = get_banner_field(banner, flat_field)
+
+        # If the field doesn't exist on the banner then ignore the record
+        if not value:
+            return False
+
+        # It must match all filters to be allowed
+        field_type = type(value)
+
+        # For lists of strings we see whether the desired value is contained in the field
+        if field_type == list or isinstance(value, basestring):
+            if check not in value:
+                return False
+        elif field_type == int:
+            if int(check) != value:
+                return False
+        elif field_type == float:
+            if float(check) != value:
+                return False
+        else:
+            # Ignore unknown types
+            pass
+
+    return True
+
+
+def async_spinner(finished):
+    spinner = itertools.cycle(['-', '/', '|', '\\'])
+    while not finished.is_set():
+        sys.stdout.write('\b{}'.format(next(spinner)))
+        sys.stdout.flush()
+        finished.wait(0.2)
diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py
new file mode 100644
index 0000000..5220339
--- /dev/null
+++ b/shodan/cli/scan.py
@@ -0,0 +1,314 @@
+import click
+import collections
+import datetime
+import shodan
+import shodan.helpers as helpers
+import socket
+import threading
+import time
+
+from shodan.cli.helpers import get_api_key, async_spinner
+from shodan.cli.settings import COLORIZE_FIELDS
+
+
+@click.group()
+def scan():
+    """Scan an IP/ netblock using Shodan."""
+    pass
+
+
+@scan.command(name='internet')
+@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True)
+@click.argument('port', type=int)
+@click.argument('protocol', type=str)
+def scan_internet(quiet, port, protocol):
+    """Scan the Internet for a specific port and protocol using the Shodan infrastructure."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        # Submit the request to Shodan
+        click.echo('Submitting Internet scan to Shodan...', nl=False)
+        scan = api.scan_internet(port, protocol)
+        click.echo('Done')
+
+        # If the requested port is part of the regular Shodan crawling, then
+        # we don't know when the scan is done so lets return immediately and
+        # let the user decide when to stop waiting for further results.
+        official_ports = api.ports()
+        if port in official_ports:
+            click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.')
+        else:
+            # Create the output file
+            filename = '{0}-{1}.json.gz'.format(port, protocol)
+            counter = 0
+            with helpers.open_file(filename, 'w') as fout:
+                click.echo('Saving results to file: {0}'.format(filename))
+
+                # Start listening for results
+                done = False
+
+                # Keep listening for results until the scan is done
+                click.echo('Waiting for data, please stand by...')
+                while not done:
+                    try:
+                        for banner in api.stream.ports([port], timeout=90):
+                            counter += 1
+                            helpers.write_banner(fout, banner)
+
+                            if not quiet:
+                                click.echo('{0:<40} {1:<20} {2}'.format(
+                                        click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
+                                        click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
+                                        ';'.join(banner['hostnames'])
+                                    )
+                                )
+                    except shodan.APIError as e:
+                        # We stop waiting for results if the scan has been processed by the crawlers and
+                        # there haven't been new results in a while
+                        if done:
+                            break
+
+                        scan = api.scan_status(scan['id'])
+                        if scan['status'] == 'DONE':
+                            done = True
+                    except socket.timeout as e:
+                        # We stop waiting for results if the scan has been processed by the crawlers and
+                        # there haven't been new results in a while
+                        if done:
+                            break
+
+                        scan = api.scan_status(scan['id'])
+                        if scan['status'] == 'DONE':
+                            done = True
+                    except Exception as e:
+                        raise click.ClickException(repr(e))
+            click.echo('Scan finished: {0} devices found'.format(counter))
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+
+@scan.command(name='protocols')
+def scan_protocols():
+    """List the protocols that you can scan with using Shodan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        protocols = api.protocols()
+
+        for name, description in iter(protocols.items()):
+            click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+
+@scan.command(name='submit')
+@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int)
+@click.option('--filename', help='Save the results in the given file.', default='', type=str)
+@click.option('--force', default=False, is_flag=True)
+@click.option('--verbose', default=False, is_flag=True)
+@click.argument('netblocks', metavar='<ip address>', nargs=-1)
+def scan_submit(wait, filename, force, verbose, netblocks):
+    """Scan an IP/ netblock using Shodan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    alert = None
+
+    # Submit the IPs for scanning
+    try:
+        # Submit the scan
+        scan = api.scan(netblocks, force=force)
+
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
+
+        click.echo('')
+        click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left']))
+
+        if verbose:
+            click.echo('# Scan ID: {}'.format(scan['id']))
+
+        # Return immediately
+        if wait <= 0:
+            click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
+        else:
+            # Setup an alert to wait for responses
+            alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks)
+
+            # Create the output file if necessary
+            filename = filename.strip()
+            fout = None
+            if filename != '':
+                # Add the appropriate extension if it's not there atm
+                if not filename.endswith('.json.gz'):
+                    filename += '.json.gz'
+                fout = helpers.open_file(filename, 'w')
+
+            # Start a spinner
+            finished_event = threading.Event()
+            progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,))
+            progress_bar_thread.start()
+
+            # Now wait a few seconds for items to get returned
+            hosts = collections.defaultdict(dict)
+            done = False
+            scan_start = time.time()
+            cache = {}
+            while not done:
+                try:
+                    for banner in api.stream.alert(aid=alert['id'], timeout=wait):
+                        ip = banner.get('ip', banner.get('ipv6', None))
+                        if not ip:
+                            continue
+
+                        # Don't show duplicate banners
+                        cache_key = '{}:{}'.format(ip, banner['port'])
+                        if cache_key not in cache:
+                            hosts[helpers.get_ip(banner)][banner['port']] = banner
+                            cache[cache_key] = True
+
+                        # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on
+                        if time.time() - scan_start >= 60:
+                            scan = api.scan_status(scan['id'])
+
+                            if verbose:
+                                click.echo('# Scan status: {}'.format(scan['status']))
+
+                            if scan['status'] == 'DONE':
+                                done = True
+                                break
+
+                except shodan.APIError as e:
+                    # If the connection timed out before the timeout, that means the streaming server
+                    # that the user tried to reach is down. In that case, lets wait briefly and try
+                    # to connect again!
+                    if (time.time() - scan_start) < wait:
+                        time.sleep(0.5)
+                        continue
+
+                    # Exit if the scan was flagged as done somehow
+                    if done:
+                        break
+
+                    scan = api.scan_status(scan['id'])
+                    if scan['status'] == 'DONE':
+                        done = True
+
+                    if verbose:
+                        click.echo('# Scan status: {}'.format(scan['status']))
+                except socket.timeout as e:
+                    # If the connection timed out before the timeout, that means the streaming server
+                    # that the user tried to reach is down. In that case, lets wait a second and try
+                    # to connect again!
+                    if (time.time() - scan_start) < wait:
+                        continue
+
+                    done = True
+                except Exception as e:
+                    finished_event.set()
+                    progress_bar_thread.join()
+                    raise click.ClickException(repr(e))
+
+            finished_event.set()
+            progress_bar_thread.join()
+
+            def print_field(name, value):
+                click.echo('  {:25s}{}'.format(name, value))
+
+            def print_banner(banner):
+                click.echo('    {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False)
+
+                if 'product' in banner:
+                    click.echo(banner['product'], nl=False)
+
+                    if 'version' in banner:
+                        click.echo(' ({})'.format(banner['version']), nl=False)
+
+                click.echo('')
+
+                # Show optional ssl info
+                if 'ssl' in banner:
+                    if 'versions' in banner['ssl']:
+                        # Only print SSL versions if they were successfully tested
+                        versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]
+                        if len(versions) > 0:
+                            click.echo('    |-- SSL Versions: {}'.format(', '.join(versions)))
+                    if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
+                        click.echo('    |-- Diffie-Hellman Parameters:')
+                        click.echo('        {:15s}{}\n        {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
+                        if 'fingerprint' in banner['ssl']['dhparams']:
+                            click.echo('        {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint']))
+
+            if hosts:
+                # Remove the remaining spinner character
+                click.echo('\b ')
+
+                for ip in sorted(hosts):
+                    host = next(iter(hosts[ip].items()))[1]
+
+                    click.echo(click.style(ip, fg='cyan'), nl=False)
+                    if 'hostnames' in host and host['hostnames']:
+                        click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False)
+                    click.echo('')
+
+                    if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']:
+                        print_field('Country', host['location']['country_name'])
+
+                        if 'city' in host['location'] and host['location']['city']:
+                            print_field('City', host['location']['city'])
+                    if 'org' in host and host['org']:
+                        print_field('Organization', host['org'])
+                    if 'os' in host and host['os']:
+                        print_field('Operating System', host['os'])
+                    click.echo('')
+
+                    # Output the vulnerabilities the host has
+                    if 'vulns' in host and len(host['vulns']) > 0:
+                        vulns = []
+                        for vuln in host['vulns']:
+                            if vuln.startswith('!'):
+                                continue
+                            if vuln.upper() == 'CVE-2014-0160':
+                                vulns.append(click.style('Heartbleed', fg='red'))
+                            else:
+                                vulns.append(click.style(vuln, fg='red'))
+
+                        if len(vulns) > 0:
+                            click.echo('  {:25s}'.format('Vulnerabilities:'), nl=False)
+
+                            for vuln in vulns:
+                                click.echo(vuln + '\t', nl=False)
+
+                            click.echo('')
+
+                    # Print all the open ports:
+                    click.echo('  Open Ports:')
+                    for port in sorted(hosts[ip]):
+                        print_banner(hosts[ip][port])
+
+                        # Save the banner in a file if necessary
+                        if fout:
+                            helpers.write_banner(fout, hosts[ip][port])
+
+                    click.echo('')
+            else:
+                # Prepend a \b to remove the spinner
+                click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.')
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    finally:
+        # Remove any alert
+        if alert:
+            api.delete_alert(alert['id'])
+
+
+@scan.command(name='status')
+@click.argument('scan_id', type=str)
+def scan_status(scan_id):
+    """Check the status of an on-demand scan."""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        scan = api.scan_status(scan_id)
+        click.echo(scan['status'])
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 16f06d4..c3357ba 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -28,6 +28,9 @@
         'corners': (1, 4, 23, 73),
         # lat top, lon left, lat bottom, lon right
         'coords': [90.0, -180.0, -90.0, 180.0],
+
+        # PyLint freaks out about the world map backslashes so ignore those warnings
+        # pylint: disable=W1401
         'data': '''
                . _..::__:  ,-"-"._       |7       ,     _,.__             
        _.___ _ _<_>`!(._`.`-.    /        _._     `_ ,_/  '  '-._.---.-.__
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 95df590..64e5937 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -93,6 +93,7 @@ def iterate_files(files, fast=False):
         # It's significantly faster at encoding/ decoding JSON but it doesn't support as
         # many options as the standard library. As such, we're mostly interested in using it for
         # decoding since reading/ parsing files will use up the most time.
+        # pylint: disable=E0401
         try:
             from ujson import loads
         except:
diff --git a/shodan/threatnet.py b/shodan/threatnet.py
index 8b609f0..97c0c7e 100644
--- a/shodan/threatnet.py
+++ b/shodan/threatnet.py
@@ -29,7 +29,7 @@ def _create_stream(self, name):
 
             if req.status_code != 200:
                 try:
-                    raise APIError(data.json()['error'])
+                    raise APIError(req.json()['error'])
                 except:
                     pass
                 raise APIError('Invalid API key or you do not have access to the Streaming API')

From 3454b818071bcf2d2888f2a8ae3dd1b8b3167c7c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 1 Sep 2018 18:35:56 -0500
Subject: [PATCH 149/263] New command: org - manage access to Shodan for your
 team

---
 shodan/__main__.py         | 10 +++--
 shodan/cli/helpers.py      | 11 ++++++
 shodan/cli/organization.py | 77 ++++++++++++++++++++++++++++++++++++++
 shodan/client.py           | 52 ++++++++++++++++++++++---
 4 files changed, 140 insertions(+), 10 deletions(-)
 create mode 100644 shodan/cli/organization.py

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 874e2d0..9d49ba9 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -76,9 +76,11 @@ def main():
 # Large subcommands are stored in separate modules
 from shodan.cli.alert import alert
 from shodan.cli.data import data
+from shodan.cli.organization import org
 from shodan.cli.scan import scan
 main.add_command(alert)
 main.add_command(data)
+main.add_command(org)
 main.add_command(scan)
 
 
@@ -134,7 +136,7 @@ def init(key):
         try:
             os.mkdir(shodan_dir)
         except OSError:
-            raise click.ClickException('Unable to create directory to store the Shodan API key (%s)' % shodan_dir)
+            raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir))
 
     # Make sure it's a valid API key
     key = key.strip()
@@ -237,7 +239,7 @@ def download(limit, filename, query):
         # Let the user know we're done
         if count < limit:
             click.echo(click.style('Notice: fewer results were saved than requested', 'yellow'))
-        click.echo(click.style('Saved %s results into file %s' % (count, filename), 'green'))
+        click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green'))
 
 
 @main.command()
@@ -480,8 +482,8 @@ def stats(limit, facets, filename, query):
             else:
                 value = str(value)
 
-            click.echo(click.style('{:28s}'.format(value), fg='cyan'), nl=False)
-            click.echo(click.style('{:12,d}'.format(item['count']), fg='green'))
+            click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False)
+            click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
 
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 711fe4c..bd29ae4 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -92,3 +92,14 @@ def async_spinner(finished):
         sys.stdout.write('\b{}'.format(next(spinner)))
         sys.stdout.flush()
         finished.wait(0.2)
+
+
+def humanize_api_plan(plan):
+    return {
+        'oss': 'Free',
+        'dev': 'Membership',
+        'basic': 'Freelancer API',
+        'plus': 'Small Business API',
+        'corp': 'Corporate API',
+        'stream-100': 'Enterprise',
+    }[plan]
diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py
new file mode 100644
index 0000000..ecdd58a
--- /dev/null
+++ b/shodan/cli/organization.py
@@ -0,0 +1,77 @@
+import click
+import requests
+import shodan
+import shodan.helpers as helpers
+
+from shodan.cli.helpers import get_api_key, humanize_api_plan
+
+
+@click.group()
+def org():
+    """Manage your organization's access to Shodan"""
+    pass
+
+
+@org.command()
+@click.option('--silent', help="Don't send a notification to the user", default=False, is_flag=True)
+@click.argument('user', metavar='<username or email>')
+def add(silent, user):
+    """Add a new member"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        api.org.add_member(user, notify=not silent)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    
+    click.secho('Successfully added the new member', fg='green')
+
+
+@org.command()
+def info():
+    """Show an overview of the organization"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+    try:
+        organization = api.org.info()
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho(organization['name'], fg='cyan')
+    click.secho('Access Level: ', nl=False, dim=True)
+    click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta')
+    click.echo('')
+    click.secho('Administrators:', dim=True)
+
+    for admin in organization['admins']:
+        click.echo(u' > {:30}\t{:30}'.format(
+            click.style(admin['username'], fg='yellow'),
+            admin['email'])
+            )
+    
+    click.echo('')
+    if organization['members']:
+        click.secho('Members:', dim=True)
+        for member in organization['members']:
+            click.echo(u' > {:30}\t{:30}'.format(
+                click.style(member['username'], fg='yellow'),
+                member['email'])
+            )
+    else:
+        click.secho('No members yet', dim=True)
+
+
+@org.command()
+@click.argument('user', metavar='<username or email>')
+def remove(user):
+    """Remove and downgrade a member"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        api.org.remove_member(user)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    
+    click.secho('Successfully removed the member', fg='green')
diff --git a/shodan/client.py b/shodan/client.py
index 0efb628..2b82ae2 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -23,6 +23,7 @@
 # pip install requests[security]
 #
 # Which will download libraries that offer more full-featured SSL classes
+# pylint: disable=E1101
 try:
     requests.packages.urllib3.disable_warnings()
 except:
@@ -135,6 +136,40 @@ def honeyscore(self, ip):
             """
             return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
 
+    class Organization:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def add_member(self, user, notify=True):
+            """Add the user to the organization.
+
+            :param user: username or email address
+            :type user: str
+            :param notify: whether or not to send the user an email notification
+            :type notify: bool
+
+            :returns: True if it succeeded and raises an Exception otherwise
+            """
+            return self.parent._request('/org/member/{}'.format(user), {
+                'notify': notify,
+            }, method='PUT')['success']
+
+        def info(self):
+            """Returns general information about the organization the current user is a member of.
+            """
+            return self.parent._request('/org', {})
+
+        def remove_member(self, user):
+            """Remove the user from the organization.
+
+            :param user: username or email address
+            :type user: str
+
+            :returns: True if it succeeded and raises an Exception otherwise
+            """
+            return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success']
+
     def __init__(self, key, proxies=None):
         """Initializes the API object.
 
@@ -149,6 +184,7 @@ def __init__(self, key, proxies=None):
         self.data = self.Data(self)
         self.exploits = self.Exploits(self)
         self.labs = self.Labs(self)
+        self.org = self.Organization(self)
         self.tools = self.Tools(self)
         self.stream = Stream(key, proxies=proxies)
         self._session = requests.Session()
@@ -177,8 +213,13 @@ def _request(self, function, params, service='shodan', method='get'):
 
         # Send the request
         try:
-            if method.lower() == 'post':
+            method = method.lower()
+            if method == 'post':
                 data = self._session.post(base_url + function, params)
+            elif method == 'put':
+                data = self._session.put(base_url + function, params=params)
+            elif method == 'delete':
+                data = self._session.delete(base_url + function, params=params)
             else:
                 data = self._session.get(base_url + function, params=params)
         except:
@@ -384,13 +425,12 @@ def search_cursor(self, query, minify=True, retries=5):
 
         :returns: A search cursor that can be used as an iterator/ generator.
         """
-        args = {
-            'query': query,
-            'minify': minify,
-        }
-
         page = 1
         tries = 0
+        results = {
+            'matches': [],
+            'total': None,
+        }
         while page == 1 or results['matches']:
             try:
                 results = self.search(query, minify=minify, page=page)

From 648d125be5ff6f68d037e9f482e3952734697f9c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 1 Sep 2018 18:38:37 -0500
Subject: [PATCH 150/263] Code cleanup

---
 shodan/cli/alert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 779c44c..a9f6ec8 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -39,8 +39,8 @@ def alert_create(name, netblock):
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
-    click.echo(click.style('Successfully created network alert!', fg='green'))
-    click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan'))
+    click.secho('Successfully created network alert!', fg='green')
+    click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 @alert.command(name='list')
 @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
@@ -69,7 +69,7 @@ def alert_list(expired):
             )
 
             if 'expired' in alert and alert['expired']:
-                click.echo(click.style('expired', fg='red'))
+                click.secho('expired', fg='red')
             else:
                 click.echo('')
     else:

From bba05c95a9d998faffa2502413420dbe52f91cf4 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 1 Sep 2018 19:57:26 -0500
Subject: [PATCH 151/263] Improve unicode handling (#78) Allow printing of
 nested properties in "shodan search"

---
 shodan/__main__.py    | 21 ++++++++-------------
 shodan/cli/helpers.py |  6 +++---
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 9d49ba9..861bce9 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -418,17 +418,18 @@ def search(color, fields, limit, separator, query):
 
         # Loop over all the fields and print the banner as a row
         for field in fields:
-            tmp = ''
-            if field in banner and banner[field]:
-                field_type = type(banner[field])
+            tmp = u''
+            value = get_banner_field(banner, field)
+            if value:
+                field_type = type(value)
 
                 # If the field is an array then merge it together
                 if field_type == list:
-                    tmp = u';'.join(banner[field])
+                    tmp = u';'.join(value)
                 elif field_type in [int, float]:
-                    tmp = u'{}'.format(banner[field])
+                    tmp = u'{}'.format(value)
                 else:
-                    tmp = escape_data(banner[field])
+                    tmp = escape_data(value)
 
                 # Colorize certain fields if the user wants it
                 if color:
@@ -476,13 +477,7 @@ def stats(limit, facets, filename, query):
         click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet))
 
         for item in results['facets'][facet]:
-            value = item['value']
-            if isinstance(value, basestring):
-                value = value.encode('ascii', errors='replace').decode('ascii')
-            else:
-                value = str(value)
-
-            click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False)
+            click.echo(click.style(u'{:28s}'.format(item['value']), fg='cyan'), nl=False)
             click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index bd29ae4..7f7d89f 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -28,9 +28,9 @@ def get_api_key():
 
 
 def escape_data(args):
-    # Ensure the provided string isn't unicode data
-    if not isinstance(args, str):
-        args = args.encode('ascii', 'replace')
+    # Make sure the string is unicode so the terminal can properly display it
+    # We do it using format() so it works across Python 2 and 3
+    args = u'{}'.format(args)
     return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
 
 

From f8f53aa94d908e20998a1d35b602d585e071184e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 1 Sep 2018 20:47:51 -0500
Subject: [PATCH 152/263] Release 1.10.0

---
 CHANGELOG.md | 6 ++++++
 setup.py     | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99a5dd8..08caae4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,12 @@
 CHANGELOG
 =========
 
+1.10.0
+------
+* New command **shodan org**: manage enterprise access to Shodan for your team
+* Improved unicode handling (#78)
+* Remove deprecated API wrapper for shodanhq.com/api
+
 1.9.1
 -----
 * The CHANGELOG is now part of the packages.
diff --git a/setup.py b/setup.py
index 88b8511..a810b7a 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.9.1',
+    version = '1.10.0',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',

From 6ad6a51f10489e5204899b75ef83759295609146 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 3 Sep 2018 23:08:26 -0500
Subject: [PATCH 153/263] Release 1.10.1 Support PUT requests in the API helper
 method

---
 CHANGELOG.md      | 4 ++++
 setup.py          | 2 +-
 shodan/helpers.py | 2 ++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08caae4..8e84249 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.10.1
+------
+* Support PUT requests in the API request helper method
+
 1.10.0
 ------
 * New command **shodan org**: manage enterprise access to Shodan for your team
diff --git a/setup.py b/setup.py
index a810b7a..5c96d32 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.10.0',
+    version = '1.10.1',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 64e5937..ed1a88a 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -51,6 +51,8 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
                                      proxies=proxies)
             elif method.lower() == 'delete':
                 data = requests.delete(base_url + function, params=params, proxies=proxies)
+            elif method.lower() == 'put':
+                data = requests.put(base_url + function, params=params, proxies=proxies)
             else:
                 data = requests.get(base_url + function, params=params, proxies=proxies)
 

From 9edd057125332e02f47cfa2659e0a7e7d39be983 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 11 Sep 2018 23:20:17 -0500
Subject: [PATCH 154/263] Show the list of authorized domains for organizations

---
 shodan/cli/organization.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py
index ecdd58a..985c38e 100644
--- a/shodan/cli/organization.py
+++ b/shodan/cli/organization.py
@@ -41,6 +41,11 @@ def info():
     click.secho(organization['name'], fg='cyan')
     click.secho('Access Level: ', nl=False, dim=True)
     click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta')
+    
+    if organization['domains']:
+        click.secho('Authorized Domains: ', nl=False, dim=True)
+        click.echo(', '.join(organization['domains']))
+    
     click.echo('')
     click.secho('Administrators:', dim=True)
 

From 3fadceff99283e806908565894913b14fe447a90 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 12 Sep 2018 16:40:39 -0500
Subject: [PATCH 155/263] Make sure all facet values are strings before
 formatting them

---
 shodan/__main__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 861bce9..339218e 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -477,7 +477,10 @@ def stats(limit, facets, filename, query):
         click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet))
 
         for item in results['facets'][facet]:
-            click.echo(click.style(u'{:28s}'.format(item['value']), fg='cyan'), nl=False)
+            # Force the value to be a string - necessary because some facet values are numbers
+            value = u'{}'.format(item['value'])
+
+            click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False)
             click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green'))
 
         click.echo('')

From ed3eadcf9e6f990379aec29f968c7edcee0bee16 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 17 Sep 2018 12:50:25 -0500
Subject: [PATCH 156/263] Release 1.10.2

---
 CHANGELOG.md | 4 ++++
 setup.py     | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e84249..a2945ec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.10.2
+------
+* Fix **shodan stats** formatting exception when faceting on **port**
+
 1.10.1
 ------
 * Support PUT requests in the API request helper method
diff --git a/setup.py b/setup.py
index 5c96d32..593a481 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.10.1',
+    version = '1.10.2',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',

From ce5a656a86442135d0a198a858795905e2b27b0a Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 18 Sep 2018 18:05:24 +0200
Subject: [PATCH 157/263] Implement feedback from lgtm tool

https://lgtm.com/projects/g/achillean/shodan-python/alerts/?mode=list
---
 CHANGELOG.md                    |  5 +++++
 shodan/__main__.py              | 10 ++--------
 shodan/cli/converter/csvc.py    |  6 +++---
 shodan/cli/converter/excel.py   |  8 ++++----
 shodan/cli/converter/geojson.py |  2 +-
 shodan/cli/converter/kml.py     |  2 +-
 shodan/cli/helpers.py           |  2 +-
 shodan/cli/organization.py      |  2 --
 shodan/cli/worldmap.py          |  2 +-
 shodan/client.py                |  8 ++++----
 shodan/helpers.py               |  6 +++---
 11 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a2945ec..063ff36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+unreleased
+----------
+* Change bare 'except:' statements to 'except Exception:' or more specific ones
+* remove unused imports
+
 1.10.2
 ------
 * Fix **shodan stats** formatting exception when faceting on **port**
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 339218e..f942080 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -26,17 +26,11 @@
 """
 
 import click
-import collections
 import csv
-import datetime
-import gzip
-import itertools
 import os
 import os.path
 import shodan
 import shodan.helpers as helpers
-import socket
-import sys
 import threading
 import requests
 import time
@@ -584,7 +578,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
     if ports:
         try:
             stream_args = [int(item.strip()) for item in ports.split(',')]
-        except:
+        except ValueError:
             raise click.ClickException('Invalid list of ports')
 
     if alert:
@@ -683,7 +677,7 @@ def _create_stream(name, args, timeout):
             quit = True
         except shodan.APIError as e:
             raise click.ClickException(e.value)
-        except:
+        except Exception:
             # For other errors lets just wait a bit and try to reconnect again
             time.sleep(1)
 
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 4e08f63..c975695 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -54,7 +54,7 @@ def process(self, files):
                     value = self.banner_field(banner, field)
                     row.append(value)
                 writer.writerow(row)
-            except:
+            except Exception:
                 pass
     
     def banner_field(self, banner, flat_field):
@@ -71,7 +71,7 @@ def banner_field(self, banner, flat_field):
                 current_obj = ','.join([str(i) for i in current_obj])
             
             return current_obj
-        except:
+        except Exception:
             pass
     
         return ''
@@ -85,4 +85,4 @@ def flatten(self, d, parent_key='', sep='.'):
                 items.extend(flatten(v, new_key, sep=sep).items())
             else:
                 items.append((new_key, v))
-        return dict(items)
\ No newline at end of file
+        return dict(items)
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index c22da90..a6b476d 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -2,7 +2,7 @@
 from .base import Converter
 from ...helpers import iterate_files, get_ip
 
-from collections import defaultdict, MutableMapping
+from collections import defaultdict
 from xlsxwriter import Workbook
 
 
@@ -90,7 +90,7 @@ def process(self, files):
                     main_sheet.write(row, col, value)
                     col += 1
                 row += 1
-            except:
+            except Exception:
                 pass
             
             # Aggregate summary information
@@ -124,7 +124,7 @@ def banner_field(self, banner, flat_field):
                 current_obj = ','.join([str(i) for i in current_obj])
             
             return current_obj
-        except:
+        except Exception:
             pass
     
-        return ''
\ No newline at end of file
+        return ''
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index 2cc55e4..8bde86f 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -53,5 +53,5 @@ def write(self, host):
             }""".format(ip, ip, lat, lon)
 
             self.fout.write(feature)
-        except:
+        except Exception:
             pass
diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py
index c8cb2cc..49938c2 100644
--- a/shodan/cli/converter/kml.py
+++ b/shodan/cli/converter/kml.py
@@ -102,5 +102,5 @@ def write(self, host):
             placemark += '</Placemark>'
 
             self.fout.write(placemark.encode('utf-8'))
-        except:
+        except Exception:
             pass
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 7f7d89f..6ef9e1b 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -51,7 +51,7 @@ def get_banner_field(banner, flat_field):
         for field in fields:
             current_obj = current_obj[field]
         return current_obj
-    except:
+    except Exception:
         pass
 
     return None
diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py
index 985c38e..50d814c 100644
--- a/shodan/cli/organization.py
+++ b/shodan/cli/organization.py
@@ -1,7 +1,5 @@
 import click
-import requests
 import shodan
-import shodan.helpers as helpers
 
 from shodan.cli.helpers import get_api_key, humanize_api_plan
 
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index c3357ba..60ef075 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -132,7 +132,7 @@ def set_data(self, data):
                 # Not all cities can be encoded in ASCII so ignore any errors
                 try:
                     desc += ' {}'.format(banner['location']['city'])
-                except:
+                except Exception:
                     pass
             
             if 'tags' in banner and banner['tags']:
diff --git a/shodan/client.py b/shodan/client.py
index 2b82ae2..1d3ff54 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -26,7 +26,7 @@
 # pylint: disable=E1101
 try:
     requests.packages.urllib3.disable_warnings()
-except:
+except Exception:
     pass
 
 # Define a basestring type if necessary for Python3 compatibility
@@ -222,7 +222,7 @@ def _request(self, function, params, service='shodan', method='get'):
                 data = self._session.delete(base_url + function, params=params)
             else:
                 data = self._session.get(base_url + function, params=params)
-        except:
+        except Exception:
             raise APIError('Unable to connect to Shodan')
 
         # Check that the API key wasn't rejected
@@ -246,7 +246,7 @@ def _request(self, function, params, service='shodan', method='get'):
         # Parse the text into JSON
         try:
             data = data.json()
-        except:
+        except ValueError:
             raise APIError('Unable to parse JSON response')
 
         # Raise an exception if an error occurred
@@ -441,7 +441,7 @@ def search_cursor(self, query, minify=True, retries=5):
                         return # exit out of the function
                 page += 1
                 tries = 0
-            except:
+            except Exception:
                 # We've retried several times but it keeps failing, so lets error out
                 if tries >= retries:
                     break
diff --git a/shodan/helpers.py b/shodan/helpers.py
index ed1a88a..2756ab0 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -58,7 +58,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
 
             # Exit out of the loop
             break
-        except:
+        except Exception:
             error = True
             tries += 1
 
@@ -69,7 +69,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     if data.status_code == 401:
         try:
             raise APIError(data.json()['error'])
-        except:
+        except (ValueError, KeyError):
             pass
         raise APIError('Invalid API key')
 
@@ -89,7 +89,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
 
 def iterate_files(files, fast=False):
     """Loop over all the records of the provided Shodan output file(s)."""
-    from json import loads
+    loads = json.loads
     if fast:
         # Try to use ujson for parsing JSON if it's available and the user requested faster throughput
         # It's significantly faster at encoding/ decoding JSON but it doesn't support as

From 3312b02075c574b25342d6fb6c960692980af4ca Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Tue, 18 Sep 2018 18:16:33 +0200
Subject: [PATCH 158/263] unix line endings everywhere

---
 CHANGELOG.md         |    1 +
 shodan/client.py     | 1126 +++++++++++++++++++++---------------------
 tests/test_shodan.py |  314 ++++++------
 3 files changed, 721 insertions(+), 720 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 063ff36..0d531ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ unreleased
 ----------
 * Change bare 'except:' statements to 'except Exception:' or more specific ones
 * remove unused imports
+* Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix
 
 1.10.2
 ------
diff --git a/shodan/client.py b/shodan/client.py
index 1d3ff54..0fd9167 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -1,563 +1,563 @@
-# -*- coding: utf-8 -*-
-"""
-shodan.client
-~~~~~~~~~~~~~
-
-This module implements the Shodan API.
-
-:copyright: (c) 2014- by John Matherly
-"""
-import time
-
-import requests
-import json
-
-from .exception import APIError
-from .helpers import api_request, create_facet_string
-from .stream import Stream
-
-
-# Try to disable the SSL warnings in urllib3 since not everybody can install
-# C extensions. If you're able to install C extensions you can try to run:
-#
-# pip install requests[security]
-#
-# Which will download libraries that offer more full-featured SSL classes
-# pylint: disable=E1101
-try:
-    requests.packages.urllib3.disable_warnings()
-except Exception:
-    pass
-
-# Define a basestring type if necessary for Python3 compatibility
-try:
-    basestring
-except NameError:
-    basestring = str
-
-
-class Shodan:
-    """Wrapper around the Shodan REST and Streaming APIs
-
-    :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io)
-    :type key: str
-    :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API.
-    :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API.
-    """
-
-    class Data:
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def list_datasets(self):
-            """Returns a list of datasets that the user has permission to download.
-
-            :returns: A list of objects where every object describes a dataset
-            """
-            return self.parent._request('/shodan/data', {})
-
-        def list_files(self, dataset):
-            """Returns a list of files that belong to the given dataset.
-
-            :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url'
-            """
-            return self.parent._request('/shodan/data/{}'.format(dataset), {})
-
-    class Tools:
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def myip(self):
-            """Get your current IP address as seen from the Internet.
-
-            :returns: str -- your IP address
-            """
-            return self.parent._request('/tools/myip', {})
-
-    class Exploits:
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def search(self, query, page=1, facets=None):
-            """Search the entire Shodan Exploits archive using the same query syntax
-            as the website.
-
-            :param query: The exploit search query; same syntax as website.
-            :type query: str
-            :param facets: A list of strings or tuples to get summary information on.
-            :type facets: str
-            :param page: The page number to access.
-            :type page: int
-            :returns: dict -- a dictionary containing the results of the search.
-            """
-            query_args = {
-                'query': query,
-                'page': page,
-            }
-            if facets:
-                query_args['facets'] = create_facet_string(facets)
-
-            return self.parent._request('/api/search', query_args, service='exploits')
-
-        def count(self, query, facets=None):
-            """Search the entire Shodan Exploits archive but only return the total # of results,
-            not the actual exploits.
-
-            :param query: The exploit search query; same syntax as website.
-            :type query: str
-            :param facets: A list of strings or tuples to get summary information on.
-            :type facets: str
-            :returns: dict -- a dictionary containing the results of the search.
-
-            """
-            query_args = {
-                'query': query,
-            }
-            if facets:
-                query_args['facets'] = create_facet_string(facets)
-
-            return self.parent._request('/api/count', query_args, service='exploits')
-
-    class Labs:
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def honeyscore(self, ip):
-            """Calculate the probability of an IP being an ICS honeypot.
-
-            :param ip: IP address of the device
-            :type ip: str
-
-            :returns: int -- honeyscore ranging from 0.0 to 1.0
-            """
-            return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
-
-    class Organization:
-
-        def __init__(self, parent):
-            self.parent = parent
-
-        def add_member(self, user, notify=True):
-            """Add the user to the organization.
-
-            :param user: username or email address
-            :type user: str
-            :param notify: whether or not to send the user an email notification
-            :type notify: bool
-
-            :returns: True if it succeeded and raises an Exception otherwise
-            """
-            return self.parent._request('/org/member/{}'.format(user), {
-                'notify': notify,
-            }, method='PUT')['success']
-
-        def info(self):
-            """Returns general information about the organization the current user is a member of.
-            """
-            return self.parent._request('/org', {})
-
-        def remove_member(self, user):
-            """Remove the user from the organization.
-
-            :param user: username or email address
-            :type user: str
-
-            :returns: True if it succeeded and raises an Exception otherwise
-            """
-            return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success']
-
-    def __init__(self, key, proxies=None):
-        """Initializes the API object.
-
-        :param key: The Shodan API key.
-        :type key: str
-        :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'}
-        :type key: dict
-        """
-        self.api_key = key
-        self.base_url = 'https://api.shodan.io'
-        self.base_exploits_url = 'https://exploits.shodan.io'
-        self.data = self.Data(self)
-        self.exploits = self.Exploits(self)
-        self.labs = self.Labs(self)
-        self.org = self.Organization(self)
-        self.tools = self.Tools(self)
-        self.stream = Stream(key, proxies=proxies)
-        self._session = requests.Session()
-        if proxies:
-            self._session.proxies.update(proxies)
-
-    def _request(self, function, params, service='shodan', method='get'):
-        """General-purpose function to create web requests to SHODAN.
-
-        Arguments:
-            function  -- name of the function you want to execute
-            params    -- dictionary of parameters for the function
-
-        Returns
-            A dictionary containing the function's results.
-
-        """
-        # Add the API key parameter automatically
-        params['key'] = self.api_key
-
-        # Determine the base_url based on which service we're interacting with
-        base_url = {
-            'shodan': self.base_url,
-            'exploits': self.base_exploits_url,
-        }.get(service, 'shodan')
-
-        # Send the request
-        try:
-            method = method.lower()
-            if method == 'post':
-                data = self._session.post(base_url + function, params)
-            elif method == 'put':
-                data = self._session.put(base_url + function, params=params)
-            elif method == 'delete':
-                data = self._session.delete(base_url + function, params=params)
-            else:
-                data = self._session.get(base_url + function, params=params)
-        except Exception:
-            raise APIError('Unable to connect to Shodan')
-
-        # Check that the API key wasn't rejected
-        if data.status_code == 401:
-            try:
-                # Return the actual error message if the API returned valid JSON
-                error = data.json()['error']
-            except Exception as e:
-                # If the response looks like HTML then it's probably the 401 page that nginx returns
-                # for 401 responses by default
-                if data.text.startswith('<'):
-                    error = 'Invalid API key'
-                else:
-                    # Otherwise lets raise the error message
-                    error = u'{}'.format(e)
-
-            raise APIError(error)
-        elif data.status_code == 403:
-            raise APIError('Access denied (403 Forbidden)')
-
-        # Parse the text into JSON
-        try:
-            data = data.json()
-        except ValueError:
-            raise APIError('Unable to parse JSON response')
-
-        # Raise an exception if an error occurred
-        if type(data) == dict and 'error' in data:
-            raise APIError(data['error'])
-
-        # Return the data
-        return data
-
-    def count(self, query, facets=None):
-        """Returns the total number of search results for the query.
-
-        :param query: Search query; identical syntax to the website
-        :type query: str
-        :param facets: (optional) A list of properties to get summary information on
-        :type facets: str
-
-        :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
-        """
-        query_args = {
-            'query': query,
-        }
-        if facets:
-            query_args['facets'] = create_facet_string(facets)
-        return self._request('/shodan/host/count', query_args)
-
-    def host(self, ips, history=False, minify=False):
-        """Get all available information on an IP.
-
-        :param ip: IP of the computer
-        :type ip: str
-        :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise.
-        :type history: bool
-        :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise.
-        :type minify: bool
-        """
-        if isinstance(ips, basestring):
-            ips = [ips]
-
-        params = {}
-        if history:
-            params['history'] = history
-        if minify:
-            params['minify'] = minify
-        return self._request('/shodan/host/%s' % ','.join(ips), params)
-
-    def info(self):
-        """Returns information about the current API key, such as a list of add-ons
-        and other features that are enabled for the current user's API plan.
-        """
-        return self._request('/api-info', {})
-
-    def ports(self):
-        """Get a list of ports that Shodan crawls
-
-        :returns: An array containing the ports that Shodan crawls for.
-        """
-        return self._request('/shodan/ports', {})
-
-    def protocols(self):
-        """Get a list of protocols that the Shodan on-demand scanning API supports.
-
-        :returns: A dictionary containing the protocol name and description.
-        """
-        return self._request('/shodan/protocols', {})
-
-    def scan(self, ips, force=False):
-        """Scan a network using Shodan
-
-        :param ips: A list of IPs or netblocks in CIDR notation or an object structured like:
-                    {
-                        "9.9.9.9": [
-                            (443, "https"),
-                            (8080, "http")
-                        ],
-                        "1.1.1.0/24": [
-                            (503, "modbus")
-                        ]
-                    }
-        :type ips: str or dict
-        :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users.
-        :type force: bool
-
-        :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left.
-        """
-        if isinstance(ips, basestring):
-            ips = [ips]
-
-        if isinstance(ips, dict):
-            networks = json.dumps(ips)
-        else:
-            networks = ','.join(ips)
-
-        params = {
-            'ips': networks,
-            'force': force,
-        }
-
-        return self._request('/shodan/scan', params, method='post')
-
-    def scan_internet(self, port, protocol):
-        """Scan a network using Shodan
-
-        :param port: The port that should get scanned.
-        :type port: int
-        :param port: The name of the protocol as returned by the protocols() method.
-        :type port: str
-
-        :returns: A dictionary with a unique ID to check on the scan progress.
-        """
-        params = {
-            'port': port,
-            'protocol': protocol,
-        }
-
-        return self._request('/shodan/scan/internet', params, method='post')
-
-    def scan_status(self, scan_id):
-        """Get the status information about a previously submitted scan.
-
-        :param id: The unique ID for the scan that was submitted
-        :type id: str
-
-        :returns: A dictionary with general information about the scan, including its status in getting processed.
-        """
-        return self._request('/shodan/scan/%s' % scan_id, {})
-
-    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
-        """Search the SHODAN database.
-
-        :param query: Search query; identical syntax to the website
-        :type query: str
-        :param page: (optional) Page number of the search results
-        :type page: int
-        :param limit: (optional) Number of results to return
-        :type limit: int
-        :param offset: (optional) Search offset to begin getting results from
-        :type offset: int
-        :param facets: (optional) A list of properties to get summary information on
-        :type facets: str
-        :param minify: (optional) Whether to minify the banner and only return the important data
-        :type minify: bool
-
-        :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
-        """
-        args = {
-            'query': query,
-            'minify': minify,
-        }
-        if limit:
-            args['limit'] = limit
-            if offset:
-                args['offset'] = offset
-        else:
-            args['page'] = page
-
-        if facets:
-            args['facets'] = create_facet_string(facets)
-
-        return self._request('/shodan/host/search', args)
-
-    def search_cursor(self, query, minify=True, retries=5):
-        """Search the SHODAN database.
-
-        This method returns an iterator that can directly be in a loop. Use it when you want to loop over
-        all of the results of a search query. But this method doesn't return a "matches" array or the "total"
-        information. And it also can't be used with facets, it's only use is to iterate over results more
-        easily.
-
-        :param query: Search query; identical syntax to the website
-        :type query: str
-        :param minify: (optional) Whether to minify the banner and only return the important data
-        :type minify: bool
-        :param retries: (optional) How often to retry the search in case it times out
-        :type minify: int
-
-        :returns: A search cursor that can be used as an iterator/ generator.
-        """
-        page = 1
-        tries = 0
-        results = {
-            'matches': [],
-            'total': None,
-        }
-        while page == 1 or results['matches']:
-            try:
-                results = self.search(query, minify=minify, page=page)
-                for banner in results['matches']:
-                    try:
-                        yield banner
-                    except GeneratorExit:
-                        return # exit out of the function
-                page += 1
-                tries = 0
-            except Exception:
-                # We've retried several times but it keeps failing, so lets error out
-                if tries >= retries:
-                    break
-
-                tries += 1
-                time.sleep(1.0) # wait 1 second if the search errored out for some reason
-
-    def search_tokens(self, query):
-        """Returns information about the search query itself (filters used etc.)
-
-        :param query: Search query; identical syntax to the website
-        :type query: str
-
-        :returns: A dictionary with 4 main properties: filters, errors, attributes and string.
-        """
-        query_args = {
-            'query': query,
-        }
-        return self._request('/shodan/host/search/tokens', query_args)
-
-    def services(self):
-        """Get a list of services that Shodan crawls
-
-        :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service.
-        """
-        return self._request('/shodan/services', {})
-
-    def queries(self, page=1, sort='timestamp', order='desc'):
-        """List the search queries that have been shared by other users.
-
-        :param page: Page number to iterate over results; each page contains 10 items
-        :type page: int
-        :param sort: Sort the list based on a property. Possible values are: votes, timestamp
-        :type sort: str
-        :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc
-        :type order: str
-
-        :returns: A list of saved search queries (dictionaries).
-        """
-        args = {
-            'page': page,
-            'sort': sort,
-            'order': order,
-        }
-        return self._request('/shodan/query', args)
-
-    def queries_search(self, query, page=1):
-        """Search the directory of saved search queries in Shodan.
-
-        :param query: The search string to look for in the search query
-        :type query: str
-        :param page: Page number to iterate over results; each page contains 10 items
-        :type page: int
-
-        :returns: A list of saved search queries (dictionaries).
-        """
-        args = {
-            'page': page,
-            'query': query,
-        }
-        return self._request('/shodan/query/search', args)
-
-    def queries_tags(self, size=10):
-        """Search the directory of saved search queries in Shodan.
-
-        :param query: The number of tags to return
-        :type page: int
-
-        :returns: A list of tags.
-        """
-        args = {
-            'size': size,
-        }
-        return self._request('/shodan/query/tags', args)
-
-    def create_alert(self, name, ip, expires=0):
-        """Search the directory of saved search queries in Shodan.
-
-        :param query: The number of tags to return
-        :type page: int
-
-        :returns: A list of tags.
-        """
-        data = {
-            'name': name,
-            'filters': {
-                'ip': ip,
-            },
-            'expires': expires,
-        }
-
-        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post',
-                               proxies=self._session.proxies)
-
-        return response
-
-    def alerts(self, aid=None, include_expired=True):
-        """List all of the active alerts that the user created."""
-        if aid:
-            func = '/shodan/alert/%s/info' % aid
-        else:
-            func = '/shodan/alert/info'
-
-        response = api_request(self.api_key, func, params={
-            'include_expired': include_expired,
-            },
-            proxies=self._session.proxies)
-
-        return response
-
-    def delete_alert(self, aid):
-        """Delete the alert with the given ID."""
-        func = '/shodan/alert/%s' % aid
-
-        response = api_request(self.api_key, func, params={}, method='delete',
-                               proxies=self._session.proxies)
-
-        return response
-
+# -*- coding: utf-8 -*-
+"""
+shodan.client
+~~~~~~~~~~~~~
+
+This module implements the Shodan API.
+
+:copyright: (c) 2014- by John Matherly
+"""
+import time
+
+import requests
+import json
+
+from .exception import APIError
+from .helpers import api_request, create_facet_string
+from .stream import Stream
+
+
+# Try to disable the SSL warnings in urllib3 since not everybody can install
+# C extensions. If you're able to install C extensions you can try to run:
+#
+# pip install requests[security]
+#
+# Which will download libraries that offer more full-featured SSL classes
+# pylint: disable=E1101
+try:
+    requests.packages.urllib3.disable_warnings()
+except Exception:
+    pass
+
+# Define a basestring type if necessary for Python3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
+
+class Shodan:
+    """Wrapper around the Shodan REST and Streaming APIs
+
+    :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io)
+    :type key: str
+    :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API.
+    :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API.
+    """
+
+    class Data:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def list_datasets(self):
+            """Returns a list of datasets that the user has permission to download.
+
+            :returns: A list of objects where every object describes a dataset
+            """
+            return self.parent._request('/shodan/data', {})
+
+        def list_files(self, dataset):
+            """Returns a list of files that belong to the given dataset.
+
+            :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url'
+            """
+            return self.parent._request('/shodan/data/{}'.format(dataset), {})
+
+    class Tools:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def myip(self):
+            """Get your current IP address as seen from the Internet.
+
+            :returns: str -- your IP address
+            """
+            return self.parent._request('/tools/myip', {})
+
+    class Exploits:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def search(self, query, page=1, facets=None):
+            """Search the entire Shodan Exploits archive using the same query syntax
+            as the website.
+
+            :param query: The exploit search query; same syntax as website.
+            :type query: str
+            :param facets: A list of strings or tuples to get summary information on.
+            :type facets: str
+            :param page: The page number to access.
+            :type page: int
+            :returns: dict -- a dictionary containing the results of the search.
+            """
+            query_args = {
+                'query': query,
+                'page': page,
+            }
+            if facets:
+                query_args['facets'] = create_facet_string(facets)
+
+            return self.parent._request('/api/search', query_args, service='exploits')
+
+        def count(self, query, facets=None):
+            """Search the entire Shodan Exploits archive but only return the total # of results,
+            not the actual exploits.
+
+            :param query: The exploit search query; same syntax as website.
+            :type query: str
+            :param facets: A list of strings or tuples to get summary information on.
+            :type facets: str
+            :returns: dict -- a dictionary containing the results of the search.
+
+            """
+            query_args = {
+                'query': query,
+            }
+            if facets:
+                query_args['facets'] = create_facet_string(facets)
+
+            return self.parent._request('/api/count', query_args, service='exploits')
+
+    class Labs:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def honeyscore(self, ip):
+            """Calculate the probability of an IP being an ICS honeypot.
+
+            :param ip: IP address of the device
+            :type ip: str
+
+            :returns: int -- honeyscore ranging from 0.0 to 1.0
+            """
+            return self.parent._request('/labs/honeyscore/{}'.format(ip), {})
+
+    class Organization:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def add_member(self, user, notify=True):
+            """Add the user to the organization.
+
+            :param user: username or email address
+            :type user: str
+            :param notify: whether or not to send the user an email notification
+            :type notify: bool
+
+            :returns: True if it succeeded and raises an Exception otherwise
+            """
+            return self.parent._request('/org/member/{}'.format(user), {
+                'notify': notify,
+            }, method='PUT')['success']
+
+        def info(self):
+            """Returns general information about the organization the current user is a member of.
+            """
+            return self.parent._request('/org', {})
+
+        def remove_member(self, user):
+            """Remove the user from the organization.
+
+            :param user: username or email address
+            :type user: str
+
+            :returns: True if it succeeded and raises an Exception otherwise
+            """
+            return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success']
+
+    def __init__(self, key, proxies=None):
+        """Initializes the API object.
+
+        :param key: The Shodan API key.
+        :type key: str
+        :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'}
+        :type key: dict
+        """
+        self.api_key = key
+        self.base_url = 'https://api.shodan.io'
+        self.base_exploits_url = 'https://exploits.shodan.io'
+        self.data = self.Data(self)
+        self.exploits = self.Exploits(self)
+        self.labs = self.Labs(self)
+        self.org = self.Organization(self)
+        self.tools = self.Tools(self)
+        self.stream = Stream(key, proxies=proxies)
+        self._session = requests.Session()
+        if proxies:
+            self._session.proxies.update(proxies)
+
+    def _request(self, function, params, service='shodan', method='get'):
+        """General-purpose function to create web requests to SHODAN.
+
+        Arguments:
+            function  -- name of the function you want to execute
+            params    -- dictionary of parameters for the function
+
+        Returns
+            A dictionary containing the function's results.
+
+        """
+        # Add the API key parameter automatically
+        params['key'] = self.api_key
+
+        # Determine the base_url based on which service we're interacting with
+        base_url = {
+            'shodan': self.base_url,
+            'exploits': self.base_exploits_url,
+        }.get(service, 'shodan')
+
+        # Send the request
+        try:
+            method = method.lower()
+            if method == 'post':
+                data = self._session.post(base_url + function, params)
+            elif method == 'put':
+                data = self._session.put(base_url + function, params=params)
+            elif method == 'delete':
+                data = self._session.delete(base_url + function, params=params)
+            else:
+                data = self._session.get(base_url + function, params=params)
+        except Exception:
+            raise APIError('Unable to connect to Shodan')
+
+        # Check that the API key wasn't rejected
+        if data.status_code == 401:
+            try:
+                # Return the actual error message if the API returned valid JSON
+                error = data.json()['error']
+            except Exception as e:
+                # If the response looks like HTML then it's probably the 401 page that nginx returns
+                # for 401 responses by default
+                if data.text.startswith('<'):
+                    error = 'Invalid API key'
+                else:
+                    # Otherwise lets raise the error message
+                    error = u'{}'.format(e)
+
+            raise APIError(error)
+        elif data.status_code == 403:
+            raise APIError('Access denied (403 Forbidden)')
+
+        # Parse the text into JSON
+        try:
+            data = data.json()
+        except ValueError:
+            raise APIError('Unable to parse JSON response')
+
+        # Raise an exception if an error occurred
+        if type(data) == dict and 'error' in data:
+            raise APIError(data['error'])
+
+        # Return the data
+        return data
+
+    def count(self, query, facets=None):
+        """Returns the total number of search results for the query.
+
+        :param query: Search query; identical syntax to the website
+        :type query: str
+        :param facets: (optional) A list of properties to get summary information on
+        :type facets: str
+
+        :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
+        """
+        query_args = {
+            'query': query,
+        }
+        if facets:
+            query_args['facets'] = create_facet_string(facets)
+        return self._request('/shodan/host/count', query_args)
+
+    def host(self, ips, history=False, minify=False):
+        """Get all available information on an IP.
+
+        :param ip: IP of the computer
+        :type ip: str
+        :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise.
+        :type history: bool
+        :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise.
+        :type minify: bool
+        """
+        if isinstance(ips, basestring):
+            ips = [ips]
+
+        params = {}
+        if history:
+            params['history'] = history
+        if minify:
+            params['minify'] = minify
+        return self._request('/shodan/host/%s' % ','.join(ips), params)
+
+    def info(self):
+        """Returns information about the current API key, such as a list of add-ons
+        and other features that are enabled for the current user's API plan.
+        """
+        return self._request('/api-info', {})
+
+    def ports(self):
+        """Get a list of ports that Shodan crawls
+
+        :returns: An array containing the ports that Shodan crawls for.
+        """
+        return self._request('/shodan/ports', {})
+
+    def protocols(self):
+        """Get a list of protocols that the Shodan on-demand scanning API supports.
+
+        :returns: A dictionary containing the protocol name and description.
+        """
+        return self._request('/shodan/protocols', {})
+
+    def scan(self, ips, force=False):
+        """Scan a network using Shodan
+
+        :param ips: A list of IPs or netblocks in CIDR notation or an object structured like:
+                    {
+                        "9.9.9.9": [
+                            (443, "https"),
+                            (8080, "http")
+                        ],
+                        "1.1.1.0/24": [
+                            (503, "modbus")
+                        ]
+                    }
+        :type ips: str or dict
+        :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users.
+        :type force: bool
+
+        :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left.
+        """
+        if isinstance(ips, basestring):
+            ips = [ips]
+
+        if isinstance(ips, dict):
+            networks = json.dumps(ips)
+        else:
+            networks = ','.join(ips)
+
+        params = {
+            'ips': networks,
+            'force': force,
+        }
+
+        return self._request('/shodan/scan', params, method='post')
+
+    def scan_internet(self, port, protocol):
+        """Scan a network using Shodan
+
+        :param port: The port that should get scanned.
+        :type port: int
+        :param port: The name of the protocol as returned by the protocols() method.
+        :type port: str
+
+        :returns: A dictionary with a unique ID to check on the scan progress.
+        """
+        params = {
+            'port': port,
+            'protocol': protocol,
+        }
+
+        return self._request('/shodan/scan/internet', params, method='post')
+
+    def scan_status(self, scan_id):
+        """Get the status information about a previously submitted scan.
+
+        :param id: The unique ID for the scan that was submitted
+        :type id: str
+
+        :returns: A dictionary with general information about the scan, including its status in getting processed.
+        """
+        return self._request('/shodan/scan/%s' % scan_id, {})
+
+    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
+        """Search the SHODAN database.
+
+        :param query: Search query; identical syntax to the website
+        :type query: str
+        :param page: (optional) Page number of the search results
+        :type page: int
+        :param limit: (optional) Number of results to return
+        :type limit: int
+        :param offset: (optional) Search offset to begin getting results from
+        :type offset: int
+        :param facets: (optional) A list of properties to get summary information on
+        :type facets: str
+        :param minify: (optional) Whether to minify the banner and only return the important data
+        :type minify: bool
+
+        :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
+        """
+        args = {
+            'query': query,
+            'minify': minify,
+        }
+        if limit:
+            args['limit'] = limit
+            if offset:
+                args['offset'] = offset
+        else:
+            args['page'] = page
+
+        if facets:
+            args['facets'] = create_facet_string(facets)
+
+        return self._request('/shodan/host/search', args)
+
+    def search_cursor(self, query, minify=True, retries=5):
+        """Search the SHODAN database.
+
+        This method returns an iterator that can directly be in a loop. Use it when you want to loop over
+        all of the results of a search query. But this method doesn't return a "matches" array or the "total"
+        information. And it also can't be used with facets, it's only use is to iterate over results more
+        easily.
+
+        :param query: Search query; identical syntax to the website
+        :type query: str
+        :param minify: (optional) Whether to minify the banner and only return the important data
+        :type minify: bool
+        :param retries: (optional) How often to retry the search in case it times out
+        :type minify: int
+
+        :returns: A search cursor that can be used as an iterator/ generator.
+        """
+        page = 1
+        tries = 0
+        results = {
+            'matches': [],
+            'total': None,
+        }
+        while page == 1 or results['matches']:
+            try:
+                results = self.search(query, minify=minify, page=page)
+                for banner in results['matches']:
+                    try:
+                        yield banner
+                    except GeneratorExit:
+                        return # exit out of the function
+                page += 1
+                tries = 0
+            except Exception:
+                # We've retried several times but it keeps failing, so lets error out
+                if tries >= retries:
+                    break
+
+                tries += 1
+                time.sleep(1.0) # wait 1 second if the search errored out for some reason
+
+    def search_tokens(self, query):
+        """Returns information about the search query itself (filters used etc.)
+
+        :param query: Search query; identical syntax to the website
+        :type query: str
+
+        :returns: A dictionary with 4 main properties: filters, errors, attributes and string.
+        """
+        query_args = {
+            'query': query,
+        }
+        return self._request('/shodan/host/search/tokens', query_args)
+
+    def services(self):
+        """Get a list of services that Shodan crawls
+
+        :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service.
+        """
+        return self._request('/shodan/services', {})
+
+    def queries(self, page=1, sort='timestamp', order='desc'):
+        """List the search queries that have been shared by other users.
+
+        :param page: Page number to iterate over results; each page contains 10 items
+        :type page: int
+        :param sort: Sort the list based on a property. Possible values are: votes, timestamp
+        :type sort: str
+        :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc
+        :type order: str
+
+        :returns: A list of saved search queries (dictionaries).
+        """
+        args = {
+            'page': page,
+            'sort': sort,
+            'order': order,
+        }
+        return self._request('/shodan/query', args)
+
+    def queries_search(self, query, page=1):
+        """Search the directory of saved search queries in Shodan.
+
+        :param query: The search string to look for in the search query
+        :type query: str
+        :param page: Page number to iterate over results; each page contains 10 items
+        :type page: int
+
+        :returns: A list of saved search queries (dictionaries).
+        """
+        args = {
+            'page': page,
+            'query': query,
+        }
+        return self._request('/shodan/query/search', args)
+
+    def queries_tags(self, size=10):
+        """Search the directory of saved search queries in Shodan.
+
+        :param query: The number of tags to return
+        :type page: int
+
+        :returns: A list of tags.
+        """
+        args = {
+            'size': size,
+        }
+        return self._request('/shodan/query/tags', args)
+
+    def create_alert(self, name, ip, expires=0):
+        """Search the directory of saved search queries in Shodan.
+
+        :param query: The number of tags to return
+        :type page: int
+
+        :returns: A list of tags.
+        """
+        data = {
+            'name': name,
+            'filters': {
+                'ip': ip,
+            },
+            'expires': expires,
+        }
+
+        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post',
+                               proxies=self._session.proxies)
+
+        return response
+
+    def alerts(self, aid=None, include_expired=True):
+        """List all of the active alerts that the user created."""
+        if aid:
+            func = '/shodan/alert/%s/info' % aid
+        else:
+            func = '/shodan/alert/info'
+
+        response = api_request(self.api_key, func, params={
+            'include_expired': include_expired,
+            },
+            proxies=self._session.proxies)
+
+        return response
+
+    def delete_alert(self, aid):
+        """Delete the alert with the given ID."""
+        func = '/shodan/alert/%s' % aid
+
+        response = api_request(self.api_key, func, params={}, method='delete',
+                               proxies=self._session.proxies)
+
+        return response
+
diff --git a/tests/test_shodan.py b/tests/test_shodan.py
index 396a059..0cdd602 100644
--- a/tests/test_shodan.py
+++ b/tests/test_shodan.py
@@ -1,157 +1,157 @@
-import unittest
-import shodan
-
-try:
-    basestring
-except NameError:
-    basestring = str
-
-
-class ShodanTests(unittest.TestCase):
-
-	api = None
-	FACETS = [
-		'port',
-		('domain', 1)
-	]
-	QUERIES = {
-		'simple': 'cisco-ios',
-		'minify': 'apache',
-		'advanced': 'apache port:443',
-		'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk',
-	}
-
-	def setUp(self):
-		self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip())
-
-	def test_search_simple(self):
-		results = self.api.search(self.QUERIES['simple'])
-
-		# Make sure the properties exist
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-
-		# Make sure no error occurred
-		self.assertNotIn('error', results)
-
-		# Make sure some values were returned
-		self.assertTrue(results['matches'])
-		self.assertTrue(results['total'])
-
-		# A regular search shouldn't have the optional info
-		self.assertNotIn('opts', results['matches'][0])
-
-	def test_search_empty(self):
-		results = self.api.search(self.QUERIES['empty'])
-		self.assertTrue(len(results['matches']) == 0)
-		self.assertEqual(results['total'], 0)
-
-	def test_search_facets(self):
-		results = self.api.search(self.QUERIES['simple'], facets=self.FACETS)
-
-		self.assertTrue(results['facets']['port'])
-		self.assertEqual(len(results['facets']['domain']), 1)
-
-	def test_count_simple(self):
-		results = self.api.count(self.QUERIES['simple'])
-
-		# Make sure the properties exist
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-
-		# Make sure no error occurred
-		self.assertNotIn('error', results)
-
-		# Make sure no values were returned
-		self.assertFalse(results['matches'])
-		self.assertTrue(results['total'])
-
-	def test_count_facets(self):
-		results = self.api.count(self.QUERIES['simple'], facets=self.FACETS)
-
-		self.assertTrue(results['facets']['port'])
-		self.assertEqual(len(results['facets']['domain']), 1)
-
-	def test_host_details(self):
-		host = self.api.host('147.228.101.7')
-
-		self.assertEqual('147.228.101.7', host['ip_str'])
-		self.assertFalse(isinstance(host['ip'], basestring))
-
-	def test_search_minify(self):
-		results = self.api.search(self.QUERIES['minify'], minify=False)
-		self.assertIn('opts', results['matches'][0])
-
-	def test_exploits_search(self):
-		results = self.api.exploits.search('apache')
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-		self.assertTrue(results['matches'])
-
-	def test_exploits_search_paging(self):
-		results = self.api.exploits.search('apache', page=1)
-		match1 = results['matches'][0]
-		results = self.api.exploits.search('apache', page=2)
-		match2 = results['matches'][0]
-
-		self.assertNotEqual(match1['_id'], match2['_id'])
-
-	def test_exploits_search_facets(self):
-		results = self.api.exploits.search('apache', facets=['source', ('author', 1)])
-		self.assertIn('facets', results)
-		self.assertTrue(results['facets']['source'])
-		self.assertTrue(len(results['facets']['author']) == 1)
-
-	def test_exploits_count(self):
-		results = self.api.exploits.count('apache')
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-		self.assertTrue(len(results['matches']) == 0)
-
-	def test_exploits_count_facets(self):
-		results = self.api.exploits.count('apache', facets=['source', ('author', 1)])
-		self.assertEqual(len(results['matches']), 0)
-		self.assertIn('facets', results)
-		self.assertTrue(results['facets']['source'])
-		self.assertTrue(len(results['facets']['author']) == 1)
-
-	# Test error responses
-	def test_invalid_key(self):
-		api = shodan.Shodan('garbage')
-		raised = False
-		try:
-			api.search('something')
-		except shodan.APIError as e:
-			raised = True
-
-		self.assertTrue(raised)
-
-	def test_invalid_host_ip(self):
-		raised = False
-		try:
-			host = self.api.host('test')
-		except shodan.APIError as e:
-			raised = True
-
-		self.assertTrue(raised)
-
-	def test_search_empty_query(self):
-		raised = False
-		try:
-			self.api.search('')
-		except shodan.APIError as e:
-			raised = True
-		self.assertTrue(raised)
-
-	def test_search_advanced_query(self):
-		# The free API plan can't use filters
-		raised = False
-		try:
-			self.api.search(self.QUERIES['advanced'])
-		except shodan.APIError as e:
-			raised = True
-		self.assertTrue(raised)
-
-
-if __name__ == '__main__':
-    unittest.main()
+import unittest
+import shodan
+
+try:
+    basestring
+except NameError:
+    basestring = str
+
+
+class ShodanTests(unittest.TestCase):
+
+	api = None
+	FACETS = [
+		'port',
+		('domain', 1)
+	]
+	QUERIES = {
+		'simple': 'cisco-ios',
+		'minify': 'apache',
+		'advanced': 'apache port:443',
+		'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk',
+	}
+
+	def setUp(self):
+		self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip())
+
+	def test_search_simple(self):
+		results = self.api.search(self.QUERIES['simple'])
+
+		# Make sure the properties exist
+		self.assertIn('matches', results)
+		self.assertIn('total', results)
+
+		# Make sure no error occurred
+		self.assertNotIn('error', results)
+
+		# Make sure some values were returned
+		self.assertTrue(results['matches'])
+		self.assertTrue(results['total'])
+
+		# A regular search shouldn't have the optional info
+		self.assertNotIn('opts', results['matches'][0])
+
+	def test_search_empty(self):
+		results = self.api.search(self.QUERIES['empty'])
+		self.assertTrue(len(results['matches']) == 0)
+		self.assertEqual(results['total'], 0)
+
+	def test_search_facets(self):
+		results = self.api.search(self.QUERIES['simple'], facets=self.FACETS)
+
+		self.assertTrue(results['facets']['port'])
+		self.assertEqual(len(results['facets']['domain']), 1)
+
+	def test_count_simple(self):
+		results = self.api.count(self.QUERIES['simple'])
+
+		# Make sure the properties exist
+		self.assertIn('matches', results)
+		self.assertIn('total', results)
+
+		# Make sure no error occurred
+		self.assertNotIn('error', results)
+
+		# Make sure no values were returned
+		self.assertFalse(results['matches'])
+		self.assertTrue(results['total'])
+
+	def test_count_facets(self):
+		results = self.api.count(self.QUERIES['simple'], facets=self.FACETS)
+
+		self.assertTrue(results['facets']['port'])
+		self.assertEqual(len(results['facets']['domain']), 1)
+
+	def test_host_details(self):
+		host = self.api.host('147.228.101.7')
+
+		self.assertEqual('147.228.101.7', host['ip_str'])
+		self.assertFalse(isinstance(host['ip'], basestring))
+
+	def test_search_minify(self):
+		results = self.api.search(self.QUERIES['minify'], minify=False)
+		self.assertIn('opts', results['matches'][0])
+
+	def test_exploits_search(self):
+		results = self.api.exploits.search('apache')
+		self.assertIn('matches', results)
+		self.assertIn('total', results)
+		self.assertTrue(results['matches'])
+
+	def test_exploits_search_paging(self):
+		results = self.api.exploits.search('apache', page=1)
+		match1 = results['matches'][0]
+		results = self.api.exploits.search('apache', page=2)
+		match2 = results['matches'][0]
+
+		self.assertNotEqual(match1['_id'], match2['_id'])
+
+	def test_exploits_search_facets(self):
+		results = self.api.exploits.search('apache', facets=['source', ('author', 1)])
+		self.assertIn('facets', results)
+		self.assertTrue(results['facets']['source'])
+		self.assertTrue(len(results['facets']['author']) == 1)
+
+	def test_exploits_count(self):
+		results = self.api.exploits.count('apache')
+		self.assertIn('matches', results)
+		self.assertIn('total', results)
+		self.assertTrue(len(results['matches']) == 0)
+
+	def test_exploits_count_facets(self):
+		results = self.api.exploits.count('apache', facets=['source', ('author', 1)])
+		self.assertEqual(len(results['matches']), 0)
+		self.assertIn('facets', results)
+		self.assertTrue(results['facets']['source'])
+		self.assertTrue(len(results['facets']['author']) == 1)
+
+	# Test error responses
+	def test_invalid_key(self):
+		api = shodan.Shodan('garbage')
+		raised = False
+		try:
+			api.search('something')
+		except shodan.APIError as e:
+			raised = True
+
+		self.assertTrue(raised)
+
+	def test_invalid_host_ip(self):
+		raised = False
+		try:
+			host = self.api.host('test')
+		except shodan.APIError as e:
+			raised = True
+
+		self.assertTrue(raised)
+
+	def test_search_empty_query(self):
+		raised = False
+		try:
+			self.api.search('')
+		except shodan.APIError as e:
+			raised = True
+		self.assertTrue(raised)
+
+	def test_search_advanced_query(self):
+		# The free API plan can't use filters
+		raised = False
+		try:
+			self.api.search(self.QUERIES['advanced'])
+		except shodan.APIError as e:
+			raised = True
+		self.assertTrue(raised)
+
+
+if __name__ == '__main__':
+    unittest.main()

From f2cb964db3f5208f7453a6d44e65402253138d67 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 21 Sep 2018 21:13:53 -0500
Subject: [PATCH 159/263] Show the list of available file formats in "shodan
 convert" (#80)

---
 shodan/__main__.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 339218e..88f4e4f 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -84,11 +84,20 @@ def main():
 main.add_command(scan)
 
 
+CONVERTERS = {
+    'kml': KmlConverter,
+    'csv': CsvConverter,
+    'geo.json': GeoJsonConverter,
+    'images': ImagesConverter,
+    'xlsx': ExcelConverter,
+}
 @main.command()
 @click.argument('input', metavar='<input file>')
-@click.argument('format', metavar='<output format>', type=click.Choice(['kml', 'csv', 'geo.json', 'images', 'xlsx']))
+@click.argument('format', metavar='<output format>', type=click.Choice(CONVERTERS.keys()))
 def convert(input, format):
-    """Convert the given input data file into a different format.
+    """Convert the given input data file into a different format. The following file formats are supported:
+
+    kml, csv, geo.json, images, xlsx
 
     Example: shodan convert data.json.gz kml
     """
@@ -107,13 +116,7 @@ def convert(input, format):
     progress_bar_thread.start()
 
     # Initialize the file converter
-    converter = {
-        'kml': KmlConverter,
-        'csv': CsvConverter,
-        'geo.json': GeoJsonConverter,
-        'images': ImagesConverter,
-        'xlsx': ExcelConverter,
-    }.get(format)(fout)
+    converter = CONVERTERS.get(format)(fout)
 
     converter.process([input])
 

From a9345073ddee623ae780d437f189a2c3147a4d9c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 21 Sep 2018 21:14:59 -0500
Subject: [PATCH 160/263] Release 1.10.3

---
 CHANGELOG.md | 5 +++--
 setup.py     | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d531ab..adce430 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,11 +1,12 @@
 CHANGELOG
 =========
 
-unreleased
-----------
+1.10.3
+------
 * Change bare 'except:' statements to 'except Exception:' or more specific ones
 * remove unused imports
 * Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix
+* List file types in **shodan convert** (#80)
 
 1.10.2
 ------
diff --git a/setup.py b/setup.py
index 593a481..125896c 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.10.2',
+    version = '1.10.3',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',

From 1025723577c07054de1737df4d4d182d85a6cbd0 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 22 Sep 2018 20:59:03 -0500
Subject: [PATCH 161/263] Code quality improvements

---
 shodan/__main__.py | 2 +-
 shodan/helpers.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index cba34d7..709a010 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -230,7 +230,7 @@ def download(limit, filename, query):
 
                     if count >= limit:
                         break
-        except:
+        except Exception:
             pass
 
         # Let the user know we're done
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 2756ab0..d3c4a1f 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -98,7 +98,7 @@ def iterate_files(files, fast=False):
         # pylint: disable=E0401
         try:
             from ujson import loads
-        except:
+        except Exception:
             pass
 
     if isinstance(files, basestring):

From da2da758cd3fad3436efc62b61b4400dc71aca80 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 4 Oct 2018 20:00:03 -0500
Subject: [PATCH 162/263] Release 1.10.4 Fix a bug when showing old banner
 records that don't have the "transport" property

---
 setup.py           | 2 +-
 shodan/cli/host.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 125896c..ef991f1 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name = 'shodan',
-    version = '1.10.3',
+    version = '1.10.4',
     description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description = README,
     long_description_content_type = 'text/x-rst',
diff --git a/shodan/cli/host.py b/shodan/cli/host.py
index fcd440f..befdc62 100644
--- a/shodan/cli/host.py
+++ b/shodan/cli/host.py
@@ -80,8 +80,9 @@ def host_print_pretty(host, history=False):
             version = '({})'.format(banner['version'])
 
         click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False)
-        click.echo('/', nl=False)
-        click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
+        if 'transport' in banner:
+            click.echo('/', nl=False)
+            click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False)
         click.echo('{} {}'.format(product, version), nl=False)
 
         if history:

From bce4564cda4e79c9e477446faf5edcb16735d147 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Sun, 7 Oct 2018 08:48:51 +0200
Subject: [PATCH 163/263] StandardError was removed in Python 3

Use Exception instead.
---
 shodan/cli/worldmap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 60ef075..fca52b4 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -177,7 +177,7 @@ def draw(self, target):
                     self.window.addstr(row, 1, det_show, attrs)
                     row += 1
                     items_to_show -= 1
-                except StandardError:
+                except Exeception:
                     # FIXME: check window size before addstr()
                     break
         self.window.overwrite(target)

From 04ca4bbfb3dc1b59b3716857819561502131ac7f Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Sun, 7 Oct 2018 09:10:35 +0200
Subject: [PATCH 164/263] Undefined name: Don't forget self when calling
 flatten()

__flatten__ is an undefined name in this context which has the potential to raise NameError at runtime.  __self.flatten()__ makes more sense here.

[flake8](http://flake8.pycqa.org) testing of https://github.com/achillean/shodan-python on Python 3.7.0

$ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__
```
./shodan/cli/helpers.py:73:52: F821 undefined name 'basestring'
        if field_type == list or isinstance(value, basestring):
                                                   ^
./shodan/cli/worldmap.py:180:24: F821 undefined name 'StandardError'
                except StandardError:
                       ^
./shodan/cli/converter/csvc.py:85:30: F821 undefined name 'flatten'
                items.extend(flatten(v, new_key, sep=sep).items())
                             ^
3     F821 undefined name 'flatten'
3
```
---
 shodan/cli/converter/csvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index c975695..bbc266c 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -82,7 +82,7 @@ def flatten(self, d, parent_key='', sep='.'):
             new_key = parent_key + sep + k if parent_key else k
             if isinstance(v, MutableMapping):
                 # pylint: disable=E0602
-                items.extend(flatten(v, new_key, sep=sep).items())
+                items.extend(self.flatten(v, new_key, sep=sep).items())
             else:
                 items.append((new_key, v))
         return dict(items)

From 7e24f4b447f764391bdc00f30de3a8abbfe75d99 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Sun, 7 Oct 2018 09:12:57 +0200
Subject: [PATCH 165/263] Remove pylint: disable=E0602

This PR fixes PyLint's complaint.
---
 shodan/cli/converter/csvc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index bbc266c..8c42254 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -81,7 +81,6 @@ def flatten(self, d, parent_key='', sep='.'):
         for k, v in d.items():
             new_key = parent_key + sep + k if parent_key else k
             if isinstance(v, MutableMapping):
-                # pylint: disable=E0602
                 items.extend(self.flatten(v, new_key, sep=sep).items())
             else:
                 items.append((new_key, v))

From aa579b732b71ca03c665b7fc91107062889b18a2 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Sun, 7 Oct 2018 09:19:49 +0200
Subject: [PATCH 166/263] Undefined name: 'basestring' was removed in Python 3

__basestring__ was removed in Python 3 but it is used on line 78 without being defined.
---
 shodan/cli/helpers.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 6ef9e1b..33cdc57 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -10,6 +10,11 @@
 
 from .settings import SHODAN_CONFIG_DIR
 
+try:
+    basestring            # Python 2
+except NameError:
+    basestring = (str, )  # Python 3
+
 def get_api_key():
     '''Returns the API key of the current logged-in user.'''
     shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)

From 0b501feba747f2a3ee9b08cb848afdbf02757fa7 Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Fri, 2 Nov 2018 10:09:18 +0100
Subject: [PATCH 167/263] add changelog for 1.10.4

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index adce430..4b7f995 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+unreleased
+----------
+
+1.10.4
+------
+* Fix a bug when showing old banner records that don't have the "transport" property
+* Code quality improvements (bare excepts)
+
 1.10.3
 ------
 * Change bare 'except:' statements to 'except Exception:' or more specific ones

From d8236dc3f152dfeb21794ea472c3f0f0203c3c86 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Mon, 5 Nov 2018 09:27:00 +0100
Subject: [PATCH 168/263] Fix a typo introduced in #82

#82 contained a typo...  Sorry about that.
---
 shodan/cli/worldmap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index fca52b4..cfe4e4a 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -177,7 +177,7 @@ def draw(self, target):
                     self.window.addstr(row, 1, det_show, attrs)
                     row += 1
                     items_to_show -= 1
-                except Exeception:
+                except Exception:
                     # FIXME: check window size before addstr()
                     break
         self.window.overwrite(target)

From 0a77e3d34b8d6e00c4363d3dc0af58b5d1b339c8 Mon Sep 17 00:00:00 2001
From: Antoine Neuenschwander <antoine.neuenschwander@switch.ch>
Date: Mon, 10 Dec 2018 14:55:21 +0100
Subject: [PATCH 169/263] Make shodan parse output separators only between
 fields.

---
 shodan/__main__.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 709a010..112834d 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -333,7 +333,7 @@ def parse(color, fields, filters, filename, separator, filenames):
             helpers.write_banner(fout, banner)
 
         # Loop over all the fields and print the banner as a row
-        for field in fields:
+        for i, field in enumerate(fields):
             tmp = u''
             value = get_banner_field(banner, field)
             if value:
@@ -351,9 +351,10 @@ def parse(color, fields, filters, filename, separator, filenames):
                 if color:
                     tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white'))
 
-                # Add the field information to the row
-                row += tmp
-            row += separator
+            # Add the field information to the row
+            if i > 0:
+                row += separator
+            row += tmp 
 
         click.echo(row)
 

From fc732ba57b449fe57fd6e1e9e278a162b1a51931 Mon Sep 17 00:00:00 2001
From: Jeremy Bae <jswbae@gmail.com>
Date: Mon, 17 Dec 2018 18:10:51 +0900
Subject: [PATCH 170/263] Add virtualenv and IntelliJ/PyCharm directories

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index eca7a0a..b719ddb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,6 @@ shodan.egg-info/*
 tmp/*
 MANIFEST
 .vscode/
-PKG-INFO
\ No newline at end of file
+PKG-INFO
+venv/*
+.idea/*
\ No newline at end of file

From 5029bb06278d64ce0685036437d5667fd2f29855 Mon Sep 17 00:00:00 2001
From: Vladimir Epifanov <voldmar@voldmar.com>
Date: Fri, 11 Jan 2019 13:41:37 +0200
Subject: [PATCH 171/263] Fix Shodan.init() type hint

`Shodan.init()` has improper type hint for `proxies` parameter which shadows type hint for `key` parameter
---
 shodan/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 0fd9167..51c33f1 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -176,7 +176,7 @@ def __init__(self, key, proxies=None):
         :param key: The Shodan API key.
         :type key: str
         :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'}
-        :type key: dict
+        :type proxies: dict
         """
         self.api_key = key
         self.base_url = 'https://api.shodan.io'

From 7f20a32d78bb59b38877e7f69dfc3619c08a25ff Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 10 Feb 2019 17:44:56 -0600
Subject: [PATCH 172/263] Include tags, timestamp and vulns information in CSV
 conversion (#85)

---
 shodan/cli/converter/csvc.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 8c42254..55dd021 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -24,9 +24,12 @@ class CsvConverter(Converter):
         'os',
         'asn',
         'port',
+        'tags',
+        'timestamp',
         'transport',
         'product',
         'version',
+        'vulns',
         
         'ssl.cipher.version',
         'ssl.cipher.bits',
@@ -48,6 +51,11 @@ def process(self, files):
         writer.writerow(self.fields)
         
         for banner in iterate_files(files):
+            # The "vulns" property can't be nicely flattened as-is so we turn
+            # it into a list before processing the banner.
+            if 'vulns' in banner:
+                banner['vulns'] = banner['vulns'].keys()
+            
             try:
                 row = []
                 for field in self.fields:

From 166597f4a9756dc38ed2077d8df73340efaa8eb4 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 10 Feb 2019 20:43:10 -0600
Subject: [PATCH 173/263] New command **shodan scan list** to list recently
 launched scans New command **shodan alert triggers** to list the available
 notification triggers New command **shodan alert enable** to enable a
 notification trigger New command **shodan alert disable** to disable a
 notification trigger Code quality improvements

---
 CHANGELOG.md                     |   6 +
 setup.py                         |  26 +--
 shodan/__main__.py               |  34 ++--
 shodan/alert.py                  |   9 -
 shodan/cli/alert.py              |  68 +++++++-
 shodan/cli/converter/__init__.py |   2 +-
 shodan/cli/converter/base.py     |   2 +-
 shodan/cli/converter/csvc.py     |  24 +--
 shodan/cli/converter/excel.py    |  26 +--
 shodan/cli/converter/geojson.py  |  20 +--
 shodan/cli/converter/images.py   |   2 +-
 shodan/cli/converter/kml.py      |  20 +--
 shodan/cli/helpers.py            |   1 +
 shodan/cli/host.py               |  10 +-
 shodan/cli/organization.py       |  12 +-
 shodan/cli/scan.py               |  36 +++-
 shodan/cli/worldmap.py           |  15 +-
 shodan/client.py                 |  45 +++--
 shodan/exception.py              |   5 +-
 shodan/helpers.py                |   8 +-
 shodan/stream.py                 |  13 +-
 shodan/threatnet.py              |   5 +-
 tests/test_shodan.py             | 284 +++++++++++++++----------------
 tox.ini                          |  14 ++
 24 files changed, 408 insertions(+), 279 deletions(-)
 delete mode 100644 shodan/alert.py
 create mode 100644 tox.ini

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b7f995..b107676 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,12 @@ CHANGELOG
 
 unreleased
 ----------
+* New command **shodan scan list** to list recently launched scans
+* New command **shodan alert triggers** to list the available notification triggers
+* New command **shodan alert enable** to enable a notification trigger
+* New command **shodan alert disable** to disable a notification trigger
+* Include timestamp, vulns and tags in CSV converter (#85)
+* Code quality improvements
 
 1.10.4
 ------
diff --git a/setup.py b/setup.py
index ef991f1..c925129 100755
--- a/setup.py
+++ b/setup.py
@@ -6,19 +6,19 @@
 README = open('README.rst', 'r').read()
 
 setup(
-    name = 'shodan',
-    version = '1.10.4',
-    description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)',
-    long_description = README,
-    long_description_content_type = 'text/x-rst',
-    author = 'John Matherly',
-    author_email = 'jmath@shodan.io',
-    url = 'http://github.com/achillean/shodan-python/tree/master',
-    packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'],
-    entry_points = {'console_scripts': ['shodan = shodan.__main__:main']},
-    install_requires = DEPENDENCIES,
-    keywords = ['security', 'network'],
-    classifiers = [
+    name='shodan',
+    version='1.10.4',
+    description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
+    long_description=README,
+    long_description_content_type='text/x-rst',
+    author='John Matherly',
+    author_email='jmath@shodan.io',
+    url='http://github.com/achillean/shodan-python/tree/master',
+    packages=['shodan', 'shodan.cli', 'shodan.cli.converter'],
+    entry_points={'console_scripts': ['shodan=shodan.__main__:main']},
+    install_requires=DEPENDENCIES,
+    keywords=['security', 'network'],
+    classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 112834d..df2b38c 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -49,6 +49,13 @@
 from click_plugins import with_plugins
 from pkg_resources import iter_entry_points
 
+# Large subcommands are stored in separate modules
+from shodan.cli.alert import alert
+from shodan.cli.data import data
+from shodan.cli.organization import org
+from shodan.cli.scan import scan
+
+
 # Make "-h" work like "--help"
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -58,7 +65,6 @@
 except NameError:
     basestring = str
 
-
 # Define the main entry point for all of our commands
 # and expose a way for 3rd-party plugins to tie into the Shodan CLI.
 @with_plugins(iter_entry_points('shodan.cli.plugins'))
@@ -67,11 +73,7 @@ def main():
     pass
 
 
-# Large subcommands are stored in separate modules
-from shodan.cli.alert import alert
-from shodan.cli.data import data
-from shodan.cli.organization import org
-from shodan.cli.scan import scan
+# Setup the large subcommands
 main.add_command(alert)
 main.add_command(data)
 main.add_command(org)
@@ -151,6 +153,7 @@ def init(key):
 
     os.chmod(keyfile, 0o600)
 
+
 @main.command()
 @click.argument('query', metavar='<search query>', nargs=-1)
 def count(query):
@@ -203,7 +206,7 @@ def download(limit, filename, query):
     try:
         total = api.count(query)['total']
         info = api.info()
-    except:
+    except Exception:
         raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.')
 
     # Print some summary information about the download request
@@ -275,7 +278,6 @@ def host(format, history, filename, save, ip):
         raise click.ClickException(e.value)
 
 
-
 @main.command()
 def info():
     """Shows general information about your account"""
@@ -308,7 +310,6 @@ def parse(color, fields, filters, filename, separator, filenames):
 
     has_filters = len(filters) > 0
 
-
     # Setup the output file handle
     fout = None
     if filename:
@@ -354,7 +355,7 @@ def parse(color, fields, filters, filename, separator, filenames):
             # Add the field information to the row
             if i > 0:
                 row += separator
-            row += tmp 
+            row += tmp
 
         click.echo(row)
 
@@ -520,7 +521,7 @@ def stats(limit, facets, filename, query):
                 if len(values) > counter:
                     has_items = True
                     row[pos] = values[counter]['value']
-                    row[pos+1] = values[counter]['count']
+                    row[pos + 1] = values[counter]['count']
 
                 pos += 2
 
@@ -546,7 +547,7 @@ def stats(limit, facets, filename, query):
 @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
 @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
 @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries,  asn, alert, compresslevel):
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -642,9 +643,9 @@ def _create_stream(name, args, timeout):
                 if datadir:
                     cur_time = timestr()
                     if cur_time != last_time:
-                            last_time = cur_time
-                            fout.close()
-                            fout = open_streaming_file(datadir, last_time)
+                        last_time = cur_time
+                        fout.close()
+                        fout = open_streaming_file(datadir, last_time)
                     helpers.write_banner(fout, banner)
 
                 # Print the banner information to stdout
@@ -707,7 +708,7 @@ def honeyscore(ip):
             click.echo(click.style('Not a honeypot', fg='green'))
 
         click.echo('Score: {}'.format(score))
-    except:
+    except Exception:
         raise click.ClickException('Unable to calculate honeyscore')
 
 
@@ -726,5 +727,6 @@ def radar():
     except Exception as e:
         raise click.ClickException(u'{}'.format(e))
 
+
 if __name__ == '__main__':
     main()
diff --git a/shodan/alert.py b/shodan/alert.py
deleted file mode 100644
index 7a89e90..0000000
--- a/shodan/alert.py
+++ /dev/null
@@ -1,9 +0,0 @@
-class Alert:
-    def __init__(self):
-        self.id = None
-        self.name = None
-        self.api_key = None
-        self.filters = None
-        self.credits = None
-        self.created = None
-        self.expires = None
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index a9f6ec8..d8b02b9 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -3,6 +3,7 @@
 
 from shodan.cli.helpers import get_api_key
 
+
 @click.group()
 def alert():
     """Manage the network alerts for your account"""
@@ -25,6 +26,7 @@ def alert_clear():
         raise click.ClickException(e.value)
     click.echo("Alerts deleted")
 
+
 @alert.command(name='create')
 @click.argument('name', metavar='<name>')
 @click.argument('netblock', metavar='<netblock>')
@@ -42,6 +44,7 @@ def alert_create(name, netblock):
     click.secho('Successfully created network alert!', fg='green')
     click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
+
 @alert.command(name='list')
 @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
 def alert_list(expired):
@@ -57,11 +60,11 @@ def alert_list(expired):
 
     if len(results) > 0:
         click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network'))
-        # click.echo('#' * 65)
+
         for alert in results:
             click.echo(
                 u'{:16} {:<30} {:<35} '.format(
-                    click.style(alert['id'],  fg='yellow'),
+                    click.style(alert['id'], fg='yellow'),
                     click.style(alert['name'], fg='cyan'),
                     click.style(', '.join(alert['filters']['ip']), fg='white')
                 ),
@@ -89,3 +92,64 @@ def alert_remove(alert_id):
     except shodan.APIError as e:
         raise click.ClickException(e.value)
     click.echo("Alert deleted")
+
+
+@alert.command(name='triggers')
+def alert_list_triggers():
+    """List all the available triggers"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        results = api.alert_triggers()
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    if len(results) > 0:
+        click.echo(u'# {:14} {:<21} {:<15s}'.format('Name', 'Description', 'Rule'))
+
+        for trigger in results:
+            click.echo(
+                u'{:16} {:<30} {:<35} '.format(
+                    click.style(trigger['name'], fg='yellow'),
+                    click.style(trigger['description'], fg='cyan'),
+                    trigger['rule']
+                )
+            )
+    else:
+        click.echo("No triggers currently available.")
+
+
+@alert.command(name='enable')
+@click.argument('alert_id', metavar='<alert ID>')
+@click.argument('trigger', metavar='<trigger name>')
+def alert_enable_trigger(alert_id, trigger):
+    """Enable a trigger for the alert"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        api.enable_alert_trigger(alert_id, trigger)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho('Successfully enabled the trigger {}'.format(trigger), color='green')
+
+
+@alert.command(name='disable')
+@click.argument('alert_id', metavar='<alert ID>')
+@click.argument('trigger', metavar='<trigger name>')
+def alert_disable_trigger(alert_id, trigger):
+    """Disable a trigger for the alert"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        api.disable_alert_trigger(alert_id, trigger)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho('Successfully disabled the trigger {}'.format(trigger), color='green')
diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py
index 507ca0b..08b068a 100644
--- a/shodan/cli/converter/__init__.py
+++ b/shodan/cli/converter/__init__.py
@@ -2,4 +2,4 @@
 from .excel import ExcelConverter
 from .geojson import GeoJsonConverter
 from .images import ImagesConverter
-from .kml import KmlConverter
\ No newline at end of file
+from .kml import KmlConverter
diff --git a/shodan/cli/converter/base.py b/shodan/cli/converter/base.py
index 9dc83c2..14b5f29 100644
--- a/shodan/cli/converter/base.py
+++ b/shodan/cli/converter/base.py
@@ -3,6 +3,6 @@ class Converter:
 
     def __init__(self, fout):
         self.fout = fout
-    
+
     def process(self, fout):
         pass
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 55dd021..39ec162 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -30,7 +30,7 @@ class CsvConverter(Converter):
         'product',
         'version',
         'vulns',
-        
+
         'ssl.cipher.version',
         'ssl.cipher.bits',
         'ssl.cipher.name',
@@ -39,23 +39,23 @@ class CsvConverter(Converter):
         'ssl.cert.serial',
         'ssl.cert.fingerprint.sha1',
         'ssl.cert.fingerprint.sha256',
-        
+
         'html',
         'title',
     ]
-    
+
     def process(self, files):
         writer = csv_writer(self.fout, dialect=excel)
-        
+
         # Write the header
         writer.writerow(self.fields)
-        
+
         for banner in iterate_files(files):
             # The "vulns" property can't be nicely flattened as-is so we turn
             # it into a list before processing the banner.
             if 'vulns' in banner:
                 banner['vulns'] = banner['vulns'].keys()
-            
+
             try:
                 row = []
                 for field in self.fields:
@@ -64,26 +64,26 @@ def process(self, files):
                 writer.writerow(row)
             except Exception:
                 pass
-    
+
     def banner_field(self, banner, flat_field):
         # The provided field is a collapsed form of the actual field
         fields = flat_field.split('.')
-    
+
         try:
             current_obj = banner
             for field in fields:
                 current_obj = current_obj[field]
-            
+
             # Convert a list into a concatenated string
             if isinstance(current_obj, list):
                 current_obj = ','.join([str(i) for i in current_obj])
-            
+
             return current_obj
         except Exception:
             pass
-    
+
         return ''
-    
+
     def flatten(self, d, parent_key='', sep='.'):
         items = []
         for k, v in d.items():
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index a6b476d..e96c24b 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -23,7 +23,7 @@ class ExcelConverter(Converter):
         'transport',
         'product',
         'version',
-        
+
         'http.server',
         'http.title',
     ]
@@ -40,7 +40,7 @@ class ExcelConverter(Converter):
         'http.server': 'Web Server',
         'http.title': 'Website Title',
     }
-    
+
     def process(self, files):
         # Get the filename from the already-open file handle
         filename = self.fout.name
@@ -55,14 +55,14 @@ def process(self, files):
         bold = workbook.add_format({
             'bold': 1,
         })
-        
+
         # Create the main worksheet where all the raw data is shown
         main_sheet = workbook.add_worksheet('Raw Data')
 
         # Write the header
-        main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently
+        main_sheet.write(0, 0, 'IP', bold)  # The IP field can be either ip_str or ipv6 so we treat it differently
         main_sheet.set_column(0, 0, 20)
-        
+
         row = 0
         col = 1
         for field in self.fields:
@@ -80,7 +80,7 @@ def process(self, files):
                 for field in self.fields:
                     value = self.banner_field(banner, field)
                     data.append(value)
-                
+
                 # Write those values to the main workbook
                 # Starting off w/ the special "IP" property
                 main_sheet.write_string(row, 0, get_ip(banner))
@@ -92,11 +92,11 @@ def process(self, files):
                 row += 1
             except Exception:
                 pass
-            
+
             # Aggregate summary information
             total += 1
             ports[banner['port']] += 1
-        
+
         summary_sheet = workbook.add_worksheet('Summary')
         summary_sheet.write(0, 0, 'Total', bold)
         summary_sheet.write(0, 1, total)
@@ -109,22 +109,22 @@ def process(self, files):
             summary_sheet.write(row, col, key)
             summary_sheet.write(row, col + 1, value)
             row += 1
-    
+
     def banner_field(self, banner, flat_field):
         # The provided field is a collapsed form of the actual field
         fields = flat_field.split('.')
-    
+
         try:
             current_obj = banner
             for field in fields:
                 current_obj = current_obj[field]
-            
+
             # Convert a list into a concatenated string
             if isinstance(current_obj, list):
                 current_obj = ','.join([str(i) for i in current_obj])
-            
+
             return current_obj
         except Exception:
             pass
-    
+
         return ''
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index 8bde86f..6126267 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -2,39 +2,39 @@
 from .base import Converter
 from ...helpers import get_ip, iterate_files
 
+
 class GeoJsonConverter(Converter):
-    
+
     def header(self):
         self.fout.write("""{
             "type": "FeatureCollection",
             "features": [
         """)
-    
+
     def footer(self):
         self.fout.write("""{ }]}""")
-    
+
     def process(self, files):
         # Write the header
         self.header()
-        
+
         hosts = {}
         for banner in iterate_files(files):
             ip = get_ip(banner)
             if not ip:
                 continue
-    
+
             if ip not in hosts:
                 hosts[ip] = banner
                 hosts[ip]['ports'] = []
-    
+
             hosts[ip]['ports'].append(banner['port'])
-    
+
         for ip, host in iter(hosts.items()):
             self.write(host)
-        
+
         self.footer()
-            
-    
+
     def write(self, host):
         try:
             ip = get_ip(host)
diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py
index b239b4d..24c68c3 100644
--- a/shodan/cli/converter/images.py
+++ b/shodan/cli/converter/images.py
@@ -14,7 +14,7 @@ class ImagesConverter(Converter):
     # special code in the Shodan CLI that relies on the "dirname" property to let
     # the user know where the images have been stored.
     dirname = None
-    
+
     def process(self, files):
         # Get the filename from the already-open file handle and use it as
         # the directory name to store the images.
diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py
index 49938c2..2cf3d44 100644
--- a/shodan/cli/converter/kml.py
+++ b/shodan/cli/converter/kml.py
@@ -2,38 +2,38 @@
 from .base import Converter
 from ...helpers import iterate_files
 
+
 class KmlConverter(Converter):
-    
+
     def header(self):
         self.fout.write("""<?xml version="1.0" encoding="UTF-8"?>
 <kml xmlns="http://www.opengis.net/kml/2.2">
   <Document>""")
-    
+
     def footer(self):
         self.fout.write("""</Document></kml>""")
-    
+
     def process(self, files):
         # Write the header
         self.header()
-        
+
         hosts = {}
         for banner in iterate_files(files):
             ip = banner.get('ip_str', banner.get('ipv6', None))
             if not ip:
                 continue
-    
+
             if ip not in hosts:
                 hosts[ip] = banner
                 hosts[ip]['ports'] = []
-    
+
             hosts[ip]['ports'].append(banner['port'])
-    
+
         for ip, host in iter(hosts.items()):
             self.write(host)
-        
+
         self.footer()
-            
-    
+
     def write(self, host):
         try:
             ip = host.get('ip_str', host.get('ipv6', None))
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 33cdc57..88cbd36 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -15,6 +15,7 @@
 except NameError:
     basestring = (str, )  # Python 3
 
+
 def get_api_key():
     '''Returns the API key of the current logged-in user.'''
     shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
diff --git a/shodan/cli/host.py b/shodan/cli/host.py
index befdc62..e90e372 100644
--- a/shodan/cli/host.py
+++ b/shodan/cli/host.py
@@ -64,9 +64,9 @@ def host_print_pretty(host, history=False):
         for port in ports:
             banner = {
                 'port': port,
-                'transport': 'tcp', # All the filtered services use TCP
-                'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner
-                'placeholder': True, # Don't store this banner when the file is saved
+                'transport': 'tcp',  # All the filtered services use TCP
+                'timestamp': host['data'][-1]['timestamp'],  # Use the timestamp of the oldest banner
+                'placeholder': True,  # Don't store this banner when the file is saved
             }
             host['data'].append(banner)
 
@@ -94,7 +94,7 @@ def host_print_pretty(host, history=False):
         # Show optional ssl info
         if 'ssl' in banner:
             if 'versions' in banner['ssl'] and banner['ssl']['versions']:
-                click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
+                click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
             if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:
                 click.echo('\t|-- Diffie-Hellman Parameters:')
                 click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator']))
@@ -119,4 +119,4 @@ def host_print_tsv(host, history=False):
 HOST_PRINT = {
     'pretty': host_print_pretty,
     'tsv': host_print_tsv,
-}
\ No newline at end of file
+}
diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py
index 50d814c..5fbb764 100644
--- a/shodan/cli/organization.py
+++ b/shodan/cli/organization.py
@@ -22,7 +22,7 @@ def add(silent, user):
         api.org.add_member(user, notify=not silent)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-    
+
     click.secho('Successfully added the new member', fg='green')
 
 
@@ -39,11 +39,11 @@ def info():
     click.secho(organization['name'], fg='cyan')
     click.secho('Access Level: ', nl=False, dim=True)
     click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta')
-    
+
     if organization['domains']:
         click.secho('Authorized Domains: ', nl=False, dim=True)
         click.echo(', '.join(organization['domains']))
-    
+
     click.echo('')
     click.secho('Administrators:', dim=True)
 
@@ -51,8 +51,8 @@ def info():
         click.echo(u' > {:30}\t{:30}'.format(
             click.style(admin['username'], fg='yellow'),
             admin['email'])
-            )
-    
+        )
+
     click.echo('')
     if organization['members']:
         click.secho('Members:', dim=True)
@@ -76,5 +76,5 @@ def remove(user):
         api.org.remove_member(user)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
-    
+
     click.secho('Successfully removed the member', fg='green')
diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py
index 5220339..4590770 100644
--- a/shodan/cli/scan.py
+++ b/shodan/cli/scan.py
@@ -17,6 +17,35 @@ def scan():
     pass
 
 
+@scan.command(name='list')
+def scan_list():
+    """Show recently launched scans"""
+    key = get_api_key()
+
+    # Get the list
+    api = shodan.Shodan(key)
+    try:
+        scans = api.scans()
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    if len(scans) > 0:
+        click.echo(u'# {} Scans Total - Showing 10 most recent scans:'.format(scans['total']))
+        click.echo(u'# {:20} {:<15} {:<10} {:<15s}'.format('Scan ID', 'Status', 'Size', 'Timestamp'))
+        # click.echo('#' * 65)
+        for scan in scans['matches'][:10]:
+            click.echo(
+                u'{:31} {:<24} {:<10} {:<15s}'.format(
+                    click.style(scan['id'], fg='yellow'),
+                    click.style(scan['status'], fg='cyan'),
+                    scan['size'],
+                    scan['created']
+                )
+            )
+    else:
+        click.echo("You haven't yet launched any scans.")
+
+
 @scan.command(name='internet')
 @click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True)
 @click.argument('port', type=int)
@@ -58,10 +87,9 @@ def scan_internet(quiet, port, protocol):
 
                             if not quiet:
                                 click.echo('{0:<40} {1:<20} {2}'.format(
-                                        click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
-                                        click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
-                                        ';'.join(banner['hostnames'])
-                                    )
+                                    click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']),
+                                    click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
+                                    ';'.join(banner['hostnames']))
                                 )
                     except shodan.APIError as e:
                         # We stop waiting for results if the scan has been processed by the crawlers and
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index cfe4e4a..767b237 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -108,14 +108,14 @@ def latlon_to_coords(self, lat, lon):
         TODO: filter out stuff that doesn't fit
         TODO: make it possible to use "zoomed" maps
         """
-        width = (self.corners[3]-self.corners[1])
-        height = (self.corners[2]-self.corners[0])
+        width = (self.corners[3] - self.corners[1])
+        height = (self.corners[2] - self.corners[0])
 
         # change to 0-180, 0-360
-        abs_lat = -lat+90
-        abs_lon = lon+180
-        x = (abs_lon/360.0)*width + self.corners[1]
-        y = (abs_lat/180.0)*height + self.corners[0]
+        abs_lat = -lat + 90
+        abs_lon = lon + 180
+        x = (abs_lon / 360.0) * width + self.corners[1]
+        y = (abs_lat / 180.0) * height + self.corners[0]
         return int(x), int(y)
 
     def set_data(self, data):
@@ -155,7 +155,7 @@ def draw(self, target):
         self.window.addstr(0, 0, self.map)
 
         # FIXME: position to be defined in map config?
-        row = self.corners[2]-6
+        row = self.corners[2] - 6
         items_to_show = 5
         for lat, lon, char, desc, attrs, color in self.data:
             # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html
@@ -257,6 +257,7 @@ def main(argv=None):
     api = Shodan(get_api_key())
     return launch_map(api)
 
+
 if __name__ == '__main__':
     import sys
     sys.exit(main())
diff --git a/shodan/client.py b/shodan/client.py
index 51c33f1..c8a2b04 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -347,6 +347,16 @@ def scan(self, ips, force=False):
 
         return self._request('/shodan/scan', params, method='post')
 
+    def scans(self, page=1):
+        """Get a list of scans submitted
+
+        :param page: Page through the list of scans 100 results at a time
+        :type page: int
+        """
+        return self._request('/shodan/scans', {
+            'page': page,
+        })
+
     def scan_internet(self, port, protocol):
         """Scan a network using Shodan
 
@@ -438,7 +448,7 @@ def search_cursor(self, query, minify=True, retries=5):
                     try:
                         yield banner
                     except GeneratorExit:
-                        return # exit out of the function
+                        return  # exit out of the function
                 page += 1
                 tries = 0
             except Exception:
@@ -447,7 +457,7 @@ def search_cursor(self, query, minify=True, retries=5):
                     break
 
                 tries += 1
-                time.sleep(1.0) # wait 1 second if the search errored out for some reason
+                time.sleep(1.0)  # wait 1 second if the search errored out for some reason
 
     def search_tokens(self, query):
         """Returns information about the search query itself (filters used etc.)
@@ -507,8 +517,8 @@ def queries_search(self, query, page=1):
     def queries_tags(self, size=10):
         """Search the directory of saved search queries in Shodan.
 
-        :param query: The number of tags to return
-        :type page: int
+        :param size: The number of tags to return
+        :type size: int
 
         :returns: A list of tags.
         """
@@ -518,12 +528,14 @@ def queries_tags(self, size=10):
         return self._request('/shodan/query/tags', args)
 
     def create_alert(self, name, ip, expires=0):
-        """Search the directory of saved search queries in Shodan.
+        """Create a network alert/ private firehose for the specified IP range(s)
 
-        :param query: The number of tags to return
-        :type page: int
+        :param name: Name of the alert
+        :type name: str
+        :param ip: Network range(s) to monitor
+        :type ip: str OR list of str
 
-        :returns: A list of tags.
+        :returns: A dict describing the alert
         """
         data = {
             'name': name,
@@ -547,8 +559,7 @@ def alerts(self, aid=None, include_expired=True):
 
         response = api_request(self.api_key, func, params={
             'include_expired': include_expired,
-            },
-            proxies=self._session.proxies)
+        }, proxies=self._session.proxies)
 
         return response
 
@@ -561,3 +572,17 @@ def delete_alert(self, aid):
 
         return response
 
+    def alert_triggers(self):
+        """Return a list of available triggers that can be enabled for alerts.
+
+        :returns: A list of triggers
+        """
+        return self._request('/shodan/alert/triggers', {})
+
+    def enable_alert_trigger(self, aid, trigger):
+        """Enable the given trigger on the alert."""
+        return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='put')
+
+    def disable_alert_trigger(self, aid, trigger):
+        """Disable the given trigger on the alert."""
+        return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete')
diff --git a/shodan/exception.py b/shodan/exception.py
index c4878b1..75b158e 100644
--- a/shodan/exception.py
+++ b/shodan/exception.py
@@ -2,11 +2,10 @@ class APIError(Exception):
     """This exception gets raised whenever a non-200 status code was returned by the Shodan API."""
     def __init__(self, value):
         self.value = value
-    
+
     def __str__(self):
         return self.value
 
 
 class APITimeout(APIError):
-	pass
-
+    pass
diff --git a/shodan/helpers.py b/shodan/helpers.py
index d3c4a1f..f289e8b 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -19,7 +19,7 @@ def create_facet_string(facets):
         if isinstance(facet, basestring):
             facet_str += facet
         else:
-            facet_str += '%s:%s'  % (facet[0], facet[1])
+            facet_str += '{}:{}'.format(facet[0], facet[1])
         facet_str += ','
     return facet_str[:-1]
 
@@ -76,7 +76,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho
     # Parse the text into JSON
     try:
         data = data.json()
-    except:
+    except Exception:
         raise APIError('Unable to parse JSON response')
 
     # Raise an exception if an error occurred
@@ -119,6 +119,7 @@ def iterate_files(files, fast=False):
             banner = loads(line)
             yield banner
 
+
 def get_screenshot(banner):
     if 'opts' in banner and 'screenshot' in banner['opts']:
         return banner['opts']['screenshot']
@@ -159,14 +160,13 @@ def humanize_bytes(bytes, precision=1):
     >>> humanize_bytes(1024*1234*1111,1)
     '1.3 GB'
     """
-
     if bytes == 1:
         return '1 byte'
     if bytes < 1024:
         return '%.*f %s' % (precision, bytes, "bytes")
 
     suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
-    multiple = 1024.0    #.0 force float on python 2
+    multiple = 1024.0  # .0 to force float on python 2
     for suffix in suffixes:
         bytes /= multiple
         if bytes < multiple:
diff --git a/shodan/stream.py b/shodan/stream.py
index 49ab633..10ccac3 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -21,7 +21,7 @@ def _create_stream(self, name, timeout=None):
 
         # The user doesn't want to use a timeout
         # If the timeout is specified as 0 then we also don't want to have a timeout
-        if ( timeout and timeout <= 0 ) or ( timeout == 0 ):
+        if (timeout and timeout <= 0) or (timeout == 0):
             timeout = None
 
         # If the user requested a timeout then we need to disable heartbeat messages
@@ -43,16 +43,16 @@ def _create_stream(self, name, timeout=None):
                 # not specific to Cloudflare.
                 if req.status_code != 524 or timeout >= 0:
                     break
-        except Exception as e:
+        except Exception:
             raise APIError('Unable to contact the Shodan Streaming API')
 
         if req.status_code != 200:
             try:
                 data = json.loads(req.text)
                 raise APIError(data['error'])
-            except APIError as e:
+            except APIError:
                 raise
-            except Exception as e:
+            except Exception:
                 pass
             raise APIError('Invalid API key or you do not have access to the Streaming API')
         if req.encoding is None:
@@ -78,9 +78,9 @@ def alert(self, aid=None, timeout=None, raw=False):
         try:
             for line in self._iter_stream(stream, raw):
                 yield line
-        except requests.exceptions.ConnectionError as e:
+        except requests.exceptions.ConnectionError:
             raise APIError('Stream timed out')
-        except ssl.SSLError as e:
+        except ssl.SSLError:
             raise APIError('Stream timed out')
 
     def asn(self, asn, raw=False, timeout=None):
@@ -123,4 +123,3 @@ def ports(self, ports, raw=False, timeout=None):
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
-
diff --git a/shodan/threatnet.py b/shodan/threatnet.py
index 97c0c7e..cad9bdd 100644
--- a/shodan/threatnet.py
+++ b/shodan/threatnet.py
@@ -24,13 +24,13 @@ def _create_stream(self, name):
             try:
                 req = requests.get(self.base_url + name, params={'key': self.parent.api_key},
                                    stream=True, proxies=self.proxies)
-            except:
+            except Exception:
                 raise APIError('Unable to contact the Shodan Streaming API')
 
             if req.status_code != 200:
                 try:
                     raise APIError(req.json()['error'])
-                except:
+                except Exception:
                     pass
                 raise APIError('Invalid API key or you do not have access to the Streaming API')
             return req
@@ -65,4 +65,3 @@ def __init__(self, key):
         self.api_key = key
         self.base_url = 'https://api.shodan.io'
         self.stream = self.Stream(self)
-
diff --git a/tests/test_shodan.py b/tests/test_shodan.py
index 0cdd602..f3405ce 100644
--- a/tests/test_shodan.py
+++ b/tests/test_shodan.py
@@ -9,148 +9,148 @@
 
 class ShodanTests(unittest.TestCase):
 
-	api = None
-	FACETS = [
-		'port',
-		('domain', 1)
-	]
-	QUERIES = {
-		'simple': 'cisco-ios',
-		'minify': 'apache',
-		'advanced': 'apache port:443',
-		'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk',
-	}
-
-	def setUp(self):
-		self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip())
-
-	def test_search_simple(self):
-		results = self.api.search(self.QUERIES['simple'])
-
-		# Make sure the properties exist
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-
-		# Make sure no error occurred
-		self.assertNotIn('error', results)
-
-		# Make sure some values were returned
-		self.assertTrue(results['matches'])
-		self.assertTrue(results['total'])
-
-		# A regular search shouldn't have the optional info
-		self.assertNotIn('opts', results['matches'][0])
-
-	def test_search_empty(self):
-		results = self.api.search(self.QUERIES['empty'])
-		self.assertTrue(len(results['matches']) == 0)
-		self.assertEqual(results['total'], 0)
-
-	def test_search_facets(self):
-		results = self.api.search(self.QUERIES['simple'], facets=self.FACETS)
-
-		self.assertTrue(results['facets']['port'])
-		self.assertEqual(len(results['facets']['domain']), 1)
-
-	def test_count_simple(self):
-		results = self.api.count(self.QUERIES['simple'])
-
-		# Make sure the properties exist
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-
-		# Make sure no error occurred
-		self.assertNotIn('error', results)
-
-		# Make sure no values were returned
-		self.assertFalse(results['matches'])
-		self.assertTrue(results['total'])
-
-	def test_count_facets(self):
-		results = self.api.count(self.QUERIES['simple'], facets=self.FACETS)
-
-		self.assertTrue(results['facets']['port'])
-		self.assertEqual(len(results['facets']['domain']), 1)
-
-	def test_host_details(self):
-		host = self.api.host('147.228.101.7')
-
-		self.assertEqual('147.228.101.7', host['ip_str'])
-		self.assertFalse(isinstance(host['ip'], basestring))
-
-	def test_search_minify(self):
-		results = self.api.search(self.QUERIES['minify'], minify=False)
-		self.assertIn('opts', results['matches'][0])
-
-	def test_exploits_search(self):
-		results = self.api.exploits.search('apache')
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-		self.assertTrue(results['matches'])
-
-	def test_exploits_search_paging(self):
-		results = self.api.exploits.search('apache', page=1)
-		match1 = results['matches'][0]
-		results = self.api.exploits.search('apache', page=2)
-		match2 = results['matches'][0]
-
-		self.assertNotEqual(match1['_id'], match2['_id'])
-
-	def test_exploits_search_facets(self):
-		results = self.api.exploits.search('apache', facets=['source', ('author', 1)])
-		self.assertIn('facets', results)
-		self.assertTrue(results['facets']['source'])
-		self.assertTrue(len(results['facets']['author']) == 1)
-
-	def test_exploits_count(self):
-		results = self.api.exploits.count('apache')
-		self.assertIn('matches', results)
-		self.assertIn('total', results)
-		self.assertTrue(len(results['matches']) == 0)
-
-	def test_exploits_count_facets(self):
-		results = self.api.exploits.count('apache', facets=['source', ('author', 1)])
-		self.assertEqual(len(results['matches']), 0)
-		self.assertIn('facets', results)
-		self.assertTrue(results['facets']['source'])
-		self.assertTrue(len(results['facets']['author']) == 1)
-
-	# Test error responses
-	def test_invalid_key(self):
-		api = shodan.Shodan('garbage')
-		raised = False
-		try:
-			api.search('something')
-		except shodan.APIError as e:
-			raised = True
-
-		self.assertTrue(raised)
-
-	def test_invalid_host_ip(self):
-		raised = False
-		try:
-			host = self.api.host('test')
-		except shodan.APIError as e:
-			raised = True
-
-		self.assertTrue(raised)
-
-	def test_search_empty_query(self):
-		raised = False
-		try:
-			self.api.search('')
-		except shodan.APIError as e:
-			raised = True
-		self.assertTrue(raised)
-
-	def test_search_advanced_query(self):
-		# The free API plan can't use filters
-		raised = False
-		try:
-			self.api.search(self.QUERIES['advanced'])
-		except shodan.APIError as e:
-			raised = True
-		self.assertTrue(raised)
+    api = None
+    FACETS = [
+        'port',
+        ('domain', 1)
+    ]
+    QUERIES = {
+        'simple': 'cisco-ios',
+        'minify': 'apache',
+        'advanced': 'apache port:443',
+        'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk',
+    }
+
+    def setUp(self):
+        self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip())
+
+    def test_search_simple(self):
+        results = self.api.search(self.QUERIES['simple'])
+
+        # Make sure the properties exist
+        self.assertIn('matches', results)
+        self.assertIn('total', results)
+
+        # Make sure no error occurred
+        self.assertNotIn('error', results)
+
+        # Make sure some values were returned
+        self.assertTrue(results['matches'])
+        self.assertTrue(results['total'])
+
+        # A regular search shouldn't have the optional info
+        self.assertNotIn('opts', results['matches'][0])
+
+    def test_search_empty(self):
+        results = self.api.search(self.QUERIES['empty'])
+        self.assertTrue(len(results['matches']) == 0)
+        self.assertEqual(results['total'], 0)
+
+    def test_search_facets(self):
+        results = self.api.search(self.QUERIES['simple'], facets=self.FACETS)
+
+        self.assertTrue(results['facets']['port'])
+        self.assertEqual(len(results['facets']['domain']), 1)
+
+    def test_count_simple(self):
+        results = self.api.count(self.QUERIES['simple'])
+
+        # Make sure the properties exist
+        self.assertIn('matches', results)
+        self.assertIn('total', results)
+
+        # Make sure no error occurred
+        self.assertNotIn('error', results)
+
+        # Make sure no values were returned
+        self.assertFalse(results['matches'])
+        self.assertTrue(results['total'])
+
+    def test_count_facets(self):
+        results = self.api.count(self.QUERIES['simple'], facets=self.FACETS)
+
+        self.assertTrue(results['facets']['port'])
+        self.assertEqual(len(results['facets']['domain']), 1)
+
+    def test_host_details(self):
+        host = self.api.host('147.228.101.7')
+
+        self.assertEqual('147.228.101.7', host['ip_str'])
+        self.assertFalse(isinstance(host['ip'], basestring))
+
+    def test_search_minify(self):
+        results = self.api.search(self.QUERIES['minify'], minify=False)
+        self.assertIn('opts', results['matches'][0])
+
+    def test_exploits_search(self):
+        results = self.api.exploits.search('apache')
+        self.assertIn('matches', results)
+        self.assertIn('total', results)
+        self.assertTrue(results['matches'])
+
+    def test_exploits_search_paging(self):
+        results = self.api.exploits.search('apache', page=1)
+        match1 = results['matches'][0]
+        results = self.api.exploits.search('apache', page=2)
+        match2 = results['matches'][0]
+
+        self.assertNotEqual(match1['_id'], match2['_id'])
+
+    def test_exploits_search_facets(self):
+        results = self.api.exploits.search('apache', facets=['source', ('author', 1)])
+        self.assertIn('facets', results)
+        self.assertTrue(results['facets']['source'])
+        self.assertTrue(len(results['facets']['author']) == 1)
+
+    def test_exploits_count(self):
+        results = self.api.exploits.count('apache')
+        self.assertIn('matches', results)
+        self.assertIn('total', results)
+        self.assertTrue(len(results['matches']) == 0)
+
+    def test_exploits_count_facets(self):
+        results = self.api.exploits.count('apache', facets=['source', ('author', 1)])
+        self.assertEqual(len(results['matches']), 0)
+        self.assertIn('facets', results)
+        self.assertTrue(results['facets']['source'])
+        self.assertTrue(len(results['facets']['author']) == 1)
+
+    # Test error responses
+    def test_invalid_key(self):
+        api = shodan.Shodan('garbage')
+        raised = False
+        try:
+            api.search('something')
+        except shodan.APIError:
+            raised = True
+
+        self.assertTrue(raised)
+
+    def test_invalid_host_ip(self):
+        raised = False
+        try:
+            self.api.host('test')
+        except shodan.APIError:
+            raised = True
+
+        self.assertTrue(raised)
+
+    def test_search_empty_query(self):
+        raised = False
+        try:
+            self.api.search('')
+        except shodan.APIError:
+            raised = True
+        self.assertTrue(raised)
+
+    def test_search_advanced_query(self):
+        # The free API plan can't use filters
+        raised = False
+        try:
+            self.api.search(self.QUERIES['advanced'])
+        except shodan.APIError:
+            raised = True
+        self.assertTrue(raised)
 
 
 if __name__ == '__main__':
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..d840b92
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,14 @@
+[flake8]
+ignore =
+    E501
+
+exclude = 
+    build,
+    docs,
+    shodan.egg-info,
+    tmp,
+
+per-file-ignores =
+    shodan/__init__.py:F401,
+    shodan/cli/converter/__init__.py:F401,
+    shodan/cli/worldmap.py:W291,W293,W605,
\ No newline at end of file

From 9fb6b7c9a5726fa23bffb1d6b134a0ffacf1be8e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 15 Feb 2019 22:10:40 -0600
Subject: [PATCH 174/263] Fix bug that caused problems parsing uncompressed
 data files in Python3

---
 shodan/helpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index f289e8b..2d976dc 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -113,7 +113,8 @@ def iterate_files(files, fast=False):
 
         for line in fin:
             # Ensure the line has been decoded into a string to prevent errors w/ Python3
-            line = line.decode('utf-8')
+            if not isinstance(line, basestring):
+                line = line.decode('utf-8')
 
             # Convert the JSON into a native Python object
             banner = loads(line)

From 23cbc6884a8ce6a3736d6d786c0adcdc19125723 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 18 Feb 2019 21:36:01 -0600
Subject: [PATCH 175/263] Show alert triggers in list view

---
 shodan/cli/alert.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index d8b02b9..112c1cf 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -71,6 +71,10 @@ def alert_list(expired):
                 nl=False
             )
 
+            if 'triggers' in alert and alert['triggers']:
+                click.secho('Triggers: ', fg='magenta', nl=False)
+                click.echo(', '.join(alert['triggers'].keys()), nl=False)
+
             if 'expired' in alert and alert['expired']:
                 click.secho('expired', fg='red')
             else:

From c798e754b715a6da42b6a7b722b27579ae877a6d Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 20 Feb 2019 01:41:15 -0600
Subject: [PATCH 176/263] Release 1.11.0 New command: shodan alert info
 Improved output of alert and trigger information

---
 CHANGELOG.md        |  5 +++-
 setup.py            |  2 +-
 shodan/cli/alert.py | 65 ++++++++++++++++++++++++++++++++++++---------
 3 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b107676..ba2044f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,14 +1,17 @@
 CHANGELOG
 =========
 
-unreleased
+1.11.0
 ----------
 * New command **shodan scan list** to list recently launched scans
 * New command **shodan alert triggers** to list the available notification triggers
 * New command **shodan alert enable** to enable a notification trigger
 * New command **shodan alert disable** to disable a notification trigger
+* New command **shodan alert info** to show details of a specific alert
 * Include timestamp, vulns and tags in CSV converter (#85)
+* Fixed bug that caused an exception when parsing uncompressed data files in Python3
 * Code quality improvements
+* Thank you for contributions from @wagner-certat, @cclauss, @opt9, @voldmar and Antoine Neuenschwander
 
 1.10.4
 ------
diff --git a/setup.py b/setup.py
index c925129..4686a81 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.10.4',
+    version='1.11.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 112c1cf..a4c2e29 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -1,6 +1,7 @@
 import click
 import shodan
 
+from operator import itemgetter
 from shodan.cli.helpers import get_api_key
 
 
@@ -45,6 +46,42 @@ def alert_create(name, netblock):
     click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 
+@alert.command(name='info')
+@click.argument('alert', metavar='<alert id>')
+def alert_info(alert):
+    """Show information about a specific alert"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        info = api.alerts(aid=alert)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho(info['name'], fg='cyan')
+    click.secho('Created: ', nl=False, dim=True)
+    click.secho(info['created'], fg='magenta')
+
+    click.secho('Notifications: ', nl=False, dim=True)
+    if 'triggers' in info and info['triggers']:
+        click.secho('enabled', fg='green')
+    else:
+        click.echo('disabled')
+
+    click.echo('')
+    click.secho('Network Range(s):', dim=True)
+
+    for network in info['filters']['ip']:
+        click.echo(u' > {}'.format(click.style(network, fg='yellow')))
+
+    click.echo('')
+    if 'triggers' in info and info['triggers']:
+        click.secho('Triggers:', dim=True)
+        for trigger in info['triggers']:
+            click.echo(u' > {}'.format(click.style(trigger, fg='yellow')))
+        click.echo('')
+
+
 @alert.command(name='list')
 @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool)
 def alert_list(expired):
@@ -100,7 +137,7 @@ def alert_remove(alert_id):
 
 @alert.command(name='triggers')
 def alert_list_triggers():
-    """List all the available triggers"""
+    """List the available notification triggers"""
     key = get_api_key()
 
     # Get the list
@@ -111,16 +148,20 @@ def alert_list_triggers():
         raise click.ClickException(e.value)
 
     if len(results) > 0:
-        click.echo(u'# {:14} {:<21} {:<15s}'.format('Name', 'Description', 'Rule'))
+        click.secho('The following triggers can be enabled on alerts:', dim=True)
+        click.echo('')
 
-        for trigger in results:
-            click.echo(
-                u'{:16} {:<30} {:<35} '.format(
-                    click.style(trigger['name'], fg='yellow'),
-                    click.style(trigger['description'], fg='cyan'),
-                    trigger['rule']
-                )
-            )
+        for trigger in sorted(results, key=itemgetter('name')):
+            click.secho('{:<12} '.format('Name'), dim=True, nl=False)
+            click.secho(trigger['name'], fg='yellow')
+
+            click.secho('{:<12} '.format('Description'), dim=True, nl=False)
+            click.secho(trigger['description'], fg='cyan')
+
+            click.secho('{:<12} '.format('Rule'), dim=True, nl=False)
+            click.echo(trigger['rule'])
+
+            click.echo('')
     else:
         click.echo("No triggers currently available.")
 
@@ -139,7 +180,7 @@ def alert_enable_trigger(alert_id, trigger):
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
-    click.secho('Successfully enabled the trigger {}'.format(trigger), color='green')
+    click.secho('Successfully enabled the trigger: {}'.format(trigger), fg='green')
 
 
 @alert.command(name='disable')
@@ -156,4 +197,4 @@ def alert_disable_trigger(alert_id, trigger):
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
-    click.secho('Successfully disabled the trigger {}'.format(trigger), color='green')
+    click.secho('Successfully disabled the trigger: {}'.format(trigger), fg='green')

From 20ea464fe067d150f7379bb9f8f07c0181761266 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 24 Feb 2019 03:58:27 -0600
Subject: [PATCH 177/263] Allow a single network alert to monitor multiple IP
 ranges (#93) Update README with information on configuring alert
 notifications

---
 CHANGELOG.md        | 6 +++++-
 README.rst          | 1 +
 setup.py            | 2 +-
 shodan/cli/alert.py | 6 +++---
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba2044f..9fd6451 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,12 @@
 CHANGELOG
 =========
 
+1.11.1
+------
+* Allow a single network alert to monitor multiple IP ranges (#93)
+
 1.11.0
-----------
+------
 * New command **shodan scan list** to list recently launched scans
 * New command **shodan alert triggers** to list the available notification triggers
 * New command **shodan alert enable** to enable a notification trigger
diff --git a/README.rst b/README.rst
index d0b1f10..610c13d 100644
--- a/README.rst
+++ b/README.rst
@@ -18,6 +18,7 @@ Features
 - `Fast/ bulk IP lookups <https://help.shodan.io/developer-fundamentals/looking-up-ip-info>`_
 - Streaming API support for real-time consumption of Shodan firehose
 - `Network alerts (aka private firehose) <https://help.shodan.io/guides/how-to-monitor-network>`_
+- `Manage Email Notifications <https://asciinema.org/a/7WvyDtNxn0YeNU70ozsxvXDmL>`_
 - Exploit search API fully implemented
 - Bulk data downloads
 - `Command-line interface <https://cli.shodan.io>`_
diff --git a/setup.py b/setup.py
index 4686a81..bb666af 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.11.0',
+    version='1.11.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index a4c2e29..1ed8572 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -30,15 +30,15 @@ def alert_clear():
 
 @alert.command(name='create')
 @click.argument('name', metavar='<name>')
-@click.argument('netblock', metavar='<netblock>')
-def alert_create(name, netblock):
+@click.argument('netblocks', metavar='<netblocks>', nargs=-1)
+def alert_create(name, netblocks):
     """Create a network alert to monitor an external network"""
     key = get_api_key()
 
     # Get the list
     api = shodan.Shodan(key)
     try:
-        alert = api.create_alert(name, netblock)
+        alert = api.create_alert(name, netblocks)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 

From 59389da25554221e67dc38aa120f44c1816a0ce1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 6 Apr 2019 18:20:12 -0500
Subject: [PATCH 178/263] Add new methods to handle ignoring/ unignoring
 trigger notifications

---
 setup.py         | 2 +-
 shodan/client.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index bb666af..3597060 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.11.1',
+    version='1.12.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index c8a2b04..fb2c87e 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -586,3 +586,11 @@ def enable_alert_trigger(self, aid, trigger):
     def disable_alert_trigger(self, aid, trigger):
         """Disable the given trigger on the alert."""
         return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete')
+
+    def ignore_alert_trigger_notification(self, aid, trigger, ip, port):
+        """Ignore trigger notifications for the provided IP and port."""
+        return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put')
+
+    def unignore_alert_trigger_notification(self, aid, trigger, ip, port):
+        """Re-enable trigger notifications for the provided IP and port"""
+        return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete')

From 6adbb630553a67df546a267c9859bb18a63a5c62 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 6 Apr 2019 18:21:29 -0500
Subject: [PATCH 179/263] Update changelog for 1.12.0 release

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9fd6451..bbc448c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.12.0
+------
+* Add new methods to ignore/ unignore trigger notifications
+
 1.11.1
 ------
 * Allow a single network alert to monitor multiple IP ranges (#93)

From e97429568c14817b9b39a9e2b9509eb5737fe8f3 Mon Sep 17 00:00:00 2001
From: Guillaume Granjus <guillaume@granjus.fr>
Date: Mon, 8 Apr 2019 10:18:14 +0200
Subject: [PATCH 180/263] Add exception raised when retry limit reached (#94)

---
 shodan/client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index fb2c87e..bc6e5af 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -431,7 +431,7 @@ def search_cursor(self, query, minify=True, retries=5):
         :param minify: (optional) Whether to minify the banner and only return the important data
         :type minify: bool
         :param retries: (optional) How often to retry the search in case it times out
-        :type minify: int
+        :type retries: int
 
         :returns: A search cursor that can be used as an iterator/ generator.
         """
@@ -454,7 +454,7 @@ def search_cursor(self, query, minify=True, retries=5):
             except Exception:
                 # We've retried several times but it keeps failing, so lets error out
                 if tries >= retries:
-                    break
+                    raise APIError('Retry limit reached ({:d})'.format(retries))
 
                 tries += 1
                 time.sleep(1.0)  # wait 1 second if the search errored out for some reason

From 6c1d6c94a278ed38a4875d58a1a345532f159d7e Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 11 Apr 2019 15:39:12 -0500
Subject: [PATCH 181/263] Explicitly close the workbook so the Excel file is
 written

---
 CHANGELOG.md                  | 4 ++++
 setup.py                      | 2 +-
 shodan/cli/converter/excel.py | 2 ++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bbc448c..04d5dcc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.12.1
+------
+* Fix Excel file conversion that resulted in empty .xlsx files
+
 1.12.0
 ------
 * Add new methods to ignore/ unignore trigger notifications
diff --git a/setup.py b/setup.py
index 3597060..4e82482 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.12.0',
+    version='1.12.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index e96c24b..24177eb 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -109,6 +109,8 @@ def process(self, files):
             summary_sheet.write(row, col, key)
             summary_sheet.write(row, col + 1, value)
             row += 1
+        
+        workbook.close()
 
     def banner_field(self, banner, flat_field):
         # The provided field is a collapsed form of the actual field

From 723fe3488d1a5ec7df3e173c5ecc3b89bf8a5bdc Mon Sep 17 00:00:00 2001
From: Koen Van Impe <koen.vanimpe@cudeso.be>
Date: Tue, 30 Apr 2019 14:03:48 +0200
Subject: [PATCH 182/263] Override environment configured settings if explicit
 proxy settings are supplied

---
 shodan/client.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shodan/client.py b/shodan/client.py
index fb2c87e..096f9d3 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -190,6 +190,7 @@ def __init__(self, key, proxies=None):
         self._session = requests.Session()
         if proxies:
             self._session.proxies.update(proxies)
+            self._session.trust_env=False
 
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.

From c4562e0182dc83e6469c2ebfd9fe0ba7d6c51eb4 Mon Sep 17 00:00:00 2001
From: Koen Van Impe <koen.vanimpe@cudeso.be>
Date: Tue, 30 Apr 2019 16:53:02 +0200
Subject: [PATCH 183/263] White space around operator

---
 shodan/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 096f9d3..61f4d64 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -190,7 +190,7 @@ def __init__(self, key, proxies=None):
         self._session = requests.Session()
         if proxies:
             self._session.proxies.update(proxies)
-            self._session.trust_env=False
+            self._session.trust_env = False
 
     def _request(self, function, params, service='shodan', method='get'):
         """General-purpose function to create web requests to SHODAN.

From d4fa8bc6e37266968642aaadeee4d4cfa0f66b6c Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 2 May 2019 17:34:45 -0500
Subject: [PATCH 184/263] New command: shodan domain Release 1.13.0

---
 CHANGELOG.md       |  5 +++++
 setup.py           |  2 +-
 shodan/__main__.py | 25 +++++++++++++++++++++++++
 shodan/client.py   | 11 +++++++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04d5dcc..5d76b17 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+1.13.0
+------
+* New command **shodan domain** to lookup a domain in Shodan's DNS database
+* Override environment configured settings if explicit proxy settings are supplied (@cudeso)
+
 1.12.1
 ------
 * Fix Excel file conversion that resulted in empty .xlsx files
diff --git a/setup.py b/setup.py
index 4e82482..164279d 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.12.1',
+    version='1.13.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index df2b38c..8e3dd17 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -125,6 +125,31 @@ def convert(input, format):
         click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green'))
 
 
+@main.command(name='domain')
+@click.argument('domain', metavar='<domain>')
+def domain_info(domain):
+    """View all available information for a domain"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        info = api.dns.domain_info(domain)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho(info['domain'].upper(), fg='green')
+    
+    click.echo('')
+    for record in info['data']:
+        click.echo(
+            '{:32}  {:14}  {}'.format(
+                click.style(record['subdomain'], fg='cyan'),
+                click.style(record['type'], fg='yellow'),
+                record['value']
+            )
+        )
+
+
 @main.command()
 @click.argument('key', metavar='<api key>')
 def init(key):
diff --git a/shodan/client.py b/shodan/client.py
index 61f4d64..cf16823 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -64,6 +64,16 @@ def list_files(self, dataset):
             """
             return self.parent._request('/shodan/data/{}'.format(dataset), {})
 
+    class Dns:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def domain_info(self, domain):
+            """Grab the DNS information for a domain.
+            """
+            return self.parent._request('/dns/domain/{}'.format(domain), {})
+
     class Tools:
 
         def __init__(self, parent):
@@ -182,6 +192,7 @@ def __init__(self, key, proxies=None):
         self.base_url = 'https://api.shodan.io'
         self.base_exploits_url = 'https://exploits.shodan.io'
         self.data = self.Data(self)
+        self.dns = self.Dns(self)
         self.exploits = self.Exploits(self)
         self.labs = self.Labs(self)
         self.org = self.Organization(self)

From 5d0fa9136d4f751bd64a5426bbf043967203b2e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gon=C3=A7alo=20Ribeiro?= <darklaw13@yahoo.co.uk>
Date: Wed, 29 May 2019 00:13:10 +0100
Subject: [PATCH 185/263] Only change api_key permissions if needed

---
 shodan/cli/helpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 88cbd36..3154591 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -27,7 +27,8 @@ def get_api_key():
         raise click.ClickException('Please run "shodan init <api key>" before using this command')
 
     # Make sure it is a read-only file
-    os.chmod(keyfile, 0o600)
+    if not oct(os.stat(keyfile).st_mode).endswith("600"):
+        os.chmod(keyfile, 0o600)
 
     with open(keyfile, 'r') as fin:
         return fin.read().strip()

From 11fc901c941d0661c5b43104dd4b7520f34cd25b Mon Sep 17 00:00:00 2001
From: Sebastian Wagner <wagner@cert.at>
Date: Mon, 1 Jul 2019 13:52:47 +0200
Subject: [PATCH 186/263] New command 'shodan version'

fixes achillean/shodan-python#104
---
 CHANGELOG.md       | 4 ++++
 shodan/__main__.py | 9 ++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5d76b17..e966335 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+unreleased
+----------
+* New command **shodan version** (#104).
+
 1.13.0
 ------
 * New command **shodan domain** to lookup a domain in Shodan's DNS database
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 8e3dd17..9b47875 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -29,6 +29,7 @@
 import csv
 import os
 import os.path
+import pkg_resources
 import shodan
 import shodan.helpers as helpers
 import threading
@@ -138,7 +139,7 @@ def domain_info(domain):
         raise click.ClickException(e.value)
 
     click.secho(info['domain'].upper(), fg='green')
-    
+
     click.echo('')
     for record in info['data']:
         click.echo(
@@ -753,5 +754,11 @@ def radar():
         raise click.ClickException(u'{}'.format(e))
 
 
+@main.command()
+def version():
+    """Print version of this tool."""
+    print(pkg_resources.get_distribution("shodan").version)
+
+
 if __name__ == '__main__':
     main()

From 68fe360ac6c4fb391a2420b2fc1aced1e5a369b9 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 19 Jul 2019 20:15:59 -0500
Subject: [PATCH 187/263] Release 1.14.0

---
 CHANGELOG.md | 3 ++-
 setup.py     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e966335..972abea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,10 @@
 CHANGELOG
 =========
 
-unreleased
+1.14.0
 ----------
 * New command **shodan version** (#104).
+* Only change api_key file permissions if needed (#103)
 
 1.13.0
 ------
diff --git a/setup.py b/setup.py
index 164279d..5bd5e84 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.13.0',
+    version='1.14.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',

From 0126de28a4a6f204e1f7e0bb24c831288469c300 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 12 Aug 2019 20:23:14 -0500
Subject: [PATCH 188/263] New option "--skip" for download command to help
 users resume a download

---
 CHANGELOG.md       |  6 +++++-
 setup.py           |  2 +-
 shodan/__main__.py |  9 +++++++--
 shodan/client.py   | 17 ++++++++++++++---
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 972abea..840da0e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,12 @@
 CHANGELOG
 =========
 
+1.15.0
+------
+* New option "--skip" for download command to help users resume a download
+
 1.14.0
-----------
+------
 * New command **shodan version** (#104).
 * Only change api_key file permissions if needed (#103)
 
diff --git a/setup.py b/setup.py
index 5bd5e84..467cd84 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.14.0',
+    version='1.15.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 9b47875..b0eeea7 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -205,9 +205,10 @@ def count(query):
 
 @main.command()
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
+@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int)
 @click.argument('filename', metavar='<filename>')
 @click.argument('query', metavar='<search query>', nargs=-1)
-def download(limit, filename, query):
+def download(limit, skip, filename, query):
     """Download search results and save them in a compressed JSON file."""
     key = get_api_key()
 
@@ -247,11 +248,15 @@ def download(limit, filename, query):
     # A limit of -1 means that we should download all the data
     if limit <= 0:
         limit = total
+    
+        # Adjust the total number of results we should expect to download if the user is skipping results
+        if skip > 0:
+            limit -= skip
 
     with helpers.open_file(filename, 'w') as fout:
         count = 0
         try:
-            cursor = api.search_cursor(query, minify=False)
+            cursor = api.search_cursor(query, minify=False, skip=skip)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
                     helpers.write_banner(fout, banner)
diff --git a/shodan/client.py b/shodan/client.py
index bc5a25e..bb38a69 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -430,7 +430,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5):
+    def search_cursor(self, query, minify=True, retries=5, skip=0):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -444,16 +444,27 @@ def search_cursor(self, query, minify=True, retries=5):
         :type minify: bool
         :param retries: (optional) How often to retry the search in case it times out
         :type retries: int
+        :param skip: (optional) Number of results to skip
+        :type skip: int
 
         :returns: A search cursor that can be used as an iterator/ generator.
         """
         page = 1
         tries = 0
+
+        # Placeholder results object to make the while loop below easier
         results = {
-            'matches': [],
+            'matches': [True],
             'total': None,
         }
-        while page == 1 or results['matches']:
+
+        # Convert the number of skipped records into a page number
+        if skip > 0:
+            # Each page returns 100 results so find the nearest page that we want
+            # the cursor to skip to.
+            page += int(skip / 100)
+
+        while results['matches']:
             try:
                 results = self.search(query, minify=minify, page=page)
                 for banner in results['matches']:

From cdf7ddec9a472e7561ce021eb3e73f7173cac506 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 21 Sep 2019 12:36:34 -0500
Subject: [PATCH 189/263] Allow users to define a list of fields to include
 when converting the data into other formats (#107)

---
 shodan/__main__.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index b0eeea7..61f341f 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -89,15 +89,23 @@ def main():
     'xlsx': ExcelConverter,
 }
 @main.command()
+@click.option('--fields', help='List of properties to output.', default=None)
 @click.argument('input', metavar='<input file>')
 @click.argument('format', metavar='<output format>', type=click.Choice(CONVERTERS.keys()))
-def convert(input, format):
+def convert(fields, input, format):
     """Convert the given input data file into a different format. The following file formats are supported:
 
     kml, csv, geo.json, images, xlsx
 
     Example: shodan convert data.json.gz kml
     """
+    # Check that the converter allows a custom list of fields
+    converter_class = CONVERTERS.get(format)
+    if fields:
+        if not hasattr(converter_class, 'fields'):
+            raise click.ClickException('File format doesnt support custom list of fields')
+        converter_class.fields = [item.strip() for item in fields.split(',')]  # Use the custom fields the user specified
+    
     # Get the basename for the input file
     basename = input.replace('.json.gz', '').replace('.json', '')
 
@@ -113,7 +121,7 @@ def convert(input, format):
     progress_bar_thread.start()
 
     # Initialize the file converter
-    converter = CONVERTERS.get(format)(fout)
+    converter = converter_class(fout)
 
     converter.process([input])
 

From 933043a7d51f3dec8ff66fe8d8833fbdbe9462a8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 21 Sep 2019 13:16:52 -0500
Subject: [PATCH 190/263] Release 1.16.0 New command option and library method
 to filter firehose based on tags

---
 CHANGELOG.md       |  5 +++++
 setup.py           |  2 +-
 shodan/__main__.py | 11 +++++++++--
 shodan/stream.py   | 11 +++++++++++
 4 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 840da0e..0fc9488 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+1.16.0
+------
+* Ability to specify list of fields to include when converting to CSV/ Excel (#107)
+* Filter the Shodan Firehose based on tags in the banner
+
 1.15.0
 ------
 * New option "--skip" for download command to help users resume a download
diff --git a/setup.py b/setup.py
index 467cd84..6dc2833 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.15.0',
+    version='1.16.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 61f341f..cc27735 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -585,8 +585,9 @@ def stats(limit, facets, filename, query):
 @click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
 @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
 @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
+@click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str)
 @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel):
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -612,9 +613,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         stream_type.append('asn')
     if alert:
         stream_type.append('alert')
+    if tags:
+        stream_type.append('tags')
 
     if len(stream_type) > 1:
-        raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.')
+        raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.')
 
     stream_args = None
 
@@ -635,6 +638,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
 
     if countries:
         stream_args = countries.split(',')
+    
+    if tags:
+        stream_args = tags.split(',')
 
     # Flatten the list of stream types
     # Possible values are:
@@ -655,6 +661,7 @@ def _create_stream(name, args, timeout):
             'asn': api.stream.asn(args, timeout=timeout),
             'countries': api.stream.countries(args, timeout=timeout),
             'ports': api.stream.ports(args, timeout=timeout),
+            'tags': api.stream.tags(args, timeout=timeout),
         }.get(name, 'all')
 
     stream = _create_stream(stream_type, stream_args, timeout=timeout)
diff --git a/shodan/stream.py b/shodan/stream.py
index 10ccac3..1feb15a 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -123,3 +123,14 @@ def ports(self, ports, raw=False, timeout=None):
         stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
+
+    def tags(self, tags, raw=False, timeout=None):
+        """
+        A filtered version of the "banners" stream to only return banners that match the tags of interest.
+
+        :param tags: A list of tags to return banner data on.
+        :type tags: string[]
+        """
+        stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout)
+        for line in self._iter_stream(stream, raw):
+            yield line

From 1a653f7e6c75c54e3ccfddb99c21ca334e7da2c6 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 21 Sep 2019 17:10:30 -0500
Subject: [PATCH 191/263] Fix bug that caused unicode error when printing
 domain information (#106)

---
 CHANGELOG.md       | 4 ++++
 shodan/__main__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0fc9488..6bcdfec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+unreleased
+----------
+* Fix bug that caused unicode error when printing domain information 
+
 1.16.0
 ------
 * Ability to specify list of fields to include when converting to CSV/ Excel (#107)
diff --git a/shodan/__main__.py b/shodan/__main__.py
index cc27735..0bcf5e3 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -151,7 +151,7 @@ def domain_info(domain):
     click.echo('')
     for record in info['data']:
         click.echo(
-            '{:32}  {:14}  {}'.format(
+            u'{:32}  {:14}  {}'.format(
                 click.style(record['subdomain'], fg='cyan'),
                 click.style(record['type'], fg='yellow'),
                 record['value']

From f2b6bc404e24173cff1a5483c42e8dd9ef153087 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 21 Sep 2019 18:44:58 -0500
Subject: [PATCH 192/263] Release 1.17.0 Add flag to let users get their IPv6
 address using "shodan myip -6" (#106)

---
 CHANGELOG.md       | 5 +++--
 setup.py           | 2 +-
 shodan/__main__.py | 8 +++++++-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6bcdfec..fe426d1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,10 @@
 CHANGELOG
 =========
 
-unreleased
+1.17.0
 ----------
-* Fix bug that caused unicode error when printing domain information 
+* Fix bug that caused unicode error when printing domain information (#106)
+* Add flag to let users get their IPv6 address **shodan myip -6**(#35)
 
 1.16.0
 ------
diff --git a/setup.py b/setup.py
index 6dc2833..eb9f224 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.16.0',
+    version='1.17.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 0bcf5e3..5ddad7a 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -400,11 +400,17 @@ def parse(color, fields, filters, filename, separator, filenames):
 
 
 @main.command()
-def myip():
+@click.option('--ipv6', '-6', is_flag=True, default=False, help='Try to use IPv6 instead of IPv4')
+def myip(ipv6):
     """Print your external IP address"""
     key = get_api_key()
 
     api = shodan.Shodan(key)
+
+    # Use the IPv6-enabled domain if requested
+    if ipv6:
+        api.base_url = 'https://apiv6.shodan.io'
+    
     try:
         click.echo(api.tools.myip())
     except shodan.APIError as e:

From a910e9043f34702f1150099844bfd557af1a9f20 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Thu, 26 Sep 2019 17:33:50 -0500
Subject: [PATCH 193/263] Add library methods for new Notifications API

---
 CHANGELOG.md     |  7 ++++-
 setup.py         |  2 +-
 shodan/client.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 74 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe426d1..c3249e3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,13 @@
 CHANGELOG
 =========
 
+1.18.0
+------
+
+* Add library methods for the new Notifications API
+
 1.17.0
-----------
+------
 * Fix bug that caused unicode error when printing domain information (#106)
 * Add flag to let users get their IPv6 address **shodan myip -6**(#35)
 
diff --git a/setup.py b/setup.py
index eb9f224..02ac89b 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.17.0',
+    version='1.18.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index bb38a69..5de2ba3 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -74,6 +74,72 @@ def domain_info(self, domain):
             """
             return self.parent._request('/dns/domain/{}'.format(domain), {})
 
+    class Notifier:
+
+        def __init__(self, parent):
+            self.parent = parent
+        
+        def create(self, provider, args, description=None):
+            """Get the settings for the specified notifier that a user has configured.
+
+            :param provider: Provider name
+            :type provider: str
+            :param args: Provider arguments
+            :type args: dict
+            :param description: Human-friendly description of the notifier
+            :type description: str
+            :returns: dict -- fields are 'success' and 'id' of the notifier
+            """
+            args['provider'] = provider
+
+            if description:
+                args['description'] = description
+            
+            return self.parent._request('/notifier', args, method='post')
+        
+        def edit(self, nid, args):
+            """Get the settings for the specified notifier that a user has configured.
+
+            :param nid: Notifier ID
+            :type nid: str
+            :param args: Provider arguments
+            :type args: dict
+            :returns: dict -- fields are 'success' and 'id' of the notifier
+            """
+            return self.parent._request('/notifier/{}'.format(nid), args, method='put')
+        
+        def get(self, nid):
+            """Get the settings for the specified notifier that a user has configured.
+
+            :param nid: Notifier ID
+            :type nid: str
+            :returns: dict -- object describing the notifier settings
+            """
+            return self.parent._request('/notifier/{}'.format(nid), {})
+
+        def list_notifiers(self):
+            """Returns a list of notifiers that the user has added.
+
+            :returns: A list of notifierse that are available on the account
+            """
+            return self.parent._request('/notifier', {})
+
+        def list_providers(self):
+            """Returns a list of supported notification providers.
+
+            :returns: A list of providers where each object describes a provider
+            """
+            return self.parent._request('/notifier/provider', {})
+        
+        def remove(self, nid):
+            """Delete the provided notifier.
+
+            :param nid: Notifier ID
+            :type nid: str
+            :returns: dict -- 'success' set to True if action succeeded
+            """
+            return self.parent._request('/notifier/{}'.format(nid), {}, method='delete')
+
     class Tools:
 
         def __init__(self, parent):
@@ -195,6 +261,7 @@ def __init__(self, key, proxies=None):
         self.dns = self.Dns(self)
         self.exploits = self.Exploits(self)
         self.labs = self.Labs(self)
+        self.notifier = self.Notifier(self)
         self.org = self.Organization(self)
         self.tools = self.Tools(self)
         self.stream = Stream(key, proxies=proxies)

From 7db73bca5ff193827d6617b7431c53969a93706b Mon Sep 17 00:00:00 2001
From: JW <jw@jw.fi>
Date: Fri, 27 Sep 2019 19:51:19 +0300
Subject: [PATCH 194/263] Add "net:ip/mask" filter for shodan parse

A simple implementation of the net:filter used in web GUI.

It uses python3's builtin ipaddress lib and only imports those libs if the net:filter is used.
---
 shodan/cli/helpers.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 3154591..a52bcb4 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -64,9 +64,26 @@ def get_banner_field(banner, flat_field):
     return None
 
 
+def filter_with_netmask(banner, netmask):
+    # filtering based on netmask is a more abstract concept than
+    # a mere check for a specific field and thus needs its own mechanism
+    # this will enable users to use the net:10.0.0.0/8 syntax they are used to
+    # to find specific networks from a big shodan download.
+    from ipaddress import ip_network, ip_address
+    network = ip_network(netmask)
+    ip_field = get_banner_field(banner, 'ip')
+    if not ip_field:
+        return False
+    banner_ip_address = ip_address(ip_field)
+    return banner_ip_address in network
+
+
 def match_filters(banner, filters):
     for args in filters:
         flat_field, check = args.split(':', 1)
+        if flat_field == 'net':
+            return filter_with_netmask(banner, check)
+
         value = get_banner_field(banner, flat_field)
 
         # If the field doesn't exist on the banner then ignore the record

From b74d8e2da5f3fa12c68d8b16b71f7c372c6c4258 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 29 Sep 2019 13:30:15 -0500
Subject: [PATCH 195/263] New method to edit the list of IPs for an existing
 network alert

---
 CHANGELOG.md     |  5 ++++-
 setup.py         |  2 +-
 shodan/client.py | 29 +++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c3249e3..8670835 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,12 @@
 CHANGELOG
 =========
 
-1.18.0
+1.19.0
 ------
+* New method to edit the list of IPs for an existing network alert
 
+1.18.0
+------
 * Add library methods for the new Notifications API
 
 1.17.0
diff --git a/setup.py b/setup.py
index 02ac89b..70381c4 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.18.0',
+    version='1.19.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index 5de2ba3..6e66259 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -640,6 +640,27 @@ def create_alert(self, name, ip, expires=0):
 
         return response
 
+    def edit_alert(self, aid, ip):
+        """Edit the IPs that should be monitored by the alert.
+
+        :param aid: Alert ID
+        :type name: str
+        :param ip: Network range(s) to monitor
+        :type ip: str OR list of str
+
+        :returns: A dict describing the alert
+        """
+        data = {
+            'filters': {
+                'ip': ip,
+            },
+        }
+
+        response = api_request(self.api_key, '/shodan/alert/{}'.format(aid), data=data, params={}, method='post',
+                               proxies=self._session.proxies)
+
+        return response
+
     def alerts(self, aid=None, include_expired=True):
         """List all of the active alerts that the user created."""
         if aid:
@@ -684,3 +705,11 @@ def ignore_alert_trigger_notification(self, aid, trigger, ip, port):
     def unignore_alert_trigger_notification(self, aid, trigger, ip, port):
         """Re-enable trigger notifications for the provided IP and port"""
         return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete')
+    
+    def add_alert_notifier(self, aid, nid):
+        """Enable the given notifier for an alert that has triggers enabled."""
+        return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put')
+    
+    def remove_alert_notifier(self, aid, nid):
+        """Remove the given notifier for an alert that has triggers enabled."""
+        return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete')

From 4c297eed9e721e9cdb08b88b1fe3a9e1335963b7 Mon Sep 17 00:00:00 2001
From: JW <jw@jw.fi>
Date: Tue, 1 Oct 2019 22:51:11 +0300
Subject: [PATCH 196/263] Moving import to top of file for pep8

---
 shodan/cli/helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index a52bcb4..4e99113 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -7,6 +7,7 @@
 import itertools
 import os
 import sys
+from ipaddress import ip_network, ip_address
 
 from .settings import SHODAN_CONFIG_DIR
 
@@ -69,7 +70,6 @@ def filter_with_netmask(banner, netmask):
     # a mere check for a specific field and thus needs its own mechanism
     # this will enable users to use the net:10.0.0.0/8 syntax they are used to
     # to find specific networks from a big shodan download.
-    from ipaddress import ip_network, ip_address
     network = ip_network(netmask)
     ip_field = get_banner_field(banner, 'ip')
     if not ip_field:

From bee4511986e3456b0b0050d93a1899ddc060a951 Mon Sep 17 00:00:00 2001
From: JW <jw@jw.fi>
Date: Sat, 5 Oct 2019 10:28:17 +0300
Subject: [PATCH 197/263] Install ipaddress for py27 compat

Using python_version marker, install ipaddress
from pypi only for python2.7 and below
---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4fa2ed6..5095f64 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,5 @@ click
 click-plugins
 colorama
 requests>=2.2.1
-XlsxWriter
\ No newline at end of file
+XlsxWriter
+ipaddress;python_version<='2.7'
\ No newline at end of file

From 2ea18d9953a92b32df78bc7d617127e233fcfb1a Mon Sep 17 00:00:00 2001
From: AaronK <aaronkaplan@users.noreply.github.com>
Date: Mon, 7 Oct 2019 18:25:19 +0200
Subject: [PATCH 198/263] Update cert-stream.rst

make the example file python3 ready
---
 docs/examples/cert-stream.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst
index e3e72c1..26bf1b4 100644
--- a/docs/examples/cert-stream.rst
+++ b/docs/examples/cert-stream.rst
@@ -24,7 +24,7 @@ information.
 	# information.
 	#
 	# Author: achillean
-
+	from __future__ import print_function
 	import shodan
 	import sys
 
@@ -35,7 +35,7 @@ information.
 	    # Setup the api
 	    api = shodan.Shodan(API_KEY)
 
-	    print 'Listening for certs...'
+	    print('Listening for certs...')
 	    for banner in api.stream.ports([443, 8443]):
 			if 'ssl' in banner:
 				# Print out all the SSL information that Shodan has collected

From 6f5a927b7f77d1f5dcfe0d87d761052a9ee98f19 Mon Sep 17 00:00:00 2001
From: AaronK <aaronkaplan@users.noreply.github.com>
Date: Wed, 9 Oct 2019 16:56:56 +0200
Subject: [PATCH 199/263] Update cert-stream.rst

as requested, we are on python 2.7+
---
 docs/examples/cert-stream.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst
index 26bf1b4..b01440e 100644
--- a/docs/examples/cert-stream.rst
+++ b/docs/examples/cert-stream.rst
@@ -24,7 +24,6 @@ information.
 	# information.
 	#
 	# Author: achillean
-	from __future__ import print_function
 	import shodan
 	import sys
 

From df8e803c7d24f2594d429d2ea6d6fbd1f2ddfa82 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 13 Nov 2019 11:13:35 -0600
Subject: [PATCH 200/263] Show SHA1 checksum in bulk data listing if available

---
 setup.py           | 2 +-
 shodan/cli/data.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 70381c4..8751b5f 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.19.0',
+    version='1.19.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/data.py b/shodan/cli/data.py
index 7cd7228..98d7852 100644
--- a/shodan/cli/data.py
+++ b/shodan/cli/data.py
@@ -27,6 +27,11 @@ def data_list(dataset):
         for file in files:
             click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False)
             click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False)
+
+            # Show the SHA1 checksum if available
+            if file.get('sha1'):
+                click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False)
+            
             click.echo('{}'.format(file['url']))
     else:
         # If no dataset was provided then show a list of all datasets

From 1f101cae4700e30611a4d9b85634bc555f8cba08 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 18 Nov 2019 19:24:32 -0600
Subject: [PATCH 201/263] New options for "shodan domain":  "-S" (save results)
 and "-D" (lookup open ports for IPs)

---
 CHANGELOG.md       |  5 +++++
 setup.py           |  2 +-
 shodan/__main__.py | 44 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8670835..8eced4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+1.20.0
+------
+* New option "-S" for **shodan domain** to save results from the lookup
+* New option "-D" for **shodan domain** to lookup open ports for IPs in the results
+
 1.19.0
 ------
 * New method to edit the list of IPs for an existing network alert
diff --git a/setup.py b/setup.py
index 8751b5f..b2cf088 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.19.1',
+    version='1.20.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 5ddad7a..d4b7aa0 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -136,7 +136,9 @@ def convert(fields, input, format):
 
 @main.command(name='domain')
 @click.argument('domain', metavar='<domain>')
-def domain_info(domain):
+@click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True)
+@click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True)
+def domain_info(domain, details, save):
     """View all available information for a domain"""
     key = get_api_key()
     api = shodan.Shodan(key)
@@ -146,6 +148,37 @@ def domain_info(domain):
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
+    # Grab the host information for any IP records that were returned
+    hosts = {}
+    if details:
+        ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']]
+        ips = set(ips)
+
+        fout = None
+        if save:
+            filename = u'{}-hosts.json.gz'.format(domain)
+            fout = helpers.open_file(filename)
+
+        for ip in ips:
+            try:
+                hosts[ip] = api.host(ip)
+
+                # Store the banners if requested
+                if fout:
+                    for banner in hosts[ip]['data']:
+                        if 'placeholder' not in banner:
+                            helpers.write_banner(fout, banner)
+            except shodan.APIError:
+                pass  # Ignore any API lookup errors as this isn't critical information
+    
+    # Save the DNS data
+    if save:
+        filename = u'{}.json.gz'.format(domain)
+        fout = helpers.open_file(filename)
+
+        for record in info['data']:
+            helpers.write_banner(fout, record)
+
     click.secho(info['domain'].upper(), fg='green')
 
     click.echo('')
@@ -155,9 +188,16 @@ def domain_info(domain):
                 click.style(record['subdomain'], fg='cyan'),
                 click.style(record['type'], fg='yellow'),
                 record['value']
-            )
+            ),
+            nl=False,
         )
 
+        if record['value'] in hosts:
+            host = hosts[record['value']]
+            click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False)
+        
+        click.echo('')
+
 
 @main.command()
 @click.argument('key', metavar='<api key>')

From da0e2ae7534c2b892a6122880e46b2b50fbe6b56 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 14 Dec 2019 17:23:07 -0600
Subject: [PATCH 202/263] Add new API methods to get list of search filters/
 facets

---
 shodan/client.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/shodan/client.py b/shodan/client.py
index 6e66259..3ed5e66 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -549,6 +549,20 @@ def search_cursor(self, query, minify=True, retries=5, skip=0):
                 tries += 1
                 time.sleep(1.0)  # wait 1 second if the search errored out for some reason
 
+    def search_facets(self):
+        """Returns a list of search facets that can be used to get aggregate information about a search query.
+
+        :returns: A list of strings where each is a facet name
+        """
+        return self._request('/shodan/host/search/facets', {})
+
+    def search_filters(self):
+        """Returns a list of search filters that are available.
+
+        :returns: A list of strings where each is a filter name
+        """
+        return self._request('/shodan/host/search/filters', {})
+
     def search_tokens(self, query):
         """Returns information about the search query itself (filters used etc.)
 

From 406abf0891f192fa2a46eff779bedbcc1841eaf8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 14 Dec 2019 17:23:47 -0600
Subject: [PATCH 203/263] Fix linting rules and errors

---
 shodan/cli/scan.py | 8 ++++----
 tox.ini            | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py
index 4590770..7ec158c 100644
--- a/shodan/cli/scan.py
+++ b/shodan/cli/scan.py
@@ -91,7 +91,7 @@ def scan_internet(quiet, port, protocol):
                                     click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']),
                                     ';'.join(banner['hostnames']))
                                 )
-                    except shodan.APIError as e:
+                    except shodan.APIError:
                         # We stop waiting for results if the scan has been processed by the crawlers and
                         # there haven't been new results in a while
                         if done:
@@ -100,7 +100,7 @@ def scan_internet(quiet, port, protocol):
                         scan = api.scan_status(scan['id'])
                         if scan['status'] == 'DONE':
                             done = True
-                    except socket.timeout as e:
+                    except socket.timeout:
                         # We stop waiting for results if the scan has been processed by the crawlers and
                         # there haven't been new results in a while
                         if done:
@@ -205,7 +205,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
                                 done = True
                                 break
 
-                except shodan.APIError as e:
+                except shodan.APIError:
                     # If the connection timed out before the timeout, that means the streaming server
                     # that the user tried to reach is down. In that case, lets wait briefly and try
                     # to connect again!
@@ -223,7 +223,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
 
                     if verbose:
                         click.echo('# Scan status: {}'.format(scan['status']))
-                except socket.timeout as e:
+                except socket.timeout:
                     # If the connection timed out before the timeout, that means the streaming server
                     # that the user tried to reach is down. In that case, lets wait a second and try
                     # to connect again!
diff --git a/tox.ini b/tox.ini
index d840b92..dd53bb5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
 [flake8]
 ignore =
-    E501
+    E501 W293
 
 exclude = 
     build,

From 9b1fe510ffb0f55a2e0f51b34f109fce904c7b16 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 14 Dec 2019 17:25:58 -0600
Subject: [PATCH 204/263] Release 1.21.0

---
 CHANGELOG.md | 4 ++++
 setup.py     | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8eced4d..5bd47a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.21.0
+------
+* New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters.
+
 1.20.0
 ------
 * New option "-S" for **shodan domain** to save results from the lookup
diff --git a/setup.py b/setup.py
index b2cf088..17337b4 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.20.0',
+    version='1.21.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',

From c73cf83e0fc92b9fa5fc8243019a99a3975ff2cc Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 25 Dec 2019 11:38:34 -0600
Subject: [PATCH 205/263] Add history and type parameters to
 Shodan.dns.domain_info() method and CLI command

---
 CHANGELOG.md       | 4 ++++
 setup.py           | 2 +-
 shodan/__main__.py | 6 ++++--
 shodan/client.py   | 9 +++++++--
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bd47a4..90c2059 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.21.1
+------
+* Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command
+
 1.21.0
 ------
 * New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters.
diff --git a/setup.py b/setup.py
index 17337b4..2148480 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.21.0',
+    version='1.21.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index d4b7aa0..9678bbb 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -138,13 +138,15 @@ def convert(fields, input, format):
 @click.argument('domain', metavar='<domain>')
 @click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True)
 @click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True)
-def domain_info(domain, details, save):
+@click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True)
+@click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None)
+def domain_info(domain, details, save, history, type):
     """View all available information for a domain"""
     key = get_api_key()
     api = shodan.Shodan(key)
 
     try:
-        info = api.dns.domain_info(domain)
+        info = api.dns.domain_info(domain, history=history, type=type)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
diff --git a/shodan/client.py b/shodan/client.py
index 3ed5e66..a83ac20 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -69,10 +69,15 @@ class Dns:
         def __init__(self, parent):
             self.parent = parent
 
-        def domain_info(self, domain):
+        def domain_info(self, domain, history=False, type=None):
             """Grab the DNS information for a domain.
             """
-            return self.parent._request('/dns/domain/{}'.format(domain), {})
+            args = {}
+            if history:
+                args['history'] = history
+            if type:
+                args['type'] = type
+            return self.parent._request('/dns/domain/{}'.format(domain), args)
 
     class Notifier:
 

From 7ef18461bae262ece63a97ee6eb9a14c465607d0 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 6 Jan 2020 23:43:22 -0600
Subject: [PATCH 206/263] Add support for paging through the domain information

---
 CHANGELOG.md     | 4 ++++
 setup.py         | 2 +-
 shodan/client.py | 6 ++++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 90c2059..91540af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.21.2
+------
+* Add support for paging through the domain information
+
 1.21.1
 ------
 * Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command
diff --git a/setup.py b/setup.py
index 2148480..b84dd71 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.21.1',
+    version='1.21.2',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index a83ac20..4116051 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -69,10 +69,12 @@ class Dns:
         def __init__(self, parent):
             self.parent = parent
 
-        def domain_info(self, domain, history=False, type=None):
+        def domain_info(self, domain, history=False, type=None, page=1):
             """Grab the DNS information for a domain.
             """
-            args = {}
+            args = {
+                'page': page,
+            }
             if history:
                 args['history'] = history
             if type:

From 9e75eb9871bad4490dbdb7c2a3130883e40c5b59 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 20 Jan 2020 13:57:44 -0600
Subject: [PATCH 207/263] Fix geo.json file converter

---
 CHANGELOG.md                    |  4 +++
 setup.py                        |  2 +-
 shodan/cli/converter/geojson.py | 49 +++++++++++++++------------------
 3 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 91540af..0d5c02d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.21.3
+------
+* Fix geo.json file converter
+
 1.21.2
 ------
 * Add support for paging through the domain information
diff --git a/setup.py b/setup.py
index b84dd71..0605303 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.21.2',
+    version='1.21.3',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index 6126267..eb9a205 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -1,4 +1,4 @@
-
+from json import dumps
 from .base import Converter
 from ...helpers import get_ip, iterate_files
 
@@ -18,40 +18,35 @@ def process(self, files):
         # Write the header
         self.header()
 
-        hosts = {}
+        # We only want to generate 1 datapoint for each IP - not per service
+        unique_hosts = set()
         for banner in iterate_files(files):
             ip = get_ip(banner)
             if not ip:
                 continue
 
-            if ip not in hosts:
-                hosts[ip] = banner
-                hosts[ip]['ports'] = []
-
-            hosts[ip]['ports'].append(banner['port'])
-
-        for ip, host in iter(hosts.items()):
-            self.write(host)
+            if ip not in unique_hosts:
+                self.write(ip, banner)
+                unique_hosts.add(ip)
 
         self.footer()
 
-    def write(self, host):
+    def write(self, ip, host):
         try:
-            ip = get_ip(host)
             lat, lon = host['location']['latitude'], host['location']['longitude']
-
-            feature = """{
-                "type": "Feature",
-                "id": "{}",
-                "properties": {
-                    "name": "{}"
-                 },
-                "geometry": {
-                    "type": "Point",
-                    "coordinates": [{}, {}]
-                }
-            }""".format(ip, ip, lat, lon)
-
-            self.fout.write(feature)
-        except Exception:
+            feature = {
+                'type': 'Feature',
+                'id': ip,
+                'properties': {
+                    'name': ip,
+                    'lat': lat,
+                    'lon': lon,
+                },
+                'geometry': {
+                    'type': 'Point',
+                    'coordinates': [lon, lat],
+                },
+            }
+            self.fout.write(dumps(feature) + ',')
+        except Exception as e:
             pass

From ba561934f7fcfce3c35270d2baee6539b2afaaf1 Mon Sep 17 00:00:00 2001
From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com>
Date: Sat, 25 Jan 2020 17:49:19 +0100
Subject: [PATCH 208/263] py 2 to 3

fix the example given in https://github.com/achillean/shodan-python/issues/114
---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 00c4d34..62744af 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -56,7 +56,7 @@ Now that we have our API object all good to go, we're ready to perform a search:
 			print('IP: {}'.format(result['ip_str']))
 			print(result['data'])
 			print('')
-	except shodan.APIError, e:
+	except shodan.APIError as e:
 		print('Error: {}'.format(e))
 
 Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which

From 67409f6eebeec28b00be0c7517d28411f1ea43c1 Mon Sep 17 00:00:00 2001
From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com>
Date: Sat, 25 Jan 2020 18:10:42 +0100
Subject: [PATCH 209/263] py 2 to 3

---
 docs/examples/query-summary.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst
index 7a60716..73a87ab 100644
--- a/docs/examples/query-summary.rst
+++ b/docs/examples/query-summary.rst
@@ -48,7 +48,7 @@ and country.
 
 	# Input validation
 	if len(sys.argv) == 1:
-	    print 'Usage: %s <search query>' % sys.argv[0]
+	    print('Usage: %s <search query>' % sys.argv[0])
 	    sys.exit(1)
 
 	try:
@@ -68,16 +68,16 @@ and country.
 	    
 	    # Print the summary info from the facets
 	    for facet in result['facets']:
-	        print FACET_TITLES[facet]
+	        print(FACET_TITLES[facet])
 	        
 	        for term in result['facets'][facet]:
-	            print '%s: %s' % (term['value'], term['count'])
+	            print('%s: %s' % (term['value'], term['count']))
 
 	        # Print an empty line between summary info
-	        print ''
+	        print('')
 	    
-	except Exception, e:
-	    print 'Error: %s' % e
+	except Exception as e:
+	    print('Error: %s' % e)
 	    sys.exit(1)
 
 	"""

From 1deba33ba6bbb0bc45dd5716c48a1f785a0a986b Mon Sep 17 00:00:00 2001
From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com>
Date: Sat, 25 Jan 2020 18:12:16 +0100
Subject: [PATCH 210/263] py 2 to 3

---
 docs/examples/gifcreator.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst
index ad9efc8..e4a43c3 100644
--- a/docs/examples/gifcreator.rst
+++ b/docs/examples/gifcreator.rst
@@ -106,7 +106,7 @@ There are a few key Shodan methods/ parameters that make the script work:
 			os.system('rm -f /tmp/gif-image-*.jpg')
 	
 			# Show a progress indicator
-			print result['ip_str']
+			print(result['ip_str'])
 
 
-The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101
\ No newline at end of file
+The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101

From a3084d3a5976e9d47e407ba073cd90f2c8122b1c Mon Sep 17 00:00:00 2001
From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com>
Date: Sat, 25 Jan 2020 18:19:02 +0100
Subject: [PATCH 211/263] py 2 to 3

---
 docs/examples/query-summary.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst
index 73a87ab..66e15fe 100644
--- a/docs/examples/query-summary.rst
+++ b/docs/examples/query-summary.rst
@@ -62,9 +62,9 @@ and country.
 	    # And it also runs faster than doing a search().
 	    result = api.count(query, facets=FACETS)
 	    
-	    print 'Shodan Summary Information'
-	    print 'Query: %s' % query
-	    print 'Total Results: %s\n' % result['total']
+	    print('Shodan Summary Information')
+	    print('Query: %s' % query)
+	    print('Total Results: %s\n' % result['total'])
 	    
 	    # Print the summary info from the facets
 	    for facet in result['facets']:

From ee93d65b75b329fbe57f4f2248ead0f695ce1ccb Mon Sep 17 00:00:00 2001
From: Fabian Affolter <mail@fabian-affolter.ch>
Date: Tue, 28 Jan 2020 09:44:48 +0100
Subject: [PATCH 212/263] Remove shebang

This is a RPM packing issue.
---
 shodan/cli/worldmap.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 767b237..30c9ec3 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 '''
 F-Secure Virus World Map console edition
 

From 2367646b8e4474ffdfd969485f989c69a4f42a22 Mon Sep 17 00:00:00 2001
From: Fabian Affolter <mail@fabian-affolter.ch>
Date: Tue, 28 Jan 2020 14:31:27 +0100
Subject: [PATCH 213/263] Point the repo directly

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 0605303..8290147 100755
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,7 @@
     long_description_content_type='text/x-rst',
     author='John Matherly',
     author_email='jmath@shodan.io',
-    url='http://github.com/achillean/shodan-python/tree/master',
+    url='https://github.com/achillean/shodan-python',
     packages=['shodan', 'shodan.cli', 'shodan.cli.converter'],
     entry_points={'console_scripts': ['shodan=shodan.__main__:main']},
     install_requires=DEPENDENCIES,

From bde444271a516a81235ed961d89e32b8f551ebce Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 17 Mar 2020 18:29:15 -0500
Subject: [PATCH 214/263] New Streaming API method: /shodan/vulns/{vulns} to
 subscribe to IPs that are vulnerable to an issue Release 1.22.0

---
 setup.py           |  2 +-
 shodan/__main__.py | 11 +++++++++--
 shodan/stream.py   | 11 +++++++++++
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 0605303..0c8882f 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.21.3',
+    version='1.22.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 9678bbb..5b1af92 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -635,7 +635,8 @@ def stats(limit, facets, filename, query):
 @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
 @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str)
 @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel):
+@click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str)
+def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -663,9 +664,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         stream_type.append('alert')
     if tags:
         stream_type.append('tags')
+    if vulns:
+        stream_type.append('vulns')
 
     if len(stream_type) > 1:
-        raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.')
+        raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.')
 
     stream_args = None
 
@@ -689,6 +692,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
     
     if tags:
         stream_args = tags.split(',')
+    
+    if vulns:
+        stream_args = vulns.split(',')
 
     # Flatten the list of stream types
     # Possible values are:
@@ -710,6 +716,7 @@ def _create_stream(name, args, timeout):
             'countries': api.stream.countries(args, timeout=timeout),
             'ports': api.stream.ports(args, timeout=timeout),
             'tags': api.stream.tags(args, timeout=timeout),
+            'vulns': api.stream.vulns(args, timeout=timeout),
         }.get(name, 'all')
 
     stream = _create_stream(stream_type, stream_args, timeout=timeout)
diff --git a/shodan/stream.py b/shodan/stream.py
index 1feb15a..15d7619 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -134,3 +134,14 @@ def tags(self, tags, raw=False, timeout=None):
         stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
+
+    def vulns(self, vulns, raw=False, timeout=None):
+        """
+        A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest.
+
+        :param vulns: A list of vulns to return banner data on.
+        :type vulns: string[]
+        """
+        stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout)
+        for line in self._iter_stream(stream, raw):
+            yield line

From 378feee4992461f8f4510034c65d4c9dfbe651b6 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 27 Mar 2020 12:12:48 -0500
Subject: [PATCH 215/263] Fix bug when converting data file to CSV using
 Python3

---
 setup.py                     | 2 +-
 shodan/cli/converter/csvc.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 0c8882f..db956d9 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 setup(
     name='shodan',
-    version='1.22.0',
+    version='1.22.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 39ec162..93b1695 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -54,7 +54,7 @@ def process(self, files):
             # The "vulns" property can't be nicely flattened as-is so we turn
             # it into a list before processing the banner.
             if 'vulns' in banner:
-                banner['vulns'] = banner['vulns'].keys()
+                banner['vulns'] = list(banner['vulns'].keys())  # Python3 returns dict_keys so we neeed to cover that to a list
 
             try:
                 row = []

From b7a9978d532094fb8dd72efaa6b6f0bb75a89ba6 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 6 Apr 2020 14:36:00 -0500
Subject: [PATCH 216/263] Update CHANGELOG to match releases

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d5c02d..de68afb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+1.22.1
+------
+* Fix bug when converting data file to CSV using Python3
+
+1.22.0
+------
+* Add support for new vulnerability streaming endpoints
+
 1.21.3
 ------
 * Fix geo.json file converter

From d91ffd8f6aa0e093c3a913bb8585add3e99c2969 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 6 Apr 2020 15:02:12 -0500
Subject: [PATCH 217/263] Add new CLI command: shodan alert domain

---
 CHANGELOG.md        |  4 ++++
 setup.py            |  4 +++-
 shodan/cli/alert.py | 30 ++++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index de68afb..50a9334 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.23.0
+------
+* Add new CLI command: shodan alert domain
+
 1.22.1
 ------
 * Fix bug when converting data file to CSV using Python3
diff --git a/setup.py b/setup.py
index db956d9..e04a96f 100755
--- a/setup.py
+++ b/setup.py
@@ -2,12 +2,14 @@
 
 from setuptools import setup
 
+
 DEPENDENCIES = open('requirements.txt', 'r').read().split('\n')
 README = open('README.rst', 'r').read()
 
+
 setup(
     name='shodan',
-    version='1.22.1',
+    version='1.23.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 1ed8572..633b2aa 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -3,6 +3,7 @@
 
 from operator import itemgetter
 from shodan.cli.helpers import get_api_key
+from time import sleep
 
 
 @click.group()
@@ -46,6 +47,35 @@ def alert_create(name, netblocks):
     click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 
+@alert.command(name='domain')
+@click.argument('domain', metavar='<domain>', type=str)
+@click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable')
+def alert_domain(domain, triggers):
+    """Create a network alert based on a domain name"""
+    key = get_api_key()
+
+    api = shodan.Shodan(key)
+    try:
+        # Grab a list of IPs for the domain
+        domain = domain.lower()
+        click.secho('Looking up domain information...', dim=True)
+        info = api.dns.domain_info(domain, type='A')
+        domain_ips = set([record['value'] for record in info['data']])
+
+        # Create the actual alert
+        click.secho('Creating alert...', dim=True)
+        alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips))
+
+        # Enable the triggers so it starts getting managed by Shodan Monitor
+        click.secho('Enabling triggers...', dim=True)
+        api.enable_alert_trigger(alert['id'], triggers)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    click.secho('Successfully created domain alert!', fg='green')
+    click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
+
+
 @alert.command(name='info')
 @click.argument('alert', metavar='<alert id>')
 def alert_info(alert):

From 31b420a590f11fd83e336f308aff499c2aa4d096 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 6 Apr 2020 15:06:42 -0500
Subject: [PATCH 218/263] Fix linting errors

---
 shodan/cli/alert.py             | 1 -
 shodan/cli/converter/geojson.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 633b2aa..0d81dc4 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -3,7 +3,6 @@
 
 from operator import itemgetter
 from shodan.cli.helpers import get_api_key
-from time import sleep
 
 
 @click.group()
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index eb9a205..3f6c975 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -48,5 +48,5 @@ def write(self, ip, host):
                 },
             }
             self.fout.write(dumps(feature) + ',')
-        except Exception as e:
+        except Exception:
             pass

From 094420d483a4be6c6d04906edfa13ab1e70489f3 Mon Sep 17 00:00:00 2001
From: Karthikeyan Singaravelan <tir.karthi@gmail.com>
Date: Sun, 3 May 2020 12:54:26 +0000
Subject: [PATCH 219/263] Import ABC from collections.abc for Python 3
 compatibility.

---
 shodan/cli/converter/csvc.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 93b1695..f51e75e 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -2,7 +2,13 @@
 from .base import Converter
 from ...helpers import iterate_files
 
-from collections import MutableMapping
+try:
+    # python 3.x: Import ABC from collections.abc
+    from collections.abc import MutableMapping
+except ImportError:
+    # Python 2.x: Import ABC from collections
+    from collections import MutableMapping
+
 from csv import writer as csv_writer, excel
 
 

From c6615d0a275ac53fadb0bb4f66fe290ffb1f9dae Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 23 Jun 2020 16:32:03 -0500
Subject: [PATCH 220/263] Fix invalid escape sequence error (#131) Improve
 ``shodan radar`` output on Python3

---
 shodan/__main__.py     | 15 ++++++++-------
 shodan/cli/worldmap.py |  5 ++---
 tox.ini                |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 5b1af92..939b53f 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -59,6 +59,13 @@
 
 # Make "-h" work like "--help"
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+CONVERTERS = {
+    'kml': KmlConverter,
+    'csv': CsvConverter,
+    'geo.json': GeoJsonConverter,
+    'images': ImagesConverter,
+    'xlsx': ExcelConverter,
+}
 
 # Define a basestring type if necessary for Python3 compatibility
 try:
@@ -66,6 +73,7 @@
 except NameError:
     basestring = str
 
+
 # Define the main entry point for all of our commands
 # and expose a way for 3rd-party plugins to tie into the Shodan CLI.
 @with_plugins(iter_entry_points('shodan.cli.plugins'))
@@ -81,13 +89,6 @@ def main():
 main.add_command(scan)
 
 
-CONVERTERS = {
-    'kml': KmlConverter,
-    'csv': CsvConverter,
-    'geo.json': GeoJsonConverter,
-    'images': ImagesConverter,
-    'xlsx': ExcelConverter,
-}
 @main.command()
 @click.option('--fields', help='List of properties to output.', default=None)
 @click.argument('input', metavar='<input file>')
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index 30c9ec3..db91e41 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -29,8 +29,7 @@
         'coords': [90.0, -180.0, -90.0, 180.0],
 
         # PyLint freaks out about the world map backslashes so ignore those warnings
-        # pylint: disable=W1401
-        'data': '''
+        'data': r'''
                . _..::__:  ,-"-"._       |7       ,     _,.__             
        _.___ _ _<_>`!(._`.`-.    /        _._     `_ ,_/  '  '-._.---.-.__
      .{     " " `-==,',._\{  \  / {)     / _ ">_,-' `                mt-2_
@@ -159,7 +158,7 @@ def draw(self, target):
         for lat, lon, char, desc, attrs, color in self.data:
             # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html
             if desc:
-                desc = desc.encode(self.encoding, 'ignore')
+                desc = desc.encode(self.encoding, 'ignore').decode()
             if items_to_show <= 0:
                 break
             char_x, char_y = self.latlon_to_coords(lat, lon)
diff --git a/tox.ini b/tox.ini
index dd53bb5..1a9f632 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,4 +11,4 @@ exclude =
 per-file-ignores =
     shodan/__init__.py:F401,
     shodan/cli/converter/__init__.py:F401,
-    shodan/cli/worldmap.py:W291,W293,W605,
\ No newline at end of file
+    shodan/cli/worldmap.py:W291,
\ No newline at end of file

From 6ad80cc48da80d3c100637cefdd6f939d5620b3a Mon Sep 17 00:00:00 2001
From: Dhaval Soneji <dhaval@soneji.xyz>
Date: Wed, 24 Jun 2020 00:11:32 +0100
Subject: [PATCH 221/263] allow user to move config dir to ~/.config

---
 shodan/cli/settings.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
index 6f27e5d..2f5a73c 100644
--- a/shodan/cli/settings.py
+++ b/shodan/cli/settings.py
@@ -1,5 +1,10 @@
+from os import path
+
+if path.exists(path.expanduser("~/.config/shodan")):
+    SHODAN_CONFIG_DIR="~/.config/shodan/"
+else:
+    SHODAN_CONFIG_DIR = '~/.shodan/'
 
-SHODAN_CONFIG_DIR = '~/.shodan/'
 COLORIZE_FIELDS = {
     'ip_str': 'green',
     'port': 'yellow',

From 1b58c7f3a6402a87e860f31a0cb197070fa7c798 Mon Sep 17 00:00:00 2001
From: Dhaval Soneji <dhaval@soneji.xyz>
Date: Wed, 24 Jun 2020 16:07:12 +0100
Subject: [PATCH 222/263] Revert "allow user to move config dir to ~/.config"

This reverts commit 6ad80cc48da80d3c100637cefdd6f939d5620b3a.
---
 shodan/cli/settings.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
index 2f5a73c..6f27e5d 100644
--- a/shodan/cli/settings.py
+++ b/shodan/cli/settings.py
@@ -1,10 +1,5 @@
-from os import path
-
-if path.exists(path.expanduser("~/.config/shodan")):
-    SHODAN_CONFIG_DIR="~/.config/shodan/"
-else:
-    SHODAN_CONFIG_DIR = '~/.shodan/'
 
+SHODAN_CONFIG_DIR = '~/.shodan/'
 COLORIZE_FIELDS = {
     'ip_str': 'green',
     'port': 'yellow',

From c01abd045b531052b8522ea532f6c6ae54d85b90 Mon Sep 17 00:00:00 2001
From: Dhaval Soneji <dhaval@soneji.xyz>
Date: Wed, 24 Jun 2020 16:09:19 +0100
Subject: [PATCH 223/263] default config dir to ~/.config/shodan, but support
 existing installations

---
 shodan/cli/settings.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
index 6f27e5d..294793c 100644
--- a/shodan/cli/settings.py
+++ b/shodan/cli/settings.py
@@ -1,5 +1,11 @@
 
-SHODAN_CONFIG_DIR = '~/.shodan/'
+from os import path
+
+if path.exists(path.expanduser("~/.shodan")):
+    SHODAN_CONFIG_DIR = '~/.shodan/'
+else:
+    SHODAN_CONFIG_DIR="~/.config/shodan/"
+
 COLORIZE_FIELDS = {
     'ip_str': 'green',
     'port': 'yellow',

From 639ca36a6747671e5d68ab9b10b054e2b31255eb Mon Sep 17 00:00:00 2001
From: Dhaval Soneji <dhaval@soneji.xyz>
Date: Wed, 24 Jun 2020 19:22:10 +0100
Subject: [PATCH 224/263] makedirs if ~/.config/shodan does not exist

---
 shodan/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 939b53f..f8ed673 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -210,7 +210,7 @@ def init(key):
     shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR)
     if not os.path.isdir(shodan_dir):
         try:
-            os.mkdir(shodan_dir)
+            os.makedirs(shodan_dir)
         except OSError:
             raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir))
 

From 56f0cdf7de8c41c47b2cfa869dd3e64bc1c1c8ce Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 7 Sep 2020 19:37:04 -0500
Subject: [PATCH 225/263] Fix bug that caused extra newlines when converting
 .json.gz data file to CSV on Windows
 (https://stackoverflow.com/questions/3191528/csv-in-python-adding-an-extra-carriage-return-on-windows)

---
 README.rst                   | 1 +
 setup.py                     | 2 +-
 shodan/cli/converter/csvc.py | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 610c13d..14f6717 100644
--- a/README.rst
+++ b/README.rst
@@ -21,6 +21,7 @@ Features
 - `Manage Email Notifications <https://asciinema.org/a/7WvyDtNxn0YeNU70ozsxvXDmL>`_
 - Exploit search API fully implemented
 - Bulk data downloads
+- Access the Shodan DNS DB to view domain information
 - `Command-line interface <https://cli.shodan.io>`_
 
 .. image:: https://cli.shodan.io/img/shodan-cli-preview.png
diff --git a/setup.py b/setup.py
index e04a96f..b1735b2 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.23.0',
+    version='1.23.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index f51e75e..20c3d03 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -51,7 +51,7 @@ class CsvConverter(Converter):
     ]
 
     def process(self, files):
-        writer = csv_writer(self.fout, dialect=excel)
+        writer = csv_writer(self.fout, dialect=excel, lineterminator='\n')
 
         # Write the header
         writer.writerow(self.fields)

From 9ebccea26867f33ddadb063af9b585a09d69fe1a Mon Sep 17 00:00:00 2001
From: malvidin <malvidin@gmail.com>
Date: Thu, 8 Oct 2020 22:40:53 +0200
Subject: [PATCH 226/263] Convert residual % formatting with str.format() Add
 API Rate limits to the Shodan class Fix human precision of bytes under 1024

---
 shodan/__main__.py     |  8 ++++----
 shodan/cli/helpers.py  |  2 +-
 shodan/cli/settings.py |  2 +-
 shodan/cli/worldmap.py |  4 ++--
 shodan/client.py       | 16 ++++++++++++----
 shodan/helpers.py      | 16 ++++++++--------
 shodan/stream.py       | 12 ++++++------
 7 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index f8ed673..bbcef07 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -288,10 +288,10 @@ def download(limit, skip, filename, query):
         raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.')
 
     # Print some summary information about the download request
-    click.echo('Search query:\t\t\t%s' % query)
-    click.echo('Total number of results:\t%s' % total)
-    click.echo('Query credits left:\t\t%s' % info['unlocked_left'])
-    click.echo('Output file:\t\t\t%s' % filename)
+    click.echo('Search query:\t\t\t{}'.format(query))
+    click.echo('Total number of results:\t{}'.format(total))
+    click.echo('Query credits left:\t\t{}'.format(info['unlocked_left']))
+    click.echo('Output file:\t\t\t{}'.format(filename))
 
     if limit > total:
         limit = total
diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py
index 4e99113..bde2f07 100644
--- a/shodan/cli/helpers.py
+++ b/shodan/cli/helpers.py
@@ -47,7 +47,7 @@ def timestr():
 
 
 def open_streaming_file(directory, timestr, compresslevel=9):
-    return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel)
+    return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel)
 
 
 def get_banner_field(banner, flat_field):
diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
index 294793c..05c1b9f 100644
--- a/shodan/cli/settings.py
+++ b/shodan/cli/settings.py
@@ -4,7 +4,7 @@
 if path.exists(path.expanduser("~/.shodan")):
     SHODAN_CONFIG_DIR = '~/.shodan/'
 else:
-    SHODAN_CONFIG_DIR="~/.config/shodan/"
+    SHODAN_CONFIG_DIR = "~/.config/shodan/"
 
 COLORIZE_FIELDS = {
     'ip_str': 'green',
diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py
index db91e41..4e09872 100755
--- a/shodan/cli/worldmap.py
+++ b/shodan/cli/worldmap.py
@@ -166,7 +166,7 @@ def draw(self, target):
                 attrs |= curses.color_pair(self.colors[color])
             self.window.addstr(char_y, char_x, char, attrs)
             if desc:
-                det_show = "%s %s" % (char, desc)
+                det_show = "{} {}".format(char, desc)
             else:
                 det_show = None
 
@@ -179,7 +179,7 @@ def draw(self, target):
                     # FIXME: check window size before addstr()
                     break
         self.window.overwrite(target)
-        self.window.leaveok(1)
+        self.window.leaveok(True)
 
 
 class MapApp(object):
diff --git a/shodan/client.py b/shodan/client.py
index 4116051..079880d 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -273,6 +273,8 @@ def __init__(self, key, proxies=None):
         self.tools = self.Tools(self)
         self.stream = Stream(key, proxies=proxies)
         self._session = requests.Session()
+        self.api_rate_limit = 1  # Requests per second
+        self._api_query_time = None
         if proxies:
             self._session.proxies.update(proxies)
             self._session.trust_env = False
@@ -297,6 +299,11 @@ def _request(self, function, params, service='shodan', method='get'):
             'exploits': self.base_exploits_url,
         }.get(service, 'shodan')
 
+        # Wait for API rate limit
+        if self._api_query_time is not None and self.api_rate_limit > 0:
+            while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time():
+                time.sleep(0.1 / self.api_rate_limit)
+
         # Send the request
         try:
             method = method.lower()
@@ -308,6 +315,7 @@ def _request(self, function, params, service='shodan', method='get'):
                 data = self._session.delete(base_url + function, params=params)
             else:
                 data = self._session.get(base_url + function, params=params)
+            self._api_query_time = time.time()
         except Exception:
             raise APIError('Unable to connect to Shodan')
 
@@ -377,7 +385,7 @@ def host(self, ips, history=False, minify=False):
             params['history'] = history
         if minify:
             params['minify'] = minify
-        return self._request('/shodan/host/%s' % ','.join(ips), params)
+        return self._request('/shodan/host/{}'.format(','.join(ips)), params)
 
     def info(self):
         """Returns information about the current API key, such as a list of add-ons
@@ -468,7 +476,7 @@ def scan_status(self, scan_id):
 
         :returns: A dictionary with general information about the scan, including its status in getting processed.
         """
-        return self._request('/shodan/scan/%s' % scan_id, {})
+        return self._request('/shodan/scan/{}'.format(scan_id), {})
 
     def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
         """Search the SHODAN database.
@@ -685,7 +693,7 @@ def edit_alert(self, aid, ip):
     def alerts(self, aid=None, include_expired=True):
         """List all of the active alerts that the user created."""
         if aid:
-            func = '/shodan/alert/%s/info' % aid
+            func = '/shodan/alert/{}/info'.format(aid)
         else:
             func = '/shodan/alert/info'
 
@@ -697,7 +705,7 @@ def alerts(self, aid=None, include_expired=True):
 
     def delete_alert(self, aid):
         """Delete the alert with the given ID."""
-        func = '/shodan/alert/%s' % aid
+        func = '/shodan/alert/{}'.format(aid)
 
         response = api_request(self.api_key, func, params={}, method='delete',
                                proxies=self._session.proxies)
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 2d976dc..378b1bb 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -142,7 +142,7 @@ def write_banner(fout, banner):
     fout.write(line.encode('utf-8'))
 
 
-def humanize_bytes(bytes, precision=1):
+def humanize_bytes(byte_count, precision=1):
     """Return a humanized string representation of a number of bytes.
     >>> humanize_bytes(1)
     '1 byte'
@@ -161,15 +161,15 @@ def humanize_bytes(bytes, precision=1):
     >>> humanize_bytes(1024*1234*1111,1)
     '1.3 GB'
     """
-    if bytes == 1:
+    if byte_count == 1:
         return '1 byte'
-    if bytes < 1024:
-        return '%.*f %s' % (precision, bytes, "bytes")
+    if byte_count < 1024:
+        '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes')
 
     suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
     multiple = 1024.0  # .0 to force float on python 2
     for suffix in suffixes:
-        bytes /= multiple
-        if bytes < multiple:
-            return '%.*f %s' % (precision, bytes, suffix)
-    return '%.*f %s' % (precision, bytes, suffix)
+        byte_count /= multiple
+        if byte_count < multiple:
+            return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix)
+    return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix)
diff --git a/shodan/stream.py b/shodan/stream.py
index 15d7619..92365dc 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -71,7 +71,7 @@ def _iter_stream(self, stream, raw):
 
     def alert(self, aid=None, timeout=None, raw=False):
         if aid:
-            stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout)
+            stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout)
         else:
             stream = self._create_stream('/shodan/alert', timeout=timeout)
 
@@ -90,7 +90,7 @@ def asn(self, asn, raw=False, timeout=None):
         :param asn: A list of ASN to return banner data on.
         :type asn: string[]
         """
-        stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout)
+        stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
 
@@ -109,7 +109,7 @@ def countries(self, countries, raw=False, timeout=None):
         :param countries: A list of countries to return banner data on.
         :type countries: string[]
         """
-        stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout)
+        stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
 
@@ -120,7 +120,7 @@ def ports(self, ports, raw=False, timeout=None):
         :param ports: A list of ports to return banner data on.
         :type ports: int[]
         """
-        stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout)
+        stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
 
@@ -131,7 +131,7 @@ def tags(self, tags, raw=False, timeout=None):
         :param tags: A list of tags to return banner data on.
         :type tags: string[]
         """
-        stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout)
+        stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line
 
@@ -142,6 +142,6 @@ def vulns(self, vulns, raw=False, timeout=None):
         :param vulns: A list of vulns to return banner data on.
         :type vulns: string[]
         """
-        stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout)
+        stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout)
         for line in self._iter_stream(stream, raw):
             yield line

From 0703499f21bd21f10b2c00d8fc470de901dc46ac Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 11 Oct 2020 17:24:39 -0500
Subject: [PATCH 227/263] Fix linting error

---
 shodan/cli/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py
index 294793c..05c1b9f 100644
--- a/shodan/cli/settings.py
+++ b/shodan/cli/settings.py
@@ -4,7 +4,7 @@
 if path.exists(path.expanduser("~/.shodan")):
     SHODAN_CONFIG_DIR = '~/.shodan/'
 else:
-    SHODAN_CONFIG_DIR="~/.config/shodan/"
+    SHODAN_CONFIG_DIR = "~/.config/shodan/"
 
 COLORIZE_FIELDS = {
     'ip_str': 'green',

From 322f169c8d0f916926e808ef6d6aa10b544fc9ac Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sun, 11 Oct 2020 17:50:10 -0500
Subject: [PATCH 228/263] Release 1.24.0 New command: shodan alert stats

---
 setup.py            |   2 +-
 shodan/cli/alert.py | 152 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b1735b2..9c6f55f 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.23.1',
+    version='1.24.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 0d81dc4..de77294 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -1,10 +1,73 @@
 import click
+import csv
 import shodan
 
+from collections import defaultdict
 from operator import itemgetter
 from shodan.cli.helpers import get_api_key
 
 
+MAX_QUERY_LENGTH = 1000
+
+
+def aggregate_facet(api, networks, facets):
+    """Merge the results from multiple facet API queries into a single result object.
+    This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit
+    into a single API call.
+    """
+    def _merge_custom_facets(lfacets, results):
+        for key in results['facets']:
+            if key not in lfacets:
+                lfacets[key] = defaultdict(int)
+
+            for item in results['facets'][key]:
+                lfacets[key][item['value']] += item['count']
+
+    # We're going to create a custom facets dict where
+    # the key is the value of a facet. Normally the facets
+    # object is a list where each item has a "value" and "count" property.
+    tmp_facets = {}
+    count = 0
+
+    query = 'net:'
+
+    for net in networks:
+        query += '{},'.format(net)
+
+        # Start running API queries if the query length is getting long
+        if len(query) > MAX_QUERY_LENGTH:
+            results = api.count(query[:-1], facets=facets)
+
+            _merge_custom_facets(tmp_facets, results)
+            count += results['total']
+            query = 'net:'
+
+    # Run any remaining search query
+    if query[-1] != ':':
+        results = api.count(query[:-1], facets=facets)
+
+        _merge_custom_facets(tmp_facets, results)
+        count += results['total']
+
+    # Convert the internal facets structure back to the one that
+    # the API returns.
+    new_facets = {}
+    for facet in tmp_facets:
+        sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True)
+        new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items]
+
+    # Make sure the facet keys exist even if there weren't any results
+    for facet, _ in facets:
+        if facet not in new_facets:
+            new_facets[facet] = []
+
+    return {
+        'matches': [],
+        'facets': new_facets,
+        'total': count,
+    }
+
+
 @click.group()
 def alert():
     """Manage the network alerts for your account"""
@@ -149,6 +212,95 @@ def alert_list(expired):
         click.echo("You haven't created any alerts yet.")
 
 
+@alert.command(name='stats')
+@click.option('--limit', help='The number of results to return.', default=10, type=int)
+@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None)
+@click.argument('facets', metavar='<facets ...>', nargs=-1)
+def alert_stats(limit, filename, facets):
+    """Show summary information about your monitored networks"""
+    # Setup Shodan
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    # Make sure the user didn't supply an empty string
+    if not facets:
+        raise click.ClickException('No facets provided')
+
+    facets = [(facet, limit) for facet in facets]
+
+    # Get the list of IPs/ networks that the user is monitoring
+    networks = set()
+    try:
+        alerts = api.alerts()
+        for alert in alerts:
+            for tmp in alert['filters']['ip']:
+                networks.add(tmp)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    # Grab the facets the user requested
+    try:
+        results = aggregate_facet(api, networks, facets)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared
+    # Print the stats tables
+    for facet in results['facets']:
+        click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet))
+
+        for item in results['facets'][facet]:
+            # Force the value to be a string - necessary because some facet values are numbers
+            value = u'{}'.format(item['value'])
+
+            click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False)
+            click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green'))
+
+        click.echo('')
+
+    # Create the output file if requested
+    fout = None
+    if filename:
+        if not filename.endswith('.csv'):
+            filename += '.csv'
+        fout = open(filename, 'w')
+        writer = csv.writer(fout, dialect=csv.excel)
+
+        # Write the header that contains the facets
+        row = []
+        for facet in results['facets']:
+            row.append(facet)
+            row.append('')
+        writer.writerow(row)
+
+        # Every facet has 2 columns (key, value)
+        counter = 0
+        has_items = True
+        while has_items:
+            # pylint: disable=W0612
+            row = ['' for i in range(len(results['facets']) * 2)]
+
+            pos = 0
+            has_items = False
+            for facet in results['facets']:
+                values = results['facets'][facet]
+
+                # Add the values for the facet into the current row
+                if len(values) > counter:
+                    has_items = True
+                    row[pos] = values[counter]['value']
+                    row[pos + 1] = values[counter]['count']
+
+                pos += 2
+
+            # Write out the row
+            if has_items:
+                writer.writerow(row)
+
+            # Move to the next row of values
+            counter += 1
+
+
 @alert.command(name='remove')
 @click.argument('alert_id', metavar='<alert ID>')
 def alert_remove(alert_id):

From 8ac631090100de6dbc53171d07918ac869a1eeb9 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 2 Nov 2020 13:46:09 -0600
Subject: [PATCH 229/263] Remove the --skip option when downloading as the API
 nolonger supports requesting arbitrary pages in the search results.

---
 shodan/__main__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index f8ed673..a98ddbd 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -256,7 +256,6 @@ def count(query):
 
 @main.command()
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
-@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int)
 @click.argument('filename', metavar='<filename>')
 @click.argument('query', metavar='<search query>', nargs=-1)
 def download(limit, skip, filename, query):

From 8195e1de58615303fe1d412d283865b1c024f0d8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 2 Nov 2020 13:46:28 -0600
Subject: [PATCH 230/263] Remove the skip parameter from the download method as
 well

---
 shodan/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index a98ddbd..d12b594 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -258,7 +258,7 @@ def count(query):
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
 @click.argument('filename', metavar='<filename>')
 @click.argument('query', metavar='<search query>', nargs=-1)
-def download(limit, skip, filename, query):
+def download(limit, filename, query):
     """Download search results and save them in a compressed JSON file."""
     key = get_api_key()
 

From 90491308971a7ba6e358d6152d7b02307964c6fe Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 2 Nov 2020 13:48:21 -0600
Subject: [PATCH 231/263] Remove the skip parameter from the search_cursor

---
 shodan/__main__.py |  6 +-----
 shodan/client.py   | 10 +---------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index d12b594..4395820 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -298,15 +298,11 @@ def download(limit, filename, query):
     # A limit of -1 means that we should download all the data
     if limit <= 0:
         limit = total
-    
-        # Adjust the total number of results we should expect to download if the user is skipping results
-        if skip > 0:
-            limit -= skip
 
     with helpers.open_file(filename, 'w') as fout:
         count = 0
         try:
-            cursor = api.search_cursor(query, minify=False, skip=skip)
+            cursor = api.search_cursor(query, minify=False)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
                     helpers.write_banner(fout, banner)
diff --git a/shodan/client.py b/shodan/client.py
index 4116051..eb66e7e 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -504,7 +504,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5, skip=0):
+    def search_cursor(self, query, minify=True, retries=5):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -518,8 +518,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0):
         :type minify: bool
         :param retries: (optional) How often to retry the search in case it times out
         :type retries: int
-        :param skip: (optional) Number of results to skip
-        :type skip: int
 
         :returns: A search cursor that can be used as an iterator/ generator.
         """
@@ -532,12 +530,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0):
             'total': None,
         }
 
-        # Convert the number of skipped records into a page number
-        if skip > 0:
-            # Each page returns 100 results so find the nearest page that we want
-            # the cursor to skip to.
-            page += int(skip / 100)
-
         while results['matches']:
             try:
                 results = self.search(query, minify=minify, page=page)

From 5eedf292d0cd4083c629a8c36acad43a63ffc472 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 25 Jan 2021 16:33:53 -0600
Subject: [PATCH 232/263] Add new CLI command: shodan alert download
 [--alert-id=] <filename>

---
 CHANGELOG.md        |  8 +++++
 setup.py            |  2 +-
 shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50a9334..2247045 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+1.25.0
+------
+* Add new CLI command: shodan alert download
+
+1.24.0
+------
+* Add new CLI command: shodan alert stats
+
 1.23.0
 ------
 * Add new CLI command: shodan alert domain
diff --git a/setup.py b/setup.py
index 9c6f55f..97616d6 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.24.0',
+    version='1.25.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index de77294..63b863d 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -4,7 +4,10 @@
 
 from collections import defaultdict
 from operator import itemgetter
+from shodan import APIError
 from shodan.cli.helpers import get_api_key
+from shodan.helpers import open_file, write_banner
+from time import sleep
 
 
 MAX_QUERY_LENGTH = 1000
@@ -138,6 +141,86 @@ def alert_domain(domain, triggers):
     click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 
+@alert.command(name='download')
+@click.argument('filename', metavar='<filename>', type=str)
+@click.option('--alert-id', help='Specific alert ID to download the data of', default=None)
+def alert_download(filename, alert_id):
+    """Download all information for monitored networks/ IPs."""
+    key = get_api_key()
+
+    api = shodan.Shodan(key)
+    ips = set()
+    networks = set()
+
+    # Helper method to process batches of IPs
+    def batch(iterable, size=1):
+        iter_length = len(iterable)
+        for ndx in range(0, iter_length, size):
+            yield iterable[ndx:min(ndx + size, iter_length)]
+
+    try:
+        # Get the list of alerts for the user
+        click.echo('Looking up alert information...')
+        if alert_id:
+            alerts = [api.alerts(aid=alert_id.strip())]
+        else:
+            alerts = api.alerts()
+        
+        click.echo('Compiling list of networks/ IPs to download...')
+        for alert in alerts:
+            for net in alert['filters']['ip']:
+                if '/' in net:
+                    networks.add(net)
+                else:
+                    ips.add(net)
+        
+        click.echo('Downloading...')
+        with open_file(filename) as fout:
+            # Check if the user is able to use batch IP lookups
+            batch_size = 1
+            if len(ips) > 0:
+                api_info = api.info()
+                if api_info['plan'] in ['corp', 'stream-100']:
+                    batch_size = 100
+            
+            # Convert it to a list so we can index into it
+            ips = list(ips)
+
+            # Grab all the IP information
+            for ip in batch(ips, size=batch_size):
+                try:
+                    click.echo(ip)
+                    results = api.host(ip)
+                    if not isinstance(results, list):
+                        results = [results]
+                    
+                    for host in results:
+                        for banner in host['data']:
+                            write_banner(fout, banner)
+                except APIError:
+                    pass
+                sleep(1)  # Slow down a bit to make sure we don't hit the rate limit
+            
+            # Grab all the network ranges
+            for net in networks:
+                try:
+                    counter = 0
+                    click.echo(net)
+                    for banner in api.search_cursor('net:{}'.format(net)):
+                        write_banner(fout, banner)
+                        
+                        # Slow down a bit to make sure we don't hit the rate limit
+                        if counter % 100 == 0:
+                            sleep(1)
+                        counter += 1
+                except APIError:
+                    pass
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+    
+    click.secho('Successfully downloaded results into: {}'.format(filename), fg='green')
+
+
 @alert.command(name='info')
 @click.argument('alert', metavar='<alert id>')
 def alert_info(alert):

From 75d4c1b37b8a553f6db73ffc28cc139febbf0d9c Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Mon, 26 Apr 2021 14:10:08 -0300
Subject: [PATCH 233/263] Adding timeout

---
 shodan/client.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index eb66e7e..397b37a 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -470,7 +470,7 @@ def scan_status(self, scan_id):
         """
         return self._request('/shodan/scan/%s' % scan_id, {})
 
-    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
+    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0):
         """Search the SHODAN database.
 
         :param query: Search query; identical syntax to the website
@@ -502,9 +502,12 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
         if facets:
             args['facets'] = create_facet_string(facets)
 
+        if timeout:
+            args['timeout'] = timeout
+
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5):
+    def search_cursor(self, query, minify=True, retries=5, timeout=0):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -532,7 +535,7 @@ def search_cursor(self, query, minify=True, retries=5):
 
         while results['matches']:
             try:
-                results = self.search(query, minify=minify, page=page)
+                results = self.search(query, minify=minify, page=page, timeout=timeout)
                 for banner in results['matches']:
                     try:
                         yield banner

From a5edaff408b8b8802c515294fdfdff8268fda957 Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Mon, 26 Apr 2021 14:31:15 -0300
Subject: [PATCH 234/263] None instead of zero

---
 shodan/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/client.py b/shodan/client.py
index 397b37a..369b039 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -470,7 +470,7 @@ def scan_status(self, scan_id):
         """
         return self._request('/shodan/scan/%s' % scan_id, {})
 
-    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0):
+    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None):
         """Search the SHODAN database.
 
         :param query: Search query; identical syntax to the website

From cdfdb865e64568fa77437174e4a27949e84bca1e Mon Sep 17 00:00:00 2001
From: Einar Felipe Lanfranco <einar-lanfranco@users.noreply.github.com>
Date: Tue, 18 May 2021 14:25:57 -0300
Subject: [PATCH 235/263] Some fixs to support big number of results

Replace all timeout=0 to timeout=None and add incremental sleep for retry
---
 shodan/client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index 369b039..10899c0 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -507,7 +507,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5, timeout=0):
+    def search_cursor(self, query, minify=True, retries=5, timeout=None):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -549,7 +549,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=0):
                     raise APIError('Retry limit reached ({:d})'.format(retries))
 
                 tries += 1
-                time.sleep(1.0)  # wait 1 second if the search errored out for some reason
+                time.sleep(tries)  # wait (1 second * retry number) if the search errored out for some reason
 
     def search_facets(self):
         """Returns a list of search facets that can be used to get aggregate information about a search query.

From 67a2585d92784b41210f8963ea93cce9d8656ae3 Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Wed, 19 May 2021 16:37:38 -0300
Subject: [PATCH 236/263] removing timeout, ther is no need for it now!

---
 shodan/client.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/shodan/client.py b/shodan/client.py
index 10899c0..02f115e 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -470,7 +470,7 @@ def scan_status(self, scan_id):
         """
         return self._request('/shodan/scan/%s' % scan_id, {})
 
-    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None):
+    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
         """Search the SHODAN database.
 
         :param query: Search query; identical syntax to the website
@@ -502,12 +502,9 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
         if facets:
             args['facets'] = create_facet_string(facets)
 
-        if timeout:
-            args['timeout'] = timeout
-
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5, timeout=None):
+    def search_cursor(self, query, minify=True, retries=5):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -535,7 +532,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=None):
 
         while results['matches']:
             try:
-                results = self.search(query, minify=minify, page=page, timeout=timeout)
+                results = self.search(query, minify=minify, page=page)
                 for banner in results['matches']:
                     try:
                         yield banner

From d1027d8e4cea56e6f00cd437eab7f2ddfde72c59 Mon Sep 17 00:00:00 2001
From: Hy Che <cvghy116@gmail.com>
Date: Tue, 28 Dec 2021 02:46:37 +0700
Subject: [PATCH 237/263] Add implementation for custom filter `shodan/custom`

Also bump version to 1.26
---
 setup.py           |  2 +-
 shodan/__main__.py | 39 +++++++++++++++++++++++----------------
 shodan/stream.py   | 19 +++++++++++++++++--
 3 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/setup.py b/setup.py
index 97616d6..6be33f3 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.25.0',
+    version='1.26.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 4395820..3700098 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -106,7 +106,7 @@ def convert(fields, input, format):
         if not hasattr(converter_class, 'fields'):
             raise click.ClickException('File format doesnt support custom list of fields')
         converter_class.fields = [item.strip() for item in fields.split(',')]  # Use the custom fields the user specified
-    
+
     # Get the basename for the input file
     basename = input.replace('.json.gz', '').replace('.json', '')
 
@@ -173,7 +173,7 @@ def domain_info(domain, details, save, history, type):
                             helpers.write_banner(fout, banner)
             except shodan.APIError:
                 pass  # Ignore any API lookup errors as this isn't critical information
-    
+
     # Save the DNS data
     if save:
         filename = u'{}.json.gz'.format(domain)
@@ -198,7 +198,7 @@ def domain_info(domain, details, save, history, type):
         if record['value'] in hosts:
             host = hosts[record['value']]
             click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False)
-        
+
         click.echo('')
 
 
@@ -448,7 +448,7 @@ def myip(ipv6):
     # Use the IPv6-enabled domain if requested
     if ipv6:
         api.base_url = 'https://apiv6.shodan.io'
-    
+
     try:
         click.echo(api.tools.myip())
     except shodan.APIError as e:
@@ -617,22 +617,23 @@ def stats(limit, facets, filename, query):
 
 
 @main.command()
-@click.option('--color/--no-color', default=True)
+@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
 @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data')
 @click.option('--separator', help='The separator between the properties of the search results.', default='\t')
-@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int)
 @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str)
-@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
-@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
-@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=0, type=int)
-@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str)
-@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
 @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str)
 @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str)
+@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str)
+@click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str)
+@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str)
 @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str)
-@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
 @click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str)
-def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns):
+@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int)
+@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int)
+@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after <timeout> seconds.', default=0, type=int)
+@click.option('--color/--no-color', default=True)
+@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True)
+def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet):
     """Stream data in real-time."""
     # Setup the Shodan API
     key = get_api_key()
@@ -662,9 +663,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
         stream_type.append('tags')
     if vulns:
         stream_type.append('vulns')
+    if custom_filters:
+        stream_type.append('custom_filters')
 
     if len(stream_type) > 1:
-        raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.')
+        raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.')
 
     stream_args = None
 
@@ -685,13 +688,16 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre
 
     if countries:
         stream_args = countries.split(',')
-    
+
     if tags:
         stream_args = tags.split(',')
-    
+
     if vulns:
         stream_args = vulns.split(',')
 
+    if custom_filters:
+        stream_args = custom_filters
+
     # Flatten the list of stream types
     # Possible values are:
     # - all
@@ -710,6 +716,7 @@ def _create_stream(name, args, timeout):
             'alert': api.stream.alert(args, timeout=timeout),
             'asn': api.stream.asn(args, timeout=timeout),
             'countries': api.stream.countries(args, timeout=timeout),
+            'custom_filters': api.stream.custom(args, timeout=timeout),
             'ports': api.stream.ports(args, timeout=timeout),
             'tags': api.stream.tags(args, timeout=timeout),
             'vulns': api.stream.vulns(args, timeout=timeout),
diff --git a/shodan/stream.py b/shodan/stream.py
index 15d7619..77ade7d 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -13,7 +13,7 @@ def __init__(self, api_key, proxies=None):
         self.api_key = api_key
         self.proxies = proxies
 
-    def _create_stream(self, name, timeout=None):
+    def _create_stream(self, name, query=None, timeout=None):
         params = {
             'key': self.api_key,
         }
@@ -27,9 +27,12 @@ def _create_stream(self, name, timeout=None):
         # If the user requested a timeout then we need to disable heartbeat messages
         # which are intended to keep stream connections alive even if there isn't any data
         # flowing through.
-        if timeout:
+        if timeout is not None:
             params['heartbeat'] = False
 
+        if query is not None:
+            params['query'] = query
+
         try:
             while True:
                 req = requests.get(stream_url, params=params, stream=True, timeout=timeout,
@@ -113,6 +116,18 @@ def countries(self, countries, raw=False, timeout=None):
         for line in self._iter_stream(stream, raw):
             yield line
 
+    def custom(self, query, raw=False, timeout=None):
+        """
+        A filtered version of the "banners" stream to only return banners that match the query of interest. The query
+        can vary and mix-match with different arguments (ports, tags, vulns, etc).
+
+        :param query: A space-separated list of key:value filters query to return banner data on.
+        :type query: string
+        """
+        stream = self._create_stream('/shodan/custom', query=query, timeout=timeout)
+        for line in self._iter_stream(stream, raw):
+            yield line
+
     def ports(self, ports, raw=False, timeout=None):
         """
         A filtered version of the "banners" stream to only return banners that match the ports of interest.

From 490fdb5e592a2be243856c21bf158a2eb01dfe48 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@surtri.com>
Date: Fri, 31 Dec 2021 16:33:06 -0600
Subject: [PATCH 238/263] Revert timeout parameter to previous behavior

---
 shodan/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index 77ade7d..eac345f 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -27,7 +27,7 @@ def _create_stream(self, name, query=None, timeout=None):
         # If the user requested a timeout then we need to disable heartbeat messages
         # which are intended to keep stream connections alive even if there isn't any data
         # flowing through.
-        if timeout is not None:
+        if timeout:
             params['heartbeat'] = False
 
         if query is not None:

From e603d462abee94f7e0a3c6780095f0f4a881af70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ch=E1=BA=BF=20V=C5=A9=20Gia=20Hy?= <cvghy116@gmail.com>
Date: Thu, 6 Jan 2022 21:53:12 +0700
Subject: [PATCH 239/263] stream: Remove set `decode_unicode`

This doesn't handle well with every unicode strings, let only json.loads() do the decode instead.
---
 shodan/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/stream.py b/shodan/stream.py
index eac345f..a47d143 100644
--- a/shodan/stream.py
+++ b/shodan/stream.py
@@ -63,7 +63,7 @@ def _create_stream(self, name, query=None, timeout=None):
         return req
 
     def _iter_stream(self, stream, raw):
-        for line in stream.iter_lines(decode_unicode=True):
+        for line in stream.iter_lines():
             # The Streaming API sends out heartbeat messages that are newlines
             # We want to ignore those messages since they don't contain any data
             if line:

From 95c6b910f7a46c3efecdab1d65085190d6404e89 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 19 Jan 2022 09:47:24 -0600
Subject: [PATCH 240/263] Release 1.26.1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index d7751d7..97c47b4 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.26.0',
+    version='1.26.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',

From 99756e2b2bd64603013be3f11caa6d406250f2a1 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 22 Feb 2022 13:53:59 -0600
Subject: [PATCH 241/263] New commands: "shodan alert export" and "shodan alert
 import" to help backup/ restore network monitoring configurations

---
 shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 63b863d..2dc3e58 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -1,5 +1,7 @@
 import click
 import csv
+import gzip
+import json
 import shodan
 
 from collections import defaultdict
@@ -221,6 +223,87 @@ def batch(iterable, size=1):
     click.secho('Successfully downloaded results into: {}'.format(filename), fg='green')
 
 
+@alert.command(name='export')
+@click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str)
+def alert_export(filename):
+    """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``."""
+    # Setup the API wrapper
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    try:
+        # Get the list of alerts for the user
+        click.echo('Looking up alert information...')
+        alerts = api.alerts()
+
+        # Create the output file
+        click.echo('Writing alerts to file: {}'.format(filename))
+        with gzip.open(filename, 'wt', encoding='utf-8') as fout:
+            json.dump(alerts, fout)
+    except Exception as e:
+        raise click.ClickException(e.value)
+    
+    click.secho('Successfully exported monitored networks', fg='green')
+
+
+@alert.command(name='import')
+@click.argument('filename', metavar='<export file>')
+def alert_import(filename):
+    """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``."""
+    # Setup the API wrapper
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    # A mapping of the old notifier IDs to the new ones
+    notifier_map = {}
+
+    try:
+        # Loading the alerts
+        click.echo('Loading alerts from: {}'.format(filename))
+        with gzip.open(filename, 'rt', encoding='utf-8') as fin:
+            alerts = json.load(fin)
+
+            for item in alerts:
+                # Create the alert
+                click.echo('Creating: {}'.format(item['name']))
+                alert = api.create_alert(item['name'], item['filters']['ip'])
+
+                # Enable any triggers
+                if item.get('triggers', {}):
+                    triggers = ','.join(item['triggers'].keys())
+
+                    api.enable_alert_trigger(alert['id'], triggers)
+
+                    # Add any whitelisted services for this trigger
+                    for trigger, info in item['triggers'].items():
+                        if info.get('ignore', []):
+                            for whitelist in info['ignore']:
+                                api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port'])
+                
+                # Enable the notifiers
+                for prev_notifier in item.get('notifiers', []):
+                    # We don't need to do anything for the default notifier as that
+                    # uses the account's email address automatically.
+                    if prev_notifier['id'] == 'default':
+                        continue
+
+                    # Get the new notifier based on the ID of the old one
+                    notifier = notifier_map.get(prev_notifier['id'])
+                    
+                    # Create the notifier if it doesn't yet exist
+                    if notifier is None:
+                        notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description'])
+
+                        # Add it to our map of old notifier IDs to new notifiers
+                        notifier_map[prev_notifier['id']] = notifier
+
+                    api.add_alert_notifier(alert['id'], notifier['id'])
+    except Exception as e:
+        raise click.ClickException(e.value)
+    
+    click.secho('Successfully imported monitored networks', fg='green')
+
+
 @alert.command(name='info')
 @click.argument('alert', metavar='<alert id>')
 def alert_info(alert):

From b32a7bc3f45f62c7ce962cea5954dd8b82fa6824 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Tue, 22 Feb 2022 17:01:05 -0600
Subject: [PATCH 242/263] Release 1.27.0

---
 CHANGELOG.md | 14 ++++++++++++++
 setup.py     |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2247045..2cc021c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,20 @@
 CHANGELOG
 =========
 
+1.27.0
+------
+* New command: ``shodan alert export`` to save the current network monitoring configuration
+* New command: ``shodan alert import`` to restore a previous network monitoring configuration
+* Automatically rate limit API requests to 1 request per second (credit to @malvidin)
+
+1.26.1
+------
+* Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON
+
+1.26.0
+------
+* Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters <query>``)
+
 1.25.0
 ------
 * Add new CLI command: shodan alert download
diff --git a/setup.py b/setup.py
index 97c47b4..942c54f 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.26.1',
+    version='1.27.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',

From 53662377e23c85926feada57b103e1a21d9ae2ee Mon Sep 17 00:00:00 2001
From: yaron-cider <100203813+yaron-cider@users.noreply.github.com>
Date: Thu, 19 May 2022 12:39:16 +0300
Subject: [PATCH 243/263] Better error handling

Raising 502 bad gateway error when receiving http status
---
 shodan/client.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shodan/client.py b/shodan/client.py
index db50b4c..f4aebc0 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -336,6 +336,8 @@ def _request(self, function, params, service='shodan', method='get'):
             raise APIError(error)
         elif data.status_code == 403:
             raise APIError('Access denied (403 Forbidden)')
+        elif data.status_code == 502:
+            raise APIError('Bad Gateway (502)')
 
         # Parse the text into JSON
         try:

From de2fd90aa9ffc8b53a6cd3f175fea50e551af0b1 Mon Sep 17 00:00:00 2001
From: Li-Heng Yu <007seadog@gmail.com>
Date: Mon, 23 May 2022 14:14:40 +0800
Subject: [PATCH 244/263] Show scan id when scanning without showing results

---
 shodan/cli/scan.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py
index 7ec158c..cfc7aab 100644
--- a/shodan/cli/scan.py
+++ b/shodan/cli/scan.py
@@ -157,6 +157,7 @@ def scan_submit(wait, filename, force, verbose, netblocks):
 
         # Return immediately
         if wait <= 0:
+            click.echo('Scan ID: {}'.format(scan['id']))
             click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.')
         else:
             # Setup an alert to wait for responses

From 7d043d74735cfaf0b0b5dc8fbc81922ba117dfea Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 9 Jul 2022 17:52:51 -0500
Subject: [PATCH 245/263] Add the ability to whitelist a specific vulnerability
 in Shodan Monitor instead of whitelisting the while IP:port

---
 CHANGELOG.md     | 7 +++++++
 setup.py         | 2 +-
 shodan/client.py | 8 +++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2cc021c..8d9f9aa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 CHANGELOG
 =========
 
+1.28.0
+------
+* Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port
+* Show scan ID when scanning without showing results (credit to @seadog007)
+* Handle bad gateway errors (credit to @yaron-cider)
+
+
 1.27.0
 ------
 * New command: ``shodan alert export`` to save the current network monitoring configuration
diff --git a/setup.py b/setup.py
index 942c54f..266ce81 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.27.0',
+    version='1.28.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index f4aebc0..70ca8f3 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -721,8 +721,14 @@ def disable_alert_trigger(self, aid, trigger):
         """Disable the given trigger on the alert."""
         return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete')
 
-    def ignore_alert_trigger_notification(self, aid, trigger, ip, port):
+    def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None):
         """Ignore trigger notifications for the provided IP and port."""
+        # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities
+        # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use
+        # a different API endpoint.
+        if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list):
+            return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put')
+        
         return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put')
 
     def unignore_alert_trigger_notification(self, aid, trigger, ip, port):

From 55d8d59bc3c82e3f35d1afbe85fe86b5c0cc75ca Mon Sep 17 00:00:00 2001
From: David xu <xqdoo00o@gmail.com>
Date: Sun, 17 Jul 2022 22:12:35 +0800
Subject: [PATCH 246/263] add missing return

---
 shodan/helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/helpers.py b/shodan/helpers.py
index 378b1bb..8900468 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -164,7 +164,7 @@ def humanize_bytes(byte_count, precision=1):
     if byte_count == 1:
         return '1 byte'
     if byte_count < 1024:
-        '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes')
+        return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes')
 
     suffixes = ['KB', 'MB', 'GB', 'TB', 'PB']
     multiple = 1024.0  # .0 to force float on python 2

From 4419d7167fad366b8942643037c427a62b0cb5bb Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Mon, 15 May 2023 09:50:54 -0700
Subject: [PATCH 247/263] Add support for the new 'fields' parameter of the
 /shodan/host/search method so we only grab the specific properties/ fields
 from the banner.

---
 setup.py           |  2 +-
 shodan/__main__.py | 11 ++++++++---
 shodan/client.py   | 11 ++++++++---
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/setup.py b/setup.py
index 266ce81..e769440 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.28.0',
+    version='1.29.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/__main__.py b/shodan/__main__.py
index 07af59c..f11a72a 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -255,10 +255,11 @@ def count(query):
 
 
 @main.command()
+@click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str)
 @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int)
 @click.argument('filename', metavar='<filename>')
 @click.argument('query', metavar='<search query>', nargs=-1)
-def download(limit, filename, query):
+def download(fields, limit, filename, query):
     """Download search results and save them in a compressed JSON file."""
     key = get_api_key()
 
@@ -276,6 +277,10 @@ def download(limit, filename, query):
     # Add the appropriate extension if it's not there atm
     if not filename.endswith('.json.gz'):
         filename += '.json.gz'
+    
+    # Strip out any whitespace in the fields and turn them into an array
+    if fields is not None:
+        fields = [item.strip() for item in fields.split(',')]
 
     # Perform the search
     api = shodan.Shodan(key)
@@ -302,7 +307,7 @@ def download(limit, filename, query):
     with helpers.open_file(filename, 'w') as fout:
         count = 0
         try:
-            cursor = api.search_cursor(query, minify=False)
+            cursor = api.search_cursor(query, minify=False, fields=fields)
             with click.progressbar(cursor, length=limit) as bar:
                 for banner in bar:
                     helpers.write_banner(fout, banner)
@@ -485,7 +490,7 @@ def search(color, fields, limit, separator, query):
     # Perform the search
     api = shodan.Shodan(key)
     try:
-        results = api.search(query, limit=limit)
+        results = api.search(query, limit=limit, minify=False, fields=fields)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
diff --git a/shodan/client.py b/shodan/client.py
index 70ca8f3..b9cb487 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -480,7 +480,7 @@ def scan_status(self, scan_id):
         """
         return self._request('/shodan/scan/{}'.format(scan_id), {})
 
-    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True):
+    def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None):
         """Search the SHODAN database.
 
         :param query: Search query; identical syntax to the website
@@ -495,6 +495,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
         :type facets: str
         :param minify: (optional) Whether to minify the banner and only return the important data
         :type minify: bool
+        :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter
+        :type fields: str
 
         :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information.
         """
@@ -511,10 +513,13 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         if facets:
             args['facets'] = create_facet_string(facets)
+        
+        if fields and isinstance(fields, list):
+            args['fields'] = ','.join(fields)
 
         return self._request('/shodan/host/search', args)
 
-    def search_cursor(self, query, minify=True, retries=5):
+    def search_cursor(self, query, minify=True, retries=5, fields=None):
         """Search the SHODAN database.
 
         This method returns an iterator that can directly be in a loop. Use it when you want to loop over
@@ -542,7 +547,7 @@ def search_cursor(self, query, minify=True, retries=5):
 
         while results['matches']:
             try:
-                results = self.search(query, minify=minify, page=page)
+                results = self.search(query, minify=minify, page=page, fields=fields)
                 for banner in results['matches']:
                     try:
                         yield banner

From 9ccc16ada4761d19ee2079a9334cac3ddbc62415 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 17 May 2023 11:00:46 -0700
Subject: [PATCH 248/263] The screenshot data has been moved to the top-level
 "screenshot" property. Update the helpers.get_screenshot() method to look in
 that location before falling back to the old opts.screenshot property.

---
 setup.py          | 2 +-
 shodan/helpers.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index e769440..3adcd8b 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.29.0',
+    version='1.29.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/helpers.py b/shodan/helpers.py
index 378b1bb..432cbeb 100644
--- a/shodan/helpers.py
+++ b/shodan/helpers.py
@@ -122,8 +122,11 @@ def iterate_files(files, fast=False):
 
 
 def get_screenshot(banner):
-    if 'opts' in banner and 'screenshot' in banner['opts']:
+    if 'screenshot' in banner and banner['screenshot']:
+        return banner['screenshot']
+    elif 'opts' in banner and 'screenshot' in banner['opts']:
         return banner['opts']['screenshot']
+    
     return None
 
 

From c97ebae69d3343da455bc5aa277c54f7cf3e115a Mon Sep 17 00:00:00 2001
From: Richard Howe <rhowe425@gmail.com>
Date: Sun, 18 Jun 2023 21:08:39 -0400
Subject: [PATCH 249/263] Added a check for file size of input, dynamically set
 workbook.use_zip64()

---
 shodan/__main__.py              | 7 ++++++-
 shodan/cli/converter/csvc.py    | 2 +-
 shodan/cli/converter/excel.py   | 6 +++++-
 shodan/cli/converter/geojson.py | 2 +-
 shodan/cli/converter/images.py  | 2 +-
 shodan/cli/converter/kml.py     | 2 +-
 6 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index f11a72a..aadb2d5 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -100,6 +100,7 @@ def convert(fields, input, format):
 
     Example: shodan convert data.json.gz kml
     """
+    file_size = 0
     # Check that the converter allows a custom list of fields
     converter_class = CONVERTERS.get(format)
     if fields:
@@ -107,6 +108,10 @@ def convert(fields, input, format):
             raise click.ClickException('File format doesnt support custom list of fields')
         converter_class.fields = [item.strip() for item in fields.split(',')]  # Use the custom fields the user specified
 
+    # Check file size of input
+    if os.path.exists(input):
+        file_size = os.path.getsize(input)
+
     # Get the basename for the input file
     basename = input.replace('.json.gz', '').replace('.json', '')
 
@@ -124,7 +129,7 @@ def convert(fields, input, format):
     # Initialize the file converter
     converter = converter_class(fout)
 
-    converter.process([input])
+    converter.process([input], file_size)
 
     finished_event.set()
     progress_bar_thread.join()
diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py
index 20c3d03..2e4e2f2 100644
--- a/shodan/cli/converter/csvc.py
+++ b/shodan/cli/converter/csvc.py
@@ -50,7 +50,7 @@ class CsvConverter(Converter):
         'title',
     ]
 
-    def process(self, files):
+    def process(self, files, file_size):
         writer = csv_writer(self.fout, dialect=excel, lineterminator='\n')
 
         # Write the header
diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index 24177eb..4db78a4 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -41,7 +41,7 @@ class ExcelConverter(Converter):
         'http.title': 'Website Title',
     }
 
-    def process(self, files):
+    def process(self, files, file_size):
         # Get the filename from the already-open file handle
         filename = self.fout.name
 
@@ -51,6 +51,10 @@ def process(self, files):
         # Create the new workbook
         workbook = Workbook(filename)
 
+        # Check if Excel file is larger than 5GB
+        if file_size > 5e9:
+            workbook.use_zip64()
+
         # Define some common styles/ formats
         bold = workbook.add_format({
             'bold': 1,
diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py
index 3f6c975..83fb935 100644
--- a/shodan/cli/converter/geojson.py
+++ b/shodan/cli/converter/geojson.py
@@ -14,7 +14,7 @@ def header(self):
     def footer(self):
         self.fout.write("""{ }]}""")
 
-    def process(self, files):
+    def process(self, files, file_size):
         # Write the header
         self.header()
 
diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py
index 24c68c3..fba9d11 100644
--- a/shodan/cli/converter/images.py
+++ b/shodan/cli/converter/images.py
@@ -15,7 +15,7 @@ class ImagesConverter(Converter):
     # the user know where the images have been stored.
     dirname = None
 
-    def process(self, files):
+    def process(self, files, file_size):
         # Get the filename from the already-open file handle and use it as
         # the directory name to store the images.
         self.dirname = self.fout.name[:-7] + '-images'
diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py
index 2cf3d44..9259ddf 100644
--- a/shodan/cli/converter/kml.py
+++ b/shodan/cli/converter/kml.py
@@ -13,7 +13,7 @@ def header(self):
     def footer(self):
         self.fout.write("""</Document></kml>""")
 
-    def process(self, files):
+    def process(self, files, file_size):
         # Write the header
         self.header()
 

From c3d6d63d4743fd71642b084ff8b2da56f37717eb Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Fri, 23 Jun 2023 16:09:18 +0700
Subject: [PATCH 250/263] Add Shodan Trends API/ CLI

---
 shodan/__main__.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++
 shodan/client.py   | 57 ++++++++++++++++++++++++++-----
 2 files changed, 131 insertions(+), 9 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index f11a72a..3a2fdf8 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -22,6 +22,7 @@
     search
     stats
     stream
+    trends
 
 """
 
@@ -35,6 +36,7 @@
 import threading
 import requests
 import time
+import json
 
 # The file converters that are used to go from .json.gz to various other formats
 from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter
@@ -799,6 +801,87 @@ def _create_stream(name, args, timeout):
             stream = _create_stream(stream_type, stream_args, timeout=timeout)
 
 
+@main.command()
+@click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None)
+@click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True)
+@click.option('--separator', help='The separator between the properties of the search results.', default='\t')
+@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str)
+@click.argument('query', metavar='<search query>', nargs=-1)
+def trends(filename, save, separator, facets, query):
+    """Search Shodan historical database"""
+    key = get_api_key()
+    api = shodan.Shodan(key)
+
+    # Create the query string out of the provided tuple
+    query = ' '.join(query).strip()
+    facets = facets.strip()
+
+    # Make sure the user didn't supply an empty query or facets
+    if query == '':
+        raise click.ClickException('Empty search query')
+
+    if facets == '':
+        raise click.ClickException('Empty search facets')
+
+    # Convert comma-separated facets string to list
+    parsed_facets = []
+    for facet in facets.split(','):
+        parts = facet.strip().split(":")
+        if len(parts) > 1:
+            parsed_facets.append((parts[0], parts[1]))
+        else:
+            parsed_facets.append((parts[0]))
+
+    # Perform the search
+    try:
+        results = api.trends.search(query, facets=parsed_facets)
+    except shodan.APIError as e:
+        raise click.ClickException(e.value)
+
+    # Error out if no results were found
+    if results['total'] == 0:
+        raise click.ClickException('No search results found')
+
+    result_facets = list(results['facets'].keys())
+
+    # Save the results first to file if user request
+    if filename or save:
+        if not filename:
+            filename = '{}-trends.json.gz'.format(query.replace(' ', '-'))
+        elif not filename.endswith('.json.gz'):
+            filename += '.json.gz'
+
+        # Create/ append to the file
+        with helpers.open_file(filename) as fout:
+            for index, match in enumerate(results['matches']):
+                # Append facet info to make up a line
+                match["facets"] = {}
+                for facet in result_facets:
+                    match["facets"][facet] = results['facets'][facet][index]['values']
+                    line = json.dumps(match) + '\n'
+                    fout.write(line.encode('utf-8'))
+
+        click.echo(click.style(u'Saved results into file {}'.format(filename), 'green'))
+
+    # We buffer the entire output so we can use click's pager functionality
+    output = u''
+
+    # Output example:
+    # 2017-06
+    #   os
+    #     Linux 3.x   384148
+    #     Windows 7 or 8      25531
+    for index, match in enumerate(results['matches']):
+        output += click.style(match['month'] + u'\n', fg='green')
+
+        for facet in result_facets:
+            output += '  ' + facet + u'\n'
+            for bucket in results['facets'][facet][index]['values']:
+                output += '    ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n'
+
+    click.echo_via_pager(output)
+
+
 @main.command()
 @click.argument('ip', metavar='<IP address>')
 def honeyscore(ip):
diff --git a/shodan/client.py b/shodan/client.py
index b9cb487..7b76fdc 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -85,7 +85,7 @@ class Notifier:
 
         def __init__(self, parent):
             self.parent = parent
-        
+
         def create(self, provider, args, description=None):
             """Get the settings for the specified notifier that a user has configured.
 
@@ -101,9 +101,9 @@ def create(self, provider, args, description=None):
 
             if description:
                 args['description'] = description
-            
+
             return self.parent._request('/notifier', args, method='post')
-        
+
         def edit(self, nid, args):
             """Get the settings for the specified notifier that a user has configured.
 
@@ -114,7 +114,7 @@ def edit(self, nid, args):
             :returns: dict -- fields are 'success' and 'id' of the notifier
             """
             return self.parent._request('/notifier/{}'.format(nid), args, method='put')
-        
+
         def get(self, nid):
             """Get the settings for the specified notifier that a user has configured.
 
@@ -137,7 +137,7 @@ def list_providers(self):
             :returns: A list of providers where each object describes a provider
             """
             return self.parent._request('/notifier/provider', {})
-        
+
         def remove(self, nid):
             """Delete the provided notifier.
 
@@ -253,6 +253,42 @@ def remove_member(self, user):
             """
             return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success']
 
+    class Trends:
+
+        def __init__(self, parent):
+            self.parent = parent
+
+        def search(self, query, facets):
+            """Search the Shodan historical database.
+
+            :param query: Search query; identical syntax to the website
+            :type query: str
+            :param facets: (optional) A list of properties to get summary information on
+            :type facets: str
+
+            :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information.
+            """
+            args = {
+                'query': query,
+                'facets': create_facet_string(facets),
+            }
+
+            return self.parent._request('/api/v1/search', args, service='trends')
+
+        def search_facets(self):
+            """This method returns a list of facets that can be used to get a breakdown of the top values for a property.
+
+            :returns: A list of strings where each is a facet name
+            """
+            return self.parent._request('/api/v1/search/facets', {}, service='trends')
+
+        def search_filters(self):
+            """This method returns a list of search filters that can be used in the search query.
+
+            :returns: A list of strings where each is a filter name
+            """
+            return self.parent._request('/api/v1/search/filters', {}, service='trends')
+
     def __init__(self, key, proxies=None):
         """Initializes the API object.
 
@@ -264,9 +300,11 @@ def __init__(self, key, proxies=None):
         self.api_key = key
         self.base_url = 'https://api.shodan.io'
         self.base_exploits_url = 'https://exploits.shodan.io'
+        self.base_trends_url = 'https://trends.shodan.io'
         self.data = self.Data(self)
         self.dns = self.Dns(self)
         self.exploits = self.Exploits(self)
+        self.trends = self.Trends(self)
         self.labs = self.Labs(self)
         self.notifier = self.Notifier(self)
         self.org = self.Organization(self)
@@ -297,6 +335,7 @@ def _request(self, function, params, service='shodan', method='get'):
         base_url = {
             'shodan': self.base_url,
             'exploits': self.base_exploits_url,
+            'trends': self.base_trends_url,
         }.get(service, 'shodan')
 
         # Wait for API rate limit
@@ -513,7 +552,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru
 
         if facets:
             args['facets'] = create_facet_string(facets)
-        
+
         if fields and isinstance(fields, list):
             args['fields'] = ','.join(fields)
 
@@ -733,17 +772,17 @@ def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None):
         # a different API endpoint.
         if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list):
             return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put')
-        
+
         return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put')
 
     def unignore_alert_trigger_notification(self, aid, trigger, ip, port):
         """Re-enable trigger notifications for the provided IP and port"""
         return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete')
-    
+
     def add_alert_notifier(self, aid, nid):
         """Enable the given notifier for an alert that has triggers enabled."""
         return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put')
-    
+
     def remove_alert_notifier(self, aid, nid):
         """Remove the given notifier for an alert that has triggers enabled."""
         return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete')

From 9cc33f5700af88067391f3bd64fbb65001a666ee Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Fri, 23 Jun 2023 16:10:47 +0700
Subject: [PATCH 251/263] Add API tests

---
 tests/test_shodan.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/tests/test_shodan.py b/tests/test_shodan.py
index f3405ce..ebe7a90 100644
--- a/tests/test_shodan.py
+++ b/tests/test_shodan.py
@@ -22,7 +22,8 @@ class ShodanTests(unittest.TestCase):
     }
 
     def setUp(self):
-        self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip())
+        with open('SHODAN-API-KEY') as f:
+            self.api = shodan.Shodan(f.read().strip())
 
     def test_search_simple(self):
         results = self.api.search(self.QUERIES['simple'])
@@ -115,6 +116,24 @@ def test_exploits_count_facets(self):
         self.assertTrue(results['facets']['source'])
         self.assertTrue(len(results['facets']['author']) == 1)
 
+    def test_trends_search(self):
+        results = self.api.trends.search('apache', facets=[('product', 10)])
+        self.assertIn('matches', results)
+        self.assertIn('facets', results)
+        self.assertIn('total', results)
+        self.assertTrue(results['matches'])
+        self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']])
+
+    def test_trends_search_filters(self):
+        results = self.api.trends.search_filters()
+        self.assertIn('has_ipv6', results)
+        self.assertNotIn('http.html', results)
+
+    def test_trends_search_facets(self):
+        results = self.api.trends.search_facets()
+        self.assertIn('product', results)
+        self.assertNotIn('cpe', results)
+
     # Test error responses
     def test_invalid_key(self):
         api = shodan.Shodan('garbage')

From 3fe0ad88b434fe480b7621f8fac20216c3578fea Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Fri, 23 Jun 2023 16:11:34 +0700
Subject: [PATCH 252/263] Bump version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3adcd8b..2546ea7 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.29.1',
+    version='1.30.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',

From 0ad61d20d4cf676269c798e573ded4439603b218 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@surtri.com>
Date: Sun, 25 Jun 2023 16:46:00 -0700
Subject: [PATCH 253/263] Change cutoff to 4GB as that's what the xlsxwriter
 documentation says

---
 shodan/cli/converter/excel.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py
index 4db78a4..2021a33 100644
--- a/shodan/cli/converter/excel.py
+++ b/shodan/cli/converter/excel.py
@@ -51,8 +51,8 @@ def process(self, files, file_size):
         # Create the new workbook
         workbook = Workbook(filename)
 
-        # Check if Excel file is larger than 5GB
-        if file_size > 5e9:
+        # Check if Excel file is larger than 4GB
+        if file_size > 4e9:
             workbook.use_zip64()
 
         # Define some common styles/ formats

From 3d1f8922cd42420eba459a5d8bd41d974ad55ec1 Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Mon, 3 Jul 2023 10:57:56 +0700
Subject: [PATCH 254/263] Better output format

---
 shodan/__main__.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 3a2fdf8..439752d 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -279,7 +279,7 @@ def download(fields, limit, filename, query):
     # Add the appropriate extension if it's not there atm
     if not filename.endswith('.json.gz'):
         filename += '.json.gz'
-    
+
     # Strip out any whitespace in the fields and turn them into an array
     if fields is not None:
         fields = [item.strip() for item in fields.split(',')]
@@ -804,10 +804,9 @@ def _create_stream(name, args, timeout):
 @main.command()
 @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None)
 @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True)
-@click.option('--separator', help='The separator between the properties of the search results.', default='\t')
 @click.option('--facets', help='List of facets to get summary information on.', required=True, type=str)
 @click.argument('query', metavar='<search query>', nargs=-1)
-def trends(filename, save, separator, facets, query):
+def trends(filename, save, facets, query):
     """Search Shodan historical database"""
     key = get_api_key()
     api = shodan.Shodan(key)
@@ -869,15 +868,17 @@ def trends(filename, save, separator, facets, query):
     # Output example:
     # 2017-06
     #   os
-    #     Linux 3.x   384148
-    #     Windows 7 or 8      25531
+    #     Linux 3.x                                                        146,502
+    #     Windows 7 or 8                                                     2,189
     for index, match in enumerate(results['matches']):
         output += click.style(match['month'] + u'\n', fg='green')
-
-        for facet in result_facets:
-            output += '  ' + facet + u'\n'
-            for bucket in results['facets'][facet][index]['values']:
-                output += '    ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n'
+        if match['count'] > 0:
+            for facet in result_facets:
+                output += click.style(u'  {}\n'.format(facet), fg='cyan')
+                for bucket in results['facets'][facet][index]['values']:
+                    output += u'    {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green'))
+        else:
+            output += u'{}\n'.format(click.style('N/A', bold=True))
 
     click.echo_via_pager(output)
 

From 9e0f4dddbc5c8737270b70c119674abe618f44b5 Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Fri, 7 Jul 2023 08:34:16 +0700
Subject: [PATCH 255/263] Make facets as required arguments

---
 shodan/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 3a81b0d..1c4a74d 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -809,7 +809,7 @@ def _create_stream(name, args, timeout):
 @main.command()
 @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None)
 @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True)
-@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str)
+@click.argument('facets', metavar='<facet>')
 @click.argument('query', metavar='<search query>', nargs=-1)
 def trends(filename, save, facets, query):
     """Search Shodan historical database"""

From a9d692b05aeea978632d17601cfec997bc8995cd Mon Sep 17 00:00:00 2001
From: Richard Howe <rhowe425@gmail.com>
Date: Tue, 11 Jul 2023 23:07:34 -0400
Subject: [PATCH 256/263] Updated implementation to monitor hostnames for
 domain-based monitoring.

---
 requirements.txt    | 3 ++-
 shodan/cli/alert.py | 9 ++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 5095f64..2692414 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ click-plugins
 colorama
 requests>=2.2.1
 XlsxWriter
-ipaddress;python_version<='2.7'
\ No newline at end of file
+ipaddress;python_version<='2.7'
+tldextract
\ No newline at end of file
diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 2dc3e58..0030589 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -3,6 +3,7 @@
 import gzip
 import json
 import shodan
+from tldextract import extract
 
 from collections import defaultdict
 from operator import itemgetter
@@ -125,9 +126,15 @@ def alert_domain(domain, triggers):
     try:
         # Grab a list of IPs for the domain
         domain = domain.lower()
+        domain_parse = extract(domain)
         click.secho('Looking up domain information...', dim=True)
         info = api.dns.domain_info(domain, type='A')
-        domain_ips = set([record['value'] for record in info['data']])
+
+        if domain_parse.subdomain:
+            domain_ips = set([record['value'] for record in info['data']
+                              if record['subdomain'] == domain_parse.subdomain])
+        else:
+            domain_ips = set([record['value'] for record in info['data']])
 
         # Create the actual alert
         click.secho('Creating alert...', dim=True)

From c90b3dd5ade683e16c7681c444790f448150a401 Mon Sep 17 00:00:00 2001
From: Richard Howe <rhowe425@gmail.com>
Date: Wed, 12 Jul 2023 22:35:36 -0400
Subject: [PATCH 257/263] Added input validation by updating click.argument for
 input parameter.

---
 shodan/__main__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 1c4a74d..8b4eb2b 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -93,7 +93,7 @@ def main():
 
 @main.command()
 @click.option('--fields', help='List of properties to output.', default=None)
-@click.argument('input', metavar='<input file>')
+@click.argument('input', metavar='<input file>', type=click.Path(exists=True))
 @click.argument('format', metavar='<output format>', type=click.Choice(CONVERTERS.keys()))
 def convert(fields, input, format):
     """Convert the given input data file into a different format. The following file formats are supported:
@@ -110,9 +110,8 @@ def convert(fields, input, format):
             raise click.ClickException('File format doesnt support custom list of fields')
         converter_class.fields = [item.strip() for item in fields.split(',')]  # Use the custom fields the user specified
 
-    # Check file size of input
-    if os.path.exists(input):
-        file_size = os.path.getsize(input)
+    # click.Path ensures that file path exists
+    file_size = os.path.getsize(input)
 
     # Get the basename for the input file
     basename = input.replace('.json.gz', '').replace('.json', '')

From 1b7cb65e7e3d4b0490aa1009d5eaac72257a7e60 Mon Sep 17 00:00:00 2001
From: Richard Howe <rhowe425@gmail.com>
Date: Thu, 13 Jul 2023 20:07:17 -0400
Subject: [PATCH 258/263] Updated CLI logic to filter out private IPs when
 creating a domain-based alert.

---
 shodan/cli/alert.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 0030589..6f82c1a 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -4,6 +4,7 @@
 import json
 import shodan
 from tldextract import extract
+from ipaddress import ip_address
 
 from collections import defaultdict
 from operator import itemgetter
@@ -120,6 +121,7 @@ def alert_create(name, netblocks):
 @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable')
 def alert_domain(domain, triggers):
     """Create a network alert based on a domain name"""
+    flag = True
     key = get_api_key()
 
     api = shodan.Shodan(key)
@@ -132,22 +134,30 @@ def alert_domain(domain, triggers):
 
         if domain_parse.subdomain:
             domain_ips = set([record['value'] for record in info['data']
-                              if record['subdomain'] == domain_parse.subdomain])
+                              if record['subdomain'] == domain_parse.subdomain and
+                              not ip_address(record['value']).is_private])
         else:
-            domain_ips = set([record['value'] for record in info['data']])
+            domain_ips = set([record['value'] for record in info['data']
+                              if not ip_address(record['value']).is_private])
 
-        # Create the actual alert
-        click.secho('Creating alert...', dim=True)
-        alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips))
+        if not domain_ips:
+            flag = False
+            click.secho('No external IPs were found to be associated with this domain. '
+                        'No alert was created.', dim=True)
+        else:
+            # Create the actual alert
+            click.secho('Creating alert...', dim=True)
+            alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips))
 
-        # Enable the triggers so it starts getting managed by Shodan Monitor
-        click.secho('Enabling triggers...', dim=True)
-        api.enable_alert_trigger(alert['id'], triggers)
+            # Enable the triggers so it starts getting managed by Shodan Monitor
+            click.secho('Enabling triggers...', dim=True)
+            api.enable_alert_trigger(alert['id'], triggers)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
-    click.secho('Successfully created domain alert!', fg='green')
-    click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
+    if flag:
+        click.secho('Successfully created domain alert!', fg='green')
+        click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 
 @alert.command(name='download')

From 3036d83a9de0efb35202d094521f1bfd152d14b2 Mon Sep 17 00:00:00 2001
From: Richard Howe <rhowe425@gmail.com>
Date: Fri, 14 Jul 2023 16:31:08 -0400
Subject: [PATCH 259/263] Updating implementation based on reviewer
 recommendations.

---
 shodan/cli/alert.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py
index 6f82c1a..1df11ea 100644
--- a/shodan/cli/alert.py
+++ b/shodan/cli/alert.py
@@ -121,7 +121,6 @@ def alert_create(name, netblocks):
 @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable')
 def alert_domain(domain, triggers):
     """Create a network alert based on a domain name"""
-    flag = True
     key = get_api_key()
 
     api = shodan.Shodan(key)
@@ -141,23 +140,21 @@ def alert_domain(domain, triggers):
                               if not ip_address(record['value']).is_private])
 
         if not domain_ips:
-            flag = False
-            click.secho('No external IPs were found to be associated with this domain. '
-                        'No alert was created.', dim=True)
-        else:
-            # Create the actual alert
-            click.secho('Creating alert...', dim=True)
-            alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips))
+            raise click.ClickException('No external IPs were found to be associated with this domain. '
+                                       'No alert was created.')
+
+        # Create the actual alert
+        click.secho('Creating alert...', dim=True)
+        alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips))
 
-            # Enable the triggers so it starts getting managed by Shodan Monitor
-            click.secho('Enabling triggers...', dim=True)
-            api.enable_alert_trigger(alert['id'], triggers)
+        # Enable the triggers so it starts getting managed by Shodan Monitor
+        click.secho('Enabling triggers...', dim=True)
+        api.enable_alert_trigger(alert['id'], triggers)
     except shodan.APIError as e:
         raise click.ClickException(e.value)
 
-    if flag:
-        click.secho('Successfully created domain alert!', fg='green')
-        click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
+    click.secho('Successfully created domain alert!', fg='green')
+    click.secho('Alert ID: {}'.format(alert['id']), fg='cyan')
 
 
 @alert.command(name='download')

From 759a8561f821bf2e970f0594200221271bcbb7a9 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Wed, 19 Jul 2023 15:55:20 -0700
Subject: [PATCH 260/263] Update "shodan host" output to show certificate
 issuer/ subject and HTTP title

---
 shodan/cli/host.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/shodan/cli/host.py b/shodan/cli/host.py
index e90e372..8ffdeed 100644
--- a/shodan/cli/host.py
+++ b/shodan/cli/host.py
@@ -91,8 +91,20 @@ def host_print_pretty(host, history=False):
             click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False)
         click.echo('')
 
+        # Show optional HTTP information
+        if 'http' in banner:
+            if 'title' in banner['http'] and banner['http']['title']:
+                click.echo('\t|-- HTTP title: {}'.format(banner['http']['title']))
+
         # Show optional ssl info
         if 'ssl' in banner:
+            if 'cert' in banner['ssl'] and banner['ssl']['cert']:
+                if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']:
+                    issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()])
+                    click.echo('\t|-- Cert Issuer: {}'.format(issuer))
+                if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']:
+                    subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()])
+                    click.echo('\t|-- Cert Subject: {}'.format(subject))
             if 'versions' in banner['ssl'] and banner['ssl']['versions']:
                 click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')])))
             if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']:

From 769e38448414c97eb7a34ecd3e88f9e0e280b2ee Mon Sep 17 00:00:00 2001
From: Thong Nguyen <thoongnv@gmail.com>
Date: Thu, 20 Jul 2023 11:50:51 +0700
Subject: [PATCH 261/263] Make Trends API facets param as optional, if not
 supply then show total query results over time

---
 shodan/__main__.py   | 54 ++++++++++++++++++++++++++++----------------
 tests/test_shodan.py |  9 +++++++-
 2 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/shodan/__main__.py b/shodan/__main__.py
index 8b4eb2b..4093b94 100644
--- a/shodan/__main__.py
+++ b/shodan/__main__.py
@@ -806,9 +806,9 @@ def _create_stream(name, args, timeout):
 
 
 @main.command()
+@click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str)
 @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None)
 @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True)
-@click.argument('facets', metavar='<facet>')
 @click.argument('query', metavar='<search query>', nargs=-1)
 def trends(filename, save, facets, query):
     """Search Shodan historical database"""
@@ -823,12 +823,12 @@ def trends(filename, save, facets, query):
     if query == '':
         raise click.ClickException('Empty search query')
 
-    if facets == '':
-        raise click.ClickException('Empty search facets')
-
     # Convert comma-separated facets string to list
     parsed_facets = []
     for facet in facets.split(','):
+        if not facet:
+            continue
+
         parts = facet.strip().split(":")
         if len(parts) > 1:
             parsed_facets.append((parts[0], parts[1]))
@@ -845,7 +845,9 @@ def trends(filename, save, facets, query):
     if results['total'] == 0:
         raise click.ClickException('No search results found')
 
-    result_facets = list(results['facets'].keys())
+    result_facets = []
+    if results.get("facets"):
+        result_facets = list(results["facets"].keys())
 
     # Save the results first to file if user request
     if filename or save:
@@ -858,31 +860,43 @@ def trends(filename, save, facets, query):
         with helpers.open_file(filename) as fout:
             for index, match in enumerate(results['matches']):
                 # Append facet info to make up a line
-                match["facets"] = {}
-                for facet in result_facets:
-                    match["facets"][facet] = results['facets'][facet][index]['values']
-                    line = json.dumps(match) + '\n'
-                    fout.write(line.encode('utf-8'))
+                if result_facets:
+                    match["facets"] = {}
+                    for facet in result_facets:
+                        match["facets"][facet] = results['facets'][facet][index]['values']
+
+                line = json.dumps(match) + '\n'
+                fout.write(line.encode('utf-8'))
 
         click.echo(click.style(u'Saved results into file {}'.format(filename), 'green'))
 
     # We buffer the entire output so we can use click's pager functionality
     output = u''
 
-    # Output example:
+    # Output examples:
+    # - Facet by os
     # 2017-06
     #   os
     #     Linux 3.x                                                        146,502
     #     Windows 7 or 8                                                     2,189
-    for index, match in enumerate(results['matches']):
-        output += click.style(match['month'] + u'\n', fg='green')
-        if match['count'] > 0:
-            for facet in result_facets:
-                output += click.style(u'  {}\n'.format(facet), fg='cyan')
-                for bucket in results['facets'][facet][index]['values']:
-                    output += u'    {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green'))
-        else:
-            output += u'{}\n'.format(click.style('N/A', bold=True))
+    #
+    # - Without facets
+    # 2017-06               19,799,459
+    # 2017-07               21,077,099
+    if result_facets:
+        for index, match in enumerate(results['matches']):
+            output += click.style(match['month'] + u'\n', fg='green')
+            if match['count'] > 0:
+                for facet in result_facets:
+                    output += click.style(u'  {}\n'.format(facet), fg='cyan')
+                    for bucket in results['facets'][facet][index]['values']:
+                        output += u'    {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green'))
+            else:
+                output += u'{}\n'.format(click.style('N/A', bold=True))
+    else:
+        # Without facets, show query total results over time
+        for index, match in enumerate(results['matches']):
+            output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green'))
 
     click.echo_via_pager(output)
 
diff --git a/tests/test_shodan.py b/tests/test_shodan.py
index ebe7a90..94ffc70 100644
--- a/tests/test_shodan.py
+++ b/tests/test_shodan.py
@@ -118,12 +118,19 @@ def test_exploits_count_facets(self):
 
     def test_trends_search(self):
         results = self.api.trends.search('apache', facets=[('product', 10)])
+        self.assertIn('total', results)
         self.assertIn('matches', results)
         self.assertIn('facets', results)
-        self.assertIn('total', results)
         self.assertTrue(results['matches'])
         self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']])
 
+        results = self.api.trends.search('apache', facets=[])
+        self.assertIn('total', results)
+        self.assertIn('matches', results)
+        self.assertNotIn('facets', results)
+        self.assertTrue(results['matches'])
+        self.assertIn('2023-06', [match['month'] for match in results['matches']])
+
     def test_trends_search_filters(self):
         results = self.api.trends.search_filters()
         self.assertIn('has_ipv6', results)

From a08353e40b2018ec0dbf04012e9a5bfbe87f55ec Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Fri, 13 Oct 2023 18:15:18 -0700
Subject: [PATCH 262/263] Improved handling of downloads to prevent it from
 exiting prematurely if the API only returns partial or missing results for a
 search results page.

---
 setup.py         |  2 +-
 shodan/client.py | 21 +++++++++++++++------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/setup.py b/setup.py
index 2546ea7..f421898 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.30.0',
+    version='1.30.1',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index 7b76fdc..ab81302 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -7,6 +7,7 @@
 
 :copyright: (c) 2014- by John Matherly
 """
+import math
 import time
 
 import requests
@@ -576,15 +577,23 @@ def search_cursor(self, query, minify=True, retries=5, fields=None):
         :returns: A search cursor that can be used as an iterator/ generator.
         """
         page = 1
+        total_pages = 0
         tries = 0
 
-        # Placeholder results object to make the while loop below easier
-        results = {
-            'matches': [True],
-            'total': None,
-        }
+        # Grab the initial page and use the total to calculate the expected number of pages
+        results = self.search(query, minify=minify, page=page, fields=fields)
+        if results['total']:
+            total_pages = int(math.ceil(results['total'] / 100))
+                    
+        for banner in results['matches']:
+            try:
+                yield banner
+            except GeneratorExit:
+                return  # exit out of the function
+        page += 1
 
-        while results['matches']:
+        # Keep iterating over the results from page 2 onwards
+        while page <= total_pages:
             try:
                 results = self.search(query, minify=minify, page=page, fields=fields)
                 for banner in results['matches']:

From 87a0688d1e5b7e4bb13ae4f5fd7cb937a671cba8 Mon Sep 17 00:00:00 2001
From: John Matherly <jmath@shodan.io>
Date: Sat, 16 Dec 2023 17:29:15 -0800
Subject: [PATCH 263/263] Consolidate all Shodan methods to use the internal
 _request method instead of sometimes using the shodan.helpers.api_request
 method. New environment variable SHODAN_API_URL that can be used to overwrite
 the base_url used for the API requests.

---
 setup.py         |  2 +-
 shodan/client.py | 30 +++++++++++++++++++-----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/setup.py b/setup.py
index f421898..53bbd9a 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 setup(
     name='shodan',
-    version='1.30.1',
+    version='1.31.0',
     description='Python library and command-line utility for Shodan (https://developer.shodan.io)',
     long_description=README,
     long_description_content_type='text/x-rst',
diff --git a/shodan/client.py b/shodan/client.py
index ab81302..21c70af 100644
--- a/shodan/client.py
+++ b/shodan/client.py
@@ -8,13 +8,14 @@
 :copyright: (c) 2014- by John Matherly
 """
 import math
+import os
 import time
 
 import requests
 import json
 
 from .exception import APIError
-from .helpers import api_request, create_facet_string
+from .helpers import create_facet_string
 from .stream import Stream
 
 
@@ -314,11 +315,15 @@ def __init__(self, key, proxies=None):
         self._session = requests.Session()
         self.api_rate_limit = 1  # Requests per second
         self._api_query_time = None
+
         if proxies:
             self._session.proxies.update(proxies)
             self._session.trust_env = False
+        
+        if os.environ.get('SHODAN_API_URL'):
+            self.base_url = os.environ.get('SHODAN_API_URL')
 
-    def _request(self, function, params, service='shodan', method='get'):
+    def _request(self, function, params, service='shodan', method='get', json_data=None):
         """General-purpose function to create web requests to SHODAN.
 
         Arguments:
@@ -348,7 +353,13 @@ def _request(self, function, params, service='shodan', method='get'):
         try:
             method = method.lower()
             if method == 'post':
-                data = self._session.post(base_url + function, params)
+                if json_data:
+                    data = self._session.post(base_url + function, params=params,
+                                            data=json.dumps(json_data),
+                                            headers={'content-type': 'application/json'},
+                        )
+                else:
+                    data = self._session.post(base_url + function, params)
             elif method == 'put':
                 data = self._session.put(base_url + function, params=params)
             elif method == 'delete':
@@ -711,8 +722,7 @@ def create_alert(self, name, ip, expires=0):
             'expires': expires,
         }
 
-        response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post',
-                               proxies=self._session.proxies)
+        response = self._request('/shodan/alert', params={}, json_data=data, method='post')
 
         return response
 
@@ -732,8 +742,7 @@ def edit_alert(self, aid, ip):
             },
         }
 
-        response = api_request(self.api_key, '/shodan/alert/{}'.format(aid), data=data, params={}, method='post',
-                               proxies=self._session.proxies)
+        response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post')
 
         return response
 
@@ -744,9 +753,9 @@ def alerts(self, aid=None, include_expired=True):
         else:
             func = '/shodan/alert/info'
 
-        response = api_request(self.api_key, func, params={
+        response = self._request(func, params={
             'include_expired': include_expired,
-        }, proxies=self._session.proxies)
+        })
 
         return response
 
@@ -754,8 +763,7 @@ def delete_alert(self, aid):
         """Delete the alert with the given ID."""
         func = '/shodan/alert/{}'.format(aid)
 
-        response = api_request(self.api_key, func, params={}, method='delete',
-                               proxies=self._session.proxies)
+        response = self._request(func, params={}, method='delete')
 
         return response