From df8e803c7d24f2594d429d2ea6d6fbd1f2ddfa82 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 13 Nov 2019 11:13:35 -0600 Subject: [PATCH 01/64] Show SHA1 checksum in bulk data listing if available --- setup.py | 2 +- shodan/cli/data.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 70381c4..8751b5f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.19.0', + version='1.19.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/data.py b/shodan/cli/data.py index 7cd7228..98d7852 100644 --- a/shodan/cli/data.py +++ b/shodan/cli/data.py @@ -27,6 +27,11 @@ def data_list(dataset): for file in files: click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + + # Show the SHA1 checksum if available + if file.get('sha1'): + click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False) + click.echo('{}'.format(file['url'])) else: # If no dataset was provided then show a list of all datasets From 1f101cae4700e30611a4d9b85634bc555f8cba08 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 18 Nov 2019 19:24:32 -0600 Subject: [PATCH 02/64] New options for "shodan domain": "-S" (save results) and "-D" (lookup open ports for IPs) --- CHANGELOG.md | 5 +++++ setup.py | 2 +- shodan/__main__.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8670835..8eced4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +1.20.0 +------ +* New option "-S" for **shodan domain** to save results from the lookup +* New option "-D" for **shodan domain** to lookup open ports for IPs in the results + 1.19.0 ------ * New method to edit the list of IPs for an existing network alert diff --git a/setup.py b/setup.py index 8751b5f..b2cf088 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.19.1', + version='1.20.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 5ddad7a..d4b7aa0 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -136,7 +136,9 @@ def convert(fields, input, format): @main.command(name='domain') @click.argument('domain', metavar='') -def domain_info(domain): +@click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) +@click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) +def domain_info(domain, details, save): """View all available information for a domain""" key = get_api_key() api = shodan.Shodan(key) @@ -146,6 +148,37 @@ def domain_info(domain): except shodan.APIError as e: raise click.ClickException(e.value) + # Grab the host information for any IP records that were returned + hosts = {} + if details: + ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']] + ips = set(ips) + + fout = None + if save: + filename = u'{}-hosts.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for ip in ips: + try: + hosts[ip] = api.host(ip) + + # Store the banners if requested + if fout: + for banner in hosts[ip]['data']: + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) + except shodan.APIError: + pass # Ignore any API lookup errors as this isn't critical information + + # Save the DNS data + if save: + filename = u'{}.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for record in info['data']: + helpers.write_banner(fout, record) + click.secho(info['domain'].upper(), fg='green') click.echo('') @@ -155,9 +188,16 @@ def domain_info(domain): click.style(record['subdomain'], fg='cyan'), click.style(record['type'], fg='yellow'), record['value'] - ) + ), + nl=False, ) + if record['value'] in hosts: + host = hosts[record['value']] + click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) + + click.echo('') + @main.command() @click.argument('key', metavar='') From da0e2ae7534c2b892a6122880e46b2b50fbe6b56 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:23:07 -0600 Subject: [PATCH 03/64] Add new API methods to get list of search filters/ facets --- shodan/client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/shodan/client.py b/shodan/client.py index 6e66259..3ed5e66 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -549,6 +549,20 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): tries += 1 time.sleep(1.0) # wait 1 second if the search errored out for some reason + def search_facets(self): + """Returns a list of search facets that can be used to get aggregate information about a search query. + + :returns: A list of strings where each is a facet name + """ + return self._request('/shodan/host/search/facets', {}) + + def search_filters(self): + """Returns a list of search filters that are available. + + :returns: A list of strings where each is a filter name + """ + return self._request('/shodan/host/search/filters', {}) + def search_tokens(self, query): """Returns information about the search query itself (filters used etc.) From 406abf0891f192fa2a46eff779bedbcc1841eaf8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:23:47 -0600 Subject: [PATCH 04/64] Fix linting rules and errors --- shodan/cli/scan.py | 8 ++++---- tox.ini | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 4590770..7ec158c 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -91,7 +91,7 @@ def scan_internet(quiet, port, protocol): click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), ';'.join(banner['hostnames'])) ) - except shodan.APIError as e: + except shodan.APIError: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -100,7 +100,7 @@ def scan_internet(quiet, port, protocol): scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True - except socket.timeout as e: + except socket.timeout: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -205,7 +205,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): done = True break - except shodan.APIError as e: + except shodan.APIError: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait briefly and try # to connect again! @@ -223,7 +223,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): if verbose: click.echo('# Scan status: {}'.format(scan['status'])) - except socket.timeout as e: + except socket.timeout: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait a second and try # to connect again! diff --git a/tox.ini b/tox.ini index d840b92..dd53bb5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [flake8] ignore = - E501 + E501 W293 exclude = build, From 9b1fe510ffb0f55a2e0f51b34f109fce904c7b16 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:25:58 -0600 Subject: [PATCH 05/64] Release 1.21.0 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eced4d..5bd47a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.0 +------ +* New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. + 1.20.0 ------ * New option "-S" for **shodan domain** to save results from the lookup diff --git a/setup.py b/setup.py index b2cf088..17337b4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.20.0', + version='1.21.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From c73cf83e0fc92b9fa5fc8243019a99a3975ff2cc Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 25 Dec 2019 11:38:34 -0600 Subject: [PATCH 06/64] Add history and type parameters to Shodan.dns.domain_info() method and CLI command --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/__main__.py | 6 ++++-- shodan/client.py | 9 +++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd47a4..90c2059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.1 +------ +* Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command + 1.21.0 ------ * New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. diff --git a/setup.py b/setup.py index 17337b4..2148480 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.0', + version='1.21.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index d4b7aa0..9678bbb 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -138,13 +138,15 @@ def convert(fields, input, format): @click.argument('domain', metavar='') @click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) @click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) -def domain_info(domain, details, save): +@click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True) +@click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None) +def domain_info(domain, details, save, history, type): """View all available information for a domain""" key = get_api_key() api = shodan.Shodan(key) try: - info = api.dns.domain_info(domain) + info = api.dns.domain_info(domain, history=history, type=type) except shodan.APIError as e: raise click.ClickException(e.value) diff --git a/shodan/client.py b/shodan/client.py index 3ed5e66..a83ac20 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -69,10 +69,15 @@ class Dns: def __init__(self, parent): self.parent = parent - def domain_info(self, domain): + def domain_info(self, domain, history=False, type=None): """Grab the DNS information for a domain. """ - return self.parent._request('/dns/domain/{}'.format(domain), {}) + args = {} + if history: + args['history'] = history + if type: + args['type'] = type + return self.parent._request('/dns/domain/{}'.format(domain), args) class Notifier: From 7ef18461bae262ece63a97ee6eb9a14c465607d0 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Jan 2020 23:43:22 -0600 Subject: [PATCH 07/64] Add support for paging through the domain information --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/client.py | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c2059..91540af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.2 +------ +* Add support for paging through the domain information + 1.21.1 ------ * Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command diff --git a/setup.py b/setup.py index 2148480..b84dd71 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.1', + version='1.21.2', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index a83ac20..4116051 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -69,10 +69,12 @@ class Dns: def __init__(self, parent): self.parent = parent - def domain_info(self, domain, history=False, type=None): + def domain_info(self, domain, history=False, type=None, page=1): """Grab the DNS information for a domain. """ - args = {} + args = { + 'page': page, + } if history: args['history'] = history if type: From 9e75eb9871bad4490dbdb7c2a3130883e40c5b59 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 20 Jan 2020 13:57:44 -0600 Subject: [PATCH 08/64] Fix geo.json file converter --- CHANGELOG.md | 4 +++ setup.py | 2 +- shodan/cli/converter/geojson.py | 49 +++++++++++++++------------------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91540af..0d5c02d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.3 +------ +* Fix geo.json file converter + 1.21.2 ------ * Add support for paging through the domain information diff --git a/setup.py b/setup.py index b84dd71..0605303 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.2', + version='1.21.3', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 6126267..eb9a205 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -1,4 +1,4 @@ - +from json import dumps from .base import Converter from ...helpers import get_ip, iterate_files @@ -18,40 +18,35 @@ def process(self, files): # Write the header self.header() - hosts = {} + # We only want to generate 1 datapoint for each IP - not per service + unique_hosts = set() for banner in iterate_files(files): ip = get_ip(banner) if not ip: continue - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) + if ip not in unique_hosts: + self.write(ip, banner) + unique_hosts.add(ip) self.footer() - def write(self, host): + def write(self, ip, host): try: - ip = get_ip(host) lat, lon = host['location']['latitude'], host['location']['longitude'] - - feature = """{ - "type": "Feature", - "id": "{}", - "properties": { - "name": "{}" - }, - "geometry": { - "type": "Point", - "coordinates": [{}, {}] - } - }""".format(ip, ip, lat, lon) - - self.fout.write(feature) - except Exception: + feature = { + 'type': 'Feature', + 'id': ip, + 'properties': { + 'name': ip, + 'lat': lat, + 'lon': lon, + }, + 'geometry': { + 'type': 'Point', + 'coordinates': [lon, lat], + }, + } + self.fout.write(dumps(feature) + ',') + except Exception as e: pass From ba561934f7fcfce3c35270d2baee6539b2afaaf1 Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 17:49:19 +0100 Subject: [PATCH 09/64] py 2 to 3 fix the example given in https://github.com/achillean/shodan-python/issues/114 --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 00c4d34..62744af 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -56,7 +56,7 @@ Now that we have our API object all good to go, we're ready to perform a search: print('IP: {}'.format(result['ip_str'])) print(result['data']) print('') - except shodan.APIError, e: + except shodan.APIError as e: print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which From 67409f6eebeec28b00be0c7517d28411f1ea43c1 Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:10:42 +0100 Subject: [PATCH 10/64] py 2 to 3 --- docs/examples/query-summary.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index 7a60716..73a87ab 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -48,7 +48,7 @@ and country. # Input validation if len(sys.argv) == 1: - print 'Usage: %s ' % sys.argv[0] + print('Usage: %s ' % sys.argv[0]) sys.exit(1) try: @@ -68,16 +68,16 @@ and country. # Print the summary info from the facets for facet in result['facets']: - print FACET_TITLES[facet] + print(FACET_TITLES[facet]) for term in result['facets'][facet]: - print '%s: %s' % (term['value'], term['count']) + print('%s: %s' % (term['value'], term['count'])) # Print an empty line between summary info - print '' + print('') - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: %s' % e) sys.exit(1) """ From 1deba33ba6bbb0bc45dd5716c48a1f785a0a986b Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:12:16 +0100 Subject: [PATCH 11/64] py 2 to 3 --- docs/examples/gifcreator.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst index ad9efc8..e4a43c3 100644 --- a/docs/examples/gifcreator.rst +++ b/docs/examples/gifcreator.rst @@ -106,7 +106,7 @@ There are a few key Shodan methods/ parameters that make the script work: os.system('rm -f /tmp/gif-image-*.jpg') # Show a progress indicator - print result['ip_str'] + print(result['ip_str']) -The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 \ No newline at end of file +The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 From a3084d3a5976e9d47e407ba073cd90f2c8122b1c Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:19:02 +0100 Subject: [PATCH 12/64] py 2 to 3 --- docs/examples/query-summary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index 73a87ab..66e15fe 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -62,9 +62,9 @@ and country. # And it also runs faster than doing a search(). result = api.count(query, facets=FACETS) - print 'Shodan Summary Information' - print 'Query: %s' % query - print 'Total Results: %s\n' % result['total'] + print('Shodan Summary Information') + print('Query: %s' % query) + print('Total Results: %s\n' % result['total']) # Print the summary info from the facets for facet in result['facets']: From ee93d65b75b329fbe57f4f2248ead0f695ce1ccb Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Tue, 28 Jan 2020 09:44:48 +0100 Subject: [PATCH 13/64] Remove shebang This is a RPM packing issue. --- shodan/cli/worldmap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 767b237..30c9ec3 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' F-Secure Virus World Map console edition From 2367646b8e4474ffdfd969485f989c69a4f42a22 Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Tue, 28 Jan 2020 14:31:27 +0100 Subject: [PATCH 14/64] Point the repo directly --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0605303..8290147 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description_content_type='text/x-rst', author='John Matherly', author_email='jmath@shodan.io', - url='http://github.com/achillean/shodan-python/tree/master', + url='https://github.com/achillean/shodan-python', packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, install_requires=DEPENDENCIES, From bde444271a516a81235ed961d89e32b8f551ebce Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 17 Mar 2020 18:29:15 -0500 Subject: [PATCH 15/64] New Streaming API method: /shodan/vulns/{vulns} to subscribe to IPs that are vulnerable to an issue Release 1.22.0 --- setup.py | 2 +- shodan/__main__.py | 11 +++++++++-- shodan/stream.py | 11 +++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0605303..0c8882f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.3', + version='1.22.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 9678bbb..5b1af92 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -635,7 +635,8 @@ def stats(limit, facets, filename, query): @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel): +@click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) +def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -663,9 +664,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('alert') if tags: stream_type.append('tags') + if vulns: + stream_type.append('vulns') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -689,6 +692,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if tags: stream_args = tags.split(',') + + if vulns: + stream_args = vulns.split(',') # Flatten the list of stream types # Possible values are: @@ -710,6 +716,7 @@ def _create_stream(name, args, timeout): 'countries': api.stream.countries(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), 'tags': api.stream.tags(args, timeout=timeout), + 'vulns': api.stream.vulns(args, timeout=timeout), }.get(name, 'all') stream = _create_stream(stream_type, stream_args, timeout=timeout) diff --git a/shodan/stream.py b/shodan/stream.py index 1feb15a..15d7619 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -134,3 +134,14 @@ def tags(self, tags, raw=False, timeout=None): stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) for line in self._iter_stream(stream, raw): yield line + + def vulns(self, vulns, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. + + :param vulns: A list of vulns to return banner data on. + :type vulns: string[] + """ + stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line From 378feee4992461f8f4510034c65d4c9dfbe651b6 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 27 Mar 2020 12:12:48 -0500 Subject: [PATCH 16/64] Fix bug when converting data file to CSV using Python3 --- setup.py | 2 +- shodan/cli/converter/csvc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0c8882f..db956d9 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.22.0', + version='1.22.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 39ec162..93b1695 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -54,7 +54,7 @@ def process(self, files): # The "vulns" property can't be nicely flattened as-is so we turn # it into a list before processing the banner. if 'vulns' in banner: - banner['vulns'] = banner['vulns'].keys() + banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list try: row = [] From b7a9978d532094fb8dd72efaa6b6f0bb75a89ba6 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 14:36:00 -0500 Subject: [PATCH 17/64] Update CHANGELOG to match releases --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5c02d..de68afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ CHANGELOG ========= +1.22.1 +------ +* Fix bug when converting data file to CSV using Python3 + +1.22.0 +------ +* Add support for new vulnerability streaming endpoints + 1.21.3 ------ * Fix geo.json file converter From d91ffd8f6aa0e093c3a913bb8585add3e99c2969 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 15:02:12 -0500 Subject: [PATCH 18/64] Add new CLI command: shodan alert domain --- CHANGELOG.md | 4 ++++ setup.py | 4 +++- shodan/cli/alert.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de68afb..50a9334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.23.0 +------ +* Add new CLI command: shodan alert domain + 1.22.1 ------ * Fix bug when converting data file to CSV using Python3 diff --git a/setup.py b/setup.py index db956d9..e04a96f 100755 --- a/setup.py +++ b/setup.py @@ -2,12 +2,14 @@ from setuptools import setup + DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') README = open('README.rst', 'r').read() + setup( name='shodan', - version='1.22.1', + version='1.23.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 1ed8572..633b2aa 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,6 +3,7 @@ from operator import itemgetter from shodan.cli.helpers import get_api_key +from time import sleep @click.group() @@ -46,6 +47,35 @@ def alert_create(name, netblocks): click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +@alert.command(name='domain') +@click.argument('domain', metavar='', type=str) +@click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') +def alert_domain(domain, triggers): + """Create a network alert based on a domain name""" + key = get_api_key() + + api = shodan.Shodan(key) + try: + # Grab a list of IPs for the domain + domain = domain.lower() + click.secho('Looking up domain information...', dim=True) + info = api.dns.domain_info(domain, type='A') + domain_ips = set([record['value'] for record in info['data']]) + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From 31b420a590f11fd83e336f308aff499c2aa4d096 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 15:06:42 -0500 Subject: [PATCH 19/64] Fix linting errors --- shodan/cli/alert.py | 1 - shodan/cli/converter/geojson.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 633b2aa..0d81dc4 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,7 +3,6 @@ from operator import itemgetter from shodan.cli.helpers import get_api_key -from time import sleep @click.group() diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index eb9a205..3f6c975 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -48,5 +48,5 @@ def write(self, ip, host): }, } self.fout.write(dumps(feature) + ',') - except Exception as e: + except Exception: pass From 094420d483a4be6c6d04906edfa13ab1e70489f3 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sun, 3 May 2020 12:54:26 +0000 Subject: [PATCH 20/64] Import ABC from collections.abc for Python 3 compatibility. --- shodan/cli/converter/csvc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 93b1695..f51e75e 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -2,7 +2,13 @@ from .base import Converter from ...helpers import iterate_files -from collections import MutableMapping +try: + # python 3.x: Import ABC from collections.abc + from collections.abc import MutableMapping +except ImportError: + # Python 2.x: Import ABC from collections + from collections import MutableMapping + from csv import writer as csv_writer, excel From c6615d0a275ac53fadb0bb4f66fe290ffb1f9dae Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 23 Jun 2020 16:32:03 -0500 Subject: [PATCH 21/64] Fix invalid escape sequence error (#131) Improve ``shodan radar`` output on Python3 --- shodan/__main__.py | 15 ++++++++------- shodan/cli/worldmap.py | 5 ++--- tox.ini | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 5b1af92..939b53f 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -59,6 +59,13 @@ # Make "-h" work like "--help" CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +CONVERTERS = { + 'kml': KmlConverter, + 'csv': CsvConverter, + 'geo.json': GeoJsonConverter, + 'images': ImagesConverter, + 'xlsx': ExcelConverter, +} # Define a basestring type if necessary for Python3 compatibility try: @@ -66,6 +73,7 @@ except NameError: basestring = str + # Define the main entry point for all of our commands # and expose a way for 3rd-party plugins to tie into the Shodan CLI. @with_plugins(iter_entry_points('shodan.cli.plugins')) @@ -81,13 +89,6 @@ def main(): main.add_command(scan) -CONVERTERS = { - 'kml': KmlConverter, - 'csv': CsvConverter, - 'geo.json': GeoJsonConverter, - 'images': ImagesConverter, - 'xlsx': ExcelConverter, -} @main.command() @click.option('--fields', help='List of properties to output.', default=None) @click.argument('input', metavar='') diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 30c9ec3..db91e41 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -29,8 +29,7 @@ 'coords': [90.0, -180.0, -90.0, 180.0], # PyLint freaks out about the world map backslashes so ignore those warnings - # pylint: disable=W1401 - 'data': ''' + 'data': r''' . _..::__: ,-"-"._ |7 , _,.__ _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ @@ -159,7 +158,7 @@ def draw(self, target): for lat, lon, char, desc, attrs, color in self.data: # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html if desc: - desc = desc.encode(self.encoding, 'ignore') + desc = desc.encode(self.encoding, 'ignore').decode() if items_to_show <= 0: break char_x, char_y = self.latlon_to_coords(lat, lon) diff --git a/tox.ini b/tox.ini index dd53bb5..1a9f632 100644 --- a/tox.ini +++ b/tox.ini @@ -11,4 +11,4 @@ exclude = per-file-ignores = shodan/__init__.py:F401, shodan/cli/converter/__init__.py:F401, - shodan/cli/worldmap.py:W291,W293,W605, \ No newline at end of file + shodan/cli/worldmap.py:W291, \ No newline at end of file From 6ad80cc48da80d3c100637cefdd6f939d5620b3a Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 00:11:32 +0100 Subject: [PATCH 22/64] allow user to move config dir to ~/.config --- shodan/cli/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 6f27e5d..2f5a73c 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,5 +1,10 @@ +from os import path + +if path.exists(path.expanduser("~/.config/shodan")): + SHODAN_CONFIG_DIR="~/.config/shodan/" +else: + SHODAN_CONFIG_DIR = '~/.shodan/' -SHODAN_CONFIG_DIR = '~/.shodan/' COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From 1b58c7f3a6402a87e860f31a0cb197070fa7c798 Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 16:07:12 +0100 Subject: [PATCH 23/64] Revert "allow user to move config dir to ~/.config" This reverts commit 6ad80cc48da80d3c100637cefdd6f939d5620b3a. --- shodan/cli/settings.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 2f5a73c..6f27e5d 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,10 +1,5 @@ -from os import path - -if path.exists(path.expanduser("~/.config/shodan")): - SHODAN_CONFIG_DIR="~/.config/shodan/" -else: - SHODAN_CONFIG_DIR = '~/.shodan/' +SHODAN_CONFIG_DIR = '~/.shodan/' COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From c01abd045b531052b8522ea532f6c6ae54d85b90 Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 16:09:19 +0100 Subject: [PATCH 24/64] default config dir to ~/.config/shodan, but support existing installations --- shodan/cli/settings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 6f27e5d..294793c 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,5 +1,11 @@ -SHODAN_CONFIG_DIR = '~/.shodan/' +from os import path + +if path.exists(path.expanduser("~/.shodan")): + SHODAN_CONFIG_DIR = '~/.shodan/' +else: + SHODAN_CONFIG_DIR="~/.config/shodan/" + COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From 639ca36a6747671e5d68ab9b10b054e2b31255eb Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 19:22:10 +0100 Subject: [PATCH 25/64] makedirs if ~/.config/shodan does not exist --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 939b53f..f8ed673 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -210,7 +210,7 @@ def init(key): shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) if not os.path.isdir(shodan_dir): try: - os.mkdir(shodan_dir) + os.makedirs(shodan_dir) except OSError: raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) From 56f0cdf7de8c41c47b2cfa869dd3e64bc1c1c8ce Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 7 Sep 2020 19:37:04 -0500 Subject: [PATCH 26/64] Fix bug that caused extra newlines when converting .json.gz data file to CSV on Windows (https://stackoverflow.com/questions/3191528/csv-in-python-adding-an-extra-carriage-return-on-windows) --- README.rst | 1 + setup.py | 2 +- shodan/cli/converter/csvc.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 610c13d..14f6717 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,7 @@ Features - `Manage Email Notifications `_ - Exploit search API fully implemented - Bulk data downloads +- Access the Shodan DNS DB to view domain information - `Command-line interface `_ .. image:: https://cli.shodan.io/img/shodan-cli-preview.png diff --git a/setup.py b/setup.py index e04a96f..b1735b2 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.23.0', + version='1.23.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index f51e75e..20c3d03 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -51,7 +51,7 @@ class CsvConverter(Converter): ] def process(self, files): - writer = csv_writer(self.fout, dialect=excel) + writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') # Write the header writer.writerow(self.fields) From 9ebccea26867f33ddadb063af9b585a09d69fe1a Mon Sep 17 00:00:00 2001 From: malvidin Date: Thu, 8 Oct 2020 22:40:53 +0200 Subject: [PATCH 27/64] Convert residual % formatting with str.format() Add API Rate limits to the Shodan class Fix human precision of bytes under 1024 --- shodan/__main__.py | 8 ++++---- shodan/cli/helpers.py | 2 +- shodan/cli/settings.py | 2 +- shodan/cli/worldmap.py | 4 ++-- shodan/client.py | 16 ++++++++++++---- shodan/helpers.py | 16 ++++++++-------- shodan/stream.py | 12 ++++++------ 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f8ed673..bbcef07 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -288,10 +288,10 @@ def download(limit, skip, filename, query): raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') # Print some summary information about the download request - click.echo('Search query:\t\t\t%s' % query) - click.echo('Total number of results:\t%s' % total) - click.echo('Query credits left:\t\t%s' % info['unlocked_left']) - click.echo('Output file:\t\t\t%s' % filename) + click.echo('Search query:\t\t\t{}'.format(query)) + click.echo('Total number of results:\t{}'.format(total)) + click.echo('Query credits left:\t\t{}'.format(info['unlocked_left'])) + click.echo('Output file:\t\t\t{}'.format(filename)) if limit > total: limit = total diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 4e99113..bde2f07 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -47,7 +47,7 @@ def timestr(): def open_streaming_file(directory, timestr, compresslevel=9): - return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel) + return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel) def get_banner_field(banner, flat_field): diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 294793c..05c1b9f 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -4,7 +4,7 @@ if path.exists(path.expanduser("~/.shodan")): SHODAN_CONFIG_DIR = '~/.shodan/' else: - SHODAN_CONFIG_DIR="~/.config/shodan/" + SHODAN_CONFIG_DIR = "~/.config/shodan/" COLORIZE_FIELDS = { 'ip_str': 'green', diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index db91e41..4e09872 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -166,7 +166,7 @@ def draw(self, target): attrs |= curses.color_pair(self.colors[color]) self.window.addstr(char_y, char_x, char, attrs) if desc: - det_show = "%s %s" % (char, desc) + det_show = "{} {}".format(char, desc) else: det_show = None @@ -179,7 +179,7 @@ def draw(self, target): # FIXME: check window size before addstr() break self.window.overwrite(target) - self.window.leaveok(1) + self.window.leaveok(True) class MapApp(object): diff --git a/shodan/client.py b/shodan/client.py index 4116051..079880d 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -273,6 +273,8 @@ def __init__(self, key, proxies=None): self.tools = self.Tools(self) self.stream = Stream(key, proxies=proxies) self._session = requests.Session() + self.api_rate_limit = 1 # Requests per second + self._api_query_time = None if proxies: self._session.proxies.update(proxies) self._session.trust_env = False @@ -297,6 +299,11 @@ def _request(self, function, params, service='shodan', method='get'): 'exploits': self.base_exploits_url, }.get(service, 'shodan') + # Wait for API rate limit + if self._api_query_time is not None and self.api_rate_limit > 0: + while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time(): + time.sleep(0.1 / self.api_rate_limit) + # Send the request try: method = method.lower() @@ -308,6 +315,7 @@ def _request(self, function, params, service='shodan', method='get'): data = self._session.delete(base_url + function, params=params) else: data = self._session.get(base_url + function, params=params) + self._api_query_time = time.time() except Exception: raise APIError('Unable to connect to Shodan') @@ -377,7 +385,7 @@ def host(self, ips, history=False, minify=False): params['history'] = history if minify: params['minify'] = minify - return self._request('/shodan/host/%s' % ','.join(ips), params) + return self._request('/shodan/host/{}'.format(','.join(ips)), params) def info(self): """Returns information about the current API key, such as a list of add-ons @@ -468,7 +476,7 @@ def scan_status(self, scan_id): :returns: A dictionary with general information about the scan, including its status in getting processed. """ - return self._request('/shodan/scan/%s' % scan_id, {}) + return self._request('/shodan/scan/{}'.format(scan_id), {}) def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): """Search the SHODAN database. @@ -685,7 +693,7 @@ def edit_alert(self, aid, ip): def alerts(self, aid=None, include_expired=True): """List all of the active alerts that the user created.""" if aid: - func = '/shodan/alert/%s/info' % aid + func = '/shodan/alert/{}/info'.format(aid) else: func = '/shodan/alert/info' @@ -697,7 +705,7 @@ def alerts(self, aid=None, include_expired=True): def delete_alert(self, aid): """Delete the alert with the given ID.""" - func = '/shodan/alert/%s' % aid + func = '/shodan/alert/{}'.format(aid) response = api_request(self.api_key, func, params={}, method='delete', proxies=self._session.proxies) diff --git a/shodan/helpers.py b/shodan/helpers.py index 2d976dc..378b1bb 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -142,7 +142,7 @@ def write_banner(fout, banner): fout.write(line.encode('utf-8')) -def humanize_bytes(bytes, precision=1): +def humanize_bytes(byte_count, precision=1): """Return a humanized string representation of a number of bytes. >>> humanize_bytes(1) '1 byte' @@ -161,15 +161,15 @@ def humanize_bytes(bytes, precision=1): >>> humanize_bytes(1024*1234*1111,1) '1.3 GB' """ - if bytes == 1: + if byte_count == 1: return '1 byte' - if bytes < 1024: - return '%.*f %s' % (precision, bytes, "bytes") + if byte_count < 1024: + '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] multiple = 1024.0 # .0 to force float on python 2 for suffix in suffixes: - bytes /= multiple - if bytes < multiple: - return '%.*f %s' % (precision, bytes, suffix) - return '%.*f %s' % (precision, bytes, suffix) + byte_count /= multiple + if byte_count < multiple: + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) diff --git a/shodan/stream.py b/shodan/stream.py index 15d7619..92365dc 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -71,7 +71,7 @@ def _iter_stream(self, stream, raw): def alert(self, aid=None, timeout=None, raw=False): if aid: - stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout) + stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout) else: stream = self._create_stream('/shodan/alert', timeout=timeout) @@ -90,7 +90,7 @@ def asn(self, asn, raw=False, timeout=None): :param asn: A list of ASN to return banner data on. :type asn: string[] """ - stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout) + stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -109,7 +109,7 @@ def countries(self, countries, raw=False, timeout=None): :param countries: A list of countries to return banner data on. :type countries: string[] """ - stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout) + stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -120,7 +120,7 @@ def ports(self, ports, raw=False, timeout=None): :param ports: A list of ports to return banner data on. :type ports: int[] """ - stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) + stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -131,7 +131,7 @@ def tags(self, tags, raw=False, timeout=None): :param tags: A list of tags to return banner data on. :type tags: string[] """ - stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) + stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -142,6 +142,6 @@ def vulns(self, vulns, raw=False, timeout=None): :param vulns: A list of vulns to return banner data on. :type vulns: string[] """ - stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout) + stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line From 0703499f21bd21f10b2c00d8fc470de901dc46ac Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 11 Oct 2020 17:24:39 -0500 Subject: [PATCH 28/64] Fix linting error --- shodan/cli/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 294793c..05c1b9f 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -4,7 +4,7 @@ if path.exists(path.expanduser("~/.shodan")): SHODAN_CONFIG_DIR = '~/.shodan/' else: - SHODAN_CONFIG_DIR="~/.config/shodan/" + SHODAN_CONFIG_DIR = "~/.config/shodan/" COLORIZE_FIELDS = { 'ip_str': 'green', From 322f169c8d0f916926e808ef6d6aa10b544fc9ac Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 11 Oct 2020 17:50:10 -0500 Subject: [PATCH 29/64] Release 1.24.0 New command: shodan alert stats --- setup.py | 2 +- shodan/cli/alert.py | 152 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1735b2..9c6f55f 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.23.1', + version='1.24.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 0d81dc4..de77294 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,10 +1,73 @@ import click +import csv import shodan +from collections import defaultdict from operator import itemgetter from shodan.cli.helpers import get_api_key +MAX_QUERY_LENGTH = 1000 + + +def aggregate_facet(api, networks, facets): + """Merge the results from multiple facet API queries into a single result object. + This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit + into a single API call. + """ + def _merge_custom_facets(lfacets, results): + for key in results['facets']: + if key not in lfacets: + lfacets[key] = defaultdict(int) + + for item in results['facets'][key]: + lfacets[key][item['value']] += item['count'] + + # We're going to create a custom facets dict where + # the key is the value of a facet. Normally the facets + # object is a list where each item has a "value" and "count" property. + tmp_facets = {} + count = 0 + + query = 'net:' + + for net in networks: + query += '{},'.format(net) + + # Start running API queries if the query length is getting long + if len(query) > MAX_QUERY_LENGTH: + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + query = 'net:' + + # Run any remaining search query + if query[-1] != ':': + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + + # Convert the internal facets structure back to the one that + # the API returns. + new_facets = {} + for facet in tmp_facets: + sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True) + new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items] + + # Make sure the facet keys exist even if there weren't any results + for facet, _ in facets: + if facet not in new_facets: + new_facets[facet] = [] + + return { + 'matches': [], + 'facets': new_facets, + 'total': count, + } + + @click.group() def alert(): """Manage the network alerts for your account""" @@ -149,6 +212,95 @@ def alert_list(expired): click.echo("You haven't created any alerts yet.") +@alert.command(name='stats') +@click.option('--limit', help='The number of results to return.', default=10, type=int) +@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) +@click.argument('facets', metavar='', nargs=-1) +def alert_stats(limit, filename, facets): + """Show summary information about your monitored networks""" + # Setup Shodan + key = get_api_key() + api = shodan.Shodan(key) + + # Make sure the user didn't supply an empty string + if not facets: + raise click.ClickException('No facets provided') + + facets = [(facet, limit) for facet in facets] + + # Get the list of IPs/ networks that the user is monitoring + networks = set() + try: + alerts = api.alerts() + for alert in alerts: + for tmp in alert['filters']['ip']: + networks.add(tmp) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the facets the user requested + try: + results = aggregate_facet(api, networks, facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared + # Print the stats tables + for facet in results['facets']: + click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) + + for item in results['facets'][facet]: + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) + + click.echo('') + + # Create the output file if requested + fout = None + if filename: + if not filename.endswith('.csv'): + filename += '.csv' + fout = open(filename, 'w') + writer = csv.writer(fout, dialect=csv.excel) + + # Write the header that contains the facets + row = [] + for facet in results['facets']: + row.append(facet) + row.append('') + writer.writerow(row) + + # Every facet has 2 columns (key, value) + counter = 0 + has_items = True + while has_items: + # pylint: disable=W0612 + row = ['' for i in range(len(results['facets']) * 2)] + + pos = 0 + has_items = False + for facet in results['facets']: + values = results['facets'][facet] + + # Add the values for the facet into the current row + if len(values) > counter: + has_items = True + row[pos] = values[counter]['value'] + row[pos + 1] = values[counter]['count'] + + pos += 2 + + # Write out the row + if has_items: + writer.writerow(row) + + # Move to the next row of values + counter += 1 + + @alert.command(name='remove') @click.argument('alert_id', metavar='') def alert_remove(alert_id): From 8ac631090100de6dbc53171d07918ac869a1eeb9 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:46:09 -0600 Subject: [PATCH 30/64] Remove the --skip option when downloading as the API nolonger supports requesting arbitrary pages in the search results. --- shodan/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f8ed673..a98ddbd 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -256,7 +256,6 @@ def count(query): @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) -@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) def download(limit, skip, filename, query): From 8195e1de58615303fe1d412d283865b1c024f0d8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:46:28 -0600 Subject: [PATCH 31/64] Remove the skip parameter from the download method as well --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index a98ddbd..d12b594 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -258,7 +258,7 @@ def count(query): @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, skip, filename, query): +def download(limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() From 90491308971a7ba6e358d6152d7b02307964c6fe Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:48:21 -0600 Subject: [PATCH 32/64] Remove the skip parameter from the search_cursor --- shodan/__main__.py | 6 +----- shodan/client.py | 10 +--------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index d12b594..4395820 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -298,15 +298,11 @@ def download(limit, filename, query): # A limit of -1 means that we should download all the data if limit <= 0: limit = total - - # Adjust the total number of results we should expect to download if the user is skipping results - if skip > 0: - limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False, skip=skip) + cursor = api.search_cursor(query, minify=False) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) diff --git a/shodan/client.py b/shodan/client.py index 4116051..eb66e7e 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -504,7 +504,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, skip=0): + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -518,8 +518,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type retries: int - :param skip: (optional) Number of results to skip - :type skip: int :returns: A search cursor that can be used as an iterator/ generator. """ @@ -532,12 +530,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): 'total': None, } - # Convert the number of skipped records into a page number - if skip > 0: - # Each page returns 100 results so find the nearest page that we want - # the cursor to skip to. - page += int(skip / 100) - while results['matches']: try: results = self.search(query, minify=minify, page=page) From 5eedf292d0cd4083c629a8c36acad43a63ffc472 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 25 Jan 2021 16:33:53 -0600 Subject: [PATCH 33/64] Add new CLI command: shodan alert download [--alert-id=] --- CHANGELOG.md | 8 +++++ setup.py | 2 +- shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a9334..2247045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ CHANGELOG ========= +1.25.0 +------ +* Add new CLI command: shodan alert download + +1.24.0 +------ +* Add new CLI command: shodan alert stats + 1.23.0 ------ * Add new CLI command: shodan alert domain diff --git a/setup.py b/setup.py index 9c6f55f..97616d6 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.24.0', + version='1.25.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index de77294..63b863d 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -4,7 +4,10 @@ from collections import defaultdict from operator import itemgetter +from shodan import APIError from shodan.cli.helpers import get_api_key +from shodan.helpers import open_file, write_banner +from time import sleep MAX_QUERY_LENGTH = 1000 @@ -138,6 +141,86 @@ def alert_domain(domain, triggers): click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +@alert.command(name='download') +@click.argument('filename', metavar='', type=str) +@click.option('--alert-id', help='Specific alert ID to download the data of', default=None) +def alert_download(filename, alert_id): + """Download all information for monitored networks/ IPs.""" + key = get_api_key() + + api = shodan.Shodan(key) + ips = set() + networks = set() + + # Helper method to process batches of IPs + def batch(iterable, size=1): + iter_length = len(iterable) + for ndx in range(0, iter_length, size): + yield iterable[ndx:min(ndx + size, iter_length)] + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + if alert_id: + alerts = [api.alerts(aid=alert_id.strip())] + else: + alerts = api.alerts() + + click.echo('Compiling list of networks/ IPs to download...') + for alert in alerts: + for net in alert['filters']['ip']: + if '/' in net: + networks.add(net) + else: + ips.add(net) + + click.echo('Downloading...') + with open_file(filename) as fout: + # Check if the user is able to use batch IP lookups + batch_size = 1 + if len(ips) > 0: + api_info = api.info() + if api_info['plan'] in ['corp', 'stream-100']: + batch_size = 100 + + # Convert it to a list so we can index into it + ips = list(ips) + + # Grab all the IP information + for ip in batch(ips, size=batch_size): + try: + click.echo(ip) + results = api.host(ip) + if not isinstance(results, list): + results = [results] + + for host in results: + for banner in host['data']: + write_banner(fout, banner) + except APIError: + pass + sleep(1) # Slow down a bit to make sure we don't hit the rate limit + + # Grab all the network ranges + for net in networks: + try: + counter = 0 + click.echo(net) + for banner in api.search_cursor('net:{}'.format(net)): + write_banner(fout, banner) + + # Slow down a bit to make sure we don't hit the rate limit + if counter % 100 == 0: + sleep(1) + counter += 1 + except APIError: + pass + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From 75d4c1b37b8a553f6db73ffc28cc139febbf0d9c Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 26 Apr 2021 14:10:08 -0300 Subject: [PATCH 34/64] Adding timeout --- shodan/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index eb66e7e..397b37a 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -502,9 +502,12 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) + if timeout: + args['timeout'] = timeout + return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, timeout=0): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -532,7 +535,7 @@ def search_cursor(self, query, minify=True, retries=5): while results['matches']: try: - results = self.search(query, minify=minify, page=page) + results = self.search(query, minify=minify, page=page, timeout=timeout) for banner in results['matches']: try: yield banner From a5edaff408b8b8802c515294fdfdff8268fda957 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 26 Apr 2021 14:31:15 -0300 Subject: [PATCH 35/64] None instead of zero --- shodan/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/client.py b/shodan/client.py index 397b37a..369b039 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None): """Search the SHODAN database. :param query: Search query; identical syntax to the website From cdfdb865e64568fa77437174e4a27949e84bca1e Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 18 May 2021 14:25:57 -0300 Subject: [PATCH 36/64] Some fixs to support big number of results Replace all timeout=0 to timeout=None and add incremental sleep for retry --- shodan/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index 369b039..10899c0 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -507,7 +507,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, timeout=0): + def search_cursor(self, query, minify=True, retries=5, timeout=None): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -549,7 +549,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=0): raise APIError('Retry limit reached ({:d})'.format(retries)) tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason + time.sleep(tries) # wait (1 second * retry number) if the search errored out for some reason def search_facets(self): """Returns a list of search facets that can be used to get aggregate information about a search query. From 67a2585d92784b41210f8963ea93cce9d8656ae3 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Wed, 19 May 2021 16:37:38 -0300 Subject: [PATCH 37/64] removing timeout, ther is no need for it now! --- shodan/client.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index 10899c0..02f115e 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -502,12 +502,9 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) - if timeout: - args['timeout'] = timeout - return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, timeout=None): + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -535,7 +532,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=None): while results['matches']: try: - results = self.search(query, minify=minify, page=page, timeout=timeout) + results = self.search(query, minify=minify, page=page) for banner in results['matches']: try: yield banner From d1027d8e4cea56e6f00cd437eab7f2ddfde72c59 Mon Sep 17 00:00:00 2001 From: Hy Che Date: Tue, 28 Dec 2021 02:46:37 +0700 Subject: [PATCH 38/64] Add implementation for custom filter `shodan/custom` Also bump version to 1.26 --- setup.py | 2 +- shodan/__main__.py | 39 +++++++++++++++++++++++---------------- shodan/stream.py | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 97616d6..6be33f3 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.25.0', + version='1.26.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 4395820..3700098 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -106,7 +106,7 @@ def convert(fields, input, format): if not hasattr(converter_class, 'fields'): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified - + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -173,7 +173,7 @@ def domain_info(domain, details, save, history, type): helpers.write_banner(fout, banner) except shodan.APIError: pass # Ignore any API lookup errors as this isn't critical information - + # Save the DNS data if save: filename = u'{}.json.gz'.format(domain) @@ -198,7 +198,7 @@ def domain_info(domain, details, save, history, type): if record['value'] in hosts: host = hosts[record['value']] click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) - + click.echo('') @@ -448,7 +448,7 @@ def myip(ipv6): # Use the IPv6-enabled domain if requested if ipv6: api.base_url = 'https://apiv6.shodan.io' - + try: click.echo(api.tools.myip()) except shodan.APIError as e: @@ -617,22 +617,23 @@ def stats(limit, facets, filename, query): @main.command() -@click.option('--color/--no-color', default=True) +@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') @click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) -@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) -@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) -@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) -@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) -@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) +@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) +@click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str) +@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) -@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) @click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns): +@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) +@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) +@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) +@click.option('--color/--no-color', default=True) +@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) +def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -662,9 +663,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('tags') if vulns: stream_type.append('vulns') + if custom_filters: + stream_type.append('custom_filters') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -685,13 +688,16 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if countries: stream_args = countries.split(',') - + if tags: stream_args = tags.split(',') - + if vulns: stream_args = vulns.split(',') + if custom_filters: + stream_args = custom_filters + # Flatten the list of stream types # Possible values are: # - all @@ -710,6 +716,7 @@ def _create_stream(name, args, timeout): 'alert': api.stream.alert(args, timeout=timeout), 'asn': api.stream.asn(args, timeout=timeout), 'countries': api.stream.countries(args, timeout=timeout), + 'custom_filters': api.stream.custom(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), 'tags': api.stream.tags(args, timeout=timeout), 'vulns': api.stream.vulns(args, timeout=timeout), diff --git a/shodan/stream.py b/shodan/stream.py index 15d7619..77ade7d 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -13,7 +13,7 @@ def __init__(self, api_key, proxies=None): self.api_key = api_key self.proxies = proxies - def _create_stream(self, name, timeout=None): + def _create_stream(self, name, query=None, timeout=None): params = { 'key': self.api_key, } @@ -27,9 +27,12 @@ def _create_stream(self, name, timeout=None): # If the user requested a timeout then we need to disable heartbeat messages # which are intended to keep stream connections alive even if there isn't any data # flowing through. - if timeout: + if timeout is not None: params['heartbeat'] = False + if query is not None: + params['query'] = query + try: while True: req = requests.get(stream_url, params=params, stream=True, timeout=timeout, @@ -113,6 +116,18 @@ def countries(self, countries, raw=False, timeout=None): for line in self._iter_stream(stream, raw): yield line + def custom(self, query, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the query of interest. The query + can vary and mix-match with different arguments (ports, tags, vulns, etc). + + :param query: A space-separated list of key:value filters query to return banner data on. + :type query: string + """ + stream = self._create_stream('/shodan/custom', query=query, timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + def ports(self, ports, raw=False, timeout=None): """ A filtered version of the "banners" stream to only return banners that match the ports of interest. From 490fdb5e592a2be243856c21bf158a2eb01dfe48 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 31 Dec 2021 16:33:06 -0600 Subject: [PATCH 39/64] Revert timeout parameter to previous behavior --- shodan/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/stream.py b/shodan/stream.py index 77ade7d..eac345f 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -27,7 +27,7 @@ def _create_stream(self, name, query=None, timeout=None): # If the user requested a timeout then we need to disable heartbeat messages # which are intended to keep stream connections alive even if there isn't any data # flowing through. - if timeout is not None: + if timeout: params['heartbeat'] = False if query is not None: From e603d462abee94f7e0a3c6780095f0f4a881af70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=E1=BA=BF=20V=C5=A9=20Gia=20Hy?= Date: Thu, 6 Jan 2022 21:53:12 +0700 Subject: [PATCH 40/64] stream: Remove set `decode_unicode` This doesn't handle well with every unicode strings, let only json.loads() do the decode instead. --- shodan/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/stream.py b/shodan/stream.py index eac345f..a47d143 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -63,7 +63,7 @@ def _create_stream(self, name, query=None, timeout=None): return req def _iter_stream(self, stream, raw): - for line in stream.iter_lines(decode_unicode=True): + for line in stream.iter_lines(): # The Streaming API sends out heartbeat messages that are newlines # We want to ignore those messages since they don't contain any data if line: From 95c6b910f7a46c3efecdab1d65085190d6404e89 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 19 Jan 2022 09:47:24 -0600 Subject: [PATCH 41/64] Release 1.26.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7751d7..97c47b4 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.26.0', + version='1.26.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 99756e2b2bd64603013be3f11caa6d406250f2a1 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 22 Feb 2022 13:53:59 -0600 Subject: [PATCH 42/64] New commands: "shodan alert export" and "shodan alert import" to help backup/ restore network monitoring configurations --- shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 63b863d..2dc3e58 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,5 +1,7 @@ import click import csv +import gzip +import json import shodan from collections import defaultdict @@ -221,6 +223,87 @@ def batch(iterable, size=1): click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') +@alert.command(name='export') +@click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str) +def alert_export(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + alerts = api.alerts() + + # Create the output file + click.echo('Writing alerts to file: {}'.format(filename)) + with gzip.open(filename, 'wt', encoding='utf-8') as fout: + json.dump(alerts, fout) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully exported monitored networks', fg='green') + + +@alert.command(name='import') +@click.argument('filename', metavar='') +def alert_import(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + # A mapping of the old notifier IDs to the new ones + notifier_map = {} + + try: + # Loading the alerts + click.echo('Loading alerts from: {}'.format(filename)) + with gzip.open(filename, 'rt', encoding='utf-8') as fin: + alerts = json.load(fin) + + for item in alerts: + # Create the alert + click.echo('Creating: {}'.format(item['name'])) + alert = api.create_alert(item['name'], item['filters']['ip']) + + # Enable any triggers + if item.get('triggers', {}): + triggers = ','.join(item['triggers'].keys()) + + api.enable_alert_trigger(alert['id'], triggers) + + # Add any whitelisted services for this trigger + for trigger, info in item['triggers'].items(): + if info.get('ignore', []): + for whitelist in info['ignore']: + api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port']) + + # Enable the notifiers + for prev_notifier in item.get('notifiers', []): + # We don't need to do anything for the default notifier as that + # uses the account's email address automatically. + if prev_notifier['id'] == 'default': + continue + + # Get the new notifier based on the ID of the old one + notifier = notifier_map.get(prev_notifier['id']) + + # Create the notifier if it doesn't yet exist + if notifier is None: + notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description']) + + # Add it to our map of old notifier IDs to new notifiers + notifier_map[prev_notifier['id']] = notifier + + api.add_alert_notifier(alert['id'], notifier['id']) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully imported monitored networks', fg='green') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From b32a7bc3f45f62c7ce962cea5954dd8b82fa6824 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 22 Feb 2022 17:01:05 -0600 Subject: [PATCH 43/64] Release 1.27.0 --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2247045..2cc021c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ CHANGELOG ========= +1.27.0 +------ +* New command: ``shodan alert export`` to save the current network monitoring configuration +* New command: ``shodan alert import`` to restore a previous network monitoring configuration +* Automatically rate limit API requests to 1 request per second (credit to @malvidin) + +1.26.1 +------ +* Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON + +1.26.0 +------ +* Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters ``) + 1.25.0 ------ * Add new CLI command: shodan alert download diff --git a/setup.py b/setup.py index 97c47b4..942c54f 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.26.1', + version='1.27.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 53662377e23c85926feada57b103e1a21d9ae2ee Mon Sep 17 00:00:00 2001 From: yaron-cider <100203813+yaron-cider@users.noreply.github.com> Date: Thu, 19 May 2022 12:39:16 +0300 Subject: [PATCH 44/64] Better error handling Raising 502 bad gateway error when receiving http status --- shodan/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shodan/client.py b/shodan/client.py index db50b4c..f4aebc0 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -336,6 +336,8 @@ def _request(self, function, params, service='shodan', method='get'): raise APIError(error) elif data.status_code == 403: raise APIError('Access denied (403 Forbidden)') + elif data.status_code == 502: + raise APIError('Bad Gateway (502)') # Parse the text into JSON try: From de2fd90aa9ffc8b53a6cd3f175fea50e551af0b1 Mon Sep 17 00:00:00 2001 From: Li-Heng Yu <007seadog@gmail.com> Date: Mon, 23 May 2022 14:14:40 +0800 Subject: [PATCH 45/64] Show scan id when scanning without showing results --- shodan/cli/scan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 7ec158c..cfc7aab 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -157,6 +157,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): # Return immediately if wait <= 0: + click.echo('Scan ID: {}'.format(scan['id'])) click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') else: # Setup an alert to wait for responses From 7d043d74735cfaf0b0b5dc8fbc81922ba117dfea Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 9 Jul 2022 17:52:51 -0500 Subject: [PATCH 46/64] Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- shodan/client.py | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc021c..8d9f9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ CHANGELOG ========= +1.28.0 +------ +* Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port +* Show scan ID when scanning without showing results (credit to @seadog007) +* Handle bad gateway errors (credit to @yaron-cider) + + 1.27.0 ------ * New command: ``shodan alert export`` to save the current network monitoring configuration diff --git a/setup.py b/setup.py index 942c54f..266ce81 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.27.0', + version='1.28.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index f4aebc0..70ca8f3 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -721,8 +721,14 @@ def disable_alert_trigger(self, aid, trigger): """Disable the given trigger on the alert.""" return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') - def ignore_alert_trigger_notification(self, aid, trigger, ip, port): + def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): """Ignore trigger notifications for the provided IP and port.""" + # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities + # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use + # a different API endpoint. + if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') def unignore_alert_trigger_notification(self, aid, trigger, ip, port): From 55d8d59bc3c82e3f35d1afbe85fe86b5c0cc75ca Mon Sep 17 00:00:00 2001 From: David xu Date: Sun, 17 Jul 2022 22:12:35 +0800 Subject: [PATCH 47/64] add missing return --- shodan/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/helpers.py b/shodan/helpers.py index 378b1bb..8900468 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -164,7 +164,7 @@ def humanize_bytes(byte_count, precision=1): if byte_count == 1: return '1 byte' if byte_count < 1024: - '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') + return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] multiple = 1024.0 # .0 to force float on python 2 From 4419d7167fad366b8942643037c427a62b0cb5bb Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 15 May 2023 09:50:54 -0700 Subject: [PATCH 48/64] Add support for the new 'fields' parameter of the /shodan/host/search method so we only grab the specific properties/ fields from the banner. --- setup.py | 2 +- shodan/__main__.py | 11 ++++++++--- shodan/client.py | 11 ++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 266ce81..e769440 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.28.0', + version='1.29.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 07af59c..f11a72a 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -255,10 +255,11 @@ def count(query): @main.command() +@click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str) @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, filename, query): +def download(fields, limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -276,6 +277,10 @@ def download(limit, filename, query): # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' + + # Strip out any whitespace in the fields and turn them into an array + if fields is not None: + fields = [item.strip() for item in fields.split(',')] # Perform the search api = shodan.Shodan(key) @@ -302,7 +307,7 @@ def download(limit, filename, query): with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False) + cursor = api.search_cursor(query, minify=False, fields=fields) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) @@ -485,7 +490,7 @@ def search(color, fields, limit, separator, query): # Perform the search api = shodan.Shodan(key) try: - results = api.search(query, limit=limit) + results = api.search(query, limit=limit, minify=False, fields=fields) except shodan.APIError as e: raise click.ClickException(e.value) diff --git a/shodan/client.py b/shodan/client.py index 70ca8f3..b9cb487 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -480,7 +480,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/{}'.format(scan_id), {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -495,6 +495,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru :type facets: str :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool + :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter + :type fields: str :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. """ @@ -511,10 +513,13 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) + + if fields and isinstance(fields, list): + args['fields'] = ','.join(fields) return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, fields=None): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -542,7 +547,7 @@ def search_cursor(self, query, minify=True, retries=5): while results['matches']: try: - results = self.search(query, minify=minify, page=page) + results = self.search(query, minify=minify, page=page, fields=fields) for banner in results['matches']: try: yield banner From 9ccc16ada4761d19ee2079a9334cac3ddbc62415 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 17 May 2023 11:00:46 -0700 Subject: [PATCH 49/64] The screenshot data has been moved to the top-level "screenshot" property. Update the helpers.get_screenshot() method to look in that location before falling back to the old opts.screenshot property. --- setup.py | 2 +- shodan/helpers.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e769440..3adcd8b 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.29.0', + version='1.29.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/helpers.py b/shodan/helpers.py index 378b1bb..432cbeb 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -122,8 +122,11 @@ def iterate_files(files, fast=False): def get_screenshot(banner): - if 'opts' in banner and 'screenshot' in banner['opts']: + if 'screenshot' in banner and banner['screenshot']: + return banner['screenshot'] + elif 'opts' in banner and 'screenshot' in banner['opts']: return banner['opts']['screenshot'] + return None From c97ebae69d3343da455bc5aa277c54f7cf3e115a Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sun, 18 Jun 2023 21:08:39 -0400 Subject: [PATCH 50/64] Added a check for file size of input, dynamically set workbook.use_zip64() --- shodan/__main__.py | 7 ++++++- shodan/cli/converter/csvc.py | 2 +- shodan/cli/converter/excel.py | 6 +++++- shodan/cli/converter/geojson.py | 2 +- shodan/cli/converter/images.py | 2 +- shodan/cli/converter/kml.py | 2 +- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f11a72a..aadb2d5 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -100,6 +100,7 @@ def convert(fields, input, format): Example: shodan convert data.json.gz kml """ + file_size = 0 # Check that the converter allows a custom list of fields converter_class = CONVERTERS.get(format) if fields: @@ -107,6 +108,10 @@ def convert(fields, input, format): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified + # Check file size of input + if os.path.exists(input): + file_size = os.path.getsize(input) + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -124,7 +129,7 @@ def convert(fields, input, format): # Initialize the file converter converter = converter_class(fout) - converter.process([input]) + converter.process([input], file_size) finished_event.set() progress_bar_thread.join() diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 20c3d03..2e4e2f2 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -50,7 +50,7 @@ class CsvConverter(Converter): 'title', ] - def process(self, files): + def process(self, files, file_size): writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') # Write the header diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index 24177eb..4db78a4 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -41,7 +41,7 @@ class ExcelConverter(Converter): 'http.title': 'Website Title', } - def process(self, files): + def process(self, files, file_size): # Get the filename from the already-open file handle filename = self.fout.name @@ -51,6 +51,10 @@ def process(self, files): # Create the new workbook workbook = Workbook(filename) + # Check if Excel file is larger than 5GB + if file_size > 5e9: + workbook.use_zip64() + # Define some common styles/ formats bold = workbook.add_format({ 'bold': 1, diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 3f6c975..83fb935 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -14,7 +14,7 @@ def header(self): def footer(self): self.fout.write("""{ }]}""") - def process(self, files): + def process(self, files, file_size): # Write the header self.header() diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py index 24c68c3..fba9d11 100644 --- a/shodan/cli/converter/images.py +++ b/shodan/cli/converter/images.py @@ -15,7 +15,7 @@ class ImagesConverter(Converter): # the user know where the images have been stored. dirname = None - def process(self, files): + def process(self, files, file_size): # Get the filename from the already-open file handle and use it as # the directory name to store the images. self.dirname = self.fout.name[:-7] + '-images' diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index 2cf3d44..9259ddf 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -13,7 +13,7 @@ def header(self): def footer(self): self.fout.write("""""") - def process(self, files): + def process(self, files, file_size): # Write the header self.header() From c3d6d63d4743fd71642b084ff8b2da56f37717eb Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:09:18 +0700 Subject: [PATCH 51/64] Add Shodan Trends API/ CLI --- shodan/__main__.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++ shodan/client.py | 57 ++++++++++++++++++++++++++----- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f11a72a..3a2fdf8 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -22,6 +22,7 @@ search stats stream + trends """ @@ -35,6 +36,7 @@ import threading import requests import time +import json # The file converters that are used to go from .json.gz to various other formats from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter @@ -799,6 +801,87 @@ def _create_stream(name, args, timeout): stream = _create_stream(stream_type, stream_args, timeout=timeout) +@main.command() +@click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) +@click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) +@click.option('--separator', help='The separator between the properties of the search results.', default='\t') +@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) +@click.argument('query', metavar='', nargs=-1) +def trends(filename, save, separator, facets, query): + """Search Shodan historical database""" + key = get_api_key() + api = shodan.Shodan(key) + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + facets = facets.strip() + + # Make sure the user didn't supply an empty query or facets + if query == '': + raise click.ClickException('Empty search query') + + if facets == '': + raise click.ClickException('Empty search facets') + + # Convert comma-separated facets string to list + parsed_facets = [] + for facet in facets.split(','): + parts = facet.strip().split(":") + if len(parts) > 1: + parsed_facets.append((parts[0], parts[1])) + else: + parsed_facets.append((parts[0])) + + # Perform the search + try: + results = api.trends.search(query, facets=parsed_facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') + + result_facets = list(results['facets'].keys()) + + # Save the results first to file if user request + if filename or save: + if not filename: + filename = '{}-trends.json.gz'.format(query.replace(' ', '-')) + elif not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Create/ append to the file + with helpers.open_file(filename) as fout: + for index, match in enumerate(results['matches']): + # Append facet info to make up a line + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) + + click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) + + # We buffer the entire output so we can use click's pager functionality + output = u'' + + # Output example: + # 2017-06 + # os + # Linux 3.x 384148 + # Windows 7 or 8 25531 + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + + for facet in result_facets: + output += ' ' + facet + u'\n' + for bucket in results['facets'][facet][index]['values']: + output += ' ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n' + + click.echo_via_pager(output) + + @main.command() @click.argument('ip', metavar='') def honeyscore(ip): diff --git a/shodan/client.py b/shodan/client.py index b9cb487..7b76fdc 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -85,7 +85,7 @@ class Notifier: def __init__(self, parent): self.parent = parent - + def create(self, provider, args, description=None): """Get the settings for the specified notifier that a user has configured. @@ -101,9 +101,9 @@ def create(self, provider, args, description=None): if description: args['description'] = description - + return self.parent._request('/notifier', args, method='post') - + def edit(self, nid, args): """Get the settings for the specified notifier that a user has configured. @@ -114,7 +114,7 @@ def edit(self, nid, args): :returns: dict -- fields are 'success' and 'id' of the notifier """ return self.parent._request('/notifier/{}'.format(nid), args, method='put') - + def get(self, nid): """Get the settings for the specified notifier that a user has configured. @@ -137,7 +137,7 @@ def list_providers(self): :returns: A list of providers where each object describes a provider """ return self.parent._request('/notifier/provider', {}) - + def remove(self, nid): """Delete the provided notifier. @@ -253,6 +253,42 @@ def remove_member(self, user): """ return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + class Trends: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, facets): + """Search the Shodan historical database. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'facets': create_facet_string(facets), + } + + return self.parent._request('/api/v1/search', args, service='trends') + + def search_facets(self): + """This method returns a list of facets that can be used to get a breakdown of the top values for a property. + + :returns: A list of strings where each is a facet name + """ + return self.parent._request('/api/v1/search/facets', {}, service='trends') + + def search_filters(self): + """This method returns a list of search filters that can be used in the search query. + + :returns: A list of strings where each is a filter name + """ + return self.parent._request('/api/v1/search/filters', {}, service='trends') + def __init__(self, key, proxies=None): """Initializes the API object. @@ -264,9 +300,11 @@ def __init__(self, key, proxies=None): self.api_key = key self.base_url = 'https://api.shodan.io' self.base_exploits_url = 'https://exploits.shodan.io' + self.base_trends_url = 'https://trends.shodan.io' self.data = self.Data(self) self.dns = self.Dns(self) self.exploits = self.Exploits(self) + self.trends = self.Trends(self) self.labs = self.Labs(self) self.notifier = self.Notifier(self) self.org = self.Organization(self) @@ -297,6 +335,7 @@ def _request(self, function, params, service='shodan', method='get'): base_url = { 'shodan': self.base_url, 'exploits': self.base_exploits_url, + 'trends': self.base_trends_url, }.get(service, 'shodan') # Wait for API rate limit @@ -513,7 +552,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) - + if fields and isinstance(fields, list): args['fields'] = ','.join(fields) @@ -733,17 +772,17 @@ def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): # a different API endpoint. if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') - + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') def unignore_alert_trigger_notification(self, aid, trigger, ip, port): """Re-enable trigger notifications for the provided IP and port""" return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') - + def add_alert_notifier(self, aid, nid): """Enable the given notifier for an alert that has triggers enabled.""" return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') - + def remove_alert_notifier(self, aid, nid): """Remove the given notifier for an alert that has triggers enabled.""" return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') From 9cc33f5700af88067391f3bd64fbb65001a666ee Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:10:47 +0700 Subject: [PATCH 52/64] Add API tests --- tests/test_shodan.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_shodan.py b/tests/test_shodan.py index f3405ce..ebe7a90 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -22,7 +22,8 @@ class ShodanTests(unittest.TestCase): } def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) + with open('SHODAN-API-KEY') as f: + self.api = shodan.Shodan(f.read().strip()) def test_search_simple(self): results = self.api.search(self.QUERIES['simple']) @@ -115,6 +116,24 @@ def test_exploits_count_facets(self): self.assertTrue(results['facets']['source']) self.assertTrue(len(results['facets']['author']) == 1) + def test_trends_search(self): + results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('matches', results) + self.assertIn('facets', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + + def test_trends_search_filters(self): + results = self.api.trends.search_filters() + self.assertIn('has_ipv6', results) + self.assertNotIn('http.html', results) + + def test_trends_search_facets(self): + results = self.api.trends.search_facets() + self.assertIn('product', results) + self.assertNotIn('cpe', results) + # Test error responses def test_invalid_key(self): api = shodan.Shodan('garbage') From 3fe0ad88b434fe480b7621f8fac20216c3578fea Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:11:34 +0700 Subject: [PATCH 53/64] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3adcd8b..2546ea7 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.29.1', + version='1.30.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 0ad61d20d4cf676269c798e573ded4439603b218 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 25 Jun 2023 16:46:00 -0700 Subject: [PATCH 54/64] Change cutoff to 4GB as that's what the xlsxwriter documentation says --- shodan/cli/converter/excel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index 4db78a4..2021a33 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -51,8 +51,8 @@ def process(self, files, file_size): # Create the new workbook workbook = Workbook(filename) - # Check if Excel file is larger than 5GB - if file_size > 5e9: + # Check if Excel file is larger than 4GB + if file_size > 4e9: workbook.use_zip64() # Define some common styles/ formats From 3d1f8922cd42420eba459a5d8bd41d974ad55ec1 Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Mon, 3 Jul 2023 10:57:56 +0700 Subject: [PATCH 55/64] Better output format --- shodan/__main__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 3a2fdf8..439752d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -279,7 +279,7 @@ def download(fields, limit, filename, query): # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' - + # Strip out any whitespace in the fields and turn them into an array if fields is not None: fields = [item.strip() for item in fields.split(',')] @@ -804,10 +804,9 @@ def _create_stream(name, args, timeout): @main.command() @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') @click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) @click.argument('query', metavar='', nargs=-1) -def trends(filename, save, separator, facets, query): +def trends(filename, save, facets, query): """Search Shodan historical database""" key = get_api_key() api = shodan.Shodan(key) @@ -869,15 +868,17 @@ def trends(filename, save, separator, facets, query): # Output example: # 2017-06 # os - # Linux 3.x 384148 - # Windows 7 or 8 25531 + # Linux 3.x 146,502 + # Windows 7 or 8 2,189 for index, match in enumerate(results['matches']): output += click.style(match['month'] + u'\n', fg='green') - - for facet in result_facets: - output += ' ' + facet + u'\n' - for bucket in results['facets'][facet][index]['values']: - output += ' ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n' + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) click.echo_via_pager(output) From 9e0f4dddbc5c8737270b70c119674abe618f44b5 Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 7 Jul 2023 08:34:16 +0700 Subject: [PATCH 56/64] Make facets as required arguments --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 3a81b0d..1c4a74d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -809,7 +809,7 @@ def _create_stream(name, args, timeout): @main.command() @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) +@click.argument('facets', metavar='') @click.argument('query', metavar='', nargs=-1) def trends(filename, save, facets, query): """Search Shodan historical database""" From a9d692b05aeea978632d17601cfec997bc8995cd Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 11 Jul 2023 23:07:34 -0400 Subject: [PATCH 57/64] Updated implementation to monitor hostnames for domain-based monitoring. --- requirements.txt | 3 ++- shodan/cli/alert.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5095f64..2692414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ click-plugins colorama requests>=2.2.1 XlsxWriter -ipaddress;python_version<='2.7' \ No newline at end of file +ipaddress;python_version<='2.7' +tldextract \ No newline at end of file diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 2dc3e58..0030589 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,6 +3,7 @@ import gzip import json import shodan +from tldextract import extract from collections import defaultdict from operator import itemgetter @@ -125,9 +126,15 @@ def alert_domain(domain, triggers): try: # Grab a list of IPs for the domain domain = domain.lower() + domain_parse = extract(domain) click.secho('Looking up domain information...', dim=True) info = api.dns.domain_info(domain, type='A') - domain_ips = set([record['value'] for record in info['data']]) + + if domain_parse.subdomain: + domain_ips = set([record['value'] for record in info['data'] + if record['subdomain'] == domain_parse.subdomain]) + else: + domain_ips = set([record['value'] for record in info['data']]) # Create the actual alert click.secho('Creating alert...', dim=True) From c90b3dd5ade683e16c7681c444790f448150a401 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Wed, 12 Jul 2023 22:35:36 -0400 Subject: [PATCH 58/64] Added input validation by updating click.argument for input parameter. --- shodan/__main__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 1c4a74d..8b4eb2b 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -93,7 +93,7 @@ def main(): @main.command() @click.option('--fields', help='List of properties to output.', default=None) -@click.argument('input', metavar='') +@click.argument('input', metavar='', type=click.Path(exists=True)) @click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) def convert(fields, input, format): """Convert the given input data file into a different format. The following file formats are supported: @@ -110,9 +110,8 @@ def convert(fields, input, format): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified - # Check file size of input - if os.path.exists(input): - file_size = os.path.getsize(input) + # click.Path ensures that file path exists + file_size = os.path.getsize(input) # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') From 1b7cb65e7e3d4b0490aa1009d5eaac72257a7e60 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Thu, 13 Jul 2023 20:07:17 -0400 Subject: [PATCH 59/64] Updated CLI logic to filter out private IPs when creating a domain-based alert. --- shodan/cli/alert.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 0030589..6f82c1a 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -4,6 +4,7 @@ import json import shodan from tldextract import extract +from ipaddress import ip_address from collections import defaultdict from operator import itemgetter @@ -120,6 +121,7 @@ def alert_create(name, netblocks): @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') def alert_domain(domain, triggers): """Create a network alert based on a domain name""" + flag = True key = get_api_key() api = shodan.Shodan(key) @@ -132,22 +134,30 @@ def alert_domain(domain, triggers): if domain_parse.subdomain: domain_ips = set([record['value'] for record in info['data'] - if record['subdomain'] == domain_parse.subdomain]) + if record['subdomain'] == domain_parse.subdomain and + not ip_address(record['value']).is_private]) else: - domain_ips = set([record['value'] for record in info['data']]) + domain_ips = set([record['value'] for record in info['data'] + if not ip_address(record['value']).is_private]) - # Create the actual alert - click.secho('Creating alert...', dim=True) - alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + if not domain_ips: + flag = False + click.secho('No external IPs were found to be associated with this domain. ' + 'No alert was created.', dim=True) + else: + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) - # Enable the triggers so it starts getting managed by Shodan Monitor - click.secho('Enabling triggers...', dim=True) - api.enable_alert_trigger(alert['id'], triggers) + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) except shodan.APIError as e: raise click.ClickException(e.value) - click.secho('Successfully created domain alert!', fg='green') - click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + if flag: + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') @alert.command(name='download') From 3036d83a9de0efb35202d094521f1bfd152d14b2 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Fri, 14 Jul 2023 16:31:08 -0400 Subject: [PATCH 60/64] Updating implementation based on reviewer recommendations. --- shodan/cli/alert.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 6f82c1a..1df11ea 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -121,7 +121,6 @@ def alert_create(name, netblocks): @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') def alert_domain(domain, triggers): """Create a network alert based on a domain name""" - flag = True key = get_api_key() api = shodan.Shodan(key) @@ -141,23 +140,21 @@ def alert_domain(domain, triggers): if not ip_address(record['value']).is_private]) if not domain_ips: - flag = False - click.secho('No external IPs were found to be associated with this domain. ' - 'No alert was created.', dim=True) - else: - # Create the actual alert - click.secho('Creating alert...', dim=True) - alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + raise click.ClickException('No external IPs were found to be associated with this domain. ' + 'No alert was created.') + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) - # Enable the triggers so it starts getting managed by Shodan Monitor - click.secho('Enabling triggers...', dim=True) - api.enable_alert_trigger(alert['id'], triggers) + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) except shodan.APIError as e: raise click.ClickException(e.value) - if flag: - click.secho('Successfully created domain alert!', fg='green') - click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') @alert.command(name='download') From 759a8561f821bf2e970f0594200221271bcbb7a9 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 19 Jul 2023 15:55:20 -0700 Subject: [PATCH 61/64] Update "shodan host" output to show certificate issuer/ subject and HTTP title --- shodan/cli/host.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shodan/cli/host.py b/shodan/cli/host.py index e90e372..8ffdeed 100644 --- a/shodan/cli/host.py +++ b/shodan/cli/host.py @@ -91,8 +91,20 @@ def host_print_pretty(host, history=False): click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) click.echo('') + # Show optional HTTP information + if 'http' in banner: + if 'title' in banner['http'] and banner['http']['title']: + click.echo('\t|-- HTTP title: {}'.format(banner['http']['title'])) + # Show optional ssl info if 'ssl' in banner: + if 'cert' in banner['ssl'] and banner['ssl']['cert']: + if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']: + issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()]) + click.echo('\t|-- Cert Issuer: {}'.format(issuer)) + if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']: + subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()]) + click.echo('\t|-- Cert Subject: {}'.format(subject)) if 'versions' in banner['ssl'] and banner['ssl']['versions']: click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: From 769e38448414c97eb7a34ecd3e88f9e0e280b2ee Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Thu, 20 Jul 2023 11:50:51 +0700 Subject: [PATCH 62/64] Make Trends API facets param as optional, if not supply then show total query results over time --- shodan/__main__.py | 54 ++++++++++++++++++++++++++++---------------- tests/test_shodan.py | 9 +++++++- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 8b4eb2b..4093b94 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -806,9 +806,9 @@ def _create_stream(name, args, timeout): @main.command() +@click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str) @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.argument('facets', metavar='') @click.argument('query', metavar='', nargs=-1) def trends(filename, save, facets, query): """Search Shodan historical database""" @@ -823,12 +823,12 @@ def trends(filename, save, facets, query): if query == '': raise click.ClickException('Empty search query') - if facets == '': - raise click.ClickException('Empty search facets') - # Convert comma-separated facets string to list parsed_facets = [] for facet in facets.split(','): + if not facet: + continue + parts = facet.strip().split(":") if len(parts) > 1: parsed_facets.append((parts[0], parts[1])) @@ -845,7 +845,9 @@ def trends(filename, save, facets, query): if results['total'] == 0: raise click.ClickException('No search results found') - result_facets = list(results['facets'].keys()) + result_facets = [] + if results.get("facets"): + result_facets = list(results["facets"].keys()) # Save the results first to file if user request if filename or save: @@ -858,31 +860,43 @@ def trends(filename, save, facets, query): with helpers.open_file(filename) as fout: for index, match in enumerate(results['matches']): # Append facet info to make up a line - match["facets"] = {} - for facet in result_facets: - match["facets"][facet] = results['facets'][facet][index]['values'] - line = json.dumps(match) + '\n' - fout.write(line.encode('utf-8')) + if result_facets: + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) # We buffer the entire output so we can use click's pager functionality output = u'' - # Output example: + # Output examples: + # - Facet by os # 2017-06 # os # Linux 3.x 146,502 # Windows 7 or 8 2,189 - for index, match in enumerate(results['matches']): - output += click.style(match['month'] + u'\n', fg='green') - if match['count'] > 0: - for facet in result_facets: - output += click.style(u' {}\n'.format(facet), fg='cyan') - for bucket in results['facets'][facet][index]['values']: - output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) - else: - output += u'{}\n'.format(click.style('N/A', bold=True)) + # + # - Without facets + # 2017-06 19,799,459 + # 2017-07 21,077,099 + if result_facets: + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) + else: + # Without facets, show query total results over time + for index, match in enumerate(results['matches']): + output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green')) click.echo_via_pager(output) diff --git a/tests/test_shodan.py b/tests/test_shodan.py index ebe7a90..94ffc70 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -118,12 +118,19 @@ def test_exploits_count_facets(self): def test_trends_search(self): results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('total', results) self.assertIn('matches', results) self.assertIn('facets', results) - self.assertIn('total', results) self.assertTrue(results['matches']) self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + results = self.api.trends.search('apache', facets=[]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertNotIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [match['month'] for match in results['matches']]) + def test_trends_search_filters(self): results = self.api.trends.search_filters() self.assertIn('has_ipv6', results) From a08353e40b2018ec0dbf04012e9a5bfbe87f55ec Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 13 Oct 2023 18:15:18 -0700 Subject: [PATCH 63/64] Improved handling of downloads to prevent it from exiting prematurely if the API only returns partial or missing results for a search results page. --- setup.py | 2 +- shodan/client.py | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 2546ea7..f421898 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.30.0', + version='1.30.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index 7b76fdc..ab81302 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -7,6 +7,7 @@ :copyright: (c) 2014- by John Matherly """ +import math import time import requests @@ -576,15 +577,23 @@ def search_cursor(self, query, minify=True, retries=5, fields=None): :returns: A search cursor that can be used as an iterator/ generator. """ page = 1 + total_pages = 0 tries = 0 - # Placeholder results object to make the while loop below easier - results = { - 'matches': [True], - 'total': None, - } + # Grab the initial page and use the total to calculate the expected number of pages + results = self.search(query, minify=minify, page=page, fields=fields) + if results['total']: + total_pages = int(math.ceil(results['total'] / 100)) + + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 - while results['matches']: + # Keep iterating over the results from page 2 onwards + while page <= total_pages: try: results = self.search(query, minify=minify, page=page, fields=fields) for banner in results['matches']: From 87a0688d1e5b7e4bb13ae4f5fd7cb937a671cba8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 16 Dec 2023 17:29:15 -0800 Subject: [PATCH 64/64] Consolidate all Shodan methods to use the internal _request method instead of sometimes using the shodan.helpers.api_request method. New environment variable SHODAN_API_URL that can be used to overwrite the base_url used for the API requests. --- setup.py | 2 +- shodan/client.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index f421898..53bbd9a 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.30.1', + version='1.31.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index ab81302..21c70af 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -8,13 +8,14 @@ :copyright: (c) 2014- by John Matherly """ import math +import os import time import requests import json from .exception import APIError -from .helpers import api_request, create_facet_string +from .helpers import create_facet_string from .stream import Stream @@ -314,11 +315,15 @@ def __init__(self, key, proxies=None): self._session = requests.Session() self.api_rate_limit = 1 # Requests per second self._api_query_time = None + if proxies: self._session.proxies.update(proxies) self._session.trust_env = False + + if os.environ.get('SHODAN_API_URL'): + self.base_url = os.environ.get('SHODAN_API_URL') - def _request(self, function, params, service='shodan', method='get'): + def _request(self, function, params, service='shodan', method='get', json_data=None): """General-purpose function to create web requests to SHODAN. Arguments: @@ -348,7 +353,13 @@ def _request(self, function, params, service='shodan', method='get'): try: method = method.lower() if method == 'post': - data = self._session.post(base_url + function, params) + if json_data: + data = self._session.post(base_url + function, params=params, + data=json.dumps(json_data), + headers={'content-type': 'application/json'}, + ) + else: + data = self._session.post(base_url + function, params) elif method == 'put': data = self._session.put(base_url + function, params=params) elif method == 'delete': @@ -711,8 +722,7 @@ def create_alert(self, name, ip, expires=0): 'expires': expires, } - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', - proxies=self._session.proxies) + response = self._request('/shodan/alert', params={}, json_data=data, method='post') return response @@ -732,8 +742,7 @@ def edit_alert(self, aid, ip): }, } - response = api_request(self.api_key, '/shodan/alert/{}'.format(aid), data=data, params={}, method='post', - proxies=self._session.proxies) + response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post') return response @@ -744,9 +753,9 @@ def alerts(self, aid=None, include_expired=True): else: func = '/shodan/alert/info' - response = api_request(self.api_key, func, params={ + response = self._request(func, params={ 'include_expired': include_expired, - }, proxies=self._session.proxies) + }) return response @@ -754,8 +763,7 @@ def delete_alert(self, aid): """Delete the alert with the given ID.""" func = '/shodan/alert/{}'.format(aid) - response = api_request(self.api_key, func, params={}, method='delete', - proxies=self._session.proxies) + response = self._request(func, params={}, method='delete') return response