From e4aadf2aa009da45a2516ff038fc9c76d705fb4f Mon Sep 17 00:00:00 2001 From: BlackVirusScript Date: Sat, 17 Mar 2018 14:42:55 -0300 Subject: [PATCH 001/148] Create `console_script` `entry_point` for `shodan` script. (Like this the console script can be executed on any platform.) --- setup.py | 2 +- bin/shodan => shodan/__main__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename bin/shodan => shodan/__main__.py (100%) mode change 100755 => 100644 diff --git a/setup.py b/setup.py index 1e4a214..8dfa7a4 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ author_email = 'jmath@shodan.io', url = 'http://github.com/achillean/shodan-python/tree/master', packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'], - scripts = ['bin/shodan'], + entry_points = {'console_scripts': ['shodan = shodan.__main__:main']}, install_requires = dependencies, keywords = ['security', 'network'], classifiers = [ diff --git a/bin/shodan b/shodan/__main__.py old mode 100755 new mode 100644 similarity index 100% rename from bin/shodan rename to shodan/__main__.py From 5977ad054901efd8ae91d96750965582240138eb Mon Sep 17 00:00:00 2001 From: BlackVirusScript Date: Fri, 23 Mar 2018 23:15:14 -0300 Subject: [PATCH 002/148] Update __main__.py --- shodan/__main__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 108acb6..f629803 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- """ Shodan CLI From 1a1c22af63968e7b49693a0f713d8d47a91645d5 Mon Sep 17 00:00:00 2001 From: Jeremy Bae Date: Wed, 23 May 2018 14:15:49 +0900 Subject: [PATCH 003/148] Add last_update time --- bin/shodan | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/shodan b/bin/shodan index 108acb6..da47cab 100755 --- a/bin/shodan +++ b/bin/shodan @@ -489,6 +489,9 @@ def host(format, history, filename, save, ip): if 'org' in host and host['org']: click.echo('{:25s}{}'.format('Organization:', host['org'])) + if 'last_update' in host and host['last_update']: + click.echo('{:25s}{}'.format('Updated:', host['last_update'])) + click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports']))) # Output the vulnerabilities the host has From 75f1ef0016f38bdf3b2e59ef9fa7650454f033c2 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 28 May 2018 06:52:57 -0500 Subject: [PATCH 004/148] Updated documentation to make it Python3 compatible and point to the new help center Prevent the worldmap from erroring out on city names that can't be printed to the terminal --- README.rst | 5 +++-- docs/examples/cert-stream.rst | 5 ++--- docs/index.rst | 6 +++++- docs/tutorial.rst | 28 ++++++++++++++-------------- shodan/cli/worldmap.py | 6 +++++- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index b53284f..0faf74e 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,8 @@ Features -------- - Search Shodan -- Streaming API support for real-time consumption of Shodan data +- Fast IP lookups +- Streaming API support for real-time consumption of Shodan firehose - Exploit search API fully implemented @@ -32,4 +33,4 @@ Or if you don't have pip installed (which you should seriously install): Documentation ------------- -Documentation is available at http://shodan.readthedocs.org/. +Documentation is available at https://shodan.readthedocs.org/ and https://help.shodan.io diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst index afa5e25..e3e72c1 100644 --- a/docs/examples/cert-stream.rst +++ b/docs/examples/cert-stream.rst @@ -39,9 +39,8 @@ information. for banner in api.stream.ports([443, 8443]): if 'ssl' in banner: # Print out all the SSL information that Shodan has collected - print banner['ssl'] + print(banner['ssl']) except Exception as e: - print 'Error: %s' % e + print('Error: {}'.format(e)) sys.exit(1) - diff --git a/docs/index.rst b/docs/index.rst index db16146..956e335 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,10 @@ Streaming API. And as a bonus it also lets you search for exploits using the Sho If you're not sure where to start simply go through the "Getting Started" section of the documentation and work your way down through the examples. +For more information about Shodan and how to use the API please visit our official help center at: + + https://help.shodan.io + Introduction ~~~~~~~~~~~~ .. toctree:: @@ -33,4 +37,4 @@ API Reference .. toctree:: :maxdepth: 2 - api \ No newline at end of file + api diff --git a/docs/tutorial.rst b/docs/tutorial.rst index e8efcb2..00c4d34 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -51,13 +51,13 @@ Now that we have our API object all good to go, we're ready to perform a search: results = api.search('apache') # Show the results - print 'Results found: %s' % results['total'] + print('Results found: {}'.format(results['total'])) for result in results['matches']: - print 'IP: %s' % result['ip_str'] - print result['data'] - print '' + print('IP: {}'.format(result['ip_str'])) + print(result['data']) + print('') except shodan.APIError, e: - print 'Error: %s' % e + print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which returns a dictionary of result information. We then print how many results were found in total, @@ -101,16 +101,16 @@ To see what Shodan has available on a specific IP we can use the :py:func:`Shoda host = api.host('217.140.75.46') # Print general info - print """ - IP: %s - Organization: %s - Operating System: %s - """ % (host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')) + print(""" + IP: {} + Organization: {} + Operating System: {} + """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a'))) # Print all banners for item in host['data']: - print """ - Port: %s - Banner: %s + print(""" + Port: {} + Banner: {} - """ % (item['port'], item['data']) \ No newline at end of file + """.format(item['port'], item['data'])) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index ad7b46a..9804aa2 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -125,7 +125,11 @@ def set_data(self, data): for banner in random.sample(data, min(len(data), 5)): desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code']) if banner['location']['city']: - desc += ' {}'.format(banner['location']['city']) + # Not all cities can be encoded in ASCII so ignore any errors + try: + desc += ' {}'.format(banner['location']['city']) + except: + pass if 'tags' in banner and banner['tags']: desc += ' / {}'.format(','.join(banner['tags'])) From f1fe2f2d4720062d1ef881584601957b7c09d753 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 28 May 2018 07:52:01 -0500 Subject: [PATCH 005/148] Add tcp/ udp information to ports when doing a host lookup (#64) --- bin/shodan | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/shodan b/bin/shodan index da47cab..7104eca 100755 --- a/bin/shodan +++ b/bin/shodan @@ -524,7 +524,9 @@ def host(format, history, filename, save, ip): if 'version' in banner and banner['version']: version = '({})'.format(banner['version']) - click.echo(click.style('{:>7d} '.format(banner['port']), fg='cyan'), nl=False) + click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) + click.echo('/', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) click.echo('{} {}'.format(product, version), nl=False) if history: From c46bb245acc522c8eec62e547f337cfbcb280ce4 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 28 May 2018 08:08:08 -0500 Subject: [PATCH 006/148] Show an error message instead of an empty screen if a search doesn't return results (#62) --- bin/shodan | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/shodan b/bin/shodan index 7104eca..994aaa6 100755 --- a/bin/shodan +++ b/bin/shodan @@ -994,6 +994,10 @@ def search(color, fields, limit, separator, query): results = api.search(query, limit=limit) except shodan.APIError as e: raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') # We buffer the entire output so we can use click's pager functionality output = '' From ca08e3f25b677259b465cc813600b446ef808c6d Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 28 May 2018 23:38:59 -0500 Subject: [PATCH 007/148] Show open ports even if the API key doesn't have access to the data on those ports (#63) --- bin/shodan | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/bin/shodan b/bin/shodan index 994aaa6..c06e0bd 100755 --- a/bin/shodan +++ b/bin/shodan @@ -515,6 +515,26 @@ def host(format, history, filename, save, ip): click.echo('') + # If the user doesn't have access to SSL/ Telnet results then we need + # to pad the host['data'] property with empty banners so they still see + # the port listed as open. (#63) + if len(host['ports']) != len(host['data']): + # Find the ports the user can't see the data for + ports = host['ports'] + for banner in host['data']: + if banner['port'] in ports: + ports.remove(banner['port']) + + # Add the placeholder banners + for port in ports: + banner = { + 'port': port, + 'transport': 'tcp', # All the filtered services use TCP + 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner + 'placeholder': True, # Don't store this banner when the file is saved + } + host['data'].append(banner) + click.echo('Ports:') for banner in sorted(host['data'], key=lambda k: k['port']): product = '' @@ -558,7 +578,8 @@ def host(format, history, filename, save, ip): fout = helpers.open_file(filename) for banner in sorted(host['data'], key=lambda k: k['port']): - helpers.write_banner(fout, banner) + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) except shodan.APIError as e: raise click.ClickException(e.value) From 4ccbe7152db100c3197da3fc597fb92becd880f5 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 29 May 2018 05:26:34 -0500 Subject: [PATCH 008/148] Release 1.8.0 Updated changelog with notable fixes/ improvements --- CHANGES => CHANGELOG.md | 14 +++++++++----- setup.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) rename CHANGES => CHANGELOG.md (71%) diff --git a/CHANGES b/CHANGELOG.md similarity index 71% rename from CHANGES rename to CHANGELOG.md index 1863461..34a1bb2 100644 --- a/CHANGES +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ -CHANGES -======= +CHANGELOG +========= -in development --------------- -* added CHANGES file +1.8.0 +----- +* Shodan CLI now installs properly on Windows (#66) +* Improved output of "shodan host" (#64, #67) +* Fixed bug that prevented an open port from being shown in "shodan host" (#63) +* No longer show an empty page if "shodan search" didn't return results (#62) +* Updated docs to make them Python3 compatible 1.7.7 ----- diff --git a/setup.py b/setup.py index 8dfa7a4..022b5a8 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = 'shodan', - version = '1.7.7', + version = '1.8.0', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', author = 'John Matherly', author_email = 'jmath@shodan.io', From f46e3836fe055330ec3e73ae74f1699b6230493d Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 31 May 2018 21:33:27 -0500 Subject: [PATCH 009/148] Minor housekeeping tasks (updated docs, metadata, license years) --- LICENSE | 2 +- README.rst | 33 ++++++++++++++++++++++++++++++++- setup.py | 9 +++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index e344a97..0af97af 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014 John Matherly +Copyright (c) 2014- John Matherly Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/README.rst b/README.rst index 0faf74e..2f4d675 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,12 @@ shodan: The official Python library for accessing Shodan ======================================================== +.. image:: https://img.shields.io/pypi/v/shodan.svg + :target: https://pypi.org/project/shodan/ + +.. image:: https://img.shields.io/github/contributors/achillean/shodan-python.svg + :target: https://github.com/achillean/shodan-python/graphs/contributors + Shodan is a search engine for Internet-connected devices. Google lets you search for websites, Shodan lets you search for devices. This library provides developers easy access to all of the data stored in Shodan in order to automate tasks and integrate into existing tools. @@ -9,10 +15,35 @@ Features -------- - Search Shodan -- Fast IP lookups +- `Fast/ bulk IP lookups `_ - Streaming API support for real-time consumption of Shodan firehose +- `Network alerts (aka private firehose) `_ - Exploit search API fully implemented +- Bulk data downloads + + +Quick Start +----------- + +```python +from shodan import Shodan + +api = Shodan('MY API KEY') + +# Lookup an IP +ipinfo = api.host('8.8.8.8') +print(ipinfo) + +# Search for websites that have been "hacked" +for banner in api.search_cursor('http.title:"hacked by"'): + print(banner) + +# Get the total number of industrial control systems services on the Internet +ics_services = api.count('tag:ics') +print('Industrial Control Systems: {}'.format(ics_services['total'])) +``` +Grab your API key from https://account.shodan.io Installation ------------ diff --git a/setup.py b/setup.py index 022b5a8..fc6cbd6 100755 --- a/setup.py +++ b/setup.py @@ -16,10 +16,19 @@ install_requires = dependencies, keywords = ['security', 'network'], classifiers = [ + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries :: Python Modules', ], ) From 04fa27c9abf6257f75c0821187fa0f04bd902df5 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 31 May 2018 21:34:31 -0500 Subject: [PATCH 010/148] Fix README to use rst --- README.rst | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 2f4d675..308428a 100644 --- a/README.rst +++ b/README.rst @@ -25,23 +25,22 @@ Features Quick Start ----------- -```python -from shodan import Shodan +.. code-block:: python + from shodan import Shodan -api = Shodan('MY API KEY') + api = Shodan('MY API KEY') -# Lookup an IP -ipinfo = api.host('8.8.8.8') -print(ipinfo) + # Lookup an IP + ipinfo = api.host('8.8.8.8') + print(ipinfo) -# Search for websites that have been "hacked" -for banner in api.search_cursor('http.title:"hacked by"'): - print(banner) + # Search for websites that have been "hacked" + for banner in api.search_cursor('http.title:"hacked by"'): + print(banner) -# Get the total number of industrial control systems services on the Internet -ics_services = api.count('tag:ics') -print('Industrial Control Systems: {}'.format(ics_services['total'])) -``` + # Get the total number of industrial control systems services on the Internet + ics_services = api.count('tag:ics') + print('Industrial Control Systems: {}'.format(ics_services['total'])) Grab your API key from https://account.shodan.io From 335c3bf822c24d77b88f3308a2eeebc2adf21a0e Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 31 May 2018 21:35:23 -0500 Subject: [PATCH 011/148] Another fix for README --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 308428a..d0a40ae 100644 --- a/README.rst +++ b/README.rst @@ -26,6 +26,7 @@ Quick Start ----------- .. code-block:: python + from shodan import Shodan api = Shodan('MY API KEY') From 88ec367e852eda0739611a59c95a15503b6ed951 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 2 Jun 2018 01:57:04 -0500 Subject: [PATCH 012/148] Disable the heartbeat messages for the Streaming API if the user requested a timeout (#70) --- shodan/stream.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/shodan/stream.py b/shodan/stream.py index 864b1c5..9864854 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -13,14 +13,25 @@ def __init__(self, api_key): self.api_key = api_key def _create_stream(self, name, timeout=None): + params = { + 'key': self.api_key, + } + stream_url = self.base_url + name + # The user doesn't want to use a timeout # If the timeout is specified as 0 then we also don't want to have a timeout if ( timeout and timeout <= 0 ) or ( timeout == 0 ): timeout = None + # If the user requested a timeout then we need to disable heartbeat messages + # which are intended to keep stream connections alive even if there isn't any data + # flowing through. + if timeout: + params['heartbeat'] = False + try: while True: - req = requests.get(self.base_url + name, params={'key': self.api_key}, stream=True, timeout=timeout) + req = requests.get(stream_url, params=params, stream=True, timeout=timeout) # Status code 524 is special to Cloudflare # It means that no data was sent from the streaming servers which caused Cloudflare @@ -46,7 +57,7 @@ def _create_stream(self, name, timeout=None): req.encoding = 'utf-8' return req - def _iter_stream(self, stream, raw, timeout=None): + def _iter_stream(self, stream, raw): for line in stream.iter_lines(decode_unicode=True): # The Streaming API sends out heartbeat messages that are newlines # We want to ignore those messages since they don't contain any data @@ -55,16 +66,6 @@ def _iter_stream(self, stream, raw, timeout=None): yield line else: yield json.loads(line) - else: - # If the user specified a timeout then we want to keep track of how long we've - # been getting heartbeat messages and exit the loop if it's been too long since - # we've seen any activity. - if timeout: - # TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke - # the ability to use inactivity timeouts (the connection timeout still works). The timeout is - # mostly needed when doing on-demand scans and wanting to temporarily consume data from a - # network alert. - pass def alert(self, aid=None, timeout=None, raw=False): if aid: From 915a5e50d846890ebd09bd68ff8af67fc7422e4b Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 2 Jun 2018 02:14:35 -0500 Subject: [PATCH 013/148] Bugfix release 1.8.1 Minor fix to check that dhparams is not empty before using it --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/__main__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34a1bb2..f97dd26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.8.1 +----- +* Fixed bug that prevented **shodan scan submit** from finishing (#70) + 1.8.0 ----- * Shodan CLI now installs properly on Windows (#66) diff --git a/setup.py b/setup.py index fc6cbd6..6520d3a 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = 'shodan', - version = '1.8.0', + version = '1.8.1', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', author = 'John Matherly', author_email = 'jmath@shodan.io', diff --git a/shodan/__main__.py b/shodan/__main__.py index 256f199..4cb1ca6 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -898,7 +898,7 @@ def print_banner(banner): versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')] if len(versions) > 0: click.echo(' |-- SSL Versions: {}'.format(', '.join(versions))) - if 'dhparams' in banner['ssl']: + if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: click.echo(' |-- Diffie-Hellman Parameters:') click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) if 'fingerprint' in banner['ssl']['dhparams']: From 9cc9988609434eb52043d38fa531611a4195cd8a Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 2 Jun 2018 02:29:54 -0500 Subject: [PATCH 014/148] Add description which will be shown on PyPi --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6520d3a..59acf83 100755 --- a/setup.py +++ b/setup.py @@ -2,18 +2,21 @@ from setuptools import setup -dependencies = open('requirements.txt', 'r').read().split('\n') +DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') +README = open('README.md', 'r').read() setup( name = 'shodan', version = '1.8.1', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', + long_description = README, + long_description_content_type = 'text/markdown', author = 'John Matherly', author_email = 'jmath@shodan.io', url = 'http://github.com/achillean/shodan-python/tree/master', packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'], entry_points = {'console_scripts': ['shodan = shodan.__main__:main']}, - install_requires = dependencies, + install_requires = DEPENDENCIES, keywords = ['security', 'network'], classifiers = [ 'Development Status :: 5 - Production/Stable', From 6d755dd3135e74b9f1ad04361b8db2a62d471e67 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 2 Jun 2018 02:32:25 -0500 Subject: [PATCH 015/148] Update description to load the README.rst --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 59acf83..9aa775c 100755 --- a/setup.py +++ b/setup.py @@ -3,14 +3,14 @@ from setuptools import setup DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') -README = open('README.md', 'r').read() +README = open('README.rst', 'r').read() setup( name = 'shodan', version = '1.8.1', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, - long_description_content_type = 'text/markdown', + long_description_content_type = 'text/x-rst', author = 'John Matherly', author_email = 'jmath@shodan.io', url = 'http://github.com/achillean/shodan-python/tree/master', From bd1fb8927f74a873b55fe3913bff048f4743ccdd Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 10 Jul 2018 11:24:40 +0200 Subject: [PATCH 016/148] Add proxy parameter fixes achillean/shodan-python#72 --- CHANGELOG.md | 4 +++ shodan/__main__.py | 4 +-- shodan/client.py | 87 ++++++++++++++++++++++++--------------------- shodan/helpers.py | 32 +++++++++-------- shodan/stream.py | 14 ++++---- shodan/threatnet.py | 12 ++++--- 6 files changed, 86 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f97dd26..7e43fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +unreleased +---------- +* New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72) + 1.8.1 ----- * Fixed bug that prevented **shodan scan submit** from finishing (#70) diff --git a/shodan/__main__.py b/shodan/__main__.py index 4cb1ca6..3a2dded 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -522,7 +522,7 @@ def host(format, history, filename, save, ip): for banner in host['data']: if banner['port'] in ports: ports.remove(banner['port']) - + # Add the placeholder banners for port in ports: banner = { @@ -1013,7 +1013,7 @@ def search(color, fields, limit, separator, query): results = api.search(query, limit=limit) except shodan.APIError as e: raise click.ClickException(e.value) - + # Error out if no results were found if results['total'] == 0: raise click.ClickException('No search results found') diff --git a/shodan/client.py b/shodan/client.py index de2cd78..6c19e48 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -43,7 +43,7 @@ class Shodan: :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. """ - + class Data: def __init__(self, parent): @@ -62,7 +62,7 @@ def list_files(self, dataset): :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' """ return self.parent._request('/shodan/data/{}'.format(dataset), {}) - + class Tools: def __init__(self, parent): @@ -74,16 +74,16 @@ def myip(self): :returns: str -- your IP address """ return self.parent._request('/tools/myip', {}) - + class Exploits: def __init__(self, parent): self.parent = parent - + def search(self, query, page=1, facets=None): """Search the entire Shodan Exploits archive using the same query syntax as the website. - + :param query: The exploit search query; same syntax as website. :type query: str :param facets: A list of strings or tuples to get summary information on. @@ -100,17 +100,17 @@ def search(self, query, page=1, facets=None): query_args['facets'] = create_facet_string(facets) return self.parent._request('/api/search', query_args, service='exploits') - + def count(self, query, facets=None): """Search the entire Shodan Exploits archive but only return the total # of results, not the actual exploits. - + :param query: The exploit search query; same syntax as website. :type query: str :param facets: A list of strings or tuples to get summary information on. :type facets: str :returns: dict -- a dictionary containing the results of the search. - + """ query_args = { 'query': query, @@ -119,7 +119,7 @@ def count(self, query, facets=None): query_args['facets'] = create_facet_string(facets) return self.parent._request('/api/count', query_args, service='exploits') - + class Labs: def __init__(self, parent): @@ -127,19 +127,21 @@ def __init__(self, parent): def honeyscore(self, ip): """Calculate the probability of an IP being an ICS honeypot. - + :param ip: IP address of the device :type ip: str :returns: int -- honeyscore ranging from 0.0 to 1.0 """ return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) - - def __init__(self, key): + + def __init__(self, key, proxies=None): """Initializes the API object. - + :param key: The Shodan API key. :type key: str + :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} + :type key: dict """ self.api_key = key self.base_url = 'https://api.shodan.io' @@ -148,23 +150,25 @@ def __init__(self, key): self.exploits = self.Exploits(self) self.labs = self.Labs(self) self.tools = self.Tools(self) - self.stream = Stream(key) + self.stream = Stream(key, proxies=proxies) self._session = requests.Session() - + if proxies: + self._session.proxies.update(proxies) + def _request(self, function, params, service='shodan', method='get'): """General-purpose function to create web requests to SHODAN. - + Arguments: function -- name of the function you want to execute params -- dictionary of parameters for the function - + Returns A dictionary containing the function's results. - + """ # Add the API key parameter automatically params['key'] = self.api_key - + # Determine the base_url based on which service we're interacting with base_url = { 'shodan': self.base_url, @@ -187,22 +191,22 @@ def _request(self, function, params, service='shodan', method='get'): error = data.json()['error'] except Exception as e: error = 'Invalid API key' - + raise APIError(error) - + # Parse the text into JSON try: data = data.json() except: raise APIError('Unable to parse JSON response') - + # Raise an exception if an error occurred if type(data) == dict and 'error' in data: raise APIError(data['error']) - + # Return the data return data - + def count(self, query, facets=None): """Returns the total number of search results for the query. @@ -210,7 +214,7 @@ def count(self, query, facets=None): :type query: str :param facets: (optional) A list of properties to get summary information on :type facets: str - + :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. """ query_args = { @@ -219,7 +223,7 @@ def count(self, query, facets=None): if facets: query_args['facets'] = create_facet_string(facets) return self._request('/shodan/host/count', query_args) - + def host(self, ips, history=False, minify=False): """Get all available information on an IP. @@ -232,14 +236,14 @@ def host(self, ips, history=False, minify=False): """ if isinstance(ips, basestring): ips = [ips] - + params = {} if history: params['history'] = history if minify: params['minify'] = minify return self._request('/shodan/host/%s' % ','.join(ips), params) - + def info(self): """Returns information about the current API key, such as a list of add-ons and other features that are enabled for the current user's API plan. @@ -281,7 +285,7 @@ def scan(self, ips, force=False): """ if isinstance(ips, basestring): ips = [ips] - + if isinstance(ips, dict): networks = json.dumps(ips) else: @@ -320,7 +324,7 @@ def scan_status(self, scan_id): :returns: A dictionary with general information about the scan, including its status in getting processed. """ return self._request('/shodan/scan/%s' % scan_id, {}) - + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): """Search the SHODAN database. @@ -336,8 +340,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru :type facets: str :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool - - :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. + + :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. """ args = { 'query': query, @@ -352,9 +356,9 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) - + return self._request('/shodan/host/search', args) - + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. @@ -369,7 +373,7 @@ def search_cursor(self, query, minify=True, retries=5): :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type minify: int - + :returns: A search cursor that can be used as an iterator/ generator. """ args = { @@ -396,13 +400,13 @@ def search_cursor(self, query, minify=True, retries=5): tries += 1 time.sleep(1.0) # wait 1 second if the search errored out for some reason - + def search_tokens(self, query): """Returns information about the search query itself (filters used etc.) :param query: Search query; identical syntax to the website :type query: str - + :returns: A dictionary with 4 main properties: filters, errors, attributes and string. """ query_args = { @@ -481,7 +485,8 @@ def create_alert(self, name, ip, expires=0): 'expires': expires, } - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post') + response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', + proxies=self._session.proxies) return response @@ -494,7 +499,8 @@ def alerts(self, aid=None, include_expired=True): response = api_request(self.api_key, func, params={ 'include_expired': include_expired, - }) + }, + proxies=self._session.proxies) return response @@ -502,7 +508,8 @@ def delete_alert(self, aid): """Delete the alert with the given ID.""" func = '/shodan/alert/%s' % aid - response = api_request(self.api_key, func, params={}, method='delete') + response = api_request(self.api_key, func, params={}, method='delete', + proxies=self._session.proxies) return response diff --git a/shodan/helpers.py b/shodan/helpers.py index dcf0afe..95df590 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -23,17 +23,19 @@ def create_facet_string(facets): facet_str += ',' return facet_str[:-1] - -def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', method='get', retries=1): + +def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', + method='get', retries=1, proxies=None): """General-purpose function to create web requests to SHODAN. - + Arguments: function -- name of the function you want to execute params -- dictionary of parameters for the function - + proxies -- a proxies array for the requests library + Returns A dictionary containing the function's results. - + """ # Add the API key parameter automatically params['key'] = key @@ -44,11 +46,13 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho while tries <= retries: try: if method.lower() == 'post': - data = requests.post(base_url + function, json.dumps(data), params=params, headers={'content-type': 'application/json'}) + data = requests.post(base_url + function, json.dumps(data), params=params, + headers={'content-type': 'application/json'}, + proxies=proxies) elif method.lower() == 'delete': - data = requests.delete(base_url + function, params=params) + data = requests.delete(base_url + function, params=params, proxies=proxies) else: - data = requests.get(base_url + function, params=params) + data = requests.get(base_url + function, params=params, proxies=proxies) # Exit out of the loop break @@ -66,17 +70,17 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho except: pass raise APIError('Invalid API key') - + # Parse the text into JSON try: data = data.json() except: raise APIError('Unable to parse JSON response') - + # Raise an exception if an error occurred if type(data) == dict and data.get('error', None): raise APIError(data['error']) - + # Return the data return data @@ -93,10 +97,10 @@ def iterate_files(files, fast=False): from ujson import loads except: pass - + if isinstance(files, basestring): files = [files] - + for filename in files: # Create a file handle depending on the filetype if filename.endswith('.gz'): @@ -157,7 +161,7 @@ def humanize_bytes(bytes, precision=1): return '1 byte' if bytes < 1024: return '%.*f %s' % (precision, bytes, "bytes") - + suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] multiple = 1024.0 #.0 force float on python 2 for suffix in suffixes: diff --git a/shodan/stream.py b/shodan/stream.py index 9864854..49ab633 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -9,29 +9,31 @@ class Stream: base_url = 'https://stream.shodan.io' - def __init__(self, api_key): + def __init__(self, api_key, proxies=None): self.api_key = api_key + self.proxies = proxies def _create_stream(self, name, timeout=None): params = { 'key': self.api_key, } stream_url = self.base_url + name - + # The user doesn't want to use a timeout # If the timeout is specified as 0 then we also don't want to have a timeout if ( timeout and timeout <= 0 ) or ( timeout == 0 ): timeout = None - + # If the user requested a timeout then we need to disable heartbeat messages # which are intended to keep stream connections alive even if there isn't any data # flowing through. if timeout: params['heartbeat'] = False - + try: while True: - req = requests.get(stream_url, params=params, stream=True, timeout=timeout) + req = requests.get(stream_url, params=params, stream=True, timeout=timeout, + proxies=self.proxies) # Status code 524 is special to Cloudflare # It means that no data was sent from the streaming servers which caused Cloudflare @@ -121,4 +123,4 @@ def ports(self, ports, raw=False, timeout=None): stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) for line in self._iter_stream(stream, raw): yield line - + diff --git a/shodan/threatnet.py b/shodan/threatnet.py index 792a924..8b609f0 100644 --- a/shodan/threatnet.py +++ b/shodan/threatnet.py @@ -11,17 +11,19 @@ class Threatnet: :type key: str :ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API. """ - + class Stream: base_url = 'https://stream.shodan.io' - def __init__(self, parent): + def __init__(self, parent, proxies=None): self.parent = parent + self.proxies = proxies def _create_stream(self, name): try: - req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True) + req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, + stream=True, proxies=self.proxies) except: raise APIError('Unable to contact the Shodan Streaming API') @@ -53,10 +55,10 @@ def activity(self): if line: banner = json.loads(line) yield banner - + def __init__(self, key): """Initializes the API object. - + :param key: The Shodan API key. :type key: str """ From c161708a3cdcec818360f5030a79c770520bcafc Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 1 Aug 2018 01:22:00 -0500 Subject: [PATCH 017/148] Release 1.9.0 --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e43fa4..840f1ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ CHANGELOG ========= -unreleased +1.9.0 ---------- * New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72) diff --git a/setup.py b/setup.py index 9aa775c..c52e5e0 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.8.1', + version = '1.9.0', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', From d8b124e64fd1ea6871b4617dcfcc7e5ae16508f4 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 1 Aug 2018 10:23:24 +0200 Subject: [PATCH 018/148] DOC: add changelog to manifest --- CHANGELOG.md | 6 +++++- MANIFEST.in | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840f1ab..5bdc40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ CHANGELOG ========= -1.9.0 +unreleased ---------- +* The CHANGELOG is now part of the packages. + +1.9.0 +----- * New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72) 1.8.1 diff --git a/MANIFEST.in b/MANIFEST.in index 5fdf9c3..4e76fba 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include AUTHORS include LICENSE include requirements.txt +inlcude CHANGELOG.md graft docs recursive-include shodan *.py From 48047b4973ddbde48bae1c2e21848f500d2ec7a1 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 1 Aug 2018 17:59:40 -0500 Subject: [PATCH 019/148] Fixed a typo --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4e76fba..4ba799c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include AUTHORS include LICENSE include requirements.txt -inlcude CHANGELOG.md +include CHANGELOG.md graft docs recursive-include shodan *.py From e6137b5b63515607bb6fc8e5a8e8ca7c7a9e7068 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 14:14:15 -0500 Subject: [PATCH 020/148] Ensure strings that can contain non-ascii characters are treated as unicode for Python2 (#78) --- shodan/__main__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 3a2dded..1c2dbe4 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -212,7 +212,7 @@ def alert_clear(): try: alerts = api.alerts() for alert in alerts: - click.echo('Removing {} ({})'.format(alert['name'], alert['id'])) + click.echo(u'Removing {} ({})'.format(alert['name'], alert['id'])) api.delete_alert(alert['id']) except shodan.APIError as e: raise click.ClickException(e.value) @@ -249,11 +249,11 @@ def alert_list(expired): raise click.ClickException(e.value) if len(results) > 0: - click.echo('# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) + click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) # click.echo('#' * 65) for alert in results: click.echo( - '{:16} {:<30} {:<35} '.format( + u'{:16} {:<30} {:<35} '.format( click.style(alert['id'], fg='yellow'), click.style(alert['name'], fg='cyan'), click.style(', '.join(alert['filters']['ip']), fg='white') @@ -326,7 +326,7 @@ def data_list(dataset): files = api.data.list_files(dataset) for file in files: - click.echo(click.style('{:20s}'.format(file['name']), fg='cyan'), nl=False) + click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) click.echo('{}'.format(file['url'])) else: @@ -473,19 +473,19 @@ def host(format, history, filename, save, ip): # General info click.echo(click.style(ip, fg='green')) if len(host['hostnames']) > 0: - click.echo('{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) + click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) if 'city' in host and host['city']: - click.echo('{:25s}{}'.format('City:', host['city'])) + click.echo(u'{:25s}{}'.format('City:', host['city'])) if 'country_name' in host and host['country_name']: - click.echo('{:25s}{}'.format('Country:', host['country_name'])) + click.echo(u'{:25s}{}'.format('Country:', host['country_name'])) if 'os' in host and host['os']: - click.echo('{:25s}{}'.format('Operating System:', host['os'])) + click.echo(u'{:25s}{}'.format('Operating System:', host['os'])) if 'org' in host and host['org']: - click.echo('{:25s}{}'.format('Organization:', host['org'])) + click.echo(u'{:25s}{}'.format('Organization:', host['org'])) if 'last_update' in host and host['last_update']: click.echo('{:25s}{}'.format('Updated:', host['last_update'])) From 743b445cf7babe9e3742bdff4406bf0b1760b7a8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 14:15:28 -0500 Subject: [PATCH 021/148] Add CHANGELOG entry for #78 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdc40e..64817d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ CHANGELOG unreleased ---------- * The CHANGELOG is now part of the packages. +* Improved unicode handling in Python2 (#78) 1.9.0 ----- From a3e5855779456ba7a6bcba05c8a4ebf2b21a9743 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 15:52:58 -0500 Subject: [PATCH 022/148] Add tsv output format for shodan host (#65) --- CHANGELOG.md | 1 + shodan/__main__.py | 97 ++---------------------------------- shodan/cli/host.py | 121 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 93 deletions(-) create mode 100644 shodan/cli/host.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 64817d4..dae37d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ unreleased ---------- * The CHANGELOG is now part of the packages. * Improved unicode handling in Python2 (#78) +* Add `tsv` output format for **shodan host** (#65) 1.9.0 ----- diff --git a/shodan/__main__.py b/shodan/__main__.py index 1c2dbe4..f247432 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -49,6 +49,7 @@ # Helper methods from shodan.cli.helpers import get_api_key +from shodan.cli.host import HOST_PRINT # Allow 3rd-parties to develop custom commands from click_plugins import with_plugins @@ -457,7 +458,7 @@ def download(limit, filename, query): @main.command() -@click.option('--format', help='The output format for the host information. Possible values are: pretty, csv, tsv. (placeholder)', default='pretty', type=str) +@click.option('--format', help='The output format for the host information. Possible values are: pretty, tsv.', default='pretty', type=click.Choice(['pretty', 'tsv'])) @click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True) @click.option('--filename', '-O', help='Save the host information in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the host information in the a file named after the IP (append if file exists).', default=False, is_flag=True) @@ -470,98 +471,8 @@ def host(format, history, filename, save, ip): try: host = api.host(ip, history=history) - # General info - click.echo(click.style(ip, fg='green')) - if len(host['hostnames']) > 0: - click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) - - if 'city' in host and host['city']: - click.echo(u'{:25s}{}'.format('City:', host['city'])) - - if 'country_name' in host and host['country_name']: - click.echo(u'{:25s}{}'.format('Country:', host['country_name'])) - - if 'os' in host and host['os']: - click.echo(u'{:25s}{}'.format('Operating System:', host['os'])) - - if 'org' in host and host['org']: - click.echo(u'{:25s}{}'.format('Organization:', host['org'])) - - if 'last_update' in host and host['last_update']: - click.echo('{:25s}{}'.format('Updated:', host['last_update'])) - - click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports']))) - - # Output the vulnerabilities the host has - if 'vulns' in host and len(host['vulns']) > 0: - vulns = [] - for vuln in host['vulns']: - if vuln.startswith('!'): - continue - if vuln.upper() == 'CVE-2014-0160': - vulns.append(click.style('Heartbleed', fg='red')) - else: - vulns.append(click.style(vuln, fg='red')) - - if len(vulns) > 0: - click.echo('{:25s}'.format('Vulnerabilities:'), nl=False) - - for vuln in vulns: - click.echo(vuln + '\t', nl=False) - - click.echo('') - - click.echo('') - - # If the user doesn't have access to SSL/ Telnet results then we need - # to pad the host['data'] property with empty banners so they still see - # the port listed as open. (#63) - if len(host['ports']) != len(host['data']): - # Find the ports the user can't see the data for - ports = host['ports'] - for banner in host['data']: - if banner['port'] in ports: - ports.remove(banner['port']) - - # Add the placeholder banners - for port in ports: - banner = { - 'port': port, - 'transport': 'tcp', # All the filtered services use TCP - 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner - 'placeholder': True, # Don't store this banner when the file is saved - } - host['data'].append(banner) - - click.echo('Ports:') - for banner in sorted(host['data'], key=lambda k: k['port']): - product = '' - version = '' - if 'product' in banner and banner['product']: - product = banner['product'] - if 'version' in banner and banner['version']: - version = '({})'.format(banner['version']) - - click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) - click.echo('/', nl=False) - click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) - click.echo('{} {}'.format(product, version), nl=False) - - if history: - # Format the timestamp to only show the year-month-day - date = banner['timestamp'][:10] - click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) - click.echo('') - - # Show optional ssl info - if 'ssl' in banner: - if 'versions' in banner['ssl'] and banner['ssl']['versions']: - click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) - if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: - click.echo('\t|-- Diffie-Hellman Parameters:') - click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) - if 'fingerprint' in banner['ssl']['dhparams']: - click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) + # Print the host information to the terminal using the user-specified format + HOST_PRINT[format](host, history=history) # Store the results if filename or save: diff --git a/shodan/cli/host.py b/shodan/cli/host.py new file mode 100644 index 0000000..fcd440f --- /dev/null +++ b/shodan/cli/host.py @@ -0,0 +1,121 @@ +# Helper methods for printing `host` information to the terminal. +import click + +from shodan.helpers import get_ip + + +def host_print_pretty(host, history=False): + """Show the host information in a user-friendly way and try to include + as much relevant information as possible.""" + # General info + click.echo(click.style(get_ip(host), fg='green')) + if len(host['hostnames']) > 0: + click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) + + if 'city' in host and host['city']: + click.echo(u'{:25s}{}'.format('City:', host['city'])) + + if 'country_name' in host and host['country_name']: + click.echo(u'{:25s}{}'.format('Country:', host['country_name'])) + + if 'os' in host and host['os']: + click.echo(u'{:25s}{}'.format('Operating System:', host['os'])) + + if 'org' in host and host['org']: + click.echo(u'{:25s}{}'.format('Organization:', host['org'])) + + if 'last_update' in host and host['last_update']: + click.echo('{:25s}{}'.format('Updated:', host['last_update'])) + + click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports']))) + + # Output the vulnerabilities the host has + if 'vulns' in host and len(host['vulns']) > 0: + vulns = [] + for vuln in host['vulns']: + if vuln.startswith('!'): + continue + if vuln.upper() == 'CVE-2014-0160': + vulns.append(click.style('Heartbleed', fg='red')) + else: + vulns.append(click.style(vuln, fg='red')) + + if len(vulns) > 0: + click.echo('{:25s}'.format('Vulnerabilities:'), nl=False) + + for vuln in vulns: + click.echo(vuln + '\t', nl=False) + + click.echo('') + + click.echo('') + + # If the user doesn't have access to SSL/ Telnet results then we need + # to pad the host['data'] property with empty banners so they still see + # the port listed as open. (#63) + if len(host['ports']) != len(host['data']): + # Find the ports the user can't see the data for + ports = host['ports'] + for banner in host['data']: + if banner['port'] in ports: + ports.remove(banner['port']) + + # Add the placeholder banners + for port in ports: + banner = { + 'port': port, + 'transport': 'tcp', # All the filtered services use TCP + 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner + 'placeholder': True, # Don't store this banner when the file is saved + } + host['data'].append(banner) + + click.echo('Ports:') + for banner in sorted(host['data'], key=lambda k: k['port']): + product = '' + version = '' + if 'product' in banner and banner['product']: + product = banner['product'] + if 'version' in banner and banner['version']: + version = '({})'.format(banner['version']) + + click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) + click.echo('/', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) + click.echo('{} {}'.format(product, version), nl=False) + + if history: + # Format the timestamp to only show the year-month-day + date = banner['timestamp'][:10] + click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) + click.echo('') + + # Show optional ssl info + if 'ssl' in banner: + if 'versions' in banner['ssl'] and banner['ssl']['versions']: + click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) + if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: + click.echo('\t|-- Diffie-Hellman Parameters:') + click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) + if 'fingerprint' in banner['ssl']['dhparams']: + click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) + + +def host_print_tsv(host, history=False): + """Show the host information in a succinct, grep-friendly manner.""" + for banner in sorted(host['data'], key=lambda k: k['port']): + click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) + click.echo('\t', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) + + if history: + # Format the timestamp to only show the year-month-day + date = banner['timestamp'][:10] + click.echo(click.style('\t({})'.format(date), fg='white', dim=True), nl=False) + click.echo('') + + +HOST_PRINT = { + 'pretty': host_print_pretty, + 'tsv': host_print_tsv, +} \ No newline at end of file From 9767c0ace95563a6857fd2d1be80e232a419aca2 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 16:13:26 -0500 Subject: [PATCH 023/148] Add information/ links for the CLI --- README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d0a40ae..c78af63 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -shodan: The official Python library for accessing Shodan -======================================================== +shodan: The official Python library and CLI for accessing Shodan +================================================================ .. image:: https://img.shields.io/pypi/v/shodan.svg :target: https://pypi.org/project/shodan/ @@ -20,6 +20,12 @@ Features - `Network alerts (aka private firehose) `_ - Exploit search API fully implemented - Bulk data downloads +- `Command-line interface `_ + +.. image:: https://asciinema.org/a/40357.png + :target: https://asciinema.org/~Shodan + :width: 400px + :align: center Quick Start From 04852200f2536e08f470d36bb968605198db2e85 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 16:19:30 -0500 Subject: [PATCH 024/148] Shorten title so it fits without wrapping and link to custom-scaled image as Github doesn't seem to honor the ":width:" modifier --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c78af63..0316ea0 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -shodan: The official Python library and CLI for accessing Shodan -================================================================ +shodan: The official Python library and CLI for Shodan +====================================================== .. image:: https://img.shields.io/pypi/v/shodan.svg :target: https://pypi.org/project/shodan/ From 6e81234088661f77c9fd312f3d4fd814ea4b3a4b Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 16:20:02 -0500 Subject: [PATCH 025/148] Update the actual link to the new image --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0316ea0..d0b1f10 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ Features - Bulk data downloads - `Command-line interface `_ -.. image:: https://asciinema.org/a/40357.png +.. image:: https://cli.shodan.io/img/shodan-cli-preview.png :target: https://asciinema.org/~Shodan :width: 400px :align: center From 5960167be753aa04e428cf8d0116e81c707555ee Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 17:49:53 -0500 Subject: [PATCH 026/148] Improved error handling around "shodan radar" (#74) --- CHANGELOG.md | 1 + shodan/__main__.py | 8 +++++++- shodan/cli/worldmap.py | 8 ++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dae37d5..73250be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ unreleased * The CHANGELOG is now part of the packages. * Improved unicode handling in Python2 (#78) * Add `tsv` output format for **shodan host** (#65) +* Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74) 1.9.0 ----- diff --git a/shodan/__main__.py b/shodan/__main__.py index f247432..3165c9a 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -1239,7 +1239,13 @@ def radar(): api = shodan.Shodan(key) from shodan.cli.worldmap import launch_map - launch_map(api) + + try: + launch_map(api) + except shodan.APIError as e: + raise click.ClickException(e.value) + except Exception as e: + raise click.ClickException(u'{}'.format(e)) def async_spinner(finished): spinner = itertools.cycle(['-', '/', '|', '\\']) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 9804aa2..16f06d4 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -18,6 +18,7 @@ import random import time +from shodan.exception import APIError from shodan.helpers import get_ip @@ -209,7 +210,7 @@ def fetch_data(self, epoch_now, force_refresh=False): break self.data = banners self.last_fetch = epoch_now - except StandardError: + except APIError: raise return refresh @@ -221,7 +222,10 @@ def run(self, scr): now = int(time.time()) refresh = self.fetch_data(now) m.set_data(self.data) - m.draw(scr) + try: + m.draw(scr) + except curses.error: + raise Exception('Terminal window too small') scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD) scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD) From f3fb662988568b00d660f54dfe820a1c2beaae89 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 18:47:26 -0500 Subject: [PATCH 027/148] Improved error messages for exceptions raised during API requests (#77) --- CHANGELOG.md | 1 + shodan/__main__.py | 2 +- shodan/client.py | 10 +++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73250be..39f33d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ unreleased * Improved unicode handling in Python2 (#78) * Add `tsv` output format for **shodan host** (#65) * Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74) +* Improved exception handling to improve debugging **shodan init** (#77) 1.9.0 ----- diff --git a/shodan/__main__.py b/shodan/__main__.py index 3165c9a..bc26f1d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -186,7 +186,7 @@ def init(key): api = shodan.Shodan(key) test = api.info() except shodan.APIError as e: - raise click.ClickException('Invalid API key') + raise click.ClickException(e.value) # Store the API key in the user's directory keyfile = shodan_dir + '/api_key' diff --git a/shodan/client.py b/shodan/client.py index 6c19e48..56a89e8 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -190,9 +190,17 @@ def _request(self, function, params, service='shodan', method='get'): # Return the actual error message if the API returned valid JSON error = data.json()['error'] except Exception as e: - error = 'Invalid API key' + # If the response looks like HTML then it's probably the 401 page that nginx returns + # for 401 responses by default + if data.text.startswith('<'): + error = 'Invalid API key' + else: + # Otherwise lets raise the error message + error = u'{}'.format(e) raise APIError(error) + else if data.status_code == 403: + raise APIError('Access denied (403 Forbidden)') # Parse the text into JSON try: From d768a0924d9355e4b250ca72808c74a3c00ec855 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 20:00:03 -0500 Subject: [PATCH 028/148] Fix typo --- shodan/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/client.py b/shodan/client.py index 56a89e8..0efb628 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -199,7 +199,7 @@ def _request(self, function, params, service='shodan', method='get'): error = u'{}'.format(e) raise APIError(error) - else if data.status_code == 403: + elif data.status_code == 403: raise APIError('Access denied (403 Forbidden)') # Parse the text into JSON From 868ac51be1ce70cf410a37ab2916278f17962cae Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 20:19:43 -0500 Subject: [PATCH 029/148] Release 1.9.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c52e5e0..88b8511 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.9.0', + version = '1.9.1', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', From 2775c5c646335da9548248083134ed41fbfcc01c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 20:22:34 -0500 Subject: [PATCH 030/148] Update CHANGELOG to match the release 1.9.1 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39f33d5..99a5dd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ CHANGELOG ========= -unreleased ----------- +1.9.1 +----- * The CHANGELOG is now part of the packages. * Improved unicode handling in Python2 (#78) * Add `tsv` output format for **shodan host** (#65) From 89e4d34c009c98cea14f93a8617bb419328b14ec Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 20:27:24 -0500 Subject: [PATCH 031/148] Remove API wrapper for the deprecated shodanhq.com website --- shodan/api.py | 222 -------------------------------------------------- 1 file changed, 222 deletions(-) delete mode 100644 shodan/api.py diff --git a/shodan/api.py b/shodan/api.py deleted file mode 100644 index 2f54f08..0000000 --- a/shodan/api.py +++ /dev/null @@ -1,222 +0,0 @@ -try: - # Python 2 - from urllib2 import urlopen - from urllib import urlencode -except: - # Python 3 - from urllib.request import urlopen - from urllib.parse import urlencode - -from json import dumps, loads - -from .exception import WebAPIError - - -__all__ = ['WebAPI'] - - -class WebAPI: - """Wrapper around the SHODAN webservices API""" - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - Arguments: - query -- exploit search query; same syntax as website - - Optional arguments: - sources -- metasploit, cve, osvdb, exploitdb - cve -- CVE identifier (ex. 2010-0432) - osvdb -- OSVDB identifier (ex. 11666) - msb -- Microsoft Security Bulletin ID (ex. MS05-030) - bid -- Bugtraq identifier (ex. 13951) - - """ - if sources: - query += ' source:%s' % (','.join(sources)) - if cve: - query += ' cve:%s' % (str(cve).strip()) - if osvdb: - query += ' osvdb:%s' % (str(osvdb).strip()) - if msb: - query += ' msb:%s' % (str(msb).strip()) - if bid: - query += ' bid:%s' % (str(bid).strip()) - return self.parent._request('api', {'q': query}, service='exploits') - - class ExploitDb: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """DEPRECATED - Download the exploit code from the ExploitDB archive. - - Arguments: - id -- ID of the ExploitDB entry - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['exploitdb']) - - def search(self, query, **kwargs): - """Search the ExploitDB archive. - - Arguments: - query -- Search terms - - Returns: - A dictionary with 2 main items: matches (list) and total (int). - """ - return self.parent.search(query, sources=['exploitdb']) - - - class Msf: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """Download a metasploit module given the fullname (id) of it. - - Arguments: - id -- fullname of the module (ex. auxiliary/admin/backupexec/dump) - - Returns: - A dictionary with the following fields: - filename -- Name of the file - content-type -- Mimetype - data -- File content - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['metasploit']) - - def search(self, query, **kwargs): - """Search for a Metasploit module. - """ - return self.parent.search(query, sources=['metasploit']) - - def __init__(self, key): - """Initializes the API object. - - Arguments: - key -- your API key - - """ - print('WARNING: This class is deprecated, please upgrade to use "shodan.Shodan()" instead of shodan.WebAPI()') - self.api_key = key - self.base_url = 'http://www.shodanhq.com/api/' - self.base_exploits_url = 'https://exploits.shodan.io/' - self.exploits = self.Exploits(self) - self.exploitdb = self.ExploitDb(self.exploits) - self.msf = self.Msf(self.exploits) - - def _request(self, function, params, service='shodan'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A JSON string containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8') - except: - raise WebAPIError('Unable to connect to Shodan') - - # Parse the text from JSON to a dict - data = loads(data) - - # Raise an exception if an error occurred - if data.get('error', None): - raise WebAPIError(data['error']) - - # Return the data - return data - - def count(self, query): - """Returns the total number of search results for the query. - """ - return self._request('count', {'q': query}) - - def locations(self, query): - """Return a break-down of all the countries and cities that the results for - the given search are located in. - """ - return self._request('locations', {'q': query}) - - def fingerprint(self, banner): - """Determine the software based on the banner. - - Arguments: - banner - HTTP banner - - Returns: - A list of software that matched the given banner. - """ - return self._request('fingerprint', {'banner': banner}) - - def host(self, ip): - """Get all available information on an IP. - - Arguments: - ip -- IP of the computer - - Returns: - All available information SHODAN has on the given IP, - subject to API key restrictions. - - """ - return self._request('host', {'ip': ip}) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('info', {}) - - def search(self, query, page=1, limit=None, offset=None): - """Search the SHODAN database. - - Arguments: - query -- search query; identical syntax to the website - - Optional arguments: - page -- page number of the search results - limit -- number of results to return - offset -- search offset to begin getting results from - - Returns: - A dictionary with 3 main items: matches, countries and total. - Visit the website for more detailed information. - - """ - args = { - 'q': query, - 'p': page, - } - if limit: - args['l'] = limit - if offset: - args['o'] = offset - - return self._request('search', args) From 2f8d21d65e6afe02fef0183b49e5bd023e8c371c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 17 Aug 2018 20:28:57 -0500 Subject: [PATCH 032/148] More code cleanup to remove deprecated classes --- shodan/__init__.py | 1 - shodan/exception.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/shodan/__init__.py b/shodan/__init__.py index 7d4b04d..bdfecaa 100644 --- a/shodan/__init__.py +++ b/shodan/__init__.py @@ -1,3 +1,2 @@ -from shodan.api import WebAPI from shodan.client import Shodan from shodan.exception import APIError diff --git a/shodan/exception.py b/shodan/exception.py index 11d89d3..c4878b1 100644 --- a/shodan/exception.py +++ b/shodan/exception.py @@ -1,11 +1,3 @@ -class WebAPIError(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - class APIError(Exception): """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" def __init__(self, value): From 64e9b6ba2c6550b68c28c9ede38dd88663789f51 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 30 Aug 2018 20:48:33 -0500 Subject: [PATCH 033/148] Major code cleanup using pylint Migrated large subcommands for the CLI into their own Python modules --- shodan/__main__.py | 582 ++------------------------------ shodan/cli/alert.py | 91 +++++ shodan/cli/converter/csvc.py | 1 + shodan/cli/converter/geojson.py | 2 +- shodan/cli/converter/kml.py | 23 +- shodan/cli/data.py | 90 +++++ shodan/cli/helpers.py | 73 +++- shodan/cli/scan.py | 314 +++++++++++++++++ shodan/cli/worldmap.py | 3 + shodan/helpers.py | 1 + shodan/threatnet.py | 2 +- 11 files changed, 603 insertions(+), 579 deletions(-) create mode 100644 shodan/cli/alert.py create mode 100644 shodan/cli/data.py create mode 100644 shodan/cli/scan.py diff --git a/shodan/__main__.py b/shodan/__main__.py index bc26f1d..874e2d0 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -48,7 +48,7 @@ from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS # Helper methods -from shodan.cli.helpers import get_api_key +from shodan.cli.helpers import async_spinner, get_api_key, escape_data, timestr, open_streaming_file, get_banner_field, match_filters from shodan.cli.host import HOST_PRINT # Allow 3rd-parties to develop custom commands @@ -65,67 +65,23 @@ basestring = str -def escape_data(args): - # Ensure the provided string isn't unicode data - if not isinstance(args, str): - args = args.encode('ascii', 'replace') - return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') - -def timestr(): - return datetime.datetime.utcnow().strftime('%Y-%m-%d') - -def open_streaming_file(directory, timestr, compresslevel=9): - return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel) - -def get_banner_field(banner, flat_field): - # The provided field is a collapsed form of the actual field - fields = flat_field.split('.') - - try: - current_obj = banner - for field in fields: - current_obj = current_obj[field] - return current_obj - except: - pass - - return None - -def match_filters(banner, filters): - for args in filters: - flat_field, check = args.split(':', 1) - value = get_banner_field(banner, flat_field) - - # If the field doesn't exist on the banner then ignore the record - if not value: - return False - - # It must match all filters to be allowed - field_type = type(value) - - # For lists of strings we see whether the desired value is contained in the field - if field_type == list or isinstance(value, basestring): - if check not in value: - return False - elif field_type == int: - if int(check) != value: - return False - elif field_type == float: - if float(check) != value: - return False - else: - # Ignore unknown types - pass - - return True - - +# Define the main entry point for all of our commands +# and expose a way for 3rd-party plugins to tie into the Shodan CLI. @with_plugins(iter_entry_points('shodan.cli.plugins')) @click.group(context_settings=CONTEXT_SETTINGS) def main(): pass +# Large subcommands are stored in separate modules +from shodan.cli.alert import alert +from shodan.cli.data import data +from shodan.cli.scan import scan +main.add_command(alert) +main.add_command(data) +main.add_command(scan) + + @main.command() @click.argument('input', metavar='') @click.argument('format', metavar='', type=click.Choice(['kml', 'csv', 'geo.json', 'images', 'xlsx'])) @@ -184,7 +140,7 @@ def init(key): key = key.strip() try: api = shodan.Shodan(key) - test = api.info() + api.info() except shodan.APIError as e: raise click.ClickException(e.value) @@ -196,95 +152,6 @@ def init(key): os.chmod(keyfile, 0o600) - -@main.group() -def alert(): - """Manage the network alerts for your account""" - pass - - -@alert.command(name='clear') -def alert_clear(): - """Remove all alerts""" - key = get_api_key() - - # Get the list - api = shodan.Shodan(key) - try: - alerts = api.alerts() - for alert in alerts: - click.echo(u'Removing {} ({})'.format(alert['name'], alert['id'])) - api.delete_alert(alert['id']) - except shodan.APIError as e: - raise click.ClickException(e.value) - click.echo("Alerts deleted") - -@alert.command(name='create') -@click.argument('name', metavar='') -@click.argument('netblock', metavar='') -def alert_create(name, netblock): - """Create a network alert to monitor an external network""" - key = get_api_key() - - # Get the list - api = shodan.Shodan(key) - try: - alert = api.create_alert(name, netblock) - except shodan.APIError as e: - raise click.ClickException(e.value) - - click.echo(click.style('Successfully created network alert!', fg='green')) - click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan')) - -@alert.command(name='list') -@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) -def alert_list(expired): - """List all the active alerts""" - key = get_api_key() - - # Get the list - api = shodan.Shodan(key) - try: - results = api.alerts(include_expired=expired) - except shodan.APIError as e: - raise click.ClickException(e.value) - - if len(results) > 0: - click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) - # click.echo('#' * 65) - for alert in results: - click.echo( - u'{:16} {:<30} {:<35} '.format( - click.style(alert['id'], fg='yellow'), - click.style(alert['name'], fg='cyan'), - click.style(', '.join(alert['filters']['ip']), fg='white') - ), - nl=False - ) - - if 'expired' in alert and alert['expired']: - click.echo(click.style('expired', fg='red')) - else: - click.echo('') - else: - click.echo("You haven't created any alerts yet.") - - -@alert.command(name='remove') -@click.argument('alert_id', metavar='') -def alert_remove(alert_id): - """Remove the specified alert""" - key = get_api_key() - - # Get the list - api = shodan.Shodan(key) - try: - results = api.delete_alert(alert_id) - except shodan.APIError as e: - raise click.ClickException(e.value) - click.echo("Alert deleted") - - @main.command() @click.argument('query', metavar='', nargs=-1) def count(query): @@ -308,90 +175,6 @@ def count(query): click.echo(results['total']) -@main.group() -def data(): - """Bulk data access to Shodan""" - pass - - -@data.command(name='list') -@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str) -def data_list(dataset): - """List available datasets or the files within those datasets.""" - # Setup the API connection - key = get_api_key() - api = shodan.Shodan(key) - - if dataset: - # Show the files within this dataset - files = api.data.list_files(dataset) - - for file in files: - click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) - click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) - click.echo('{}'.format(file['url'])) - else: - # If no dataset was provided then show a list of all datasets - datasets = api.data.list_datasets() - - for ds in datasets: - click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False) - click.echo('{}'.format(ds['description'])) - - -@data.command(name='download') -@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int) -@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.') -@click.argument('dataset', metavar='') -@click.argument('name', metavar='') -def data_download(chunksize, filename, dataset, name): - # Setup the API connection - key = get_api_key() - api = shodan.Shodan(key) - - # Get the file object that the user requested which will contain the URL and total file size - file = None - try: - files = api.data.list_files(dataset) - for tmp in files: - if tmp['name'] == name: - file = tmp - break - except shodan.APIError as e: - raise click.ClickException(e.value) - - # The file isn't available - if not file: - raise click.ClickException('File not found') - - # Start downloading the file - response = requests.get(file['url'], stream=True) - - # Figure out the size of the file based on the headers - filesize = response.headers.get('content-length', None) - if not filesize: - # Fall back to using the filesize provided by the API - filesize = file['size'] - else: - filesize = int(filesize) - - chunk_size = 1024 - limit = filesize / chunk_size - - # Create a default filename based on the dataset and the filename within that dataset - if not filename: - filename = '{}-{}'.format(dataset, name) - - # Open the output file and start writing to it in chunks - with open(filename, 'wb') as fout: - with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar: - for chunk in bar: - if chunk: - fout.write(chunk) - - click.echo(click.style('Download completed: {}'.format(filename), 'green')) - - @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @@ -514,7 +297,7 @@ def info(): @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') @click.option('--filters', '-f', help='Filter the results for specific values using key:value pairs.', multiple=True) @click.option('--filename', '-O', help='Save the filtered results in the given file (append if file exists).') -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') +@click.option('--separator', help='The separator between the properties of the search results.', default=u'\t') @click.argument('filenames', metavar='', type=click.Path(exists=True), nargs=-1) def parse(color, fields, filters, filename, separator, filenames): """Extract information out of compressed JSON files.""" @@ -540,7 +323,7 @@ def parse(color, fields, filters, filename, separator, filenames): fout = helpers.open_file(filename) for banner in helpers.iterate_files(filenames): - row = '' + row = u'' # Validate the banner against any provided filters if has_filters and not match_filters(banner, filters): @@ -552,16 +335,16 @@ def parse(color, fields, filters, filename, separator, filenames): # Loop over all the fields and print the banner as a row for field in fields: - tmp = '' + tmp = u'' value = get_banner_field(banner, field) if value: field_type = type(value) # If the field is an array then merge it together if field_type == list: - tmp = ';'.join(value) + tmp = u';'.join(value) elif field_type in [int, float]: - tmp = str(value) + tmp = u'{}'.format(value) else: tmp = escape_data(value) @@ -588,309 +371,6 @@ def myip(): raise click.ClickException(e.value) -@main.group() -def scan(): - """Scan an IP/ netblock using Shodan.""" - pass - - -@scan.command(name='internet') -@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) -@click.argument('port', type=int) -@click.argument('protocol', type=str) -def scan_internet(quiet, port, protocol): - """Scan the Internet for a specific port and protocol using the Shodan infrastructure.""" - key = get_api_key() - api = shodan.Shodan(key) - - try: - # Submit the request to Shodan - click.echo('Submitting Internet scan to Shodan...', nl=False) - scan = api.scan_internet(port, protocol) - click.echo('Done') - - # If the requested port is part of the regular Shodan crawling, then - # we don't know when the scan is done so lets return immediately and - # let the user decide when to stop waiting for further results. - official_ports = api.ports() - if port in official_ports: - click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.') - else: - # Create the output file - filename = '{0}-{1}.json.gz'.format(port, protocol) - counter = 0 - with helpers.open_file(filename, 'w') as fout: - click.echo('Saving results to file: {0}'.format(filename)) - - # Start listening for results - done = False - - # Keep listening for results until the scan is done - click.echo('Waiting for data, please stand by...') - while not done: - try: - for banner in api.stream.ports([port], timeout=90): - counter += 1 - helpers.write_banner(fout, banner) - - if not quiet: - click.echo('{0:<40} {1:<20} {2}'.format( - click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), - click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), - ';'.join(banner['hostnames']) - ) - ) - except shodan.APIError as e: - # We stop waiting for results if the scan has been processed by the crawlers and - # there haven't been new results in a while - if done: - break - - scan = api.scan_status(scan['id']) - if scan['status'] == 'DONE': - done = True - except socket.timeout as e: - # We stop waiting for results if the scan has been processed by the crawlers and - # there haven't been new results in a while - if done: - break - - scan = api.scan_status(scan['id']) - if scan['status'] == 'DONE': - done = True - except Exception as e: - raise click.ClickException(repr(e)) - click.echo('Scan finished: {0} devices found'.format(counter)) - except shodan.APIError as e: - raise click.ClickException(e.value) - - -@scan.command(name='protocols') -def scan_protocols(): - """List the protocols that you can scan with using Shodan.""" - key = get_api_key() - api = shodan.Shodan(key) - try: - protocols = api.protocols() - - for name, description in iter(protocols.items()): - click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description) - except shodan.APIError as e: - raise click.ClickException(e.value) - - -@scan.command(name='submit') -@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int) -@click.option('--filename', help='Save the results in the given file.', default='', type=str) -@click.option('--force', default=False, is_flag=True) -@click.option('--verbose', default=False, is_flag=True) -@click.argument('netblocks', metavar='', nargs=-1) -def scan_submit(wait, filename, force, verbose, netblocks): - """Scan an IP/ netblock using Shodan.""" - key = get_api_key() - api = shodan.Shodan(key) - alert = None - - # Submit the IPs for scanning - try: - # Submit the scan - scan = api.scan(netblocks, force=force) - - now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') - - click.echo('') - click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left'])) - - if verbose: - click.echo('# Scan ID: {}'.format(scan['id'])) - - # Return immediately - if wait <= 0: - click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') - else: - # Setup an alert to wait for responses - alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks) - - # Create the output file if necessary - filename = filename.strip() - fout = None - if filename != '': - # Add the appropriate extension if it's not there atm - if not filename.endswith('.json.gz'): - filename += '.json.gz' - fout = helpers.open_file(filename, 'w') - - # Start a spinner - finished_event = threading.Event() - progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) - progress_bar_thread.start() - - # Now wait a few seconds for items to get returned - hosts = collections.defaultdict(dict) - done = False - scan_start = time.time() - cache = {} - while not done: - try: - for banner in api.stream.alert(aid=alert['id'], timeout=wait): - ip = banner.get('ip', banner.get('ipv6', None)) - if not ip: - continue - - # Don't show duplicate banners - cache_key = '{}:{}'.format(ip, banner['port']) - if cache_key not in cache: - hosts[helpers.get_ip(banner)][banner['port']] = banner - cache[cache_key] = True - - # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on - if time.time() - scan_start >= 60: - scan = api.scan_status(scan['id']) - - if verbose: - click.echo('# Scan status: {}'.format(scan['status'])) - - if scan['status'] == 'DONE': - done = True - break - - except shodan.APIError as e: - # If the connection timed out before the timeout, that means the streaming server - # that the user tried to reach is down. In that case, lets wait briefly and try - # to connect again! - if (time.time() - scan_start) < wait: - time.sleep(0.5) - continue - - # Exit if the scan was flagged as done somehow - if done: - break - - scan = api.scan_status(scan['id']) - if scan['status'] == 'DONE': - done = True - - if verbose: - click.echo('# Scan status: {}'.format(scan['status'])) - except socket.timeout as e: - # If the connection timed out before the timeout, that means the streaming server - # that the user tried to reach is down. In that case, lets wait a second and try - # to connect again! - if (time.time() - scan_start) < wait: - continue - - done = True - except Exception as e: - finished_event.set() - progress_bar_thread.join() - raise click.ClickException(repr(e)) - - finished_event.set() - progress_bar_thread.join() - - def print_field(name, value): - click.echo(' {:25s}{}'.format(name, value)) - - def print_banner(banner): - click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False) - - if 'product' in banner: - click.echo(banner['product'], nl=False) - - if 'version' in banner: - click.echo(' ({})'.format(banner['version']), nl=False) - - click.echo('') - - # Show optional ssl info - if 'ssl' in banner: - if 'versions' in banner['ssl']: - # Only print SSL versions if they were successfully tested - versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')] - if len(versions) > 0: - click.echo(' |-- SSL Versions: {}'.format(', '.join(versions))) - if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: - click.echo(' |-- Diffie-Hellman Parameters:') - click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) - if 'fingerprint' in banner['ssl']['dhparams']: - click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) - - if hosts: - # Remove the remaining spinner character - click.echo('\b ') - - for ip in sorted(hosts): - host = next(iter(hosts[ip].items()))[1] - - click.echo(click.style(ip, fg='cyan'), nl=False) - if 'hostnames' in host and host['hostnames']: - click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False) - click.echo('') - - if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']: - print_field('Country', host['location']['country_name']) - - if 'city' in host['location'] and host['location']['city']: - print_field('City', host['location']['city']) - if 'org' in host and host['org']: - print_field('Organization', host['org']) - if 'os' in host and host['os']: - print_field('Operating System', host['os']) - click.echo('') - - # Output the vulnerabilities the host has - if 'vulns' in host and len(host['vulns']) > 0: - vulns = [] - for vuln in host['vulns']: - if vuln.startswith('!'): - continue - if vuln.upper() == 'CVE-2014-0160': - vulns.append(click.style('Heartbleed', fg='red')) - else: - vulns.append(click.style(vuln, fg='red')) - - if len(vulns) > 0: - click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False) - - for vuln in vulns: - click.echo(vuln + '\t', nl=False) - - click.echo('') - - # Print all the open ports: - click.echo(' Open Ports:') - for port in sorted(hosts[ip]): - print_banner(hosts[ip][port]) - - # Save the banner in a file if necessary - if fout: - helpers.write_banner(fout, hosts[ip][port]) - - click.echo('') - else: - # Prepend a \b to remove the spinner - click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.') - except shodan.APIError as e: - raise click.ClickException(e.value) - finally: - # Remove any alert - if alert: - api.delete_alert(alert['id']) - - -@scan.command(name='status') -@click.argument('scan_id', type=str) -def scan_status(scan_id): - """Check the status of an on-demand scan.""" - key = get_api_key() - api = shodan.Shodan(key) - try: - scan = api.scan_status(scan_id) - click.echo(scan['status']) - except shodan.APIError as e: - raise click.ClickException(e.value) - - @main.command() @click.option('--color/--no-color', default=True) @click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data') @@ -930,9 +410,9 @@ def search(color, fields, limit, separator, query): raise click.ClickException('No search results found') # We buffer the entire output so we can use click's pager functionality - output = '' + output = u'' for banner in results['matches']: - row = '' + row = u'' # Loop over all the fields and print the banner as a row for field in fields: @@ -942,9 +422,9 @@ def search(color, fields, limit, separator, query): # If the field is an array then merge it together if field_type == list: - tmp = ';'.join(banner[field]) + tmp = u';'.join(banner[field]) elif field_type in [int, float]: - tmp = str(banner[field]) + tmp = u'{}'.format(banner[field]) else: tmp = escape_data(banner[field]) @@ -957,7 +437,7 @@ def search(color, fields, limit, separator, query): row += separator # click.echo(out + separator, nl=False) - output += row + '\n' + output += row + u'\n' # click.echo('') click.echo_via_pager(output) @@ -1030,6 +510,7 @@ def stats(limit, facets, filename, query): counter = 0 has_items = True while has_items: + # pylint: disable=W0612 row = ['' for i in range(len(results['facets']) * 2)] pos = 0 @@ -1170,20 +651,20 @@ def _create_stream(name, args, timeout): # Print the banner information to stdout if not quiet: - row = '' + row = u'' # Loop over all the fields and print the banner as a row for field in fields: - tmp = '' + tmp = u'' value = get_banner_field(banner, field) if value: field_type = type(value) # If the field is an array then merge it together if field_type == list: - tmp = ';'.join(value) + tmp = u';'.join(value) elif field_type in [int, float]: - tmp = str(value) + tmp = u'{}'.format(value) else: tmp = escape_data(value) @@ -1247,12 +728,5 @@ def radar(): except Exception as e: raise click.ClickException(u'{}'.format(e)) -def async_spinner(finished): - spinner = itertools.cycle(['-', '/', '|', '\\']) - while not finished.is_set(): - sys.stdout.write('\b{}'.format(next(spinner))) - sys.stdout.flush() - finished.wait(0.2) - if __name__ == '__main__': main() diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py new file mode 100644 index 0000000..779c44c --- /dev/null +++ b/shodan/cli/alert.py @@ -0,0 +1,91 @@ +import click +import shodan + +from shodan.cli.helpers import get_api_key + +@click.group() +def alert(): + """Manage the network alerts for your account""" + pass + + +@alert.command(name='clear') +def alert_clear(): + """Remove all alerts""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + alerts = api.alerts() + for alert in alerts: + click.echo(u'Removing {} ({})'.format(alert['name'], alert['id'])) + api.delete_alert(alert['id']) + except shodan.APIError as e: + raise click.ClickException(e.value) + click.echo("Alerts deleted") + +@alert.command(name='create') +@click.argument('name', metavar='') +@click.argument('netblock', metavar='') +def alert_create(name, netblock): + """Create a network alert to monitor an external network""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + alert = api.create_alert(name, netblock) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.echo(click.style('Successfully created network alert!', fg='green')) + click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan')) + +@alert.command(name='list') +@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) +def alert_list(expired): + """List all the active alerts""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + results = api.alerts(include_expired=expired) + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(results) > 0: + click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) + # click.echo('#' * 65) + for alert in results: + click.echo( + u'{:16} {:<30} {:<35} '.format( + click.style(alert['id'], fg='yellow'), + click.style(alert['name'], fg='cyan'), + click.style(', '.join(alert['filters']['ip']), fg='white') + ), + nl=False + ) + + if 'expired' in alert and alert['expired']: + click.echo(click.style('expired', fg='red')) + else: + click.echo('') + else: + click.echo("You haven't created any alerts yet.") + + +@alert.command(name='remove') +@click.argument('alert_id', metavar='') +def alert_remove(alert_id): + """Remove the specified alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.delete_alert(alert_id) + except shodan.APIError as e: + raise click.ClickException(e.value) + click.echo("Alert deleted") diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 6e22b62..4e08f63 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -81,6 +81,7 @@ def flatten(self, d, parent_key='', sep='.'): for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, MutableMapping): + # pylint: disable=E0602 items.extend(flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 8d6d7d0..2cc55e4 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -53,5 +53,5 @@ def write(self, host): }""".format(ip, ip, lat, lon) self.fout.write(feature) - except Exception as e: + except: pass diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index 7bcac11..c8cb2cc 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -45,27 +45,6 @@ def write(self, host): if 'hostnames' in host and host['hostnames']: placemark += ''.format(host['hostnames'][0]) - test = """ - - - - - - - - - - - - - - - -
CityAlbuquerque
CountryUnited States
OrganizationNexcess.net L.L.C.
-

Ports

-
    - """ - placemark += '

    Ports

      ' for port in host['ports']: @@ -123,5 +102,5 @@ def write(self, host): placemark += '' self.fout.write(placemark.encode('utf-8')) - except Exception as e: + except: pass diff --git a/shodan/cli/data.py b/shodan/cli/data.py new file mode 100644 index 0000000..7cd7228 --- /dev/null +++ b/shodan/cli/data.py @@ -0,0 +1,90 @@ +import click +import requests +import shodan +import shodan.helpers as helpers + +from shodan.cli.helpers import get_api_key + + +@click.group() +def data(): + """Bulk data access to Shodan""" + pass + + +@data.command(name='list') +@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str) +def data_list(dataset): + """List available datasets or the files within those datasets.""" + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + if dataset: + # Show the files within this dataset + files = api.data.list_files(dataset) + + for file in files: + click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) + click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + click.echo('{}'.format(file['url'])) + else: + # If no dataset was provided then show a list of all datasets + datasets = api.data.list_datasets() + + for ds in datasets: + click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False) + click.echo('{}'.format(ds['description'])) + + +@data.command(name='download') +@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int) +@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.') +@click.argument('dataset', metavar='') +@click.argument('name', metavar='') +def data_download(chunksize, filename, dataset, name): + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + # Get the file object that the user requested which will contain the URL and total file size + file = None + try: + files = api.data.list_files(dataset) + for tmp in files: + if tmp['name'] == name: + file = tmp + break + except shodan.APIError as e: + raise click.ClickException(e.value) + + # The file isn't available + if not file: + raise click.ClickException('File not found') + + # Start downloading the file + response = requests.get(file['url'], stream=True) + + # Figure out the size of the file based on the headers + filesize = response.headers.get('content-length', None) + if not filesize: + # Fall back to using the filesize provided by the API + filesize = file['size'] + else: + filesize = int(filesize) + + chunk_size = 1024 + limit = filesize / chunk_size + + # Create a default filename based on the dataset and the filename within that dataset + if not filename: + filename = '{}-{}'.format(dataset, name) + + # Open the output file and start writing to it in chunks + with open(filename, 'wb') as fout: + with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar: + for chunk in bar: + if chunk: + fout.write(chunk) + + click.echo(click.style('Download completed: {}'.format(filename), 'green')) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 24f7b83..711fe4c 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -1,8 +1,12 @@ ''' -Helper methods to create your own CLI commands. +Helper methods used across the CLI commands. ''' import click +import datetime +import gzip +import itertools import os +import sys from .settings import SHODAN_CONFIG_DIR @@ -21,3 +25,70 @@ def get_api_key(): with open(keyfile, 'r') as fin: return fin.read().strip() + + +def escape_data(args): + # Ensure the provided string isn't unicode data + if not isinstance(args, str): + args = args.encode('ascii', 'replace') + return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + + +def timestr(): + return datetime.datetime.utcnow().strftime('%Y-%m-%d') + + +def open_streaming_file(directory, timestr, compresslevel=9): + return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel) + + +def get_banner_field(banner, flat_field): + # The provided field is a collapsed form of the actual field + fields = flat_field.split('.') + + try: + current_obj = banner + for field in fields: + current_obj = current_obj[field] + return current_obj + except: + pass + + return None + + +def match_filters(banner, filters): + for args in filters: + flat_field, check = args.split(':', 1) + value = get_banner_field(banner, flat_field) + + # If the field doesn't exist on the banner then ignore the record + if not value: + return False + + # It must match all filters to be allowed + field_type = type(value) + + # For lists of strings we see whether the desired value is contained in the field + if field_type == list or isinstance(value, basestring): + if check not in value: + return False + elif field_type == int: + if int(check) != value: + return False + elif field_type == float: + if float(check) != value: + return False + else: + # Ignore unknown types + pass + + return True + + +def async_spinner(finished): + spinner = itertools.cycle(['-', '/', '|', '\\']) + while not finished.is_set(): + sys.stdout.write('\b{}'.format(next(spinner))) + sys.stdout.flush() + finished.wait(0.2) diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py new file mode 100644 index 0000000..5220339 --- /dev/null +++ b/shodan/cli/scan.py @@ -0,0 +1,314 @@ +import click +import collections +import datetime +import shodan +import shodan.helpers as helpers +import socket +import threading +import time + +from shodan.cli.helpers import get_api_key, async_spinner +from shodan.cli.settings import COLORIZE_FIELDS + + +@click.group() +def scan(): + """Scan an IP/ netblock using Shodan.""" + pass + + +@scan.command(name='internet') +@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) +@click.argument('port', type=int) +@click.argument('protocol', type=str) +def scan_internet(quiet, port, protocol): + """Scan the Internet for a specific port and protocol using the Shodan infrastructure.""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Submit the request to Shodan + click.echo('Submitting Internet scan to Shodan...', nl=False) + scan = api.scan_internet(port, protocol) + click.echo('Done') + + # If the requested port is part of the regular Shodan crawling, then + # we don't know when the scan is done so lets return immediately and + # let the user decide when to stop waiting for further results. + official_ports = api.ports() + if port in official_ports: + click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.') + else: + # Create the output file + filename = '{0}-{1}.json.gz'.format(port, protocol) + counter = 0 + with helpers.open_file(filename, 'w') as fout: + click.echo('Saving results to file: {0}'.format(filename)) + + # Start listening for results + done = False + + # Keep listening for results until the scan is done + click.echo('Waiting for data, please stand by...') + while not done: + try: + for banner in api.stream.ports([port], timeout=90): + counter += 1 + helpers.write_banner(fout, banner) + + if not quiet: + click.echo('{0:<40} {1:<20} {2}'.format( + click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), + click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), + ';'.join(banner['hostnames']) + ) + ) + except shodan.APIError as e: + # We stop waiting for results if the scan has been processed by the crawlers and + # there haven't been new results in a while + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + except socket.timeout as e: + # We stop waiting for results if the scan has been processed by the crawlers and + # there haven't been new results in a while + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + except Exception as e: + raise click.ClickException(repr(e)) + click.echo('Scan finished: {0} devices found'.format(counter)) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@scan.command(name='protocols') +def scan_protocols(): + """List the protocols that you can scan with using Shodan.""" + key = get_api_key() + api = shodan.Shodan(key) + try: + protocols = api.protocols() + + for name, description in iter(protocols.items()): + click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@scan.command(name='submit') +@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int) +@click.option('--filename', help='Save the results in the given file.', default='', type=str) +@click.option('--force', default=False, is_flag=True) +@click.option('--verbose', default=False, is_flag=True) +@click.argument('netblocks', metavar='', nargs=-1) +def scan_submit(wait, filename, force, verbose, netblocks): + """Scan an IP/ netblock using Shodan.""" + key = get_api_key() + api = shodan.Shodan(key) + alert = None + + # Submit the IPs for scanning + try: + # Submit the scan + scan = api.scan(netblocks, force=force) + + now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + + click.echo('') + click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left'])) + + if verbose: + click.echo('# Scan ID: {}'.format(scan['id'])) + + # Return immediately + if wait <= 0: + click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') + else: + # Setup an alert to wait for responses + alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks) + + # Create the output file if necessary + filename = filename.strip() + fout = None + if filename != '': + # Add the appropriate extension if it's not there atm + if not filename.endswith('.json.gz'): + filename += '.json.gz' + fout = helpers.open_file(filename, 'w') + + # Start a spinner + finished_event = threading.Event() + progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) + progress_bar_thread.start() + + # Now wait a few seconds for items to get returned + hosts = collections.defaultdict(dict) + done = False + scan_start = time.time() + cache = {} + while not done: + try: + for banner in api.stream.alert(aid=alert['id'], timeout=wait): + ip = banner.get('ip', banner.get('ipv6', None)) + if not ip: + continue + + # Don't show duplicate banners + cache_key = '{}:{}'.format(ip, banner['port']) + if cache_key not in cache: + hosts[helpers.get_ip(banner)][banner['port']] = banner + cache[cache_key] = True + + # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on + if time.time() - scan_start >= 60: + scan = api.scan_status(scan['id']) + + if verbose: + click.echo('# Scan status: {}'.format(scan['status'])) + + if scan['status'] == 'DONE': + done = True + break + + except shodan.APIError as e: + # If the connection timed out before the timeout, that means the streaming server + # that the user tried to reach is down. In that case, lets wait briefly and try + # to connect again! + if (time.time() - scan_start) < wait: + time.sleep(0.5) + continue + + # Exit if the scan was flagged as done somehow + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + + if verbose: + click.echo('# Scan status: {}'.format(scan['status'])) + except socket.timeout as e: + # If the connection timed out before the timeout, that means the streaming server + # that the user tried to reach is down. In that case, lets wait a second and try + # to connect again! + if (time.time() - scan_start) < wait: + continue + + done = True + except Exception as e: + finished_event.set() + progress_bar_thread.join() + raise click.ClickException(repr(e)) + + finished_event.set() + progress_bar_thread.join() + + def print_field(name, value): + click.echo(' {:25s}{}'.format(name, value)) + + def print_banner(banner): + click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False) + + if 'product' in banner: + click.echo(banner['product'], nl=False) + + if 'version' in banner: + click.echo(' ({})'.format(banner['version']), nl=False) + + click.echo('') + + # Show optional ssl info + if 'ssl' in banner: + if 'versions' in banner['ssl']: + # Only print SSL versions if they were successfully tested + versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')] + if len(versions) > 0: + click.echo(' |-- SSL Versions: {}'.format(', '.join(versions))) + if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: + click.echo(' |-- Diffie-Hellman Parameters:') + click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) + if 'fingerprint' in banner['ssl']['dhparams']: + click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) + + if hosts: + # Remove the remaining spinner character + click.echo('\b ') + + for ip in sorted(hosts): + host = next(iter(hosts[ip].items()))[1] + + click.echo(click.style(ip, fg='cyan'), nl=False) + if 'hostnames' in host and host['hostnames']: + click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False) + click.echo('') + + if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']: + print_field('Country', host['location']['country_name']) + + if 'city' in host['location'] and host['location']['city']: + print_field('City', host['location']['city']) + if 'org' in host and host['org']: + print_field('Organization', host['org']) + if 'os' in host and host['os']: + print_field('Operating System', host['os']) + click.echo('') + + # Output the vulnerabilities the host has + if 'vulns' in host and len(host['vulns']) > 0: + vulns = [] + for vuln in host['vulns']: + if vuln.startswith('!'): + continue + if vuln.upper() == 'CVE-2014-0160': + vulns.append(click.style('Heartbleed', fg='red')) + else: + vulns.append(click.style(vuln, fg='red')) + + if len(vulns) > 0: + click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False) + + for vuln in vulns: + click.echo(vuln + '\t', nl=False) + + click.echo('') + + # Print all the open ports: + click.echo(' Open Ports:') + for port in sorted(hosts[ip]): + print_banner(hosts[ip][port]) + + # Save the banner in a file if necessary + if fout: + helpers.write_banner(fout, hosts[ip][port]) + + click.echo('') + else: + # Prepend a \b to remove the spinner + click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.') + except shodan.APIError as e: + raise click.ClickException(e.value) + finally: + # Remove any alert + if alert: + api.delete_alert(alert['id']) + + +@scan.command(name='status') +@click.argument('scan_id', type=str) +def scan_status(scan_id): + """Check the status of an on-demand scan.""" + key = get_api_key() + api = shodan.Shodan(key) + try: + scan = api.scan_status(scan_id) + click.echo(scan['status']) + except shodan.APIError as e: + raise click.ClickException(e.value) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 16f06d4..c3357ba 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -28,6 +28,9 @@ 'corners': (1, 4, 23, 73), # lat top, lon left, lat bottom, lon right 'coords': [90.0, -180.0, -90.0, 180.0], + + # PyLint freaks out about the world map backslashes so ignore those warnings + # pylint: disable=W1401 'data': ''' . _..::__: ,-"-"._ |7 , _,.__ _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ diff --git a/shodan/helpers.py b/shodan/helpers.py index 95df590..64e5937 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -93,6 +93,7 @@ def iterate_files(files, fast=False): # It's significantly faster at encoding/ decoding JSON but it doesn't support as # many options as the standard library. As such, we're mostly interested in using it for # decoding since reading/ parsing files will use up the most time. + # pylint: disable=E0401 try: from ujson import loads except: diff --git a/shodan/threatnet.py b/shodan/threatnet.py index 8b609f0..97c0c7e 100644 --- a/shodan/threatnet.py +++ b/shodan/threatnet.py @@ -29,7 +29,7 @@ def _create_stream(self, name): if req.status_code != 200: try: - raise APIError(data.json()['error']) + raise APIError(req.json()['error']) except: pass raise APIError('Invalid API key or you do not have access to the Streaming API') From 3454b818071bcf2d2888f2a8ae3dd1b8b3167c7c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 1 Sep 2018 18:35:56 -0500 Subject: [PATCH 034/148] New command: org - manage access to Shodan for your team --- shodan/__main__.py | 10 +++-- shodan/cli/helpers.py | 11 ++++++ shodan/cli/organization.py | 77 ++++++++++++++++++++++++++++++++++++++ shodan/client.py | 52 ++++++++++++++++++++++--- 4 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 shodan/cli/organization.py diff --git a/shodan/__main__.py b/shodan/__main__.py index 874e2d0..9d49ba9 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -76,9 +76,11 @@ def main(): # Large subcommands are stored in separate modules from shodan.cli.alert import alert from shodan.cli.data import data +from shodan.cli.organization import org from shodan.cli.scan import scan main.add_command(alert) main.add_command(data) +main.add_command(org) main.add_command(scan) @@ -134,7 +136,7 @@ def init(key): try: os.mkdir(shodan_dir) except OSError: - raise click.ClickException('Unable to create directory to store the Shodan API key (%s)' % shodan_dir) + raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) # Make sure it's a valid API key key = key.strip() @@ -237,7 +239,7 @@ def download(limit, filename, query): # Let the user know we're done if count < limit: click.echo(click.style('Notice: fewer results were saved than requested', 'yellow')) - click.echo(click.style('Saved %s results into file %s' % (count, filename), 'green')) + click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green')) @main.command() @@ -480,8 +482,8 @@ def stats(limit, facets, filename, query): else: value = str(value) - click.echo(click.style('{:28s}'.format(value), fg='cyan'), nl=False) - click.echo(click.style('{:12,d}'.format(item['count']), fg='green')) + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) click.echo('') diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 711fe4c..bd29ae4 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -92,3 +92,14 @@ def async_spinner(finished): sys.stdout.write('\b{}'.format(next(spinner))) sys.stdout.flush() finished.wait(0.2) + + +def humanize_api_plan(plan): + return { + 'oss': 'Free', + 'dev': 'Membership', + 'basic': 'Freelancer API', + 'plus': 'Small Business API', + 'corp': 'Corporate API', + 'stream-100': 'Enterprise', + }[plan] diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py new file mode 100644 index 0000000..ecdd58a --- /dev/null +++ b/shodan/cli/organization.py @@ -0,0 +1,77 @@ +import click +import requests +import shodan +import shodan.helpers as helpers + +from shodan.cli.helpers import get_api_key, humanize_api_plan + + +@click.group() +def org(): + """Manage your organization's access to Shodan""" + pass + + +@org.command() +@click.option('--silent', help="Don't send a notification to the user", default=False, is_flag=True) +@click.argument('user', metavar='') +def add(silent, user): + """Add a new member""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + api.org.add_member(user, notify=not silent) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully added the new member', fg='green') + + +@org.command() +def info(): + """Show an overview of the organization""" + key = get_api_key() + api = shodan.Shodan(key) + try: + organization = api.org.info() + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(organization['name'], fg='cyan') + click.secho('Access Level: ', nl=False, dim=True) + click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') + click.echo('') + click.secho('Administrators:', dim=True) + + for admin in organization['admins']: + click.echo(u' > {:30}\t{:30}'.format( + click.style(admin['username'], fg='yellow'), + admin['email']) + ) + + click.echo('') + if organization['members']: + click.secho('Members:', dim=True) + for member in organization['members']: + click.echo(u' > {:30}\t{:30}'.format( + click.style(member['username'], fg='yellow'), + member['email']) + ) + else: + click.secho('No members yet', dim=True) + + +@org.command() +@click.argument('user', metavar='') +def remove(user): + """Remove and downgrade a member""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + api.org.remove_member(user) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully removed the member', fg='green') diff --git a/shodan/client.py b/shodan/client.py index 0efb628..2b82ae2 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -23,6 +23,7 @@ # pip install requests[security] # # Which will download libraries that offer more full-featured SSL classes +# pylint: disable=E1101 try: requests.packages.urllib3.disable_warnings() except: @@ -135,6 +136,40 @@ def honeyscore(self, ip): """ return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) + class Organization: + + def __init__(self, parent): + self.parent = parent + + def add_member(self, user, notify=True): + """Add the user to the organization. + + :param user: username or email address + :type user: str + :param notify: whether or not to send the user an email notification + :type notify: bool + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), { + 'notify': notify, + }, method='PUT')['success'] + + def info(self): + """Returns general information about the organization the current user is a member of. + """ + return self.parent._request('/org', {}) + + def remove_member(self, user): + """Remove the user from the organization. + + :param user: username or email address + :type user: str + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + def __init__(self, key, proxies=None): """Initializes the API object. @@ -149,6 +184,7 @@ def __init__(self, key, proxies=None): self.data = self.Data(self) self.exploits = self.Exploits(self) self.labs = self.Labs(self) + self.org = self.Organization(self) self.tools = self.Tools(self) self.stream = Stream(key, proxies=proxies) self._session = requests.Session() @@ -177,8 +213,13 @@ def _request(self, function, params, service='shodan', method='get'): # Send the request try: - if method.lower() == 'post': + method = method.lower() + if method == 'post': data = self._session.post(base_url + function, params) + elif method == 'put': + data = self._session.put(base_url + function, params=params) + elif method == 'delete': + data = self._session.delete(base_url + function, params=params) else: data = self._session.get(base_url + function, params=params) except: @@ -384,13 +425,12 @@ def search_cursor(self, query, minify=True, retries=5): :returns: A search cursor that can be used as an iterator/ generator. """ - args = { - 'query': query, - 'minify': minify, - } - page = 1 tries = 0 + results = { + 'matches': [], + 'total': None, + } while page == 1 or results['matches']: try: results = self.search(query, minify=minify, page=page) From 648d125be5ff6f68d037e9f482e3952734697f9c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 1 Sep 2018 18:38:37 -0500 Subject: [PATCH 035/148] Code cleanup --- shodan/cli/alert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 779c44c..a9f6ec8 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -39,8 +39,8 @@ def alert_create(name, netblock): except shodan.APIError as e: raise click.ClickException(e.value) - click.echo(click.style('Successfully created network alert!', fg='green')) - click.echo(click.style('Alert ID: {}'.format(alert['id']), fg='cyan')) + click.secho('Successfully created network alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') @alert.command(name='list') @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) @@ -69,7 +69,7 @@ def alert_list(expired): ) if 'expired' in alert and alert['expired']: - click.echo(click.style('expired', fg='red')) + click.secho('expired', fg='red') else: click.echo('') else: From bba05c95a9d998faffa2502413420dbe52f91cf4 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 1 Sep 2018 19:57:26 -0500 Subject: [PATCH 036/148] Improve unicode handling (#78) Allow printing of nested properties in "shodan search" --- shodan/__main__.py | 21 ++++++++------------- shodan/cli/helpers.py | 6 +++--- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 9d49ba9..861bce9 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -418,17 +418,18 @@ def search(color, fields, limit, separator, query): # Loop over all the fields and print the banner as a row for field in fields: - tmp = '' - if field in banner and banner[field]: - field_type = type(banner[field]) + tmp = u'' + value = get_banner_field(banner, field) + if value: + field_type = type(value) # If the field is an array then merge it together if field_type == list: - tmp = u';'.join(banner[field]) + tmp = u';'.join(value) elif field_type in [int, float]: - tmp = u'{}'.format(banner[field]) + tmp = u'{}'.format(value) else: - tmp = escape_data(banner[field]) + tmp = escape_data(value) # Colorize certain fields if the user wants it if color: @@ -476,13 +477,7 @@ def stats(limit, facets, filename, query): click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) for item in results['facets'][facet]: - value = item['value'] - if isinstance(value, basestring): - value = value.encode('ascii', errors='replace').decode('ascii') - else: - value = str(value) - - click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:28s}'.format(item['value']), fg='cyan'), nl=False) click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) click.echo('') diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index bd29ae4..7f7d89f 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -28,9 +28,9 @@ def get_api_key(): def escape_data(args): - # Ensure the provided string isn't unicode data - if not isinstance(args, str): - args = args.encode('ascii', 'replace') + # Make sure the string is unicode so the terminal can properly display it + # We do it using format() so it works across Python 2 and 3 + args = u'{}'.format(args) return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') From f8f53aa94d908e20998a1d35b602d585e071184e Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 1 Sep 2018 20:47:51 -0500 Subject: [PATCH 037/148] Release 1.10.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99a5dd8..08caae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ CHANGELOG ========= +1.10.0 +------ +* New command **shodan org**: manage enterprise access to Shodan for your team +* Improved unicode handling (#78) +* Remove deprecated API wrapper for shodanhq.com/api + 1.9.1 ----- * The CHANGELOG is now part of the packages. diff --git a/setup.py b/setup.py index 88b8511..a810b7a 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.9.1', + version = '1.10.0', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', From 6ad6a51f10489e5204899b75ef83759295609146 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 3 Sep 2018 23:08:26 -0500 Subject: [PATCH 038/148] Release 1.10.1 Support PUT requests in the API helper method --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/helpers.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08caae4..8e84249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.10.1 +------ +* Support PUT requests in the API request helper method + 1.10.0 ------ * New command **shodan org**: manage enterprise access to Shodan for your team diff --git a/setup.py b/setup.py index a810b7a..5c96d32 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.10.0', + version = '1.10.1', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', diff --git a/shodan/helpers.py b/shodan/helpers.py index 64e5937..ed1a88a 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -51,6 +51,8 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho proxies=proxies) elif method.lower() == 'delete': data = requests.delete(base_url + function, params=params, proxies=proxies) + elif method.lower() == 'put': + data = requests.put(base_url + function, params=params, proxies=proxies) else: data = requests.get(base_url + function, params=params, proxies=proxies) From 9edd057125332e02f47cfa2659e0a7e7d39be983 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 11 Sep 2018 23:20:17 -0500 Subject: [PATCH 039/148] Show the list of authorized domains for organizations --- shodan/cli/organization.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py index ecdd58a..985c38e 100644 --- a/shodan/cli/organization.py +++ b/shodan/cli/organization.py @@ -41,6 +41,11 @@ def info(): click.secho(organization['name'], fg='cyan') click.secho('Access Level: ', nl=False, dim=True) click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') + + if organization['domains']: + click.secho('Authorized Domains: ', nl=False, dim=True) + click.echo(', '.join(organization['domains'])) + click.echo('') click.secho('Administrators:', dim=True) From 3fadceff99283e806908565894913b14fe447a90 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 12 Sep 2018 16:40:39 -0500 Subject: [PATCH 040/148] Make sure all facet values are strings before formatting them --- shodan/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 861bce9..339218e 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -477,7 +477,10 @@ def stats(limit, facets, filename, query): click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) for item in results['facets'][facet]: - click.echo(click.style(u'{:28s}'.format(item['value']), fg='cyan'), nl=False) + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) click.echo('') From ed3eadcf9e6f990379aec29f968c7edcee0bee16 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 17 Sep 2018 12:50:25 -0500 Subject: [PATCH 041/148] Release 1.10.2 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e84249..a2945ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.10.2 +------ +* Fix **shodan stats** formatting exception when faceting on **port** + 1.10.1 ------ * Support PUT requests in the API request helper method diff --git a/setup.py b/setup.py index 5c96d32..593a481 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.10.1', + version = '1.10.2', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', From ce5a656a86442135d0a198a858795905e2b27b0a Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 18 Sep 2018 18:05:24 +0200 Subject: [PATCH 042/148] Implement feedback from lgtm tool https://lgtm.com/projects/g/achillean/shodan-python/alerts/?mode=list --- CHANGELOG.md | 5 +++++ shodan/__main__.py | 10 ++-------- shodan/cli/converter/csvc.py | 6 +++--- shodan/cli/converter/excel.py | 8 ++++---- shodan/cli/converter/geojson.py | 2 +- shodan/cli/converter/kml.py | 2 +- shodan/cli/helpers.py | 2 +- shodan/cli/organization.py | 2 -- shodan/cli/worldmap.py | 2 +- shodan/client.py | 8 ++++---- shodan/helpers.py | 6 +++--- 11 files changed, 25 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2945ec..063ff36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +unreleased +---------- +* Change bare 'except:' statements to 'except Exception:' or more specific ones +* remove unused imports + 1.10.2 ------ * Fix **shodan stats** formatting exception when faceting on **port** diff --git a/shodan/__main__.py b/shodan/__main__.py index 339218e..f942080 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -26,17 +26,11 @@ """ import click -import collections import csv -import datetime -import gzip -import itertools import os import os.path import shodan import shodan.helpers as helpers -import socket -import sys import threading import requests import time @@ -584,7 +578,7 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if ports: try: stream_args = [int(item.strip()) for item in ports.split(',')] - except: + except ValueError: raise click.ClickException('Invalid list of ports') if alert: @@ -683,7 +677,7 @@ def _create_stream(name, args, timeout): quit = True except shodan.APIError as e: raise click.ClickException(e.value) - except: + except Exception: # For other errors lets just wait a bit and try to reconnect again time.sleep(1) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 4e08f63..c975695 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -54,7 +54,7 @@ def process(self, files): value = self.banner_field(banner, field) row.append(value) writer.writerow(row) - except: + except Exception: pass def banner_field(self, banner, flat_field): @@ -71,7 +71,7 @@ def banner_field(self, banner, flat_field): current_obj = ','.join([str(i) for i in current_obj]) return current_obj - except: + except Exception: pass return '' @@ -85,4 +85,4 @@ def flatten(self, d, parent_key='', sep='.'): items.extend(flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) - return dict(items) \ No newline at end of file + return dict(items) diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index c22da90..a6b476d 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -2,7 +2,7 @@ from .base import Converter from ...helpers import iterate_files, get_ip -from collections import defaultdict, MutableMapping +from collections import defaultdict from xlsxwriter import Workbook @@ -90,7 +90,7 @@ def process(self, files): main_sheet.write(row, col, value) col += 1 row += 1 - except: + except Exception: pass # Aggregate summary information @@ -124,7 +124,7 @@ def banner_field(self, banner, flat_field): current_obj = ','.join([str(i) for i in current_obj]) return current_obj - except: + except Exception: pass - return '' \ No newline at end of file + return '' diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 2cc55e4..8bde86f 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -53,5 +53,5 @@ def write(self, host): }""".format(ip, ip, lat, lon) self.fout.write(feature) - except: + except Exception: pass diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index c8cb2cc..49938c2 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -102,5 +102,5 @@ def write(self, host): placemark += '' self.fout.write(placemark.encode('utf-8')) - except: + except Exception: pass diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 7f7d89f..6ef9e1b 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -51,7 +51,7 @@ def get_banner_field(banner, flat_field): for field in fields: current_obj = current_obj[field] return current_obj - except: + except Exception: pass return None diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py index 985c38e..50d814c 100644 --- a/shodan/cli/organization.py +++ b/shodan/cli/organization.py @@ -1,7 +1,5 @@ import click -import requests import shodan -import shodan.helpers as helpers from shodan.cli.helpers import get_api_key, humanize_api_plan diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index c3357ba..60ef075 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -132,7 +132,7 @@ def set_data(self, data): # Not all cities can be encoded in ASCII so ignore any errors try: desc += ' {}'.format(banner['location']['city']) - except: + except Exception: pass if 'tags' in banner and banner['tags']: diff --git a/shodan/client.py b/shodan/client.py index 2b82ae2..1d3ff54 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -26,7 +26,7 @@ # pylint: disable=E1101 try: requests.packages.urllib3.disable_warnings() -except: +except Exception: pass # Define a basestring type if necessary for Python3 compatibility @@ -222,7 +222,7 @@ def _request(self, function, params, service='shodan', method='get'): data = self._session.delete(base_url + function, params=params) else: data = self._session.get(base_url + function, params=params) - except: + except Exception: raise APIError('Unable to connect to Shodan') # Check that the API key wasn't rejected @@ -246,7 +246,7 @@ def _request(self, function, params, service='shodan', method='get'): # Parse the text into JSON try: data = data.json() - except: + except ValueError: raise APIError('Unable to parse JSON response') # Raise an exception if an error occurred @@ -441,7 +441,7 @@ def search_cursor(self, query, minify=True, retries=5): return # exit out of the function page += 1 tries = 0 - except: + except Exception: # We've retried several times but it keeps failing, so lets error out if tries >= retries: break diff --git a/shodan/helpers.py b/shodan/helpers.py index ed1a88a..2756ab0 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -58,7 +58,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho # Exit out of the loop break - except: + except Exception: error = True tries += 1 @@ -69,7 +69,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho if data.status_code == 401: try: raise APIError(data.json()['error']) - except: + except (ValueError, KeyError): pass raise APIError('Invalid API key') @@ -89,7 +89,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho def iterate_files(files, fast=False): """Loop over all the records of the provided Shodan output file(s).""" - from json import loads + loads = json.loads if fast: # Try to use ujson for parsing JSON if it's available and the user requested faster throughput # It's significantly faster at encoding/ decoding JSON but it doesn't support as From 3312b02075c574b25342d6fb6c960692980af4ca Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 18 Sep 2018 18:16:33 +0200 Subject: [PATCH 043/148] unix line endings everywhere --- CHANGELOG.md | 1 + shodan/client.py | 1126 +++++++++++++++++++++--------------------- tests/test_shodan.py | 314 ++++++------ 3 files changed, 721 insertions(+), 720 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 063ff36..0d531ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ unreleased ---------- * Change bare 'except:' statements to 'except Exception:' or more specific ones * remove unused imports +* Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix 1.10.2 ------ diff --git a/shodan/client.py b/shodan/client.py index 1d3ff54..0fd9167 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -1,563 +1,563 @@ -# -*- coding: utf-8 -*- -""" -shodan.client -~~~~~~~~~~~~~ - -This module implements the Shodan API. - -:copyright: (c) 2014- by John Matherly -""" -import time - -import requests -import json - -from .exception import APIError -from .helpers import api_request, create_facet_string -from .stream import Stream - - -# Try to disable the SSL warnings in urllib3 since not everybody can install -# C extensions. If you're able to install C extensions you can try to run: -# -# pip install requests[security] -# -# Which will download libraries that offer more full-featured SSL classes -# pylint: disable=E1101 -try: - requests.packages.urllib3.disable_warnings() -except Exception: - pass - -# Define a basestring type if necessary for Python3 compatibility -try: - basestring -except NameError: - basestring = str - - -class Shodan: - """Wrapper around the Shodan REST and Streaming APIs - - :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) - :type key: str - :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. - :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. - """ - - class Data: - - def __init__(self, parent): - self.parent = parent - - def list_datasets(self): - """Returns a list of datasets that the user has permission to download. - - :returns: A list of objects where every object describes a dataset - """ - return self.parent._request('/shodan/data', {}) - - def list_files(self, dataset): - """Returns a list of files that belong to the given dataset. - - :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' - """ - return self.parent._request('/shodan/data/{}'.format(dataset), {}) - - class Tools: - - def __init__(self, parent): - self.parent = parent - - def myip(self): - """Get your current IP address as seen from the Internet. - - :returns: str -- your IP address - """ - return self.parent._request('/tools/myip', {}) - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, page=1, facets=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :param page: The page number to access. - :type page: int - :returns: dict -- a dictionary containing the results of the search. - """ - query_args = { - 'query': query, - 'page': page, - } - if facets: - query_args['facets'] = create_facet_string(facets) - - return self.parent._request('/api/search', query_args, service='exploits') - - def count(self, query, facets=None): - """Search the entire Shodan Exploits archive but only return the total # of results, - not the actual exploits. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :returns: dict -- a dictionary containing the results of the search. - - """ - query_args = { - 'query': query, - } - if facets: - query_args['facets'] = create_facet_string(facets) - - return self.parent._request('/api/count', query_args, service='exploits') - - class Labs: - - def __init__(self, parent): - self.parent = parent - - def honeyscore(self, ip): - """Calculate the probability of an IP being an ICS honeypot. - - :param ip: IP address of the device - :type ip: str - - :returns: int -- honeyscore ranging from 0.0 to 1.0 - """ - return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) - - class Organization: - - def __init__(self, parent): - self.parent = parent - - def add_member(self, user, notify=True): - """Add the user to the organization. - - :param user: username or email address - :type user: str - :param notify: whether or not to send the user an email notification - :type notify: bool - - :returns: True if it succeeded and raises an Exception otherwise - """ - return self.parent._request('/org/member/{}'.format(user), { - 'notify': notify, - }, method='PUT')['success'] - - def info(self): - """Returns general information about the organization the current user is a member of. - """ - return self.parent._request('/org', {}) - - def remove_member(self, user): - """Remove the user from the organization. - - :param user: username or email address - :type user: str - - :returns: True if it succeeded and raises an Exception otherwise - """ - return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] - - def __init__(self, key, proxies=None): - """Initializes the API object. - - :param key: The Shodan API key. - :type key: str - :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} - :type key: dict - """ - self.api_key = key - self.base_url = 'https://api.shodan.io' - self.base_exploits_url = 'https://exploits.shodan.io' - self.data = self.Data(self) - self.exploits = self.Exploits(self) - self.labs = self.Labs(self) - self.org = self.Organization(self) - self.tools = self.Tools(self) - self.stream = Stream(key, proxies=proxies) - self._session = requests.Session() - if proxies: - self._session.proxies.update(proxies) - - def _request(self, function, params, service='shodan', method='get'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A dictionary containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - method = method.lower() - if method == 'post': - data = self._session.post(base_url + function, params) - elif method == 'put': - data = self._session.put(base_url + function, params=params) - elif method == 'delete': - data = self._session.delete(base_url + function, params=params) - else: - data = self._session.get(base_url + function, params=params) - except Exception: - raise APIError('Unable to connect to Shodan') - - # Check that the API key wasn't rejected - if data.status_code == 401: - try: - # Return the actual error message if the API returned valid JSON - error = data.json()['error'] - except Exception as e: - # If the response looks like HTML then it's probably the 401 page that nginx returns - # for 401 responses by default - if data.text.startswith('<'): - error = 'Invalid API key' - else: - # Otherwise lets raise the error message - error = u'{}'.format(e) - - raise APIError(error) - elif data.status_code == 403: - raise APIError('Access denied (403 Forbidden)') - - # Parse the text into JSON - try: - data = data.json() - except ValueError: - raise APIError('Unable to parse JSON response') - - # Raise an exception if an error occurred - if type(data) == dict and 'error' in data: - raise APIError(data['error']) - - # Return the data - return data - - def count(self, query, facets=None): - """Returns the total number of search results for the query. - - :param query: Search query; identical syntax to the website - :type query: str - :param facets: (optional) A list of properties to get summary information on - :type facets: str - - :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - query_args = { - 'query': query, - } - if facets: - query_args['facets'] = create_facet_string(facets) - return self._request('/shodan/host/count', query_args) - - def host(self, ips, history=False, minify=False): - """Get all available information on an IP. - - :param ip: IP of the computer - :type ip: str - :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. - :type history: bool - :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise. - :type minify: bool - """ - if isinstance(ips, basestring): - ips = [ips] - - params = {} - if history: - params['history'] = history - if minify: - params['minify'] = minify - return self._request('/shodan/host/%s' % ','.join(ips), params) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('/api-info', {}) - - def ports(self): - """Get a list of ports that Shodan crawls - - :returns: An array containing the ports that Shodan crawls for. - """ - return self._request('/shodan/ports', {}) - - def protocols(self): - """Get a list of protocols that the Shodan on-demand scanning API supports. - - :returns: A dictionary containing the protocol name and description. - """ - return self._request('/shodan/protocols', {}) - - def scan(self, ips, force=False): - """Scan a network using Shodan - - :param ips: A list of IPs or netblocks in CIDR notation or an object structured like: - { - "9.9.9.9": [ - (443, "https"), - (8080, "http") - ], - "1.1.1.0/24": [ - (503, "modbus") - ] - } - :type ips: str or dict - :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users. - :type force: bool - - :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. - """ - if isinstance(ips, basestring): - ips = [ips] - - if isinstance(ips, dict): - networks = json.dumps(ips) - else: - networks = ','.join(ips) - - params = { - 'ips': networks, - 'force': force, - } - - return self._request('/shodan/scan', params, method='post') - - def scan_internet(self, port, protocol): - """Scan a network using Shodan - - :param port: The port that should get scanned. - :type port: int - :param port: The name of the protocol as returned by the protocols() method. - :type port: str - - :returns: A dictionary with a unique ID to check on the scan progress. - """ - params = { - 'port': port, - 'protocol': protocol, - } - - return self._request('/shodan/scan/internet', params, method='post') - - def scan_status(self, scan_id): - """Get the status information about a previously submitted scan. - - :param id: The unique ID for the scan that was submitted - :type id: str - - :returns: A dictionary with general information about the scan, including its status in getting processed. - """ - return self._request('/shodan/scan/%s' % scan_id, {}) - - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): - """Search the SHODAN database. - - :param query: Search query; identical syntax to the website - :type query: str - :param page: (optional) Page number of the search results - :type page: int - :param limit: (optional) Number of results to return - :type limit: int - :param offset: (optional) Search offset to begin getting results from - :type offset: int - :param facets: (optional) A list of properties to get summary information on - :type facets: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - - :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - args = { - 'query': query, - 'minify': minify, - } - if limit: - args['limit'] = limit - if offset: - args['offset'] = offset - else: - args['page'] = page - - if facets: - args['facets'] = create_facet_string(facets) - - return self._request('/shodan/host/search', args) - - def search_cursor(self, query, minify=True, retries=5): - """Search the SHODAN database. - - This method returns an iterator that can directly be in a loop. Use it when you want to loop over - all of the results of a search query. But this method doesn't return a "matches" array or the "total" - information. And it also can't be used with facets, it's only use is to iterate over results more - easily. - - :param query: Search query; identical syntax to the website - :type query: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - :param retries: (optional) How often to retry the search in case it times out - :type minify: int - - :returns: A search cursor that can be used as an iterator/ generator. - """ - page = 1 - tries = 0 - results = { - 'matches': [], - 'total': None, - } - while page == 1 or results['matches']: - try: - results = self.search(query, minify=minify, page=page) - for banner in results['matches']: - try: - yield banner - except GeneratorExit: - return # exit out of the function - page += 1 - tries = 0 - except Exception: - # We've retried several times but it keeps failing, so lets error out - if tries >= retries: - break - - tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason - - def search_tokens(self, query): - """Returns information about the search query itself (filters used etc.) - - :param query: Search query; identical syntax to the website - :type query: str - - :returns: A dictionary with 4 main properties: filters, errors, attributes and string. - """ - query_args = { - 'query': query, - } - return self._request('/shodan/host/search/tokens', query_args) - - def services(self): - """Get a list of services that Shodan crawls - - :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. - """ - return self._request('/shodan/services', {}) - - def queries(self, page=1, sort='timestamp', order='desc'): - """List the search queries that have been shared by other users. - - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - :param sort: Sort the list based on a property. Possible values are: votes, timestamp - :type sort: str - :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc - :type order: str - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'sort': sort, - 'order': order, - } - return self._request('/shodan/query', args) - - def queries_search(self, query, page=1): - """Search the directory of saved search queries in Shodan. - - :param query: The search string to look for in the search query - :type query: str - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'query': query, - } - return self._request('/shodan/query/search', args) - - def queries_tags(self, size=10): - """Search the directory of saved search queries in Shodan. - - :param query: The number of tags to return - :type page: int - - :returns: A list of tags. - """ - args = { - 'size': size, - } - return self._request('/shodan/query/tags', args) - - def create_alert(self, name, ip, expires=0): - """Search the directory of saved search queries in Shodan. - - :param query: The number of tags to return - :type page: int - - :returns: A list of tags. - """ - data = { - 'name': name, - 'filters': { - 'ip': ip, - }, - 'expires': expires, - } - - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', - proxies=self._session.proxies) - - return response - - def alerts(self, aid=None, include_expired=True): - """List all of the active alerts that the user created.""" - if aid: - func = '/shodan/alert/%s/info' % aid - else: - func = '/shodan/alert/info' - - response = api_request(self.api_key, func, params={ - 'include_expired': include_expired, - }, - proxies=self._session.proxies) - - return response - - def delete_alert(self, aid): - """Delete the alert with the given ID.""" - func = '/shodan/alert/%s' % aid - - response = api_request(self.api_key, func, params={}, method='delete', - proxies=self._session.proxies) - - return response - +# -*- coding: utf-8 -*- +""" +shodan.client +~~~~~~~~~~~~~ + +This module implements the Shodan API. + +:copyright: (c) 2014- by John Matherly +""" +import time + +import requests +import json + +from .exception import APIError +from .helpers import api_request, create_facet_string +from .stream import Stream + + +# Try to disable the SSL warnings in urllib3 since not everybody can install +# C extensions. If you're able to install C extensions you can try to run: +# +# pip install requests[security] +# +# Which will download libraries that offer more full-featured SSL classes +# pylint: disable=E1101 +try: + requests.packages.urllib3.disable_warnings() +except Exception: + pass + +# Define a basestring type if necessary for Python3 compatibility +try: + basestring +except NameError: + basestring = str + + +class Shodan: + """Wrapper around the Shodan REST and Streaming APIs + + :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) + :type key: str + :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. + :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. + """ + + class Data: + + def __init__(self, parent): + self.parent = parent + + def list_datasets(self): + """Returns a list of datasets that the user has permission to download. + + :returns: A list of objects where every object describes a dataset + """ + return self.parent._request('/shodan/data', {}) + + def list_files(self, dataset): + """Returns a list of files that belong to the given dataset. + + :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' + """ + return self.parent._request('/shodan/data/{}'.format(dataset), {}) + + class Tools: + + def __init__(self, parent): + self.parent = parent + + def myip(self): + """Get your current IP address as seen from the Internet. + + :returns: str -- your IP address + """ + return self.parent._request('/tools/myip', {}) + + class Exploits: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, page=1, facets=None): + """Search the entire Shodan Exploits archive using the same query syntax + as the website. + + :param query: The exploit search query; same syntax as website. + :type query: str + :param facets: A list of strings or tuples to get summary information on. + :type facets: str + :param page: The page number to access. + :type page: int + :returns: dict -- a dictionary containing the results of the search. + """ + query_args = { + 'query': query, + 'page': page, + } + if facets: + query_args['facets'] = create_facet_string(facets) + + return self.parent._request('/api/search', query_args, service='exploits') + + def count(self, query, facets=None): + """Search the entire Shodan Exploits archive but only return the total # of results, + not the actual exploits. + + :param query: The exploit search query; same syntax as website. + :type query: str + :param facets: A list of strings or tuples to get summary information on. + :type facets: str + :returns: dict -- a dictionary containing the results of the search. + + """ + query_args = { + 'query': query, + } + if facets: + query_args['facets'] = create_facet_string(facets) + + return self.parent._request('/api/count', query_args, service='exploits') + + class Labs: + + def __init__(self, parent): + self.parent = parent + + def honeyscore(self, ip): + """Calculate the probability of an IP being an ICS honeypot. + + :param ip: IP address of the device + :type ip: str + + :returns: int -- honeyscore ranging from 0.0 to 1.0 + """ + return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) + + class Organization: + + def __init__(self, parent): + self.parent = parent + + def add_member(self, user, notify=True): + """Add the user to the organization. + + :param user: username or email address + :type user: str + :param notify: whether or not to send the user an email notification + :type notify: bool + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), { + 'notify': notify, + }, method='PUT')['success'] + + def info(self): + """Returns general information about the organization the current user is a member of. + """ + return self.parent._request('/org', {}) + + def remove_member(self, user): + """Remove the user from the organization. + + :param user: username or email address + :type user: str + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + + def __init__(self, key, proxies=None): + """Initializes the API object. + + :param key: The Shodan API key. + :type key: str + :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} + :type key: dict + """ + self.api_key = key + self.base_url = 'https://api.shodan.io' + self.base_exploits_url = 'https://exploits.shodan.io' + self.data = self.Data(self) + self.exploits = self.Exploits(self) + self.labs = self.Labs(self) + self.org = self.Organization(self) + self.tools = self.Tools(self) + self.stream = Stream(key, proxies=proxies) + self._session = requests.Session() + if proxies: + self._session.proxies.update(proxies) + + def _request(self, function, params, service='shodan', method='get'): + """General-purpose function to create web requests to SHODAN. + + Arguments: + function -- name of the function you want to execute + params -- dictionary of parameters for the function + + Returns + A dictionary containing the function's results. + + """ + # Add the API key parameter automatically + params['key'] = self.api_key + + # Determine the base_url based on which service we're interacting with + base_url = { + 'shodan': self.base_url, + 'exploits': self.base_exploits_url, + }.get(service, 'shodan') + + # Send the request + try: + method = method.lower() + if method == 'post': + data = self._session.post(base_url + function, params) + elif method == 'put': + data = self._session.put(base_url + function, params=params) + elif method == 'delete': + data = self._session.delete(base_url + function, params=params) + else: + data = self._session.get(base_url + function, params=params) + except Exception: + raise APIError('Unable to connect to Shodan') + + # Check that the API key wasn't rejected + if data.status_code == 401: + try: + # Return the actual error message if the API returned valid JSON + error = data.json()['error'] + except Exception as e: + # If the response looks like HTML then it's probably the 401 page that nginx returns + # for 401 responses by default + if data.text.startswith('<'): + error = 'Invalid API key' + else: + # Otherwise lets raise the error message + error = u'{}'.format(e) + + raise APIError(error) + elif data.status_code == 403: + raise APIError('Access denied (403 Forbidden)') + + # Parse the text into JSON + try: + data = data.json() + except ValueError: + raise APIError('Unable to parse JSON response') + + # Raise an exception if an error occurred + if type(data) == dict and 'error' in data: + raise APIError(data['error']) + + # Return the data + return data + + def count(self, query, facets=None): + """Returns the total number of search results for the query. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. + """ + query_args = { + 'query': query, + } + if facets: + query_args['facets'] = create_facet_string(facets) + return self._request('/shodan/host/count', query_args) + + def host(self, ips, history=False, minify=False): + """Get all available information on an IP. + + :param ip: IP of the computer + :type ip: str + :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. + :type history: bool + :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise. + :type minify: bool + """ + if isinstance(ips, basestring): + ips = [ips] + + params = {} + if history: + params['history'] = history + if minify: + params['minify'] = minify + return self._request('/shodan/host/%s' % ','.join(ips), params) + + def info(self): + """Returns information about the current API key, such as a list of add-ons + and other features that are enabled for the current user's API plan. + """ + return self._request('/api-info', {}) + + def ports(self): + """Get a list of ports that Shodan crawls + + :returns: An array containing the ports that Shodan crawls for. + """ + return self._request('/shodan/ports', {}) + + def protocols(self): + """Get a list of protocols that the Shodan on-demand scanning API supports. + + :returns: A dictionary containing the protocol name and description. + """ + return self._request('/shodan/protocols', {}) + + def scan(self, ips, force=False): + """Scan a network using Shodan + + :param ips: A list of IPs or netblocks in CIDR notation or an object structured like: + { + "9.9.9.9": [ + (443, "https"), + (8080, "http") + ], + "1.1.1.0/24": [ + (503, "modbus") + ] + } + :type ips: str or dict + :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users. + :type force: bool + + :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. + """ + if isinstance(ips, basestring): + ips = [ips] + + if isinstance(ips, dict): + networks = json.dumps(ips) + else: + networks = ','.join(ips) + + params = { + 'ips': networks, + 'force': force, + } + + return self._request('/shodan/scan', params, method='post') + + def scan_internet(self, port, protocol): + """Scan a network using Shodan + + :param port: The port that should get scanned. + :type port: int + :param port: The name of the protocol as returned by the protocols() method. + :type port: str + + :returns: A dictionary with a unique ID to check on the scan progress. + """ + params = { + 'port': port, + 'protocol': protocol, + } + + return self._request('/shodan/scan/internet', params, method='post') + + def scan_status(self, scan_id): + """Get the status information about a previously submitted scan. + + :param id: The unique ID for the scan that was submitted + :type id: str + + :returns: A dictionary with general information about the scan, including its status in getting processed. + """ + return self._request('/shodan/scan/%s' % scan_id, {}) + + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + """Search the SHODAN database. + + :param query: Search query; identical syntax to the website + :type query: str + :param page: (optional) Page number of the search results + :type page: int + :param limit: (optional) Number of results to return + :type limit: int + :param offset: (optional) Search offset to begin getting results from + :type offset: int + :param facets: (optional) A list of properties to get summary information on + :type facets: str + :param minify: (optional) Whether to minify the banner and only return the important data + :type minify: bool + + :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'minify': minify, + } + if limit: + args['limit'] = limit + if offset: + args['offset'] = offset + else: + args['page'] = page + + if facets: + args['facets'] = create_facet_string(facets) + + return self._request('/shodan/host/search', args) + + def search_cursor(self, query, minify=True, retries=5): + """Search the SHODAN database. + + This method returns an iterator that can directly be in a loop. Use it when you want to loop over + all of the results of a search query. But this method doesn't return a "matches" array or the "total" + information. And it also can't be used with facets, it's only use is to iterate over results more + easily. + + :param query: Search query; identical syntax to the website + :type query: str + :param minify: (optional) Whether to minify the banner and only return the important data + :type minify: bool + :param retries: (optional) How often to retry the search in case it times out + :type minify: int + + :returns: A search cursor that can be used as an iterator/ generator. + """ + page = 1 + tries = 0 + results = { + 'matches': [], + 'total': None, + } + while page == 1 or results['matches']: + try: + results = self.search(query, minify=minify, page=page) + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 + tries = 0 + except Exception: + # We've retried several times but it keeps failing, so lets error out + if tries >= retries: + break + + tries += 1 + time.sleep(1.0) # wait 1 second if the search errored out for some reason + + def search_tokens(self, query): + """Returns information about the search query itself (filters used etc.) + + :param query: Search query; identical syntax to the website + :type query: str + + :returns: A dictionary with 4 main properties: filters, errors, attributes and string. + """ + query_args = { + 'query': query, + } + return self._request('/shodan/host/search/tokens', query_args) + + def services(self): + """Get a list of services that Shodan crawls + + :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. + """ + return self._request('/shodan/services', {}) + + def queries(self, page=1, sort='timestamp', order='desc'): + """List the search queries that have been shared by other users. + + :param page: Page number to iterate over results; each page contains 10 items + :type page: int + :param sort: Sort the list based on a property. Possible values are: votes, timestamp + :type sort: str + :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc + :type order: str + + :returns: A list of saved search queries (dictionaries). + """ + args = { + 'page': page, + 'sort': sort, + 'order': order, + } + return self._request('/shodan/query', args) + + def queries_search(self, query, page=1): + """Search the directory of saved search queries in Shodan. + + :param query: The search string to look for in the search query + :type query: str + :param page: Page number to iterate over results; each page contains 10 items + :type page: int + + :returns: A list of saved search queries (dictionaries). + """ + args = { + 'page': page, + 'query': query, + } + return self._request('/shodan/query/search', args) + + def queries_tags(self, size=10): + """Search the directory of saved search queries in Shodan. + + :param query: The number of tags to return + :type page: int + + :returns: A list of tags. + """ + args = { + 'size': size, + } + return self._request('/shodan/query/tags', args) + + def create_alert(self, name, ip, expires=0): + """Search the directory of saved search queries in Shodan. + + :param query: The number of tags to return + :type page: int + + :returns: A list of tags. + """ + data = { + 'name': name, + 'filters': { + 'ip': ip, + }, + 'expires': expires, + } + + response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', + proxies=self._session.proxies) + + return response + + def alerts(self, aid=None, include_expired=True): + """List all of the active alerts that the user created.""" + if aid: + func = '/shodan/alert/%s/info' % aid + else: + func = '/shodan/alert/info' + + response = api_request(self.api_key, func, params={ + 'include_expired': include_expired, + }, + proxies=self._session.proxies) + + return response + + def delete_alert(self, aid): + """Delete the alert with the given ID.""" + func = '/shodan/alert/%s' % aid + + response = api_request(self.api_key, func, params={}, method='delete', + proxies=self._session.proxies) + + return response + diff --git a/tests/test_shodan.py b/tests/test_shodan.py index 396a059..0cdd602 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -1,157 +1,157 @@ -import unittest -import shodan - -try: - basestring -except NameError: - basestring = str - - -class ShodanTests(unittest.TestCase): - - api = None - FACETS = [ - 'port', - ('domain', 1) - ] - QUERIES = { - 'simple': 'cisco-ios', - 'minify': 'apache', - 'advanced': 'apache port:443', - 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', - } - - def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) - - def test_search_simple(self): - results = self.api.search(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure some values were returned - self.assertTrue(results['matches']) - self.assertTrue(results['total']) - - # A regular search shouldn't have the optional info - self.assertNotIn('opts', results['matches'][0]) - - def test_search_empty(self): - results = self.api.search(self.QUERIES['empty']) - self.assertTrue(len(results['matches']) == 0) - self.assertEqual(results['total'], 0) - - def test_search_facets(self): - results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_count_simple(self): - results = self.api.count(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure no values were returned - self.assertFalse(results['matches']) - self.assertTrue(results['total']) - - def test_count_facets(self): - results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_host_details(self): - host = self.api.host('147.228.101.7') - - self.assertEqual('147.228.101.7', host['ip_str']) - self.assertFalse(isinstance(host['ip'], basestring)) - - def test_search_minify(self): - results = self.api.search(self.QUERIES['minify'], minify=False) - self.assertIn('opts', results['matches'][0]) - - def test_exploits_search(self): - results = self.api.exploits.search('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(results['matches']) - - def test_exploits_search_paging(self): - results = self.api.exploits.search('apache', page=1) - match1 = results['matches'][0] - results = self.api.exploits.search('apache', page=2) - match2 = results['matches'][0] - - self.assertNotEqual(match1['_id'], match2['_id']) - - def test_exploits_search_facets(self): - results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - def test_exploits_count(self): - results = self.api.exploits.count('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(len(results['matches']) == 0) - - def test_exploits_count_facets(self): - results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) - self.assertEqual(len(results['matches']), 0) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - # Test error responses - def test_invalid_key(self): - api = shodan.Shodan('garbage') - raised = False - try: - api.search('something') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_invalid_host_ip(self): - raised = False - try: - host = self.api.host('test') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_search_empty_query(self): - raised = False - try: - self.api.search('') - except shodan.APIError as e: - raised = True - self.assertTrue(raised) - - def test_search_advanced_query(self): - # The free API plan can't use filters - raised = False - try: - self.api.search(self.QUERIES['advanced']) - except shodan.APIError as e: - raised = True - self.assertTrue(raised) - - -if __name__ == '__main__': - unittest.main() +import unittest +import shodan + +try: + basestring +except NameError: + basestring = str + + +class ShodanTests(unittest.TestCase): + + api = None + FACETS = [ + 'port', + ('domain', 1) + ] + QUERIES = { + 'simple': 'cisco-ios', + 'minify': 'apache', + 'advanced': 'apache port:443', + 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', + } + + def setUp(self): + self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) + + def test_search_simple(self): + results = self.api.search(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure some values were returned + self.assertTrue(results['matches']) + self.assertTrue(results['total']) + + # A regular search shouldn't have the optional info + self.assertNotIn('opts', results['matches'][0]) + + def test_search_empty(self): + results = self.api.search(self.QUERIES['empty']) + self.assertTrue(len(results['matches']) == 0) + self.assertEqual(results['total'], 0) + + def test_search_facets(self): + results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_count_simple(self): + results = self.api.count(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure no values were returned + self.assertFalse(results['matches']) + self.assertTrue(results['total']) + + def test_count_facets(self): + results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_host_details(self): + host = self.api.host('147.228.101.7') + + self.assertEqual('147.228.101.7', host['ip_str']) + self.assertFalse(isinstance(host['ip'], basestring)) + + def test_search_minify(self): + results = self.api.search(self.QUERIES['minify'], minify=False) + self.assertIn('opts', results['matches'][0]) + + def test_exploits_search(self): + results = self.api.exploits.search('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + + def test_exploits_search_paging(self): + results = self.api.exploits.search('apache', page=1) + match1 = results['matches'][0] + results = self.api.exploits.search('apache', page=2) + match2 = results['matches'][0] + + self.assertNotEqual(match1['_id'], match2['_id']) + + def test_exploits_search_facets(self): + results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_exploits_count(self): + results = self.api.exploits.count('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(len(results['matches']) == 0) + + def test_exploits_count_facets(self): + results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) + self.assertEqual(len(results['matches']), 0) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + # Test error responses + def test_invalid_key(self): + api = shodan.Shodan('garbage') + raised = False + try: + api.search('something') + except shodan.APIError as e: + raised = True + + self.assertTrue(raised) + + def test_invalid_host_ip(self): + raised = False + try: + host = self.api.host('test') + except shodan.APIError as e: + raised = True + + self.assertTrue(raised) + + def test_search_empty_query(self): + raised = False + try: + self.api.search('') + except shodan.APIError as e: + raised = True + self.assertTrue(raised) + + def test_search_advanced_query(self): + # The free API plan can't use filters + raised = False + try: + self.api.search(self.QUERIES['advanced']) + except shodan.APIError as e: + raised = True + self.assertTrue(raised) + + +if __name__ == '__main__': + unittest.main() From f2cb964db3f5208f7453a6d44e65402253138d67 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 21 Sep 2018 21:13:53 -0500 Subject: [PATCH 044/148] Show the list of available file formats in "shodan convert" (#80) --- shodan/__main__.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 339218e..88f4e4f 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -84,11 +84,20 @@ def main(): main.add_command(scan) +CONVERTERS = { + 'kml': KmlConverter, + 'csv': CsvConverter, + 'geo.json': GeoJsonConverter, + 'images': ImagesConverter, + 'xlsx': ExcelConverter, +} @main.command() @click.argument('input', metavar='') -@click.argument('format', metavar='', type=click.Choice(['kml', 'csv', 'geo.json', 'images', 'xlsx'])) +@click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) def convert(input, format): - """Convert the given input data file into a different format. + """Convert the given input data file into a different format. The following file formats are supported: + + kml, csv, geo.json, images, xlsx Example: shodan convert data.json.gz kml """ @@ -107,13 +116,7 @@ def convert(input, format): progress_bar_thread.start() # Initialize the file converter - converter = { - 'kml': KmlConverter, - 'csv': CsvConverter, - 'geo.json': GeoJsonConverter, - 'images': ImagesConverter, - 'xlsx': ExcelConverter, - }.get(format)(fout) + converter = CONVERTERS.get(format)(fout) converter.process([input]) From a9345073ddee623ae780d437f189a2c3147a4d9c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 21 Sep 2018 21:14:59 -0500 Subject: [PATCH 045/148] Release 1.10.3 --- CHANGELOG.md | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d531ab..adce430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ CHANGELOG ========= -unreleased ----------- +1.10.3 +------ * Change bare 'except:' statements to 'except Exception:' or more specific ones * remove unused imports * Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix +* List file types in **shodan convert** (#80) 1.10.2 ------ diff --git a/setup.py b/setup.py index 593a481..125896c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.10.2', + version = '1.10.3', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', From 1025723577c07054de1737df4d4d182d85a6cbd0 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 22 Sep 2018 20:59:03 -0500 Subject: [PATCH 046/148] Code quality improvements --- shodan/__main__.py | 2 +- shodan/helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index cba34d7..709a010 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -230,7 +230,7 @@ def download(limit, filename, query): if count >= limit: break - except: + except Exception: pass # Let the user know we're done diff --git a/shodan/helpers.py b/shodan/helpers.py index 2756ab0..d3c4a1f 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -98,7 +98,7 @@ def iterate_files(files, fast=False): # pylint: disable=E0401 try: from ujson import loads - except: + except Exception: pass if isinstance(files, basestring): From da2da758cd3fad3436efc62b61b4400dc71aca80 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 4 Oct 2018 20:00:03 -0500 Subject: [PATCH 047/148] Release 1.10.4 Fix a bug when showing old banner records that don't have the "transport" property --- setup.py | 2 +- shodan/cli/host.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 125896c..ef991f1 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'shodan', - version = '1.10.3', + version = '1.10.4', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description = README, long_description_content_type = 'text/x-rst', diff --git a/shodan/cli/host.py b/shodan/cli/host.py index fcd440f..befdc62 100644 --- a/shodan/cli/host.py +++ b/shodan/cli/host.py @@ -80,8 +80,9 @@ def host_print_pretty(host, history=False): version = '({})'.format(banner['version']) click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) - click.echo('/', nl=False) - click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) + if 'transport' in banner: + click.echo('/', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) click.echo('{} {}'.format(product, version), nl=False) if history: From bce4564cda4e79c9e477446faf5edcb16735d147 Mon Sep 17 00:00:00 2001 From: cclauss Date: Sun, 7 Oct 2018 08:48:51 +0200 Subject: [PATCH 048/148] StandardError was removed in Python 3 Use Exception instead. --- shodan/cli/worldmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 60ef075..fca52b4 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -177,7 +177,7 @@ def draw(self, target): self.window.addstr(row, 1, det_show, attrs) row += 1 items_to_show -= 1 - except StandardError: + except Exeception: # FIXME: check window size before addstr() break self.window.overwrite(target) From 04ca4bbfb3dc1b59b3716857819561502131ac7f Mon Sep 17 00:00:00 2001 From: cclauss Date: Sun, 7 Oct 2018 09:10:35 +0200 Subject: [PATCH 049/148] Undefined name: Don't forget self when calling flatten() __flatten__ is an undefined name in this context which has the potential to raise NameError at runtime. __self.flatten()__ makes more sense here. [flake8](http://flake8.pycqa.org) testing of https://github.com/achillean/shodan-python on Python 3.7.0 $ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__ ``` ./shodan/cli/helpers.py:73:52: F821 undefined name 'basestring' if field_type == list or isinstance(value, basestring): ^ ./shodan/cli/worldmap.py:180:24: F821 undefined name 'StandardError' except StandardError: ^ ./shodan/cli/converter/csvc.py:85:30: F821 undefined name 'flatten' items.extend(flatten(v, new_key, sep=sep).items()) ^ 3 F821 undefined name 'flatten' 3 ``` --- shodan/cli/converter/csvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index c975695..bbc266c 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -82,7 +82,7 @@ def flatten(self, d, parent_key='', sep='.'): new_key = parent_key + sep + k if parent_key else k if isinstance(v, MutableMapping): # pylint: disable=E0602 - items.extend(flatten(v, new_key, sep=sep).items()) + items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) From 7e24f4b447f764391bdc00f30de3a8abbfe75d99 Mon Sep 17 00:00:00 2001 From: cclauss Date: Sun, 7 Oct 2018 09:12:57 +0200 Subject: [PATCH 050/148] Remove pylint: disable=E0602 This PR fixes PyLint's complaint. --- shodan/cli/converter/csvc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index bbc266c..8c42254 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -81,7 +81,6 @@ def flatten(self, d, parent_key='', sep='.'): for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, MutableMapping): - # pylint: disable=E0602 items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) From aa579b732b71ca03c665b7fc91107062889b18a2 Mon Sep 17 00:00:00 2001 From: cclauss Date: Sun, 7 Oct 2018 09:19:49 +0200 Subject: [PATCH 051/148] Undefined name: 'basestring' was removed in Python 3 __basestring__ was removed in Python 3 but it is used on line 78 without being defined. --- shodan/cli/helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 6ef9e1b..33cdc57 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -10,6 +10,11 @@ from .settings import SHODAN_CONFIG_DIR +try: + basestring # Python 2 +except NameError: + basestring = (str, ) # Python 3 + def get_api_key(): '''Returns the API key of the current logged-in user.''' shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) From 0b501feba747f2a3ee9b08cb848afdbf02757fa7 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 2 Nov 2018 10:09:18 +0100 Subject: [PATCH 052/148] add changelog for 1.10.4 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index adce430..4b7f995 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ CHANGELOG ========= +unreleased +---------- + +1.10.4 +------ +* Fix a bug when showing old banner records that don't have the "transport" property +* Code quality improvements (bare excepts) + 1.10.3 ------ * Change bare 'except:' statements to 'except Exception:' or more specific ones From d8236dc3f152dfeb21794ea472c3f0f0203c3c86 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 5 Nov 2018 09:27:00 +0100 Subject: [PATCH 053/148] Fix a typo introduced in #82 #82 contained a typo... Sorry about that. --- shodan/cli/worldmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index fca52b4..cfe4e4a 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -177,7 +177,7 @@ def draw(self, target): self.window.addstr(row, 1, det_show, attrs) row += 1 items_to_show -= 1 - except Exeception: + except Exception: # FIXME: check window size before addstr() break self.window.overwrite(target) From 0a77e3d34b8d6e00c4363d3dc0af58b5d1b339c8 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 10 Dec 2018 14:55:21 +0100 Subject: [PATCH 054/148] Make shodan parse output separators only between fields. --- shodan/__main__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 709a010..112834d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -333,7 +333,7 @@ def parse(color, fields, filters, filename, separator, filenames): helpers.write_banner(fout, banner) # Loop over all the fields and print the banner as a row - for field in fields: + for i, field in enumerate(fields): tmp = u'' value = get_banner_field(banner, field) if value: @@ -351,9 +351,10 @@ def parse(color, fields, filters, filename, separator, filenames): if color: tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) - # Add the field information to the row - row += tmp - row += separator + # Add the field information to the row + if i > 0: + row += separator + row += tmp click.echo(row) From fc732ba57b449fe57fd6e1e9e278a162b1a51931 Mon Sep 17 00:00:00 2001 From: Jeremy Bae Date: Mon, 17 Dec 2018 18:10:51 +0900 Subject: [PATCH 055/148] Add virtualenv and IntelliJ/PyCharm directories --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index eca7a0a..b719ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ shodan.egg-info/* tmp/* MANIFEST .vscode/ -PKG-INFO \ No newline at end of file +PKG-INFO +venv/* +.idea/* \ No newline at end of file From 5029bb06278d64ce0685036437d5667fd2f29855 Mon Sep 17 00:00:00 2001 From: Vladimir Epifanov Date: Fri, 11 Jan 2019 13:41:37 +0200 Subject: [PATCH 056/148] Fix Shodan.init() type hint `Shodan.init()` has improper type hint for `proxies` parameter which shadows type hint for `key` parameter --- shodan/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/client.py b/shodan/client.py index 0fd9167..51c33f1 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -176,7 +176,7 @@ def __init__(self, key, proxies=None): :param key: The Shodan API key. :type key: str :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} - :type key: dict + :type proxies: dict """ self.api_key = key self.base_url = 'https://api.shodan.io' From 7f20a32d78bb59b38877e7f69dfc3619c08a25ff Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 10 Feb 2019 17:44:56 -0600 Subject: [PATCH 057/148] Include tags, timestamp and vulns information in CSV conversion (#85) --- shodan/cli/converter/csvc.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 8c42254..55dd021 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -24,9 +24,12 @@ class CsvConverter(Converter): 'os', 'asn', 'port', + 'tags', + 'timestamp', 'transport', 'product', 'version', + 'vulns', 'ssl.cipher.version', 'ssl.cipher.bits', @@ -48,6 +51,11 @@ def process(self, files): writer.writerow(self.fields) for banner in iterate_files(files): + # The "vulns" property can't be nicely flattened as-is so we turn + # it into a list before processing the banner. + if 'vulns' in banner: + banner['vulns'] = banner['vulns'].keys() + try: row = [] for field in self.fields: From 166597f4a9756dc38ed2077d8df73340efaa8eb4 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 10 Feb 2019 20:43:10 -0600 Subject: [PATCH 058/148] New command **shodan scan list** to list recently launched scans New command **shodan alert triggers** to list the available notification triggers New command **shodan alert enable** to enable a notification trigger New command **shodan alert disable** to disable a notification trigger Code quality improvements --- CHANGELOG.md | 6 + setup.py | 26 +-- shodan/__main__.py | 34 ++-- shodan/alert.py | 9 - shodan/cli/alert.py | 68 +++++++- shodan/cli/converter/__init__.py | 2 +- shodan/cli/converter/base.py | 2 +- shodan/cli/converter/csvc.py | 24 +-- shodan/cli/converter/excel.py | 26 +-- shodan/cli/converter/geojson.py | 20 +-- shodan/cli/converter/images.py | 2 +- shodan/cli/converter/kml.py | 20 +-- shodan/cli/helpers.py | 1 + shodan/cli/host.py | 10 +- shodan/cli/organization.py | 12 +- shodan/cli/scan.py | 36 +++- shodan/cli/worldmap.py | 15 +- shodan/client.py | 45 +++-- shodan/exception.py | 5 +- shodan/helpers.py | 8 +- shodan/stream.py | 13 +- shodan/threatnet.py | 5 +- tests/test_shodan.py | 284 +++++++++++++++---------------- tox.ini | 14 ++ 24 files changed, 408 insertions(+), 279 deletions(-) delete mode 100644 shodan/alert.py create mode 100644 tox.ini diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7f995..b107676 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ CHANGELOG unreleased ---------- +* New command **shodan scan list** to list recently launched scans +* New command **shodan alert triggers** to list the available notification triggers +* New command **shodan alert enable** to enable a notification trigger +* New command **shodan alert disable** to disable a notification trigger +* Include timestamp, vulns and tags in CSV converter (#85) +* Code quality improvements 1.10.4 ------ diff --git a/setup.py b/setup.py index ef991f1..c925129 100755 --- a/setup.py +++ b/setup.py @@ -6,19 +6,19 @@ README = open('README.rst', 'r').read() setup( - name = 'shodan', - version = '1.10.4', - description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', - long_description = README, - long_description_content_type = 'text/x-rst', - author = 'John Matherly', - author_email = 'jmath@shodan.io', - url = 'http://github.com/achillean/shodan-python/tree/master', - packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'], - entry_points = {'console_scripts': ['shodan = shodan.__main__:main']}, - install_requires = DEPENDENCIES, - keywords = ['security', 'network'], - classifiers = [ + name='shodan', + version='1.10.4', + description='Python library and command-line utility for Shodan (https://developer.shodan.io)', + long_description=README, + long_description_content_type='text/x-rst', + author='John Matherly', + author_email='jmath@shodan.io', + url='http://github.com/achillean/shodan-python/tree/master', + packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], + entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, + install_requires=DEPENDENCIES, + keywords=['security', 'network'], + classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', diff --git a/shodan/__main__.py b/shodan/__main__.py index 112834d..df2b38c 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -49,6 +49,13 @@ from click_plugins import with_plugins from pkg_resources import iter_entry_points +# Large subcommands are stored in separate modules +from shodan.cli.alert import alert +from shodan.cli.data import data +from shodan.cli.organization import org +from shodan.cli.scan import scan + + # Make "-h" work like "--help" CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -58,7 +65,6 @@ except NameError: basestring = str - # Define the main entry point for all of our commands # and expose a way for 3rd-party plugins to tie into the Shodan CLI. @with_plugins(iter_entry_points('shodan.cli.plugins')) @@ -67,11 +73,7 @@ def main(): pass -# Large subcommands are stored in separate modules -from shodan.cli.alert import alert -from shodan.cli.data import data -from shodan.cli.organization import org -from shodan.cli.scan import scan +# Setup the large subcommands main.add_command(alert) main.add_command(data) main.add_command(org) @@ -151,6 +153,7 @@ def init(key): os.chmod(keyfile, 0o600) + @main.command() @click.argument('query', metavar='', nargs=-1) def count(query): @@ -203,7 +206,7 @@ def download(limit, filename, query): try: total = api.count(query)['total'] info = api.info() - except: + except Exception: raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') # Print some summary information about the download request @@ -275,7 +278,6 @@ def host(format, history, filename, save, ip): raise click.ClickException(e.value) - @main.command() def info(): """Shows general information about your account""" @@ -308,7 +310,6 @@ def parse(color, fields, filters, filename, separator, filenames): has_filters = len(filters) > 0 - # Setup the output file handle fout = None if filename: @@ -354,7 +355,7 @@ def parse(color, fields, filters, filename, separator, filenames): # Add the field information to the row if i > 0: row += separator - row += tmp + row += tmp click.echo(row) @@ -520,7 +521,7 @@ def stats(limit, facets, filename, query): if len(values) > counter: has_items = True row[pos] = values[counter]['value'] - row[pos+1] = values[counter]['count'] + row[pos + 1] = values[counter]['count'] pos += 2 @@ -546,7 +547,7 @@ def stats(limit, facets, filename, query): @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel): +def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -642,9 +643,9 @@ def _create_stream(name, args, timeout): if datadir: cur_time = timestr() if cur_time != last_time: - last_time = cur_time - fout.close() - fout = open_streaming_file(datadir, last_time) + last_time = cur_time + fout.close() + fout = open_streaming_file(datadir, last_time) helpers.write_banner(fout, banner) # Print the banner information to stdout @@ -707,7 +708,7 @@ def honeyscore(ip): click.echo(click.style('Not a honeypot', fg='green')) click.echo('Score: {}'.format(score)) - except: + except Exception: raise click.ClickException('Unable to calculate honeyscore') @@ -726,5 +727,6 @@ def radar(): except Exception as e: raise click.ClickException(u'{}'.format(e)) + if __name__ == '__main__': main() diff --git a/shodan/alert.py b/shodan/alert.py deleted file mode 100644 index 7a89e90..0000000 --- a/shodan/alert.py +++ /dev/null @@ -1,9 +0,0 @@ -class Alert: - def __init__(self): - self.id = None - self.name = None - self.api_key = None - self.filters = None - self.credits = None - self.created = None - self.expires = None diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index a9f6ec8..d8b02b9 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,6 +3,7 @@ from shodan.cli.helpers import get_api_key + @click.group() def alert(): """Manage the network alerts for your account""" @@ -25,6 +26,7 @@ def alert_clear(): raise click.ClickException(e.value) click.echo("Alerts deleted") + @alert.command(name='create') @click.argument('name', metavar='') @click.argument('netblock', metavar='') @@ -42,6 +44,7 @@ def alert_create(name, netblock): click.secho('Successfully created network alert!', fg='green') click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + @alert.command(name='list') @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) def alert_list(expired): @@ -57,11 +60,11 @@ def alert_list(expired): if len(results) > 0: click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) - # click.echo('#' * 65) + for alert in results: click.echo( u'{:16} {:<30} {:<35} '.format( - click.style(alert['id'], fg='yellow'), + click.style(alert['id'], fg='yellow'), click.style(alert['name'], fg='cyan'), click.style(', '.join(alert['filters']['ip']), fg='white') ), @@ -89,3 +92,64 @@ def alert_remove(alert_id): except shodan.APIError as e: raise click.ClickException(e.value) click.echo("Alert deleted") + + +@alert.command(name='triggers') +def alert_list_triggers(): + """List all the available triggers""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + results = api.alert_triggers() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(results) > 0: + click.echo(u'# {:14} {:<21} {:<15s}'.format('Name', 'Description', 'Rule')) + + for trigger in results: + click.echo( + u'{:16} {:<30} {:<35} '.format( + click.style(trigger['name'], fg='yellow'), + click.style(trigger['description'], fg='cyan'), + trigger['rule'] + ) + ) + else: + click.echo("No triggers currently available.") + + +@alert.command(name='enable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_enable_trigger(alert_id, trigger): + """Enable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.enable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully enabled the trigger {}'.format(trigger), color='green') + + +@alert.command(name='disable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_disable_trigger(alert_id, trigger): + """Disable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.disable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully disabled the trigger {}'.format(trigger), color='green') diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py index 507ca0b..08b068a 100644 --- a/shodan/cli/converter/__init__.py +++ b/shodan/cli/converter/__init__.py @@ -2,4 +2,4 @@ from .excel import ExcelConverter from .geojson import GeoJsonConverter from .images import ImagesConverter -from .kml import KmlConverter \ No newline at end of file +from .kml import KmlConverter diff --git a/shodan/cli/converter/base.py b/shodan/cli/converter/base.py index 9dc83c2..14b5f29 100644 --- a/shodan/cli/converter/base.py +++ b/shodan/cli/converter/base.py @@ -3,6 +3,6 @@ class Converter: def __init__(self, fout): self.fout = fout - + def process(self, fout): pass diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 55dd021..39ec162 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -30,7 +30,7 @@ class CsvConverter(Converter): 'product', 'version', 'vulns', - + 'ssl.cipher.version', 'ssl.cipher.bits', 'ssl.cipher.name', @@ -39,23 +39,23 @@ class CsvConverter(Converter): 'ssl.cert.serial', 'ssl.cert.fingerprint.sha1', 'ssl.cert.fingerprint.sha256', - + 'html', 'title', ] - + def process(self, files): writer = csv_writer(self.fout, dialect=excel) - + # Write the header writer.writerow(self.fields) - + for banner in iterate_files(files): # The "vulns" property can't be nicely flattened as-is so we turn # it into a list before processing the banner. if 'vulns' in banner: banner['vulns'] = banner['vulns'].keys() - + try: row = [] for field in self.fields: @@ -64,26 +64,26 @@ def process(self, files): writer.writerow(row) except Exception: pass - + def banner_field(self, banner, flat_field): # The provided field is a collapsed form of the actual field fields = flat_field.split('.') - + try: current_obj = banner for field in fields: current_obj = current_obj[field] - + # Convert a list into a concatenated string if isinstance(current_obj, list): current_obj = ','.join([str(i) for i in current_obj]) - + return current_obj except Exception: pass - + return '' - + def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index a6b476d..e96c24b 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -23,7 +23,7 @@ class ExcelConverter(Converter): 'transport', 'product', 'version', - + 'http.server', 'http.title', ] @@ -40,7 +40,7 @@ class ExcelConverter(Converter): 'http.server': 'Web Server', 'http.title': 'Website Title', } - + def process(self, files): # Get the filename from the already-open file handle filename = self.fout.name @@ -55,14 +55,14 @@ def process(self, files): bold = workbook.add_format({ 'bold': 1, }) - + # Create the main worksheet where all the raw data is shown main_sheet = workbook.add_worksheet('Raw Data') # Write the header - main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently + main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently main_sheet.set_column(0, 0, 20) - + row = 0 col = 1 for field in self.fields: @@ -80,7 +80,7 @@ def process(self, files): for field in self.fields: value = self.banner_field(banner, field) data.append(value) - + # Write those values to the main workbook # Starting off w/ the special "IP" property main_sheet.write_string(row, 0, get_ip(banner)) @@ -92,11 +92,11 @@ def process(self, files): row += 1 except Exception: pass - + # Aggregate summary information total += 1 ports[banner['port']] += 1 - + summary_sheet = workbook.add_worksheet('Summary') summary_sheet.write(0, 0, 'Total', bold) summary_sheet.write(0, 1, total) @@ -109,22 +109,22 @@ def process(self, files): summary_sheet.write(row, col, key) summary_sheet.write(row, col + 1, value) row += 1 - + def banner_field(self, banner, flat_field): # The provided field is a collapsed form of the actual field fields = flat_field.split('.') - + try: current_obj = banner for field in fields: current_obj = current_obj[field] - + # Convert a list into a concatenated string if isinstance(current_obj, list): current_obj = ','.join([str(i) for i in current_obj]) - + return current_obj except Exception: pass - + return '' diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 8bde86f..6126267 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -2,39 +2,39 @@ from .base import Converter from ...helpers import get_ip, iterate_files + class GeoJsonConverter(Converter): - + def header(self): self.fout.write("""{ "type": "FeatureCollection", "features": [ """) - + def footer(self): self.fout.write("""{ }]}""") - + def process(self, files): # Write the header self.header() - + hosts = {} for banner in iterate_files(files): ip = get_ip(banner) if not ip: continue - + if ip not in hosts: hosts[ip] = banner hosts[ip]['ports'] = [] - + hosts[ip]['ports'].append(banner['port']) - + for ip, host in iter(hosts.items()): self.write(host) - + self.footer() - - + def write(self, host): try: ip = get_ip(host) diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py index b239b4d..24c68c3 100644 --- a/shodan/cli/converter/images.py +++ b/shodan/cli/converter/images.py @@ -14,7 +14,7 @@ class ImagesConverter(Converter): # special code in the Shodan CLI that relies on the "dirname" property to let # the user know where the images have been stored. dirname = None - + def process(self, files): # Get the filename from the already-open file handle and use it as # the directory name to store the images. diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index 49938c2..2cf3d44 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -2,38 +2,38 @@ from .base import Converter from ...helpers import iterate_files + class KmlConverter(Converter): - + def header(self): self.fout.write(""" """) - + def footer(self): self.fout.write("""""") - + def process(self, files): # Write the header self.header() - + hosts = {} for banner in iterate_files(files): ip = banner.get('ip_str', banner.get('ipv6', None)) if not ip: continue - + if ip not in hosts: hosts[ip] = banner hosts[ip]['ports'] = [] - + hosts[ip]['ports'].append(banner['port']) - + for ip, host in iter(hosts.items()): self.write(host) - + self.footer() - - + def write(self, host): try: ip = host.get('ip_str', host.get('ipv6', None)) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 33cdc57..88cbd36 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -15,6 +15,7 @@ except NameError: basestring = (str, ) # Python 3 + def get_api_key(): '''Returns the API key of the current logged-in user.''' shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) diff --git a/shodan/cli/host.py b/shodan/cli/host.py index befdc62..e90e372 100644 --- a/shodan/cli/host.py +++ b/shodan/cli/host.py @@ -64,9 +64,9 @@ def host_print_pretty(host, history=False): for port in ports: banner = { 'port': port, - 'transport': 'tcp', # All the filtered services use TCP - 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner - 'placeholder': True, # Don't store this banner when the file is saved + 'transport': 'tcp', # All the filtered services use TCP + 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner + 'placeholder': True, # Don't store this banner when the file is saved } host['data'].append(banner) @@ -94,7 +94,7 @@ def host_print_pretty(host, history=False): # Show optional ssl info if 'ssl' in banner: if 'versions' in banner['ssl'] and banner['ssl']['versions']: - click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) + click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: click.echo('\t|-- Diffie-Hellman Parameters:') click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) @@ -119,4 +119,4 @@ def host_print_tsv(host, history=False): HOST_PRINT = { 'pretty': host_print_pretty, 'tsv': host_print_tsv, -} \ No newline at end of file +} diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py index 50d814c..5fbb764 100644 --- a/shodan/cli/organization.py +++ b/shodan/cli/organization.py @@ -22,7 +22,7 @@ def add(silent, user): api.org.add_member(user, notify=not silent) except shodan.APIError as e: raise click.ClickException(e.value) - + click.secho('Successfully added the new member', fg='green') @@ -39,11 +39,11 @@ def info(): click.secho(organization['name'], fg='cyan') click.secho('Access Level: ', nl=False, dim=True) click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') - + if organization['domains']: click.secho('Authorized Domains: ', nl=False, dim=True) click.echo(', '.join(organization['domains'])) - + click.echo('') click.secho('Administrators:', dim=True) @@ -51,8 +51,8 @@ def info(): click.echo(u' > {:30}\t{:30}'.format( click.style(admin['username'], fg='yellow'), admin['email']) - ) - + ) + click.echo('') if organization['members']: click.secho('Members:', dim=True) @@ -76,5 +76,5 @@ def remove(user): api.org.remove_member(user) except shodan.APIError as e: raise click.ClickException(e.value) - + click.secho('Successfully removed the member', fg='green') diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 5220339..4590770 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -17,6 +17,35 @@ def scan(): pass +@scan.command(name='list') +def scan_list(): + """Show recently launched scans""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + scans = api.scans() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(scans) > 0: + click.echo(u'# {} Scans Total - Showing 10 most recent scans:'.format(scans['total'])) + click.echo(u'# {:20} {:<15} {:<10} {:<15s}'.format('Scan ID', 'Status', 'Size', 'Timestamp')) + # click.echo('#' * 65) + for scan in scans['matches'][:10]: + click.echo( + u'{:31} {:<24} {:<10} {:<15s}'.format( + click.style(scan['id'], fg='yellow'), + click.style(scan['status'], fg='cyan'), + scan['size'], + scan['created'] + ) + ) + else: + click.echo("You haven't yet launched any scans.") + + @scan.command(name='internet') @click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) @click.argument('port', type=int) @@ -58,10 +87,9 @@ def scan_internet(quiet, port, protocol): if not quiet: click.echo('{0:<40} {1:<20} {2}'.format( - click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), - click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), - ';'.join(banner['hostnames']) - ) + click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), + click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), + ';'.join(banner['hostnames'])) ) except shodan.APIError as e: # We stop waiting for results if the scan has been processed by the crawlers and diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index cfe4e4a..767b237 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -108,14 +108,14 @@ def latlon_to_coords(self, lat, lon): TODO: filter out stuff that doesn't fit TODO: make it possible to use "zoomed" maps """ - width = (self.corners[3]-self.corners[1]) - height = (self.corners[2]-self.corners[0]) + width = (self.corners[3] - self.corners[1]) + height = (self.corners[2] - self.corners[0]) # change to 0-180, 0-360 - abs_lat = -lat+90 - abs_lon = lon+180 - x = (abs_lon/360.0)*width + self.corners[1] - y = (abs_lat/180.0)*height + self.corners[0] + abs_lat = -lat + 90 + abs_lon = lon + 180 + x = (abs_lon / 360.0) * width + self.corners[1] + y = (abs_lat / 180.0) * height + self.corners[0] return int(x), int(y) def set_data(self, data): @@ -155,7 +155,7 @@ def draw(self, target): self.window.addstr(0, 0, self.map) # FIXME: position to be defined in map config? - row = self.corners[2]-6 + row = self.corners[2] - 6 items_to_show = 5 for lat, lon, char, desc, attrs, color in self.data: # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html @@ -257,6 +257,7 @@ def main(argv=None): api = Shodan(get_api_key()) return launch_map(api) + if __name__ == '__main__': import sys sys.exit(main()) diff --git a/shodan/client.py b/shodan/client.py index 51c33f1..c8a2b04 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -347,6 +347,16 @@ def scan(self, ips, force=False): return self._request('/shodan/scan', params, method='post') + def scans(self, page=1): + """Get a list of scans submitted + + :param page: Page through the list of scans 100 results at a time + :type page: int + """ + return self._request('/shodan/scans', { + 'page': page, + }) + def scan_internet(self, port, protocol): """Scan a network using Shodan @@ -438,7 +448,7 @@ def search_cursor(self, query, minify=True, retries=5): try: yield banner except GeneratorExit: - return # exit out of the function + return # exit out of the function page += 1 tries = 0 except Exception: @@ -447,7 +457,7 @@ def search_cursor(self, query, minify=True, retries=5): break tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason + time.sleep(1.0) # wait 1 second if the search errored out for some reason def search_tokens(self, query): """Returns information about the search query itself (filters used etc.) @@ -507,8 +517,8 @@ def queries_search(self, query, page=1): def queries_tags(self, size=10): """Search the directory of saved search queries in Shodan. - :param query: The number of tags to return - :type page: int + :param size: The number of tags to return + :type size: int :returns: A list of tags. """ @@ -518,12 +528,14 @@ def queries_tags(self, size=10): return self._request('/shodan/query/tags', args) def create_alert(self, name, ip, expires=0): - """Search the directory of saved search queries in Shodan. + """Create a network alert/ private firehose for the specified IP range(s) - :param query: The number of tags to return - :type page: int + :param name: Name of the alert + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str - :returns: A list of tags. + :returns: A dict describing the alert """ data = { 'name': name, @@ -547,8 +559,7 @@ def alerts(self, aid=None, include_expired=True): response = api_request(self.api_key, func, params={ 'include_expired': include_expired, - }, - proxies=self._session.proxies) + }, proxies=self._session.proxies) return response @@ -561,3 +572,17 @@ def delete_alert(self, aid): return response + def alert_triggers(self): + """Return a list of available triggers that can be enabled for alerts. + + :returns: A list of triggers + """ + return self._request('/shodan/alert/triggers', {}) + + def enable_alert_trigger(self, aid, trigger): + """Enable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='put') + + def disable_alert_trigger(self, aid, trigger): + """Disable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') diff --git a/shodan/exception.py b/shodan/exception.py index c4878b1..75b158e 100644 --- a/shodan/exception.py +++ b/shodan/exception.py @@ -2,11 +2,10 @@ class APIError(Exception): """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" def __init__(self, value): self.value = value - + def __str__(self): return self.value class APITimeout(APIError): - pass - + pass diff --git a/shodan/helpers.py b/shodan/helpers.py index d3c4a1f..f289e8b 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -19,7 +19,7 @@ def create_facet_string(facets): if isinstance(facet, basestring): facet_str += facet else: - facet_str += '%s:%s' % (facet[0], facet[1]) + facet_str += '{}:{}'.format(facet[0], facet[1]) facet_str += ',' return facet_str[:-1] @@ -76,7 +76,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho # Parse the text into JSON try: data = data.json() - except: + except Exception: raise APIError('Unable to parse JSON response') # Raise an exception if an error occurred @@ -119,6 +119,7 @@ def iterate_files(files, fast=False): banner = loads(line) yield banner + def get_screenshot(banner): if 'opts' in banner and 'screenshot' in banner['opts']: return banner['opts']['screenshot'] @@ -159,14 +160,13 @@ def humanize_bytes(bytes, precision=1): >>> humanize_bytes(1024*1234*1111,1) '1.3 GB' """ - if bytes == 1: return '1 byte' if bytes < 1024: return '%.*f %s' % (precision, bytes, "bytes") suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] - multiple = 1024.0 #.0 force float on python 2 + multiple = 1024.0 # .0 to force float on python 2 for suffix in suffixes: bytes /= multiple if bytes < multiple: diff --git a/shodan/stream.py b/shodan/stream.py index 49ab633..10ccac3 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -21,7 +21,7 @@ def _create_stream(self, name, timeout=None): # The user doesn't want to use a timeout # If the timeout is specified as 0 then we also don't want to have a timeout - if ( timeout and timeout <= 0 ) or ( timeout == 0 ): + if (timeout and timeout <= 0) or (timeout == 0): timeout = None # If the user requested a timeout then we need to disable heartbeat messages @@ -43,16 +43,16 @@ def _create_stream(self, name, timeout=None): # not specific to Cloudflare. if req.status_code != 524 or timeout >= 0: break - except Exception as e: + except Exception: raise APIError('Unable to contact the Shodan Streaming API') if req.status_code != 200: try: data = json.loads(req.text) raise APIError(data['error']) - except APIError as e: + except APIError: raise - except Exception as e: + except Exception: pass raise APIError('Invalid API key or you do not have access to the Streaming API') if req.encoding is None: @@ -78,9 +78,9 @@ def alert(self, aid=None, timeout=None, raw=False): try: for line in self._iter_stream(stream, raw): yield line - except requests.exceptions.ConnectionError as e: + except requests.exceptions.ConnectionError: raise APIError('Stream timed out') - except ssl.SSLError as e: + except ssl.SSLError: raise APIError('Stream timed out') def asn(self, asn, raw=False, timeout=None): @@ -123,4 +123,3 @@ def ports(self, ports, raw=False, timeout=None): stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) for line in self._iter_stream(stream, raw): yield line - diff --git a/shodan/threatnet.py b/shodan/threatnet.py index 97c0c7e..cad9bdd 100644 --- a/shodan/threatnet.py +++ b/shodan/threatnet.py @@ -24,13 +24,13 @@ def _create_stream(self, name): try: req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True, proxies=self.proxies) - except: + except Exception: raise APIError('Unable to contact the Shodan Streaming API') if req.status_code != 200: try: raise APIError(req.json()['error']) - except: + except Exception: pass raise APIError('Invalid API key or you do not have access to the Streaming API') return req @@ -65,4 +65,3 @@ def __init__(self, key): self.api_key = key self.base_url = 'https://api.shodan.io' self.stream = self.Stream(self) - diff --git a/tests/test_shodan.py b/tests/test_shodan.py index 0cdd602..f3405ce 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -9,148 +9,148 @@ class ShodanTests(unittest.TestCase): - api = None - FACETS = [ - 'port', - ('domain', 1) - ] - QUERIES = { - 'simple': 'cisco-ios', - 'minify': 'apache', - 'advanced': 'apache port:443', - 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', - } - - def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) - - def test_search_simple(self): - results = self.api.search(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure some values were returned - self.assertTrue(results['matches']) - self.assertTrue(results['total']) - - # A regular search shouldn't have the optional info - self.assertNotIn('opts', results['matches'][0]) - - def test_search_empty(self): - results = self.api.search(self.QUERIES['empty']) - self.assertTrue(len(results['matches']) == 0) - self.assertEqual(results['total'], 0) - - def test_search_facets(self): - results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_count_simple(self): - results = self.api.count(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure no values were returned - self.assertFalse(results['matches']) - self.assertTrue(results['total']) - - def test_count_facets(self): - results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_host_details(self): - host = self.api.host('147.228.101.7') - - self.assertEqual('147.228.101.7', host['ip_str']) - self.assertFalse(isinstance(host['ip'], basestring)) - - def test_search_minify(self): - results = self.api.search(self.QUERIES['minify'], minify=False) - self.assertIn('opts', results['matches'][0]) - - def test_exploits_search(self): - results = self.api.exploits.search('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(results['matches']) - - def test_exploits_search_paging(self): - results = self.api.exploits.search('apache', page=1) - match1 = results['matches'][0] - results = self.api.exploits.search('apache', page=2) - match2 = results['matches'][0] - - self.assertNotEqual(match1['_id'], match2['_id']) - - def test_exploits_search_facets(self): - results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - def test_exploits_count(self): - results = self.api.exploits.count('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(len(results['matches']) == 0) - - def test_exploits_count_facets(self): - results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) - self.assertEqual(len(results['matches']), 0) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - # Test error responses - def test_invalid_key(self): - api = shodan.Shodan('garbage') - raised = False - try: - api.search('something') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_invalid_host_ip(self): - raised = False - try: - host = self.api.host('test') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_search_empty_query(self): - raised = False - try: - self.api.search('') - except shodan.APIError as e: - raised = True - self.assertTrue(raised) - - def test_search_advanced_query(self): - # The free API plan can't use filters - raised = False - try: - self.api.search(self.QUERIES['advanced']) - except shodan.APIError as e: - raised = True - self.assertTrue(raised) + api = None + FACETS = [ + 'port', + ('domain', 1) + ] + QUERIES = { + 'simple': 'cisco-ios', + 'minify': 'apache', + 'advanced': 'apache port:443', + 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', + } + + def setUp(self): + self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) + + def test_search_simple(self): + results = self.api.search(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure some values were returned + self.assertTrue(results['matches']) + self.assertTrue(results['total']) + + # A regular search shouldn't have the optional info + self.assertNotIn('opts', results['matches'][0]) + + def test_search_empty(self): + results = self.api.search(self.QUERIES['empty']) + self.assertTrue(len(results['matches']) == 0) + self.assertEqual(results['total'], 0) + + def test_search_facets(self): + results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_count_simple(self): + results = self.api.count(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure no values were returned + self.assertFalse(results['matches']) + self.assertTrue(results['total']) + + def test_count_facets(self): + results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_host_details(self): + host = self.api.host('147.228.101.7') + + self.assertEqual('147.228.101.7', host['ip_str']) + self.assertFalse(isinstance(host['ip'], basestring)) + + def test_search_minify(self): + results = self.api.search(self.QUERIES['minify'], minify=False) + self.assertIn('opts', results['matches'][0]) + + def test_exploits_search(self): + results = self.api.exploits.search('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + + def test_exploits_search_paging(self): + results = self.api.exploits.search('apache', page=1) + match1 = results['matches'][0] + results = self.api.exploits.search('apache', page=2) + match2 = results['matches'][0] + + self.assertNotEqual(match1['_id'], match2['_id']) + + def test_exploits_search_facets(self): + results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_exploits_count(self): + results = self.api.exploits.count('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(len(results['matches']) == 0) + + def test_exploits_count_facets(self): + results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) + self.assertEqual(len(results['matches']), 0) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + # Test error responses + def test_invalid_key(self): + api = shodan.Shodan('garbage') + raised = False + try: + api.search('something') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_invalid_host_ip(self): + raised = False + try: + self.api.host('test') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_search_empty_query(self): + raised = False + try: + self.api.search('') + except shodan.APIError: + raised = True + self.assertTrue(raised) + + def test_search_advanced_query(self): + # The free API plan can't use filters + raised = False + try: + self.api.search(self.QUERIES['advanced']) + except shodan.APIError: + raised = True + self.assertTrue(raised) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d840b92 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[flake8] +ignore = + E501 + +exclude = + build, + docs, + shodan.egg-info, + tmp, + +per-file-ignores = + shodan/__init__.py:F401, + shodan/cli/converter/__init__.py:F401, + shodan/cli/worldmap.py:W291,W293,W605, \ No newline at end of file From 9fb6b7c9a5726fa23bffb1d6b134a0ffacf1be8e Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 15 Feb 2019 22:10:40 -0600 Subject: [PATCH 059/148] Fix bug that caused problems parsing uncompressed data files in Python3 --- shodan/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shodan/helpers.py b/shodan/helpers.py index f289e8b..2d976dc 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -113,7 +113,8 @@ def iterate_files(files, fast=False): for line in fin: # Ensure the line has been decoded into a string to prevent errors w/ Python3 - line = line.decode('utf-8') + if not isinstance(line, basestring): + line = line.decode('utf-8') # Convert the JSON into a native Python object banner = loads(line) From 23cbc6884a8ce6a3736d6d786c0adcdc19125723 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 18 Feb 2019 21:36:01 -0600 Subject: [PATCH 060/148] Show alert triggers in list view --- shodan/cli/alert.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index d8b02b9..112c1cf 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -71,6 +71,10 @@ def alert_list(expired): nl=False ) + if 'triggers' in alert and alert['triggers']: + click.secho('Triggers: ', fg='magenta', nl=False) + click.echo(', '.join(alert['triggers'].keys()), nl=False) + if 'expired' in alert and alert['expired']: click.secho('expired', fg='red') else: From c798e754b715a6da42b6a7b722b27579ae877a6d Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 20 Feb 2019 01:41:15 -0600 Subject: [PATCH 061/148] Release 1.11.0 New command: shodan alert info Improved output of alert and trigger information --- CHANGELOG.md | 5 +++- setup.py | 2 +- shodan/cli/alert.py | 65 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b107676..ba2044f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,17 @@ CHANGELOG ========= -unreleased +1.11.0 ---------- * New command **shodan scan list** to list recently launched scans * New command **shodan alert triggers** to list the available notification triggers * New command **shodan alert enable** to enable a notification trigger * New command **shodan alert disable** to disable a notification trigger +* New command **shodan alert info** to show details of a specific alert * Include timestamp, vulns and tags in CSV converter (#85) +* Fixed bug that caused an exception when parsing uncompressed data files in Python3 * Code quality improvements +* Thank you for contributions from @wagner-certat, @cclauss, @opt9, @voldmar and Antoine Neuenschwander 1.10.4 ------ diff --git a/setup.py b/setup.py index c925129..4686a81 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.10.4', + version='1.11.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 112c1cf..a4c2e29 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,6 +1,7 @@ import click import shodan +from operator import itemgetter from shodan.cli.helpers import get_api_key @@ -45,6 +46,42 @@ def alert_create(name, netblock): click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +@alert.command(name='info') +@click.argument('alert', metavar='') +def alert_info(alert): + """Show information about a specific alert""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.alerts(aid=alert) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(info['name'], fg='cyan') + click.secho('Created: ', nl=False, dim=True) + click.secho(info['created'], fg='magenta') + + click.secho('Notifications: ', nl=False, dim=True) + if 'triggers' in info and info['triggers']: + click.secho('enabled', fg='green') + else: + click.echo('disabled') + + click.echo('') + click.secho('Network Range(s):', dim=True) + + for network in info['filters']['ip']: + click.echo(u' > {}'.format(click.style(network, fg='yellow'))) + + click.echo('') + if 'triggers' in info and info['triggers']: + click.secho('Triggers:', dim=True) + for trigger in info['triggers']: + click.echo(u' > {}'.format(click.style(trigger, fg='yellow'))) + click.echo('') + + @alert.command(name='list') @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) def alert_list(expired): @@ -100,7 +137,7 @@ def alert_remove(alert_id): @alert.command(name='triggers') def alert_list_triggers(): - """List all the available triggers""" + """List the available notification triggers""" key = get_api_key() # Get the list @@ -111,16 +148,20 @@ def alert_list_triggers(): raise click.ClickException(e.value) if len(results) > 0: - click.echo(u'# {:14} {:<21} {:<15s}'.format('Name', 'Description', 'Rule')) + click.secho('The following triggers can be enabled on alerts:', dim=True) + click.echo('') - for trigger in results: - click.echo( - u'{:16} {:<30} {:<35} '.format( - click.style(trigger['name'], fg='yellow'), - click.style(trigger['description'], fg='cyan'), - trigger['rule'] - ) - ) + for trigger in sorted(results, key=itemgetter('name')): + click.secho('{:<12} '.format('Name'), dim=True, nl=False) + click.secho(trigger['name'], fg='yellow') + + click.secho('{:<12} '.format('Description'), dim=True, nl=False) + click.secho(trigger['description'], fg='cyan') + + click.secho('{:<12} '.format('Rule'), dim=True, nl=False) + click.echo(trigger['rule']) + + click.echo('') else: click.echo("No triggers currently available.") @@ -139,7 +180,7 @@ def alert_enable_trigger(alert_id, trigger): except shodan.APIError as e: raise click.ClickException(e.value) - click.secho('Successfully enabled the trigger {}'.format(trigger), color='green') + click.secho('Successfully enabled the trigger: {}'.format(trigger), fg='green') @alert.command(name='disable') @@ -156,4 +197,4 @@ def alert_disable_trigger(alert_id, trigger): except shodan.APIError as e: raise click.ClickException(e.value) - click.secho('Successfully disabled the trigger {}'.format(trigger), color='green') + click.secho('Successfully disabled the trigger: {}'.format(trigger), fg='green') From 20ea464fe067d150f7379bb9f8f07c0181761266 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 24 Feb 2019 03:58:27 -0600 Subject: [PATCH 062/148] Allow a single network alert to monitor multiple IP ranges (#93) Update README with information on configuring alert notifications --- CHANGELOG.md | 6 +++++- README.rst | 1 + setup.py | 2 +- shodan/cli/alert.py | 6 +++--- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba2044f..9fd6451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ CHANGELOG ========= +1.11.1 +------ +* Allow a single network alert to monitor multiple IP ranges (#93) + 1.11.0 ----------- +------ * New command **shodan scan list** to list recently launched scans * New command **shodan alert triggers** to list the available notification triggers * New command **shodan alert enable** to enable a notification trigger diff --git a/README.rst b/README.rst index d0b1f10..610c13d 100644 --- a/README.rst +++ b/README.rst @@ -18,6 +18,7 @@ Features - `Fast/ bulk IP lookups `_ - Streaming API support for real-time consumption of Shodan firehose - `Network alerts (aka private firehose) `_ +- `Manage Email Notifications `_ - Exploit search API fully implemented - Bulk data downloads - `Command-line interface `_ diff --git a/setup.py b/setup.py index 4686a81..bb666af 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.11.0', + version='1.11.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index a4c2e29..1ed8572 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -30,15 +30,15 @@ def alert_clear(): @alert.command(name='create') @click.argument('name', metavar='') -@click.argument('netblock', metavar='') -def alert_create(name, netblock): +@click.argument('netblocks', metavar='', nargs=-1) +def alert_create(name, netblocks): """Create a network alert to monitor an external network""" key = get_api_key() # Get the list api = shodan.Shodan(key) try: - alert = api.create_alert(name, netblock) + alert = api.create_alert(name, netblocks) except shodan.APIError as e: raise click.ClickException(e.value) From 59389da25554221e67dc38aa120f44c1816a0ce1 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 6 Apr 2019 18:20:12 -0500 Subject: [PATCH 063/148] Add new methods to handle ignoring/ unignoring trigger notifications --- setup.py | 2 +- shodan/client.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bb666af..3597060 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.11.1', + version='1.12.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index c8a2b04..fb2c87e 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -586,3 +586,11 @@ def enable_alert_trigger(self, aid, trigger): def disable_alert_trigger(self, aid, trigger): """Disable the given trigger on the alert.""" return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') + + def ignore_alert_trigger_notification(self, aid, trigger, ip, port): + """Ignore trigger notifications for the provided IP and port.""" + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') + + def unignore_alert_trigger_notification(self, aid, trigger, ip, port): + """Re-enable trigger notifications for the provided IP and port""" + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') From 6adbb630553a67df546a267c9859bb18a63a5c62 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 6 Apr 2019 18:21:29 -0500 Subject: [PATCH 064/148] Update changelog for 1.12.0 release --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd6451..bbc448c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.12.0 +------ +* Add new methods to ignore/ unignore trigger notifications + 1.11.1 ------ * Allow a single network alert to monitor multiple IP ranges (#93) From e97429568c14817b9b39a9e2b9509eb5737fe8f3 Mon Sep 17 00:00:00 2001 From: Guillaume Granjus Date: Mon, 8 Apr 2019 10:18:14 +0200 Subject: [PATCH 065/148] Add exception raised when retry limit reached (#94) --- shodan/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index fb2c87e..bc6e5af 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -431,7 +431,7 @@ def search_cursor(self, query, minify=True, retries=5): :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool :param retries: (optional) How often to retry the search in case it times out - :type minify: int + :type retries: int :returns: A search cursor that can be used as an iterator/ generator. """ @@ -454,7 +454,7 @@ def search_cursor(self, query, minify=True, retries=5): except Exception: # We've retried several times but it keeps failing, so lets error out if tries >= retries: - break + raise APIError('Retry limit reached ({:d})'.format(retries)) tries += 1 time.sleep(1.0) # wait 1 second if the search errored out for some reason From 6c1d6c94a278ed38a4875d58a1a345532f159d7e Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 11 Apr 2019 15:39:12 -0500 Subject: [PATCH 066/148] Explicitly close the workbook so the Excel file is written --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/cli/converter/excel.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc448c..04d5dcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.12.1 +------ +* Fix Excel file conversion that resulted in empty .xlsx files + 1.12.0 ------ * Add new methods to ignore/ unignore trigger notifications diff --git a/setup.py b/setup.py index 3597060..4e82482 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.12.0', + version='1.12.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index e96c24b..24177eb 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -109,6 +109,8 @@ def process(self, files): summary_sheet.write(row, col, key) summary_sheet.write(row, col + 1, value) row += 1 + + workbook.close() def banner_field(self, banner, flat_field): # The provided field is a collapsed form of the actual field From 723fe3488d1a5ec7df3e173c5ecc3b89bf8a5bdc Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 30 Apr 2019 14:03:48 +0200 Subject: [PATCH 067/148] Override environment configured settings if explicit proxy settings are supplied --- shodan/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shodan/client.py b/shodan/client.py index fb2c87e..096f9d3 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -190,6 +190,7 @@ def __init__(self, key, proxies=None): self._session = requests.Session() if proxies: self._session.proxies.update(proxies) + self._session.trust_env=False def _request(self, function, params, service='shodan', method='get'): """General-purpose function to create web requests to SHODAN. From c4562e0182dc83e6469c2ebfd9fe0ba7d6c51eb4 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 30 Apr 2019 16:53:02 +0200 Subject: [PATCH 068/148] White space around operator --- shodan/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/client.py b/shodan/client.py index 096f9d3..61f4d64 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -190,7 +190,7 @@ def __init__(self, key, proxies=None): self._session = requests.Session() if proxies: self._session.proxies.update(proxies) - self._session.trust_env=False + self._session.trust_env = False def _request(self, function, params, service='shodan', method='get'): """General-purpose function to create web requests to SHODAN. From d4fa8bc6e37266968642aaadeee4d4cfa0f66b6c Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 2 May 2019 17:34:45 -0500 Subject: [PATCH 069/148] New command: shodan domain Release 1.13.0 --- CHANGELOG.md | 5 +++++ setup.py | 2 +- shodan/__main__.py | 25 +++++++++++++++++++++++++ shodan/client.py | 11 +++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d5dcc..5d76b17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +1.13.0 +------ +* New command **shodan domain** to lookup a domain in Shodan's DNS database +* Override environment configured settings if explicit proxy settings are supplied (@cudeso) + 1.12.1 ------ * Fix Excel file conversion that resulted in empty .xlsx files diff --git a/setup.py b/setup.py index 4e82482..164279d 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.12.1', + version='1.13.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index df2b38c..8e3dd17 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -125,6 +125,31 @@ def convert(input, format): click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green')) +@main.command(name='domain') +@click.argument('domain', metavar='') +def domain_info(domain): + """View all available information for a domain""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.dns.domain_info(domain) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(info['domain'].upper(), fg='green') + + click.echo('') + for record in info['data']: + click.echo( + '{:32} {:14} {}'.format( + click.style(record['subdomain'], fg='cyan'), + click.style(record['type'], fg='yellow'), + record['value'] + ) + ) + + @main.command() @click.argument('key', metavar='') def init(key): diff --git a/shodan/client.py b/shodan/client.py index 61f4d64..cf16823 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -64,6 +64,16 @@ def list_files(self, dataset): """ return self.parent._request('/shodan/data/{}'.format(dataset), {}) + class Dns: + + def __init__(self, parent): + self.parent = parent + + def domain_info(self, domain): + """Grab the DNS information for a domain. + """ + return self.parent._request('/dns/domain/{}'.format(domain), {}) + class Tools: def __init__(self, parent): @@ -182,6 +192,7 @@ def __init__(self, key, proxies=None): self.base_url = 'https://api.shodan.io' self.base_exploits_url = 'https://exploits.shodan.io' self.data = self.Data(self) + self.dns = self.Dns(self) self.exploits = self.Exploits(self) self.labs = self.Labs(self) self.org = self.Organization(self) From 5d0fa9136d4f751bd64a5426bbf043967203b2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Ribeiro?= Date: Wed, 29 May 2019 00:13:10 +0100 Subject: [PATCH 070/148] Only change api_key permissions if needed --- shodan/cli/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 88cbd36..3154591 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -27,7 +27,8 @@ def get_api_key(): raise click.ClickException('Please run "shodan init " before using this command') # Make sure it is a read-only file - os.chmod(keyfile, 0o600) + if not oct(os.stat(keyfile).st_mode).endswith("600"): + os.chmod(keyfile, 0o600) with open(keyfile, 'r') as fin: return fin.read().strip() From 11fc901c941d0661c5b43104dd4b7520f34cd25b Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 1 Jul 2019 13:52:47 +0200 Subject: [PATCH 071/148] New command 'shodan version' fixes achillean/shodan-python#104 --- CHANGELOG.md | 4 ++++ shodan/__main__.py | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d76b17..e966335 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +unreleased +---------- +* New command **shodan version** (#104). + 1.13.0 ------ * New command **shodan domain** to lookup a domain in Shodan's DNS database diff --git a/shodan/__main__.py b/shodan/__main__.py index 8e3dd17..9b47875 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -29,6 +29,7 @@ import csv import os import os.path +import pkg_resources import shodan import shodan.helpers as helpers import threading @@ -138,7 +139,7 @@ def domain_info(domain): raise click.ClickException(e.value) click.secho(info['domain'].upper(), fg='green') - + click.echo('') for record in info['data']: click.echo( @@ -753,5 +754,11 @@ def radar(): raise click.ClickException(u'{}'.format(e)) +@main.command() +def version(): + """Print version of this tool.""" + print(pkg_resources.get_distribution("shodan").version) + + if __name__ == '__main__': main() From 68fe360ac6c4fb391a2420b2fc1aced1e5a369b9 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 19 Jul 2019 20:15:59 -0500 Subject: [PATCH 072/148] Release 1.14.0 --- CHANGELOG.md | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e966335..972abea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ CHANGELOG ========= -unreleased +1.14.0 ---------- * New command **shodan version** (#104). +* Only change api_key file permissions if needed (#103) 1.13.0 ------ diff --git a/setup.py b/setup.py index 164279d..5bd5e84 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.13.0', + version='1.14.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 0126de28a4a6f204e1f7e0bb24c831288469c300 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 12 Aug 2019 20:23:14 -0500 Subject: [PATCH 073/148] New option "--skip" for download command to help users resume a download --- CHANGELOG.md | 6 +++++- setup.py | 2 +- shodan/__main__.py | 9 +++++++-- shodan/client.py | 17 ++++++++++++++--- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 972abea..840da0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ CHANGELOG ========= +1.15.0 +------ +* New option "--skip" for download command to help users resume a download + 1.14.0 ----------- +------ * New command **shodan version** (#104). * Only change api_key file permissions if needed (#103) diff --git a/setup.py b/setup.py index 5bd5e84..467cd84 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.14.0', + version='1.15.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 9b47875..b0eeea7 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -205,9 +205,10 @@ def count(query): @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) +@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, filename, query): +def download(limit, skip, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -247,11 +248,15 @@ def download(limit, filename, query): # A limit of -1 means that we should download all the data if limit <= 0: limit = total + + # Adjust the total number of results we should expect to download if the user is skipping results + if skip > 0: + limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False) + cursor = api.search_cursor(query, minify=False, skip=skip) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) diff --git a/shodan/client.py b/shodan/client.py index bc5a25e..bb38a69 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -430,7 +430,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, skip=0): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -444,16 +444,27 @@ def search_cursor(self, query, minify=True, retries=5): :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type retries: int + :param skip: (optional) Number of results to skip + :type skip: int :returns: A search cursor that can be used as an iterator/ generator. """ page = 1 tries = 0 + + # Placeholder results object to make the while loop below easier results = { - 'matches': [], + 'matches': [True], 'total': None, } - while page == 1 or results['matches']: + + # Convert the number of skipped records into a page number + if skip > 0: + # Each page returns 100 results so find the nearest page that we want + # the cursor to skip to. + page += int(skip / 100) + + while results['matches']: try: results = self.search(query, minify=minify, page=page) for banner in results['matches']: From cdf7ddec9a472e7561ce021eb3e73f7173cac506 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 21 Sep 2019 12:36:34 -0500 Subject: [PATCH 074/148] Allow users to define a list of fields to include when converting the data into other formats (#107) --- shodan/__main__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index b0eeea7..61f341f 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -89,15 +89,23 @@ def main(): 'xlsx': ExcelConverter, } @main.command() +@click.option('--fields', help='List of properties to output.', default=None) @click.argument('input', metavar='') @click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) -def convert(input, format): +def convert(fields, input, format): """Convert the given input data file into a different format. The following file formats are supported: kml, csv, geo.json, images, xlsx Example: shodan convert data.json.gz kml """ + # Check that the converter allows a custom list of fields + converter_class = CONVERTERS.get(format) + if fields: + if not hasattr(converter_class, 'fields'): + raise click.ClickException('File format doesnt support custom list of fields') + converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -113,7 +121,7 @@ def convert(input, format): progress_bar_thread.start() # Initialize the file converter - converter = CONVERTERS.get(format)(fout) + converter = converter_class(fout) converter.process([input]) From 933043a7d51f3dec8ff66fe8d8833fbdbe9462a8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 21 Sep 2019 13:16:52 -0500 Subject: [PATCH 075/148] Release 1.16.0 New command option and library method to filter firehose based on tags --- CHANGELOG.md | 5 +++++ setup.py | 2 +- shodan/__main__.py | 11 +++++++++-- shodan/stream.py | 11 +++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840da0e..0fc9488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +1.16.0 +------ +* Ability to specify list of fields to include when converting to CSV/ Excel (#107) +* Filter the Shodan Firehose based on tags in the banner + 1.15.0 ------ * New option "--skip" for download command to help users resume a download diff --git a/setup.py b/setup.py index 467cd84..6dc2833 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.15.0', + version='1.16.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 61f341f..cc27735 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -585,8 +585,9 @@ def stats(limit, facets, filename, query): @click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) +@click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel): +def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -612,9 +613,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('asn') if alert: stream_type.append('alert') + if tags: + stream_type.append('tags') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -635,6 +638,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if countries: stream_args = countries.split(',') + + if tags: + stream_args = tags.split(',') # Flatten the list of stream types # Possible values are: @@ -655,6 +661,7 @@ def _create_stream(name, args, timeout): 'asn': api.stream.asn(args, timeout=timeout), 'countries': api.stream.countries(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), + 'tags': api.stream.tags(args, timeout=timeout), }.get(name, 'all') stream = _create_stream(stream_type, stream_args, timeout=timeout) diff --git a/shodan/stream.py b/shodan/stream.py index 10ccac3..1feb15a 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -123,3 +123,14 @@ def ports(self, ports, raw=False, timeout=None): stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) for line in self._iter_stream(stream, raw): yield line + + def tags(self, tags, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the tags of interest. + + :param tags: A list of tags to return banner data on. + :type tags: string[] + """ + stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line From 1a653f7e6c75c54e3ccfddb99c21ca334e7da2c6 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 21 Sep 2019 17:10:30 -0500 Subject: [PATCH 076/148] Fix bug that caused unicode error when printing domain information (#106) --- CHANGELOG.md | 4 ++++ shodan/__main__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fc9488..6bcdfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +unreleased +---------- +* Fix bug that caused unicode error when printing domain information + 1.16.0 ------ * Ability to specify list of fields to include when converting to CSV/ Excel (#107) diff --git a/shodan/__main__.py b/shodan/__main__.py index cc27735..0bcf5e3 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -151,7 +151,7 @@ def domain_info(domain): click.echo('') for record in info['data']: click.echo( - '{:32} {:14} {}'.format( + u'{:32} {:14} {}'.format( click.style(record['subdomain'], fg='cyan'), click.style(record['type'], fg='yellow'), record['value'] From f2b6bc404e24173cff1a5483c42e8dd9ef153087 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 21 Sep 2019 18:44:58 -0500 Subject: [PATCH 077/148] Release 1.17.0 Add flag to let users get their IPv6 address using "shodan myip -6" (#106) --- CHANGELOG.md | 5 +++-- setup.py | 2 +- shodan/__main__.py | 8 +++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bcdfec..fe426d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ CHANGELOG ========= -unreleased +1.17.0 ---------- -* Fix bug that caused unicode error when printing domain information +* Fix bug that caused unicode error when printing domain information (#106) +* Add flag to let users get their IPv6 address **shodan myip -6**(#35) 1.16.0 ------ diff --git a/setup.py b/setup.py index 6dc2833..eb9f224 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.16.0', + version='1.17.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 0bcf5e3..5ddad7a 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -400,11 +400,17 @@ def parse(color, fields, filters, filename, separator, filenames): @main.command() -def myip(): +@click.option('--ipv6', '-6', is_flag=True, default=False, help='Try to use IPv6 instead of IPv4') +def myip(ipv6): """Print your external IP address""" key = get_api_key() api = shodan.Shodan(key) + + # Use the IPv6-enabled domain if requested + if ipv6: + api.base_url = 'https://apiv6.shodan.io' + try: click.echo(api.tools.myip()) except shodan.APIError as e: From a910e9043f34702f1150099844bfd557af1a9f20 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Thu, 26 Sep 2019 17:33:50 -0500 Subject: [PATCH 078/148] Add library methods for new Notifications API --- CHANGELOG.md | 7 ++++- setup.py | 2 +- shodan/client.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe426d1..c3249e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ CHANGELOG ========= +1.18.0 +------ + +* Add library methods for the new Notifications API + 1.17.0 ----------- +------ * Fix bug that caused unicode error when printing domain information (#106) * Add flag to let users get their IPv6 address **shodan myip -6**(#35) diff --git a/setup.py b/setup.py index eb9f224..02ac89b 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.17.0', + version='1.18.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index bb38a69..5de2ba3 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -74,6 +74,72 @@ def domain_info(self, domain): """ return self.parent._request('/dns/domain/{}'.format(domain), {}) + class Notifier: + + def __init__(self, parent): + self.parent = parent + + def create(self, provider, args, description=None): + """Get the settings for the specified notifier that a user has configured. + + :param provider: Provider name + :type provider: str + :param args: Provider arguments + :type args: dict + :param description: Human-friendly description of the notifier + :type description: str + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + args['provider'] = provider + + if description: + args['description'] = description + + return self.parent._request('/notifier', args, method='post') + + def edit(self, nid, args): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :param args: Provider arguments + :type args: dict + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + return self.parent._request('/notifier/{}'.format(nid), args, method='put') + + def get(self, nid): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- object describing the notifier settings + """ + return self.parent._request('/notifier/{}'.format(nid), {}) + + def list_notifiers(self): + """Returns a list of notifiers that the user has added. + + :returns: A list of notifierse that are available on the account + """ + return self.parent._request('/notifier', {}) + + def list_providers(self): + """Returns a list of supported notification providers. + + :returns: A list of providers where each object describes a provider + """ + return self.parent._request('/notifier/provider', {}) + + def remove(self, nid): + """Delete the provided notifier. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- 'success' set to True if action succeeded + """ + return self.parent._request('/notifier/{}'.format(nid), {}, method='delete') + class Tools: def __init__(self, parent): @@ -195,6 +261,7 @@ def __init__(self, key, proxies=None): self.dns = self.Dns(self) self.exploits = self.Exploits(self) self.labs = self.Labs(self) + self.notifier = self.Notifier(self) self.org = self.Organization(self) self.tools = self.Tools(self) self.stream = Stream(key, proxies=proxies) From 7db73bca5ff193827d6617b7431c53969a93706b Mon Sep 17 00:00:00 2001 From: JW Date: Fri, 27 Sep 2019 19:51:19 +0300 Subject: [PATCH 079/148] Add "net:ip/mask" filter for shodan parse A simple implementation of the net:filter used in web GUI. It uses python3's builtin ipaddress lib and only imports those libs if the net:filter is used. --- shodan/cli/helpers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 3154591..a52bcb4 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -64,9 +64,26 @@ def get_banner_field(banner, flat_field): return None +def filter_with_netmask(banner, netmask): + # filtering based on netmask is a more abstract concept than + # a mere check for a specific field and thus needs its own mechanism + # this will enable users to use the net:10.0.0.0/8 syntax they are used to + # to find specific networks from a big shodan download. + from ipaddress import ip_network, ip_address + network = ip_network(netmask) + ip_field = get_banner_field(banner, 'ip') + if not ip_field: + return False + banner_ip_address = ip_address(ip_field) + return banner_ip_address in network + + def match_filters(banner, filters): for args in filters: flat_field, check = args.split(':', 1) + if flat_field == 'net': + return filter_with_netmask(banner, check) + value = get_banner_field(banner, flat_field) # If the field doesn't exist on the banner then ignore the record From b74d8e2da5f3fa12c68d8b16b71f7c372c6c4258 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 29 Sep 2019 13:30:15 -0500 Subject: [PATCH 080/148] New method to edit the list of IPs for an existing network alert --- CHANGELOG.md | 5 ++++- setup.py | 2 +- shodan/client.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3249e3..8670835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ CHANGELOG ========= -1.18.0 +1.19.0 ------ +* New method to edit the list of IPs for an existing network alert +1.18.0 +------ * Add library methods for the new Notifications API 1.17.0 diff --git a/setup.py b/setup.py index 02ac89b..70381c4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.18.0', + version='1.19.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index 5de2ba3..6e66259 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -640,6 +640,27 @@ def create_alert(self, name, ip, expires=0): return response + def edit_alert(self, aid, ip): + """Edit the IPs that should be monitored by the alert. + + :param aid: Alert ID + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str + + :returns: A dict describing the alert + """ + data = { + 'filters': { + 'ip': ip, + }, + } + + response = api_request(self.api_key, '/shodan/alert/{}'.format(aid), data=data, params={}, method='post', + proxies=self._session.proxies) + + return response + def alerts(self, aid=None, include_expired=True): """List all of the active alerts that the user created.""" if aid: @@ -684,3 +705,11 @@ def ignore_alert_trigger_notification(self, aid, trigger, ip, port): def unignore_alert_trigger_notification(self, aid, trigger, ip, port): """Re-enable trigger notifications for the provided IP and port""" return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') + + def add_alert_notifier(self, aid, nid): + """Enable the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') + + def remove_alert_notifier(self, aid, nid): + """Remove the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') From 4c297eed9e721e9cdb08b88b1fe3a9e1335963b7 Mon Sep 17 00:00:00 2001 From: JW Date: Tue, 1 Oct 2019 22:51:11 +0300 Subject: [PATCH 081/148] Moving import to top of file for pep8 --- shodan/cli/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index a52bcb4..4e99113 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -7,6 +7,7 @@ import itertools import os import sys +from ipaddress import ip_network, ip_address from .settings import SHODAN_CONFIG_DIR @@ -69,7 +70,6 @@ def filter_with_netmask(banner, netmask): # a mere check for a specific field and thus needs its own mechanism # this will enable users to use the net:10.0.0.0/8 syntax they are used to # to find specific networks from a big shodan download. - from ipaddress import ip_network, ip_address network = ip_network(netmask) ip_field = get_banner_field(banner, 'ip') if not ip_field: From bee4511986e3456b0b0050d93a1899ddc060a951 Mon Sep 17 00:00:00 2001 From: JW Date: Sat, 5 Oct 2019 10:28:17 +0300 Subject: [PATCH 082/148] Install ipaddress for py27 compat Using python_version marker, install ipaddress from pypi only for python2.7 and below --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fa2ed6..5095f64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ click click-plugins colorama requests>=2.2.1 -XlsxWriter \ No newline at end of file +XlsxWriter +ipaddress;python_version<='2.7' \ No newline at end of file From 2ea18d9953a92b32df78bc7d617127e233fcfb1a Mon Sep 17 00:00:00 2001 From: AaronK Date: Mon, 7 Oct 2019 18:25:19 +0200 Subject: [PATCH 083/148] Update cert-stream.rst make the example file python3 ready --- docs/examples/cert-stream.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst index e3e72c1..26bf1b4 100644 --- a/docs/examples/cert-stream.rst +++ b/docs/examples/cert-stream.rst @@ -24,7 +24,7 @@ information. # information. # # Author: achillean - + from __future__ import print_function import shodan import sys @@ -35,7 +35,7 @@ information. # Setup the api api = shodan.Shodan(API_KEY) - print 'Listening for certs...' + print('Listening for certs...') for banner in api.stream.ports([443, 8443]): if 'ssl' in banner: # Print out all the SSL information that Shodan has collected From 6f5a927b7f77d1f5dcfe0d87d761052a9ee98f19 Mon Sep 17 00:00:00 2001 From: AaronK Date: Wed, 9 Oct 2019 16:56:56 +0200 Subject: [PATCH 084/148] Update cert-stream.rst as requested, we are on python 2.7+ --- docs/examples/cert-stream.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst index 26bf1b4..b01440e 100644 --- a/docs/examples/cert-stream.rst +++ b/docs/examples/cert-stream.rst @@ -24,7 +24,6 @@ information. # information. # # Author: achillean - from __future__ import print_function import shodan import sys From df8e803c7d24f2594d429d2ea6d6fbd1f2ddfa82 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 13 Nov 2019 11:13:35 -0600 Subject: [PATCH 085/148] Show SHA1 checksum in bulk data listing if available --- setup.py | 2 +- shodan/cli/data.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 70381c4..8751b5f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.19.0', + version='1.19.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/data.py b/shodan/cli/data.py index 7cd7228..98d7852 100644 --- a/shodan/cli/data.py +++ b/shodan/cli/data.py @@ -27,6 +27,11 @@ def data_list(dataset): for file in files: click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + + # Show the SHA1 checksum if available + if file.get('sha1'): + click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False) + click.echo('{}'.format(file['url'])) else: # If no dataset was provided then show a list of all datasets From 1f101cae4700e30611a4d9b85634bc555f8cba08 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 18 Nov 2019 19:24:32 -0600 Subject: [PATCH 086/148] New options for "shodan domain": "-S" (save results) and "-D" (lookup open ports for IPs) --- CHANGELOG.md | 5 +++++ setup.py | 2 +- shodan/__main__.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8670835..8eced4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +1.20.0 +------ +* New option "-S" for **shodan domain** to save results from the lookup +* New option "-D" for **shodan domain** to lookup open ports for IPs in the results + 1.19.0 ------ * New method to edit the list of IPs for an existing network alert diff --git a/setup.py b/setup.py index 8751b5f..b2cf088 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.19.1', + version='1.20.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 5ddad7a..d4b7aa0 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -136,7 +136,9 @@ def convert(fields, input, format): @main.command(name='domain') @click.argument('domain', metavar='') -def domain_info(domain): +@click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) +@click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) +def domain_info(domain, details, save): """View all available information for a domain""" key = get_api_key() api = shodan.Shodan(key) @@ -146,6 +148,37 @@ def domain_info(domain): except shodan.APIError as e: raise click.ClickException(e.value) + # Grab the host information for any IP records that were returned + hosts = {} + if details: + ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']] + ips = set(ips) + + fout = None + if save: + filename = u'{}-hosts.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for ip in ips: + try: + hosts[ip] = api.host(ip) + + # Store the banners if requested + if fout: + for banner in hosts[ip]['data']: + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) + except shodan.APIError: + pass # Ignore any API lookup errors as this isn't critical information + + # Save the DNS data + if save: + filename = u'{}.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for record in info['data']: + helpers.write_banner(fout, record) + click.secho(info['domain'].upper(), fg='green') click.echo('') @@ -155,9 +188,16 @@ def domain_info(domain): click.style(record['subdomain'], fg='cyan'), click.style(record['type'], fg='yellow'), record['value'] - ) + ), + nl=False, ) + if record['value'] in hosts: + host = hosts[record['value']] + click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) + + click.echo('') + @main.command() @click.argument('key', metavar='') From da0e2ae7534c2b892a6122880e46b2b50fbe6b56 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:23:07 -0600 Subject: [PATCH 087/148] Add new API methods to get list of search filters/ facets --- shodan/client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/shodan/client.py b/shodan/client.py index 6e66259..3ed5e66 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -549,6 +549,20 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): tries += 1 time.sleep(1.0) # wait 1 second if the search errored out for some reason + def search_facets(self): + """Returns a list of search facets that can be used to get aggregate information about a search query. + + :returns: A list of strings where each is a facet name + """ + return self._request('/shodan/host/search/facets', {}) + + def search_filters(self): + """Returns a list of search filters that are available. + + :returns: A list of strings where each is a filter name + """ + return self._request('/shodan/host/search/filters', {}) + def search_tokens(self, query): """Returns information about the search query itself (filters used etc.) From 406abf0891f192fa2a46eff779bedbcc1841eaf8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:23:47 -0600 Subject: [PATCH 088/148] Fix linting rules and errors --- shodan/cli/scan.py | 8 ++++---- tox.ini | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 4590770..7ec158c 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -91,7 +91,7 @@ def scan_internet(quiet, port, protocol): click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), ';'.join(banner['hostnames'])) ) - except shodan.APIError as e: + except shodan.APIError: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -100,7 +100,7 @@ def scan_internet(quiet, port, protocol): scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True - except socket.timeout as e: + except socket.timeout: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -205,7 +205,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): done = True break - except shodan.APIError as e: + except shodan.APIError: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait briefly and try # to connect again! @@ -223,7 +223,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): if verbose: click.echo('# Scan status: {}'.format(scan['status'])) - except socket.timeout as e: + except socket.timeout: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait a second and try # to connect again! diff --git a/tox.ini b/tox.ini index d840b92..dd53bb5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [flake8] ignore = - E501 + E501 W293 exclude = build, From 9b1fe510ffb0f55a2e0f51b34f109fce904c7b16 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 14 Dec 2019 17:25:58 -0600 Subject: [PATCH 089/148] Release 1.21.0 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eced4d..5bd47a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.0 +------ +* New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. + 1.20.0 ------ * New option "-S" for **shodan domain** to save results from the lookup diff --git a/setup.py b/setup.py index b2cf088..17337b4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.20.0', + version='1.21.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From c73cf83e0fc92b9fa5fc8243019a99a3975ff2cc Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 25 Dec 2019 11:38:34 -0600 Subject: [PATCH 090/148] Add history and type parameters to Shodan.dns.domain_info() method and CLI command --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/__main__.py | 6 ++++-- shodan/client.py | 9 +++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd47a4..90c2059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.1 +------ +* Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command + 1.21.0 ------ * New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. diff --git a/setup.py b/setup.py index 17337b4..2148480 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.0', + version='1.21.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index d4b7aa0..9678bbb 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -138,13 +138,15 @@ def convert(fields, input, format): @click.argument('domain', metavar='') @click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) @click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) -def domain_info(domain, details, save): +@click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True) +@click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None) +def domain_info(domain, details, save, history, type): """View all available information for a domain""" key = get_api_key() api = shodan.Shodan(key) try: - info = api.dns.domain_info(domain) + info = api.dns.domain_info(domain, history=history, type=type) except shodan.APIError as e: raise click.ClickException(e.value) diff --git a/shodan/client.py b/shodan/client.py index 3ed5e66..a83ac20 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -69,10 +69,15 @@ class Dns: def __init__(self, parent): self.parent = parent - def domain_info(self, domain): + def domain_info(self, domain, history=False, type=None): """Grab the DNS information for a domain. """ - return self.parent._request('/dns/domain/{}'.format(domain), {}) + args = {} + if history: + args['history'] = history + if type: + args['type'] = type + return self.parent._request('/dns/domain/{}'.format(domain), args) class Notifier: From 7ef18461bae262ece63a97ee6eb9a14c465607d0 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Jan 2020 23:43:22 -0600 Subject: [PATCH 091/148] Add support for paging through the domain information --- CHANGELOG.md | 4 ++++ setup.py | 2 +- shodan/client.py | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c2059..91540af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.2 +------ +* Add support for paging through the domain information + 1.21.1 ------ * Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command diff --git a/setup.py b/setup.py index 2148480..b84dd71 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.1', + version='1.21.2', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index a83ac20..4116051 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -69,10 +69,12 @@ class Dns: def __init__(self, parent): self.parent = parent - def domain_info(self, domain, history=False, type=None): + def domain_info(self, domain, history=False, type=None, page=1): """Grab the DNS information for a domain. """ - args = {} + args = { + 'page': page, + } if history: args['history'] = history if type: From 9e75eb9871bad4490dbdb7c2a3130883e40c5b59 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 20 Jan 2020 13:57:44 -0600 Subject: [PATCH 092/148] Fix geo.json file converter --- CHANGELOG.md | 4 +++ setup.py | 2 +- shodan/cli/converter/geojson.py | 49 +++++++++++++++------------------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91540af..0d5c02d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.21.3 +------ +* Fix geo.json file converter + 1.21.2 ------ * Add support for paging through the domain information diff --git a/setup.py b/setup.py index b84dd71..0605303 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.2', + version='1.21.3', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 6126267..eb9a205 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -1,4 +1,4 @@ - +from json import dumps from .base import Converter from ...helpers import get_ip, iterate_files @@ -18,40 +18,35 @@ def process(self, files): # Write the header self.header() - hosts = {} + # We only want to generate 1 datapoint for each IP - not per service + unique_hosts = set() for banner in iterate_files(files): ip = get_ip(banner) if not ip: continue - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) + if ip not in unique_hosts: + self.write(ip, banner) + unique_hosts.add(ip) self.footer() - def write(self, host): + def write(self, ip, host): try: - ip = get_ip(host) lat, lon = host['location']['latitude'], host['location']['longitude'] - - feature = """{ - "type": "Feature", - "id": "{}", - "properties": { - "name": "{}" - }, - "geometry": { - "type": "Point", - "coordinates": [{}, {}] - } - }""".format(ip, ip, lat, lon) - - self.fout.write(feature) - except Exception: + feature = { + 'type': 'Feature', + 'id': ip, + 'properties': { + 'name': ip, + 'lat': lat, + 'lon': lon, + }, + 'geometry': { + 'type': 'Point', + 'coordinates': [lon, lat], + }, + } + self.fout.write(dumps(feature) + ',') + except Exception as e: pass From ba561934f7fcfce3c35270d2baee6539b2afaaf1 Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 17:49:19 +0100 Subject: [PATCH 093/148] py 2 to 3 fix the example given in https://github.com/achillean/shodan-python/issues/114 --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 00c4d34..62744af 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -56,7 +56,7 @@ Now that we have our API object all good to go, we're ready to perform a search: print('IP: {}'.format(result['ip_str'])) print(result['data']) print('') - except shodan.APIError, e: + except shodan.APIError as e: print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which From 67409f6eebeec28b00be0c7517d28411f1ea43c1 Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:10:42 +0100 Subject: [PATCH 094/148] py 2 to 3 --- docs/examples/query-summary.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index 7a60716..73a87ab 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -48,7 +48,7 @@ and country. # Input validation if len(sys.argv) == 1: - print 'Usage: %s ' % sys.argv[0] + print('Usage: %s ' % sys.argv[0]) sys.exit(1) try: @@ -68,16 +68,16 @@ and country. # Print the summary info from the facets for facet in result['facets']: - print FACET_TITLES[facet] + print(FACET_TITLES[facet]) for term in result['facets'][facet]: - print '%s: %s' % (term['value'], term['count']) + print('%s: %s' % (term['value'], term['count'])) # Print an empty line between summary info - print '' + print('') - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: %s' % e) sys.exit(1) """ From 1deba33ba6bbb0bc45dd5716c48a1f785a0a986b Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:12:16 +0100 Subject: [PATCH 095/148] py 2 to 3 --- docs/examples/gifcreator.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst index ad9efc8..e4a43c3 100644 --- a/docs/examples/gifcreator.rst +++ b/docs/examples/gifcreator.rst @@ -106,7 +106,7 @@ There are a few key Shodan methods/ parameters that make the script work: os.system('rm -f /tmp/gif-image-*.jpg') # Show a progress indicator - print result['ip_str'] + print(result['ip_str']) -The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 \ No newline at end of file +The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 From a3084d3a5976e9d47e407ba073cd90f2c8122b1c Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sat, 25 Jan 2020 18:19:02 +0100 Subject: [PATCH 096/148] py 2 to 3 --- docs/examples/query-summary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index 73a87ab..66e15fe 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -62,9 +62,9 @@ and country. # And it also runs faster than doing a search(). result = api.count(query, facets=FACETS) - print 'Shodan Summary Information' - print 'Query: %s' % query - print 'Total Results: %s\n' % result['total'] + print('Shodan Summary Information') + print('Query: %s' % query) + print('Total Results: %s\n' % result['total']) # Print the summary info from the facets for facet in result['facets']: From ee93d65b75b329fbe57f4f2248ead0f695ce1ccb Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Tue, 28 Jan 2020 09:44:48 +0100 Subject: [PATCH 097/148] Remove shebang This is a RPM packing issue. --- shodan/cli/worldmap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 767b237..30c9ec3 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' F-Secure Virus World Map console edition From 2367646b8e4474ffdfd969485f989c69a4f42a22 Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Tue, 28 Jan 2020 14:31:27 +0100 Subject: [PATCH 098/148] Point the repo directly --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0605303..8290147 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description_content_type='text/x-rst', author='John Matherly', author_email='jmath@shodan.io', - url='http://github.com/achillean/shodan-python/tree/master', + url='https://github.com/achillean/shodan-python', packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, install_requires=DEPENDENCIES, From bde444271a516a81235ed961d89e32b8f551ebce Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 17 Mar 2020 18:29:15 -0500 Subject: [PATCH 099/148] New Streaming API method: /shodan/vulns/{vulns} to subscribe to IPs that are vulnerable to an issue Release 1.22.0 --- setup.py | 2 +- shodan/__main__.py | 11 +++++++++-- shodan/stream.py | 11 +++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0605303..0c8882f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.21.3', + version='1.22.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 9678bbb..5b1af92 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -635,7 +635,8 @@ def stats(limit, facets, filename, query): @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel): +@click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) +def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -663,9 +664,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('alert') if tags: stream_type.append('tags') + if vulns: + stream_type.append('vulns') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -689,6 +692,9 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if tags: stream_args = tags.split(',') + + if vulns: + stream_args = vulns.split(',') # Flatten the list of stream types # Possible values are: @@ -710,6 +716,7 @@ def _create_stream(name, args, timeout): 'countries': api.stream.countries(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), 'tags': api.stream.tags(args, timeout=timeout), + 'vulns': api.stream.vulns(args, timeout=timeout), }.get(name, 'all') stream = _create_stream(stream_type, stream_args, timeout=timeout) diff --git a/shodan/stream.py b/shodan/stream.py index 1feb15a..15d7619 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -134,3 +134,14 @@ def tags(self, tags, raw=False, timeout=None): stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) for line in self._iter_stream(stream, raw): yield line + + def vulns(self, vulns, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. + + :param vulns: A list of vulns to return banner data on. + :type vulns: string[] + """ + stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line From 378feee4992461f8f4510034c65d4c9dfbe651b6 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 27 Mar 2020 12:12:48 -0500 Subject: [PATCH 100/148] Fix bug when converting data file to CSV using Python3 --- setup.py | 2 +- shodan/cli/converter/csvc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0c8882f..db956d9 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.22.0', + version='1.22.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 39ec162..93b1695 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -54,7 +54,7 @@ def process(self, files): # The "vulns" property can't be nicely flattened as-is so we turn # it into a list before processing the banner. if 'vulns' in banner: - banner['vulns'] = banner['vulns'].keys() + banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list try: row = [] From b7a9978d532094fb8dd72efaa6b6f0bb75a89ba6 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 14:36:00 -0500 Subject: [PATCH 101/148] Update CHANGELOG to match releases --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5c02d..de68afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ CHANGELOG ========= +1.22.1 +------ +* Fix bug when converting data file to CSV using Python3 + +1.22.0 +------ +* Add support for new vulnerability streaming endpoints + 1.21.3 ------ * Fix geo.json file converter From d91ffd8f6aa0e093c3a913bb8585add3e99c2969 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 15:02:12 -0500 Subject: [PATCH 102/148] Add new CLI command: shodan alert domain --- CHANGELOG.md | 4 ++++ setup.py | 4 +++- shodan/cli/alert.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de68afb..50a9334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.23.0 +------ +* Add new CLI command: shodan alert domain + 1.22.1 ------ * Fix bug when converting data file to CSV using Python3 diff --git a/setup.py b/setup.py index db956d9..e04a96f 100755 --- a/setup.py +++ b/setup.py @@ -2,12 +2,14 @@ from setuptools import setup + DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') README = open('README.rst', 'r').read() + setup( name='shodan', - version='1.22.1', + version='1.23.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 1ed8572..633b2aa 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,6 +3,7 @@ from operator import itemgetter from shodan.cli.helpers import get_api_key +from time import sleep @click.group() @@ -46,6 +47,35 @@ def alert_create(name, netblocks): click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +@alert.command(name='domain') +@click.argument('domain', metavar='', type=str) +@click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') +def alert_domain(domain, triggers): + """Create a network alert based on a domain name""" + key = get_api_key() + + api = shodan.Shodan(key) + try: + # Grab a list of IPs for the domain + domain = domain.lower() + click.secho('Looking up domain information...', dim=True) + info = api.dns.domain_info(domain, type='A') + domain_ips = set([record['value'] for record in info['data']]) + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From 31b420a590f11fd83e336f308aff499c2aa4d096 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 6 Apr 2020 15:06:42 -0500 Subject: [PATCH 103/148] Fix linting errors --- shodan/cli/alert.py | 1 - shodan/cli/converter/geojson.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 633b2aa..0d81dc4 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,7 +3,6 @@ from operator import itemgetter from shodan.cli.helpers import get_api_key -from time import sleep @click.group() diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index eb9a205..3f6c975 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -48,5 +48,5 @@ def write(self, ip, host): }, } self.fout.write(dumps(feature) + ',') - except Exception as e: + except Exception: pass From 094420d483a4be6c6d04906edfa13ab1e70489f3 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sun, 3 May 2020 12:54:26 +0000 Subject: [PATCH 104/148] Import ABC from collections.abc for Python 3 compatibility. --- shodan/cli/converter/csvc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 93b1695..f51e75e 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -2,7 +2,13 @@ from .base import Converter from ...helpers import iterate_files -from collections import MutableMapping +try: + # python 3.x: Import ABC from collections.abc + from collections.abc import MutableMapping +except ImportError: + # Python 2.x: Import ABC from collections + from collections import MutableMapping + from csv import writer as csv_writer, excel From c6615d0a275ac53fadb0bb4f66fe290ffb1f9dae Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 23 Jun 2020 16:32:03 -0500 Subject: [PATCH 105/148] Fix invalid escape sequence error (#131) Improve ``shodan radar`` output on Python3 --- shodan/__main__.py | 15 ++++++++------- shodan/cli/worldmap.py | 5 ++--- tox.ini | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 5b1af92..939b53f 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -59,6 +59,13 @@ # Make "-h" work like "--help" CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +CONVERTERS = { + 'kml': KmlConverter, + 'csv': CsvConverter, + 'geo.json': GeoJsonConverter, + 'images': ImagesConverter, + 'xlsx': ExcelConverter, +} # Define a basestring type if necessary for Python3 compatibility try: @@ -66,6 +73,7 @@ except NameError: basestring = str + # Define the main entry point for all of our commands # and expose a way for 3rd-party plugins to tie into the Shodan CLI. @with_plugins(iter_entry_points('shodan.cli.plugins')) @@ -81,13 +89,6 @@ def main(): main.add_command(scan) -CONVERTERS = { - 'kml': KmlConverter, - 'csv': CsvConverter, - 'geo.json': GeoJsonConverter, - 'images': ImagesConverter, - 'xlsx': ExcelConverter, -} @main.command() @click.option('--fields', help='List of properties to output.', default=None) @click.argument('input', metavar='') diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 30c9ec3..db91e41 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -29,8 +29,7 @@ 'coords': [90.0, -180.0, -90.0, 180.0], # PyLint freaks out about the world map backslashes so ignore those warnings - # pylint: disable=W1401 - 'data': ''' + 'data': r''' . _..::__: ,-"-"._ |7 , _,.__ _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ @@ -159,7 +158,7 @@ def draw(self, target): for lat, lon, char, desc, attrs, color in self.data: # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html if desc: - desc = desc.encode(self.encoding, 'ignore') + desc = desc.encode(self.encoding, 'ignore').decode() if items_to_show <= 0: break char_x, char_y = self.latlon_to_coords(lat, lon) diff --git a/tox.ini b/tox.ini index dd53bb5..1a9f632 100644 --- a/tox.ini +++ b/tox.ini @@ -11,4 +11,4 @@ exclude = per-file-ignores = shodan/__init__.py:F401, shodan/cli/converter/__init__.py:F401, - shodan/cli/worldmap.py:W291,W293,W605, \ No newline at end of file + shodan/cli/worldmap.py:W291, \ No newline at end of file From 6ad80cc48da80d3c100637cefdd6f939d5620b3a Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 00:11:32 +0100 Subject: [PATCH 106/148] allow user to move config dir to ~/.config --- shodan/cli/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 6f27e5d..2f5a73c 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,5 +1,10 @@ +from os import path + +if path.exists(path.expanduser("~/.config/shodan")): + SHODAN_CONFIG_DIR="~/.config/shodan/" +else: + SHODAN_CONFIG_DIR = '~/.shodan/' -SHODAN_CONFIG_DIR = '~/.shodan/' COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From 1b58c7f3a6402a87e860f31a0cb197070fa7c798 Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 16:07:12 +0100 Subject: [PATCH 107/148] Revert "allow user to move config dir to ~/.config" This reverts commit 6ad80cc48da80d3c100637cefdd6f939d5620b3a. --- shodan/cli/settings.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 2f5a73c..6f27e5d 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,10 +1,5 @@ -from os import path - -if path.exists(path.expanduser("~/.config/shodan")): - SHODAN_CONFIG_DIR="~/.config/shodan/" -else: - SHODAN_CONFIG_DIR = '~/.shodan/' +SHODAN_CONFIG_DIR = '~/.shodan/' COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From c01abd045b531052b8522ea532f6c6ae54d85b90 Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 16:09:19 +0100 Subject: [PATCH 108/148] default config dir to ~/.config/shodan, but support existing installations --- shodan/cli/settings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 6f27e5d..294793c 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,5 +1,11 @@ -SHODAN_CONFIG_DIR = '~/.shodan/' +from os import path + +if path.exists(path.expanduser("~/.shodan")): + SHODAN_CONFIG_DIR = '~/.shodan/' +else: + SHODAN_CONFIG_DIR="~/.config/shodan/" + COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', From 639ca36a6747671e5d68ab9b10b054e2b31255eb Mon Sep 17 00:00:00 2001 From: Dhaval Soneji Date: Wed, 24 Jun 2020 19:22:10 +0100 Subject: [PATCH 109/148] makedirs if ~/.config/shodan does not exist --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 939b53f..f8ed673 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -210,7 +210,7 @@ def init(key): shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) if not os.path.isdir(shodan_dir): try: - os.mkdir(shodan_dir) + os.makedirs(shodan_dir) except OSError: raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) From 56f0cdf7de8c41c47b2cfa869dd3e64bc1c1c8ce Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 7 Sep 2020 19:37:04 -0500 Subject: [PATCH 110/148] Fix bug that caused extra newlines when converting .json.gz data file to CSV on Windows (https://stackoverflow.com/questions/3191528/csv-in-python-adding-an-extra-carriage-return-on-windows) --- README.rst | 1 + setup.py | 2 +- shodan/cli/converter/csvc.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 610c13d..14f6717 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,7 @@ Features - `Manage Email Notifications `_ - Exploit search API fully implemented - Bulk data downloads +- Access the Shodan DNS DB to view domain information - `Command-line interface `_ .. image:: https://cli.shodan.io/img/shodan-cli-preview.png diff --git a/setup.py b/setup.py index e04a96f..b1735b2 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.23.0', + version='1.23.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index f51e75e..20c3d03 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -51,7 +51,7 @@ class CsvConverter(Converter): ] def process(self, files): - writer = csv_writer(self.fout, dialect=excel) + writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') # Write the header writer.writerow(self.fields) From 9ebccea26867f33ddadb063af9b585a09d69fe1a Mon Sep 17 00:00:00 2001 From: malvidin Date: Thu, 8 Oct 2020 22:40:53 +0200 Subject: [PATCH 111/148] Convert residual % formatting with str.format() Add API Rate limits to the Shodan class Fix human precision of bytes under 1024 --- shodan/__main__.py | 8 ++++---- shodan/cli/helpers.py | 2 +- shodan/cli/settings.py | 2 +- shodan/cli/worldmap.py | 4 ++-- shodan/client.py | 16 ++++++++++++---- shodan/helpers.py | 16 ++++++++-------- shodan/stream.py | 12 ++++++------ 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f8ed673..bbcef07 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -288,10 +288,10 @@ def download(limit, skip, filename, query): raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') # Print some summary information about the download request - click.echo('Search query:\t\t\t%s' % query) - click.echo('Total number of results:\t%s' % total) - click.echo('Query credits left:\t\t%s' % info['unlocked_left']) - click.echo('Output file:\t\t\t%s' % filename) + click.echo('Search query:\t\t\t{}'.format(query)) + click.echo('Total number of results:\t{}'.format(total)) + click.echo('Query credits left:\t\t{}'.format(info['unlocked_left'])) + click.echo('Output file:\t\t\t{}'.format(filename)) if limit > total: limit = total diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 4e99113..bde2f07 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -47,7 +47,7 @@ def timestr(): def open_streaming_file(directory, timestr, compresslevel=9): - return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel) + return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel) def get_banner_field(banner, flat_field): diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 294793c..05c1b9f 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -4,7 +4,7 @@ if path.exists(path.expanduser("~/.shodan")): SHODAN_CONFIG_DIR = '~/.shodan/' else: - SHODAN_CONFIG_DIR="~/.config/shodan/" + SHODAN_CONFIG_DIR = "~/.config/shodan/" COLORIZE_FIELDS = { 'ip_str': 'green', diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index db91e41..4e09872 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -166,7 +166,7 @@ def draw(self, target): attrs |= curses.color_pair(self.colors[color]) self.window.addstr(char_y, char_x, char, attrs) if desc: - det_show = "%s %s" % (char, desc) + det_show = "{} {}".format(char, desc) else: det_show = None @@ -179,7 +179,7 @@ def draw(self, target): # FIXME: check window size before addstr() break self.window.overwrite(target) - self.window.leaveok(1) + self.window.leaveok(True) class MapApp(object): diff --git a/shodan/client.py b/shodan/client.py index 4116051..079880d 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -273,6 +273,8 @@ def __init__(self, key, proxies=None): self.tools = self.Tools(self) self.stream = Stream(key, proxies=proxies) self._session = requests.Session() + self.api_rate_limit = 1 # Requests per second + self._api_query_time = None if proxies: self._session.proxies.update(proxies) self._session.trust_env = False @@ -297,6 +299,11 @@ def _request(self, function, params, service='shodan', method='get'): 'exploits': self.base_exploits_url, }.get(service, 'shodan') + # Wait for API rate limit + if self._api_query_time is not None and self.api_rate_limit > 0: + while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time(): + time.sleep(0.1 / self.api_rate_limit) + # Send the request try: method = method.lower() @@ -308,6 +315,7 @@ def _request(self, function, params, service='shodan', method='get'): data = self._session.delete(base_url + function, params=params) else: data = self._session.get(base_url + function, params=params) + self._api_query_time = time.time() except Exception: raise APIError('Unable to connect to Shodan') @@ -377,7 +385,7 @@ def host(self, ips, history=False, minify=False): params['history'] = history if minify: params['minify'] = minify - return self._request('/shodan/host/%s' % ','.join(ips), params) + return self._request('/shodan/host/{}'.format(','.join(ips)), params) def info(self): """Returns information about the current API key, such as a list of add-ons @@ -468,7 +476,7 @@ def scan_status(self, scan_id): :returns: A dictionary with general information about the scan, including its status in getting processed. """ - return self._request('/shodan/scan/%s' % scan_id, {}) + return self._request('/shodan/scan/{}'.format(scan_id), {}) def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): """Search the SHODAN database. @@ -685,7 +693,7 @@ def edit_alert(self, aid, ip): def alerts(self, aid=None, include_expired=True): """List all of the active alerts that the user created.""" if aid: - func = '/shodan/alert/%s/info' % aid + func = '/shodan/alert/{}/info'.format(aid) else: func = '/shodan/alert/info' @@ -697,7 +705,7 @@ def alerts(self, aid=None, include_expired=True): def delete_alert(self, aid): """Delete the alert with the given ID.""" - func = '/shodan/alert/%s' % aid + func = '/shodan/alert/{}'.format(aid) response = api_request(self.api_key, func, params={}, method='delete', proxies=self._session.proxies) diff --git a/shodan/helpers.py b/shodan/helpers.py index 2d976dc..378b1bb 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -142,7 +142,7 @@ def write_banner(fout, banner): fout.write(line.encode('utf-8')) -def humanize_bytes(bytes, precision=1): +def humanize_bytes(byte_count, precision=1): """Return a humanized string representation of a number of bytes. >>> humanize_bytes(1) '1 byte' @@ -161,15 +161,15 @@ def humanize_bytes(bytes, precision=1): >>> humanize_bytes(1024*1234*1111,1) '1.3 GB' """ - if bytes == 1: + if byte_count == 1: return '1 byte' - if bytes < 1024: - return '%.*f %s' % (precision, bytes, "bytes") + if byte_count < 1024: + '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] multiple = 1024.0 # .0 to force float on python 2 for suffix in suffixes: - bytes /= multiple - if bytes < multiple: - return '%.*f %s' % (precision, bytes, suffix) - return '%.*f %s' % (precision, bytes, suffix) + byte_count /= multiple + if byte_count < multiple: + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) diff --git a/shodan/stream.py b/shodan/stream.py index 15d7619..92365dc 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -71,7 +71,7 @@ def _iter_stream(self, stream, raw): def alert(self, aid=None, timeout=None, raw=False): if aid: - stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout) + stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout) else: stream = self._create_stream('/shodan/alert', timeout=timeout) @@ -90,7 +90,7 @@ def asn(self, asn, raw=False, timeout=None): :param asn: A list of ASN to return banner data on. :type asn: string[] """ - stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout) + stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -109,7 +109,7 @@ def countries(self, countries, raw=False, timeout=None): :param countries: A list of countries to return banner data on. :type countries: string[] """ - stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout) + stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -120,7 +120,7 @@ def ports(self, ports, raw=False, timeout=None): :param ports: A list of ports to return banner data on. :type ports: int[] """ - stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) + stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -131,7 +131,7 @@ def tags(self, tags, raw=False, timeout=None): :param tags: A list of tags to return banner data on. :type tags: string[] """ - stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) + stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -142,6 +142,6 @@ def vulns(self, vulns, raw=False, timeout=None): :param vulns: A list of vulns to return banner data on. :type vulns: string[] """ - stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout) + stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line From 0703499f21bd21f10b2c00d8fc470de901dc46ac Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 11 Oct 2020 17:24:39 -0500 Subject: [PATCH 112/148] Fix linting error --- shodan/cli/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 294793c..05c1b9f 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -4,7 +4,7 @@ if path.exists(path.expanduser("~/.shodan")): SHODAN_CONFIG_DIR = '~/.shodan/' else: - SHODAN_CONFIG_DIR="~/.config/shodan/" + SHODAN_CONFIG_DIR = "~/.config/shodan/" COLORIZE_FIELDS = { 'ip_str': 'green', From 322f169c8d0f916926e808ef6d6aa10b544fc9ac Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 11 Oct 2020 17:50:10 -0500 Subject: [PATCH 113/148] Release 1.24.0 New command: shodan alert stats --- setup.py | 2 +- shodan/cli/alert.py | 152 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1735b2..9c6f55f 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.23.1', + version='1.24.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 0d81dc4..de77294 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,10 +1,73 @@ import click +import csv import shodan +from collections import defaultdict from operator import itemgetter from shodan.cli.helpers import get_api_key +MAX_QUERY_LENGTH = 1000 + + +def aggregate_facet(api, networks, facets): + """Merge the results from multiple facet API queries into a single result object. + This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit + into a single API call. + """ + def _merge_custom_facets(lfacets, results): + for key in results['facets']: + if key not in lfacets: + lfacets[key] = defaultdict(int) + + for item in results['facets'][key]: + lfacets[key][item['value']] += item['count'] + + # We're going to create a custom facets dict where + # the key is the value of a facet. Normally the facets + # object is a list where each item has a "value" and "count" property. + tmp_facets = {} + count = 0 + + query = 'net:' + + for net in networks: + query += '{},'.format(net) + + # Start running API queries if the query length is getting long + if len(query) > MAX_QUERY_LENGTH: + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + query = 'net:' + + # Run any remaining search query + if query[-1] != ':': + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + + # Convert the internal facets structure back to the one that + # the API returns. + new_facets = {} + for facet in tmp_facets: + sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True) + new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items] + + # Make sure the facet keys exist even if there weren't any results + for facet, _ in facets: + if facet not in new_facets: + new_facets[facet] = [] + + return { + 'matches': [], + 'facets': new_facets, + 'total': count, + } + + @click.group() def alert(): """Manage the network alerts for your account""" @@ -149,6 +212,95 @@ def alert_list(expired): click.echo("You haven't created any alerts yet.") +@alert.command(name='stats') +@click.option('--limit', help='The number of results to return.', default=10, type=int) +@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) +@click.argument('facets', metavar='', nargs=-1) +def alert_stats(limit, filename, facets): + """Show summary information about your monitored networks""" + # Setup Shodan + key = get_api_key() + api = shodan.Shodan(key) + + # Make sure the user didn't supply an empty string + if not facets: + raise click.ClickException('No facets provided') + + facets = [(facet, limit) for facet in facets] + + # Get the list of IPs/ networks that the user is monitoring + networks = set() + try: + alerts = api.alerts() + for alert in alerts: + for tmp in alert['filters']['ip']: + networks.add(tmp) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the facets the user requested + try: + results = aggregate_facet(api, networks, facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared + # Print the stats tables + for facet in results['facets']: + click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) + + for item in results['facets'][facet]: + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) + + click.echo('') + + # Create the output file if requested + fout = None + if filename: + if not filename.endswith('.csv'): + filename += '.csv' + fout = open(filename, 'w') + writer = csv.writer(fout, dialect=csv.excel) + + # Write the header that contains the facets + row = [] + for facet in results['facets']: + row.append(facet) + row.append('') + writer.writerow(row) + + # Every facet has 2 columns (key, value) + counter = 0 + has_items = True + while has_items: + # pylint: disable=W0612 + row = ['' for i in range(len(results['facets']) * 2)] + + pos = 0 + has_items = False + for facet in results['facets']: + values = results['facets'][facet] + + # Add the values for the facet into the current row + if len(values) > counter: + has_items = True + row[pos] = values[counter]['value'] + row[pos + 1] = values[counter]['count'] + + pos += 2 + + # Write out the row + if has_items: + writer.writerow(row) + + # Move to the next row of values + counter += 1 + + @alert.command(name='remove') @click.argument('alert_id', metavar='') def alert_remove(alert_id): From 8ac631090100de6dbc53171d07918ac869a1eeb9 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:46:09 -0600 Subject: [PATCH 114/148] Remove the --skip option when downloading as the API nolonger supports requesting arbitrary pages in the search results. --- shodan/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f8ed673..a98ddbd 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -256,7 +256,6 @@ def count(query): @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) -@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) def download(limit, skip, filename, query): From 8195e1de58615303fe1d412d283865b1c024f0d8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:46:28 -0600 Subject: [PATCH 115/148] Remove the skip parameter from the download method as well --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index a98ddbd..d12b594 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -258,7 +258,7 @@ def count(query): @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, skip, filename, query): +def download(limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() From 90491308971a7ba6e358d6152d7b02307964c6fe Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 2 Nov 2020 13:48:21 -0600 Subject: [PATCH 116/148] Remove the skip parameter from the search_cursor --- shodan/__main__.py | 6 +----- shodan/client.py | 10 +--------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index d12b594..4395820 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -298,15 +298,11 @@ def download(limit, filename, query): # A limit of -1 means that we should download all the data if limit <= 0: limit = total - - # Adjust the total number of results we should expect to download if the user is skipping results - if skip > 0: - limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False, skip=skip) + cursor = api.search_cursor(query, minify=False) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) diff --git a/shodan/client.py b/shodan/client.py index 4116051..eb66e7e 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -504,7 +504,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, skip=0): + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -518,8 +518,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type retries: int - :param skip: (optional) Number of results to skip - :type skip: int :returns: A search cursor that can be used as an iterator/ generator. """ @@ -532,12 +530,6 @@ def search_cursor(self, query, minify=True, retries=5, skip=0): 'total': None, } - # Convert the number of skipped records into a page number - if skip > 0: - # Each page returns 100 results so find the nearest page that we want - # the cursor to skip to. - page += int(skip / 100) - while results['matches']: try: results = self.search(query, minify=minify, page=page) From 5eedf292d0cd4083c629a8c36acad43a63ffc472 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 25 Jan 2021 16:33:53 -0600 Subject: [PATCH 117/148] Add new CLI command: shodan alert download [--alert-id=] --- CHANGELOG.md | 8 +++++ setup.py | 2 +- shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a9334..2247045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ CHANGELOG ========= +1.25.0 +------ +* Add new CLI command: shodan alert download + +1.24.0 +------ +* Add new CLI command: shodan alert stats + 1.23.0 ------ * Add new CLI command: shodan alert domain diff --git a/setup.py b/setup.py index 9c6f55f..97616d6 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.24.0', + version='1.25.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index de77294..63b863d 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -4,7 +4,10 @@ from collections import defaultdict from operator import itemgetter +from shodan import APIError from shodan.cli.helpers import get_api_key +from shodan.helpers import open_file, write_banner +from time import sleep MAX_QUERY_LENGTH = 1000 @@ -138,6 +141,86 @@ def alert_domain(domain, triggers): click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +@alert.command(name='download') +@click.argument('filename', metavar='', type=str) +@click.option('--alert-id', help='Specific alert ID to download the data of', default=None) +def alert_download(filename, alert_id): + """Download all information for monitored networks/ IPs.""" + key = get_api_key() + + api = shodan.Shodan(key) + ips = set() + networks = set() + + # Helper method to process batches of IPs + def batch(iterable, size=1): + iter_length = len(iterable) + for ndx in range(0, iter_length, size): + yield iterable[ndx:min(ndx + size, iter_length)] + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + if alert_id: + alerts = [api.alerts(aid=alert_id.strip())] + else: + alerts = api.alerts() + + click.echo('Compiling list of networks/ IPs to download...') + for alert in alerts: + for net in alert['filters']['ip']: + if '/' in net: + networks.add(net) + else: + ips.add(net) + + click.echo('Downloading...') + with open_file(filename) as fout: + # Check if the user is able to use batch IP lookups + batch_size = 1 + if len(ips) > 0: + api_info = api.info() + if api_info['plan'] in ['corp', 'stream-100']: + batch_size = 100 + + # Convert it to a list so we can index into it + ips = list(ips) + + # Grab all the IP information + for ip in batch(ips, size=batch_size): + try: + click.echo(ip) + results = api.host(ip) + if not isinstance(results, list): + results = [results] + + for host in results: + for banner in host['data']: + write_banner(fout, banner) + except APIError: + pass + sleep(1) # Slow down a bit to make sure we don't hit the rate limit + + # Grab all the network ranges + for net in networks: + try: + counter = 0 + click.echo(net) + for banner in api.search_cursor('net:{}'.format(net)): + write_banner(fout, banner) + + # Slow down a bit to make sure we don't hit the rate limit + if counter % 100 == 0: + sleep(1) + counter += 1 + except APIError: + pass + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From 75d4c1b37b8a553f6db73ffc28cc139febbf0d9c Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 26 Apr 2021 14:10:08 -0300 Subject: [PATCH 118/148] Adding timeout --- shodan/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index eb66e7e..397b37a 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -502,9 +502,12 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) + if timeout: + args['timeout'] = timeout + return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, timeout=0): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -532,7 +535,7 @@ def search_cursor(self, query, minify=True, retries=5): while results['matches']: try: - results = self.search(query, minify=minify, page=page) + results = self.search(query, minify=minify, page=page, timeout=timeout) for banner in results['matches']: try: yield banner From a5edaff408b8b8802c515294fdfdff8268fda957 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 26 Apr 2021 14:31:15 -0300 Subject: [PATCH 119/148] None instead of zero --- shodan/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/client.py b/shodan/client.py index 397b37a..369b039 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=0): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None): """Search the SHODAN database. :param query: Search query; identical syntax to the website From cdfdb865e64568fa77437174e4a27949e84bca1e Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 18 May 2021 14:25:57 -0300 Subject: [PATCH 120/148] Some fixs to support big number of results Replace all timeout=0 to timeout=None and add incremental sleep for retry --- shodan/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index 369b039..10899c0 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -507,7 +507,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, timeout=0): + def search_cursor(self, query, minify=True, retries=5, timeout=None): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -549,7 +549,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=0): raise APIError('Retry limit reached ({:d})'.format(retries)) tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason + time.sleep(tries) # wait (1 second * retry number) if the search errored out for some reason def search_facets(self): """Returns a list of search facets that can be used to get aggregate information about a search query. From 67a2585d92784b41210f8963ea93cce9d8656ae3 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Wed, 19 May 2021 16:37:38 -0300 Subject: [PATCH 121/148] removing timeout, ther is no need for it now! --- shodan/client.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/shodan/client.py b/shodan/client.py index 10899c0..02f115e 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -470,7 +470,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/%s' % scan_id, {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, timeout=None): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -502,12 +502,9 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) - if timeout: - args['timeout'] = timeout - return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, timeout=None): + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -535,7 +532,7 @@ def search_cursor(self, query, minify=True, retries=5, timeout=None): while results['matches']: try: - results = self.search(query, minify=minify, page=page, timeout=timeout) + results = self.search(query, minify=minify, page=page) for banner in results['matches']: try: yield banner From d1027d8e4cea56e6f00cd437eab7f2ddfde72c59 Mon Sep 17 00:00:00 2001 From: Hy Che Date: Tue, 28 Dec 2021 02:46:37 +0700 Subject: [PATCH 122/148] Add implementation for custom filter `shodan/custom` Also bump version to 1.26 --- setup.py | 2 +- shodan/__main__.py | 39 +++++++++++++++++++++++---------------- shodan/stream.py | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 97616d6..6be33f3 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.25.0', + version='1.26.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 4395820..3700098 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -106,7 +106,7 @@ def convert(fields, input, format): if not hasattr(converter_class, 'fields'): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified - + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -173,7 +173,7 @@ def domain_info(domain, details, save, history, type): helpers.write_banner(fout, banner) except shodan.APIError: pass # Ignore any API lookup errors as this isn't critical information - + # Save the DNS data if save: filename = u'{}.json.gz'.format(domain) @@ -198,7 +198,7 @@ def domain_info(domain, details, save, history, type): if record['value'] in hosts: host = hosts[record['value']] click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) - + click.echo('') @@ -448,7 +448,7 @@ def myip(ipv6): # Use the IPv6-enabled domain if requested if ipv6: api.base_url = 'https://apiv6.shodan.io' - + try: click.echo(api.tools.myip()) except shodan.APIError as e: @@ -617,22 +617,23 @@ def stats(limit, facets, filename, query): @main.command() -@click.option('--color/--no-color', default=True) +@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') @click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) -@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) -@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) -@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) -@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) -@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) +@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) +@click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str) +@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) -@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) @click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns): +@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) +@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) +@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) +@click.option('--color/--no-color', default=True) +@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) +def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -662,9 +663,11 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('tags') if vulns: stream_type.append('vulns') + if custom_filters: + stream_type.append('custom_filters') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -685,13 +688,16 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if countries: stream_args = countries.split(',') - + if tags: stream_args = tags.split(',') - + if vulns: stream_args = vulns.split(',') + if custom_filters: + stream_args = custom_filters + # Flatten the list of stream types # Possible values are: # - all @@ -710,6 +716,7 @@ def _create_stream(name, args, timeout): 'alert': api.stream.alert(args, timeout=timeout), 'asn': api.stream.asn(args, timeout=timeout), 'countries': api.stream.countries(args, timeout=timeout), + 'custom_filters': api.stream.custom(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), 'tags': api.stream.tags(args, timeout=timeout), 'vulns': api.stream.vulns(args, timeout=timeout), diff --git a/shodan/stream.py b/shodan/stream.py index 15d7619..77ade7d 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -13,7 +13,7 @@ def __init__(self, api_key, proxies=None): self.api_key = api_key self.proxies = proxies - def _create_stream(self, name, timeout=None): + def _create_stream(self, name, query=None, timeout=None): params = { 'key': self.api_key, } @@ -27,9 +27,12 @@ def _create_stream(self, name, timeout=None): # If the user requested a timeout then we need to disable heartbeat messages # which are intended to keep stream connections alive even if there isn't any data # flowing through. - if timeout: + if timeout is not None: params['heartbeat'] = False + if query is not None: + params['query'] = query + try: while True: req = requests.get(stream_url, params=params, stream=True, timeout=timeout, @@ -113,6 +116,18 @@ def countries(self, countries, raw=False, timeout=None): for line in self._iter_stream(stream, raw): yield line + def custom(self, query, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the query of interest. The query + can vary and mix-match with different arguments (ports, tags, vulns, etc). + + :param query: A space-separated list of key:value filters query to return banner data on. + :type query: string + """ + stream = self._create_stream('/shodan/custom', query=query, timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + def ports(self, ports, raw=False, timeout=None): """ A filtered version of the "banners" stream to only return banners that match the ports of interest. From 490fdb5e592a2be243856c21bf158a2eb01dfe48 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 31 Dec 2021 16:33:06 -0600 Subject: [PATCH 123/148] Revert timeout parameter to previous behavior --- shodan/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/stream.py b/shodan/stream.py index 77ade7d..eac345f 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -27,7 +27,7 @@ def _create_stream(self, name, query=None, timeout=None): # If the user requested a timeout then we need to disable heartbeat messages # which are intended to keep stream connections alive even if there isn't any data # flowing through. - if timeout is not None: + if timeout: params['heartbeat'] = False if query is not None: From e603d462abee94f7e0a3c6780095f0f4a881af70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=E1=BA=BF=20V=C5=A9=20Gia=20Hy?= Date: Thu, 6 Jan 2022 21:53:12 +0700 Subject: [PATCH 124/148] stream: Remove set `decode_unicode` This doesn't handle well with every unicode strings, let only json.loads() do the decode instead. --- shodan/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/stream.py b/shodan/stream.py index eac345f..a47d143 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -63,7 +63,7 @@ def _create_stream(self, name, query=None, timeout=None): return req def _iter_stream(self, stream, raw): - for line in stream.iter_lines(decode_unicode=True): + for line in stream.iter_lines(): # The Streaming API sends out heartbeat messages that are newlines # We want to ignore those messages since they don't contain any data if line: From 95c6b910f7a46c3efecdab1d65085190d6404e89 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 19 Jan 2022 09:47:24 -0600 Subject: [PATCH 125/148] Release 1.26.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7751d7..97c47b4 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.26.0', + version='1.26.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 99756e2b2bd64603013be3f11caa6d406250f2a1 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 22 Feb 2022 13:53:59 -0600 Subject: [PATCH 126/148] New commands: "shodan alert export" and "shodan alert import" to help backup/ restore network monitoring configurations --- shodan/cli/alert.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 63b863d..2dc3e58 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,5 +1,7 @@ import click import csv +import gzip +import json import shodan from collections import defaultdict @@ -221,6 +223,87 @@ def batch(iterable, size=1): click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') +@alert.command(name='export') +@click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str) +def alert_export(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + alerts = api.alerts() + + # Create the output file + click.echo('Writing alerts to file: {}'.format(filename)) + with gzip.open(filename, 'wt', encoding='utf-8') as fout: + json.dump(alerts, fout) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully exported monitored networks', fg='green') + + +@alert.command(name='import') +@click.argument('filename', metavar='') +def alert_import(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + # A mapping of the old notifier IDs to the new ones + notifier_map = {} + + try: + # Loading the alerts + click.echo('Loading alerts from: {}'.format(filename)) + with gzip.open(filename, 'rt', encoding='utf-8') as fin: + alerts = json.load(fin) + + for item in alerts: + # Create the alert + click.echo('Creating: {}'.format(item['name'])) + alert = api.create_alert(item['name'], item['filters']['ip']) + + # Enable any triggers + if item.get('triggers', {}): + triggers = ','.join(item['triggers'].keys()) + + api.enable_alert_trigger(alert['id'], triggers) + + # Add any whitelisted services for this trigger + for trigger, info in item['triggers'].items(): + if info.get('ignore', []): + for whitelist in info['ignore']: + api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port']) + + # Enable the notifiers + for prev_notifier in item.get('notifiers', []): + # We don't need to do anything for the default notifier as that + # uses the account's email address automatically. + if prev_notifier['id'] == 'default': + continue + + # Get the new notifier based on the ID of the old one + notifier = notifier_map.get(prev_notifier['id']) + + # Create the notifier if it doesn't yet exist + if notifier is None: + notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description']) + + # Add it to our map of old notifier IDs to new notifiers + notifier_map[prev_notifier['id']] = notifier + + api.add_alert_notifier(alert['id'], notifier['id']) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully imported monitored networks', fg='green') + + @alert.command(name='info') @click.argument('alert', metavar='') def alert_info(alert): From b32a7bc3f45f62c7ce962cea5954dd8b82fa6824 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Tue, 22 Feb 2022 17:01:05 -0600 Subject: [PATCH 127/148] Release 1.27.0 --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2247045..2cc021c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ CHANGELOG ========= +1.27.0 +------ +* New command: ``shodan alert export`` to save the current network monitoring configuration +* New command: ``shodan alert import`` to restore a previous network monitoring configuration +* Automatically rate limit API requests to 1 request per second (credit to @malvidin) + +1.26.1 +------ +* Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON + +1.26.0 +------ +* Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters ``) + 1.25.0 ------ * Add new CLI command: shodan alert download diff --git a/setup.py b/setup.py index 97c47b4..942c54f 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.26.1', + version='1.27.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 53662377e23c85926feada57b103e1a21d9ae2ee Mon Sep 17 00:00:00 2001 From: yaron-cider <100203813+yaron-cider@users.noreply.github.com> Date: Thu, 19 May 2022 12:39:16 +0300 Subject: [PATCH 128/148] Better error handling Raising 502 bad gateway error when receiving http status --- shodan/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shodan/client.py b/shodan/client.py index db50b4c..f4aebc0 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -336,6 +336,8 @@ def _request(self, function, params, service='shodan', method='get'): raise APIError(error) elif data.status_code == 403: raise APIError('Access denied (403 Forbidden)') + elif data.status_code == 502: + raise APIError('Bad Gateway (502)') # Parse the text into JSON try: From de2fd90aa9ffc8b53a6cd3f175fea50e551af0b1 Mon Sep 17 00:00:00 2001 From: Li-Heng Yu <007seadog@gmail.com> Date: Mon, 23 May 2022 14:14:40 +0800 Subject: [PATCH 129/148] Show scan id when scanning without showing results --- shodan/cli/scan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 7ec158c..cfc7aab 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -157,6 +157,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): # Return immediately if wait <= 0: + click.echo('Scan ID: {}'.format(scan['id'])) click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') else: # Setup an alert to wait for responses From 7d043d74735cfaf0b0b5dc8fbc81922ba117dfea Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 9 Jul 2022 17:52:51 -0500 Subject: [PATCH 130/148] Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- shodan/client.py | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc021c..8d9f9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ CHANGELOG ========= +1.28.0 +------ +* Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port +* Show scan ID when scanning without showing results (credit to @seadog007) +* Handle bad gateway errors (credit to @yaron-cider) + + 1.27.0 ------ * New command: ``shodan alert export`` to save the current network monitoring configuration diff --git a/setup.py b/setup.py index 942c54f..266ce81 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.27.0', + version='1.28.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index f4aebc0..70ca8f3 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -721,8 +721,14 @@ def disable_alert_trigger(self, aid, trigger): """Disable the given trigger on the alert.""" return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') - def ignore_alert_trigger_notification(self, aid, trigger, ip, port): + def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): """Ignore trigger notifications for the provided IP and port.""" + # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities + # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use + # a different API endpoint. + if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') def unignore_alert_trigger_notification(self, aid, trigger, ip, port): From 55d8d59bc3c82e3f35d1afbe85fe86b5c0cc75ca Mon Sep 17 00:00:00 2001 From: David xu Date: Sun, 17 Jul 2022 22:12:35 +0800 Subject: [PATCH 131/148] add missing return --- shodan/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/helpers.py b/shodan/helpers.py index 378b1bb..8900468 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -164,7 +164,7 @@ def humanize_bytes(byte_count, precision=1): if byte_count == 1: return '1 byte' if byte_count < 1024: - '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') + return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] multiple = 1024.0 # .0 to force float on python 2 From 4419d7167fad366b8942643037c427a62b0cb5bb Mon Sep 17 00:00:00 2001 From: John Matherly Date: Mon, 15 May 2023 09:50:54 -0700 Subject: [PATCH 132/148] Add support for the new 'fields' parameter of the /shodan/host/search method so we only grab the specific properties/ fields from the banner. --- setup.py | 2 +- shodan/__main__.py | 11 ++++++++--- shodan/client.py | 11 ++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 266ce81..e769440 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.28.0', + version='1.29.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/__main__.py b/shodan/__main__.py index 07af59c..f11a72a 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -255,10 +255,11 @@ def count(query): @main.command() +@click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str) @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, filename, query): +def download(fields, limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -276,6 +277,10 @@ def download(limit, filename, query): # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' + + # Strip out any whitespace in the fields and turn them into an array + if fields is not None: + fields = [item.strip() for item in fields.split(',')] # Perform the search api = shodan.Shodan(key) @@ -302,7 +307,7 @@ def download(limit, filename, query): with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False) + cursor = api.search_cursor(query, minify=False, fields=fields) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) @@ -485,7 +490,7 @@ def search(color, fields, limit, separator, query): # Perform the search api = shodan.Shodan(key) try: - results = api.search(query, limit=limit) + results = api.search(query, limit=limit, minify=False, fields=fields) except shodan.APIError as e: raise click.ClickException(e.value) diff --git a/shodan/client.py b/shodan/client.py index 70ca8f3..b9cb487 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -480,7 +480,7 @@ def scan_status(self, scan_id): """ return self._request('/shodan/scan/{}'.format(scan_id), {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -495,6 +495,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru :type facets: str :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool + :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter + :type fields: str :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. """ @@ -511,10 +513,13 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) + + if fields and isinstance(fields, list): + args['fields'] = ','.join(fields) return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, fields=None): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -542,7 +547,7 @@ def search_cursor(self, query, minify=True, retries=5): while results['matches']: try: - results = self.search(query, minify=minify, page=page) + results = self.search(query, minify=minify, page=page, fields=fields) for banner in results['matches']: try: yield banner From 9ccc16ada4761d19ee2079a9334cac3ddbc62415 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 17 May 2023 11:00:46 -0700 Subject: [PATCH 133/148] The screenshot data has been moved to the top-level "screenshot" property. Update the helpers.get_screenshot() method to look in that location before falling back to the old opts.screenshot property. --- setup.py | 2 +- shodan/helpers.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e769440..3adcd8b 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.29.0', + version='1.29.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/helpers.py b/shodan/helpers.py index 378b1bb..432cbeb 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -122,8 +122,11 @@ def iterate_files(files, fast=False): def get_screenshot(banner): - if 'opts' in banner and 'screenshot' in banner['opts']: + if 'screenshot' in banner and banner['screenshot']: + return banner['screenshot'] + elif 'opts' in banner and 'screenshot' in banner['opts']: return banner['opts']['screenshot'] + return None From c97ebae69d3343da455bc5aa277c54f7cf3e115a Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sun, 18 Jun 2023 21:08:39 -0400 Subject: [PATCH 134/148] Added a check for file size of input, dynamically set workbook.use_zip64() --- shodan/__main__.py | 7 ++++++- shodan/cli/converter/csvc.py | 2 +- shodan/cli/converter/excel.py | 6 +++++- shodan/cli/converter/geojson.py | 2 +- shodan/cli/converter/images.py | 2 +- shodan/cli/converter/kml.py | 2 +- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f11a72a..aadb2d5 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -100,6 +100,7 @@ def convert(fields, input, format): Example: shodan convert data.json.gz kml """ + file_size = 0 # Check that the converter allows a custom list of fields converter_class = CONVERTERS.get(format) if fields: @@ -107,6 +108,10 @@ def convert(fields, input, format): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified + # Check file size of input + if os.path.exists(input): + file_size = os.path.getsize(input) + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -124,7 +129,7 @@ def convert(fields, input, format): # Initialize the file converter converter = converter_class(fout) - converter.process([input]) + converter.process([input], file_size) finished_event.set() progress_bar_thread.join() diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index 20c3d03..2e4e2f2 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -50,7 +50,7 @@ class CsvConverter(Converter): 'title', ] - def process(self, files): + def process(self, files, file_size): writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') # Write the header diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index 24177eb..4db78a4 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -41,7 +41,7 @@ class ExcelConverter(Converter): 'http.title': 'Website Title', } - def process(self, files): + def process(self, files, file_size): # Get the filename from the already-open file handle filename = self.fout.name @@ -51,6 +51,10 @@ def process(self, files): # Create the new workbook workbook = Workbook(filename) + # Check if Excel file is larger than 5GB + if file_size > 5e9: + workbook.use_zip64() + # Define some common styles/ formats bold = workbook.add_format({ 'bold': 1, diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 3f6c975..83fb935 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -14,7 +14,7 @@ def header(self): def footer(self): self.fout.write("""{ }]}""") - def process(self, files): + def process(self, files, file_size): # Write the header self.header() diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py index 24c68c3..fba9d11 100644 --- a/shodan/cli/converter/images.py +++ b/shodan/cli/converter/images.py @@ -15,7 +15,7 @@ class ImagesConverter(Converter): # the user know where the images have been stored. dirname = None - def process(self, files): + def process(self, files, file_size): # Get the filename from the already-open file handle and use it as # the directory name to store the images. self.dirname = self.fout.name[:-7] + '-images' diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index 2cf3d44..9259ddf 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -13,7 +13,7 @@ def header(self): def footer(self): self.fout.write("""""") - def process(self, files): + def process(self, files, file_size): # Write the header self.header() From c3d6d63d4743fd71642b084ff8b2da56f37717eb Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:09:18 +0700 Subject: [PATCH 135/148] Add Shodan Trends API/ CLI --- shodan/__main__.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++ shodan/client.py | 57 ++++++++++++++++++++++++++----- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index f11a72a..3a2fdf8 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -22,6 +22,7 @@ search stats stream + trends """ @@ -35,6 +36,7 @@ import threading import requests import time +import json # The file converters that are used to go from .json.gz to various other formats from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter @@ -799,6 +801,87 @@ def _create_stream(name, args, timeout): stream = _create_stream(stream_type, stream_args, timeout=timeout) +@main.command() +@click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) +@click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) +@click.option('--separator', help='The separator between the properties of the search results.', default='\t') +@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) +@click.argument('query', metavar='', nargs=-1) +def trends(filename, save, separator, facets, query): + """Search Shodan historical database""" + key = get_api_key() + api = shodan.Shodan(key) + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + facets = facets.strip() + + # Make sure the user didn't supply an empty query or facets + if query == '': + raise click.ClickException('Empty search query') + + if facets == '': + raise click.ClickException('Empty search facets') + + # Convert comma-separated facets string to list + parsed_facets = [] + for facet in facets.split(','): + parts = facet.strip().split(":") + if len(parts) > 1: + parsed_facets.append((parts[0], parts[1])) + else: + parsed_facets.append((parts[0])) + + # Perform the search + try: + results = api.trends.search(query, facets=parsed_facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') + + result_facets = list(results['facets'].keys()) + + # Save the results first to file if user request + if filename or save: + if not filename: + filename = '{}-trends.json.gz'.format(query.replace(' ', '-')) + elif not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Create/ append to the file + with helpers.open_file(filename) as fout: + for index, match in enumerate(results['matches']): + # Append facet info to make up a line + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) + + click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) + + # We buffer the entire output so we can use click's pager functionality + output = u'' + + # Output example: + # 2017-06 + # os + # Linux 3.x 384148 + # Windows 7 or 8 25531 + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + + for facet in result_facets: + output += ' ' + facet + u'\n' + for bucket in results['facets'][facet][index]['values']: + output += ' ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n' + + click.echo_via_pager(output) + + @main.command() @click.argument('ip', metavar='') def honeyscore(ip): diff --git a/shodan/client.py b/shodan/client.py index b9cb487..7b76fdc 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -85,7 +85,7 @@ class Notifier: def __init__(self, parent): self.parent = parent - + def create(self, provider, args, description=None): """Get the settings for the specified notifier that a user has configured. @@ -101,9 +101,9 @@ def create(self, provider, args, description=None): if description: args['description'] = description - + return self.parent._request('/notifier', args, method='post') - + def edit(self, nid, args): """Get the settings for the specified notifier that a user has configured. @@ -114,7 +114,7 @@ def edit(self, nid, args): :returns: dict -- fields are 'success' and 'id' of the notifier """ return self.parent._request('/notifier/{}'.format(nid), args, method='put') - + def get(self, nid): """Get the settings for the specified notifier that a user has configured. @@ -137,7 +137,7 @@ def list_providers(self): :returns: A list of providers where each object describes a provider """ return self.parent._request('/notifier/provider', {}) - + def remove(self, nid): """Delete the provided notifier. @@ -253,6 +253,42 @@ def remove_member(self, user): """ return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + class Trends: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, facets): + """Search the Shodan historical database. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'facets': create_facet_string(facets), + } + + return self.parent._request('/api/v1/search', args, service='trends') + + def search_facets(self): + """This method returns a list of facets that can be used to get a breakdown of the top values for a property. + + :returns: A list of strings where each is a facet name + """ + return self.parent._request('/api/v1/search/facets', {}, service='trends') + + def search_filters(self): + """This method returns a list of search filters that can be used in the search query. + + :returns: A list of strings where each is a filter name + """ + return self.parent._request('/api/v1/search/filters', {}, service='trends') + def __init__(self, key, proxies=None): """Initializes the API object. @@ -264,9 +300,11 @@ def __init__(self, key, proxies=None): self.api_key = key self.base_url = 'https://api.shodan.io' self.base_exploits_url = 'https://exploits.shodan.io' + self.base_trends_url = 'https://trends.shodan.io' self.data = self.Data(self) self.dns = self.Dns(self) self.exploits = self.Exploits(self) + self.trends = self.Trends(self) self.labs = self.Labs(self) self.notifier = self.Notifier(self) self.org = self.Organization(self) @@ -297,6 +335,7 @@ def _request(self, function, params, service='shodan', method='get'): base_url = { 'shodan': self.base_url, 'exploits': self.base_exploits_url, + 'trends': self.base_trends_url, }.get(service, 'shodan') # Wait for API rate limit @@ -513,7 +552,7 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) - + if fields and isinstance(fields, list): args['fields'] = ','.join(fields) @@ -733,17 +772,17 @@ def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): # a different API endpoint. if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') - + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') def unignore_alert_trigger_notification(self, aid, trigger, ip, port): """Re-enable trigger notifications for the provided IP and port""" return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') - + def add_alert_notifier(self, aid, nid): """Enable the given notifier for an alert that has triggers enabled.""" return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') - + def remove_alert_notifier(self, aid, nid): """Remove the given notifier for an alert that has triggers enabled.""" return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') From 9cc33f5700af88067391f3bd64fbb65001a666ee Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:10:47 +0700 Subject: [PATCH 136/148] Add API tests --- tests/test_shodan.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_shodan.py b/tests/test_shodan.py index f3405ce..ebe7a90 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -22,7 +22,8 @@ class ShodanTests(unittest.TestCase): } def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) + with open('SHODAN-API-KEY') as f: + self.api = shodan.Shodan(f.read().strip()) def test_search_simple(self): results = self.api.search(self.QUERIES['simple']) @@ -115,6 +116,24 @@ def test_exploits_count_facets(self): self.assertTrue(results['facets']['source']) self.assertTrue(len(results['facets']['author']) == 1) + def test_trends_search(self): + results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('matches', results) + self.assertIn('facets', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + + def test_trends_search_filters(self): + results = self.api.trends.search_filters() + self.assertIn('has_ipv6', results) + self.assertNotIn('http.html', results) + + def test_trends_search_facets(self): + results = self.api.trends.search_facets() + self.assertIn('product', results) + self.assertNotIn('cpe', results) + # Test error responses def test_invalid_key(self): api = shodan.Shodan('garbage') From 3fe0ad88b434fe480b7621f8fac20216c3578fea Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 23 Jun 2023 16:11:34 +0700 Subject: [PATCH 137/148] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3adcd8b..2546ea7 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.29.1', + version='1.30.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', From 0ad61d20d4cf676269c798e573ded4439603b218 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sun, 25 Jun 2023 16:46:00 -0700 Subject: [PATCH 138/148] Change cutoff to 4GB as that's what the xlsxwriter documentation says --- shodan/cli/converter/excel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index 4db78a4..2021a33 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -51,8 +51,8 @@ def process(self, files, file_size): # Create the new workbook workbook = Workbook(filename) - # Check if Excel file is larger than 5GB - if file_size > 5e9: + # Check if Excel file is larger than 4GB + if file_size > 4e9: workbook.use_zip64() # Define some common styles/ formats From 3d1f8922cd42420eba459a5d8bd41d974ad55ec1 Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Mon, 3 Jul 2023 10:57:56 +0700 Subject: [PATCH 139/148] Better output format --- shodan/__main__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 3a2fdf8..439752d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -279,7 +279,7 @@ def download(fields, limit, filename, query): # Add the appropriate extension if it's not there atm if not filename.endswith('.json.gz'): filename += '.json.gz' - + # Strip out any whitespace in the fields and turn them into an array if fields is not None: fields = [item.strip() for item in fields.split(',')] @@ -804,10 +804,9 @@ def _create_stream(name, args, timeout): @main.command() @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') @click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) @click.argument('query', metavar='', nargs=-1) -def trends(filename, save, separator, facets, query): +def trends(filename, save, facets, query): """Search Shodan historical database""" key = get_api_key() api = shodan.Shodan(key) @@ -869,15 +868,17 @@ def trends(filename, save, separator, facets, query): # Output example: # 2017-06 # os - # Linux 3.x 384148 - # Windows 7 or 8 25531 + # Linux 3.x 146,502 + # Windows 7 or 8 2,189 for index, match in enumerate(results['matches']): output += click.style(match['month'] + u'\n', fg='green') - - for facet in result_facets: - output += ' ' + facet + u'\n' - for bucket in results['facets'][facet][index]['values']: - output += ' ' + str(bucket['value']) + separator + str(bucket['count']) + u'\n' + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) click.echo_via_pager(output) From 9e0f4dddbc5c8737270b70c119674abe618f44b5 Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Fri, 7 Jul 2023 08:34:16 +0700 Subject: [PATCH 140/148] Make facets as required arguments --- shodan/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 3a81b0d..1c4a74d 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -809,7 +809,7 @@ def _create_stream(name, args, timeout): @main.command() @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.option('--facets', help='List of facets to get summary information on.', required=True, type=str) +@click.argument('facets', metavar='') @click.argument('query', metavar='', nargs=-1) def trends(filename, save, facets, query): """Search Shodan historical database""" From a9d692b05aeea978632d17601cfec997bc8995cd Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 11 Jul 2023 23:07:34 -0400 Subject: [PATCH 141/148] Updated implementation to monitor hostnames for domain-based monitoring. --- requirements.txt | 3 ++- shodan/cli/alert.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5095f64..2692414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ click-plugins colorama requests>=2.2.1 XlsxWriter -ipaddress;python_version<='2.7' \ No newline at end of file +ipaddress;python_version<='2.7' +tldextract \ No newline at end of file diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 2dc3e58..0030589 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -3,6 +3,7 @@ import gzip import json import shodan +from tldextract import extract from collections import defaultdict from operator import itemgetter @@ -125,9 +126,15 @@ def alert_domain(domain, triggers): try: # Grab a list of IPs for the domain domain = domain.lower() + domain_parse = extract(domain) click.secho('Looking up domain information...', dim=True) info = api.dns.domain_info(domain, type='A') - domain_ips = set([record['value'] for record in info['data']]) + + if domain_parse.subdomain: + domain_ips = set([record['value'] for record in info['data'] + if record['subdomain'] == domain_parse.subdomain]) + else: + domain_ips = set([record['value'] for record in info['data']]) # Create the actual alert click.secho('Creating alert...', dim=True) From c90b3dd5ade683e16c7681c444790f448150a401 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Wed, 12 Jul 2023 22:35:36 -0400 Subject: [PATCH 142/148] Added input validation by updating click.argument for input parameter. --- shodan/__main__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 1c4a74d..8b4eb2b 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -93,7 +93,7 @@ def main(): @main.command() @click.option('--fields', help='List of properties to output.', default=None) -@click.argument('input', metavar='') +@click.argument('input', metavar='', type=click.Path(exists=True)) @click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) def convert(fields, input, format): """Convert the given input data file into a different format. The following file formats are supported: @@ -110,9 +110,8 @@ def convert(fields, input, format): raise click.ClickException('File format doesnt support custom list of fields') converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified - # Check file size of input - if os.path.exists(input): - file_size = os.path.getsize(input) + # click.Path ensures that file path exists + file_size = os.path.getsize(input) # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') From 1b7cb65e7e3d4b0490aa1009d5eaac72257a7e60 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Thu, 13 Jul 2023 20:07:17 -0400 Subject: [PATCH 143/148] Updated CLI logic to filter out private IPs when creating a domain-based alert. --- shodan/cli/alert.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 0030589..6f82c1a 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -4,6 +4,7 @@ import json import shodan from tldextract import extract +from ipaddress import ip_address from collections import defaultdict from operator import itemgetter @@ -120,6 +121,7 @@ def alert_create(name, netblocks): @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') def alert_domain(domain, triggers): """Create a network alert based on a domain name""" + flag = True key = get_api_key() api = shodan.Shodan(key) @@ -132,22 +134,30 @@ def alert_domain(domain, triggers): if domain_parse.subdomain: domain_ips = set([record['value'] for record in info['data'] - if record['subdomain'] == domain_parse.subdomain]) + if record['subdomain'] == domain_parse.subdomain and + not ip_address(record['value']).is_private]) else: - domain_ips = set([record['value'] for record in info['data']]) + domain_ips = set([record['value'] for record in info['data'] + if not ip_address(record['value']).is_private]) - # Create the actual alert - click.secho('Creating alert...', dim=True) - alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + if not domain_ips: + flag = False + click.secho('No external IPs were found to be associated with this domain. ' + 'No alert was created.', dim=True) + else: + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) - # Enable the triggers so it starts getting managed by Shodan Monitor - click.secho('Enabling triggers...', dim=True) - api.enable_alert_trigger(alert['id'], triggers) + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) except shodan.APIError as e: raise click.ClickException(e.value) - click.secho('Successfully created domain alert!', fg='green') - click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + if flag: + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') @alert.command(name='download') From 3036d83a9de0efb35202d094521f1bfd152d14b2 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Fri, 14 Jul 2023 16:31:08 -0400 Subject: [PATCH 144/148] Updating implementation based on reviewer recommendations. --- shodan/cli/alert.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index 6f82c1a..1df11ea 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -121,7 +121,6 @@ def alert_create(name, netblocks): @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') def alert_domain(domain, triggers): """Create a network alert based on a domain name""" - flag = True key = get_api_key() api = shodan.Shodan(key) @@ -141,23 +140,21 @@ def alert_domain(domain, triggers): if not ip_address(record['value']).is_private]) if not domain_ips: - flag = False - click.secho('No external IPs were found to be associated with this domain. ' - 'No alert was created.', dim=True) - else: - # Create the actual alert - click.secho('Creating alert...', dim=True) - alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + raise click.ClickException('No external IPs were found to be associated with this domain. ' + 'No alert was created.') + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) - # Enable the triggers so it starts getting managed by Shodan Monitor - click.secho('Enabling triggers...', dim=True) - api.enable_alert_trigger(alert['id'], triggers) + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) except shodan.APIError as e: raise click.ClickException(e.value) - if flag: - click.secho('Successfully created domain alert!', fg='green') - click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') @alert.command(name='download') From 759a8561f821bf2e970f0594200221271bcbb7a9 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Wed, 19 Jul 2023 15:55:20 -0700 Subject: [PATCH 145/148] Update "shodan host" output to show certificate issuer/ subject and HTTP title --- shodan/cli/host.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shodan/cli/host.py b/shodan/cli/host.py index e90e372..8ffdeed 100644 --- a/shodan/cli/host.py +++ b/shodan/cli/host.py @@ -91,8 +91,20 @@ def host_print_pretty(host, history=False): click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) click.echo('') + # Show optional HTTP information + if 'http' in banner: + if 'title' in banner['http'] and banner['http']['title']: + click.echo('\t|-- HTTP title: {}'.format(banner['http']['title'])) + # Show optional ssl info if 'ssl' in banner: + if 'cert' in banner['ssl'] and banner['ssl']['cert']: + if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']: + issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()]) + click.echo('\t|-- Cert Issuer: {}'.format(issuer)) + if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']: + subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()]) + click.echo('\t|-- Cert Subject: {}'.format(subject)) if 'versions' in banner['ssl'] and banner['ssl']['versions']: click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: From 769e38448414c97eb7a34ecd3e88f9e0e280b2ee Mon Sep 17 00:00:00 2001 From: Thong Nguyen Date: Thu, 20 Jul 2023 11:50:51 +0700 Subject: [PATCH 146/148] Make Trends API facets param as optional, if not supply then show total query results over time --- shodan/__main__.py | 54 ++++++++++++++++++++++++++++---------------- tests/test_shodan.py | 9 +++++++- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/shodan/__main__.py b/shodan/__main__.py index 8b4eb2b..4093b94 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -806,9 +806,9 @@ def _create_stream(name, args, timeout): @main.command() +@click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str) @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) -@click.argument('facets', metavar='') @click.argument('query', metavar='', nargs=-1) def trends(filename, save, facets, query): """Search Shodan historical database""" @@ -823,12 +823,12 @@ def trends(filename, save, facets, query): if query == '': raise click.ClickException('Empty search query') - if facets == '': - raise click.ClickException('Empty search facets') - # Convert comma-separated facets string to list parsed_facets = [] for facet in facets.split(','): + if not facet: + continue + parts = facet.strip().split(":") if len(parts) > 1: parsed_facets.append((parts[0], parts[1])) @@ -845,7 +845,9 @@ def trends(filename, save, facets, query): if results['total'] == 0: raise click.ClickException('No search results found') - result_facets = list(results['facets'].keys()) + result_facets = [] + if results.get("facets"): + result_facets = list(results["facets"].keys()) # Save the results first to file if user request if filename or save: @@ -858,31 +860,43 @@ def trends(filename, save, facets, query): with helpers.open_file(filename) as fout: for index, match in enumerate(results['matches']): # Append facet info to make up a line - match["facets"] = {} - for facet in result_facets: - match["facets"][facet] = results['facets'][facet][index]['values'] - line = json.dumps(match) + '\n' - fout.write(line.encode('utf-8')) + if result_facets: + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) # We buffer the entire output so we can use click's pager functionality output = u'' - # Output example: + # Output examples: + # - Facet by os # 2017-06 # os # Linux 3.x 146,502 # Windows 7 or 8 2,189 - for index, match in enumerate(results['matches']): - output += click.style(match['month'] + u'\n', fg='green') - if match['count'] > 0: - for facet in result_facets: - output += click.style(u' {}\n'.format(facet), fg='cyan') - for bucket in results['facets'][facet][index]['values']: - output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) - else: - output += u'{}\n'.format(click.style('N/A', bold=True)) + # + # - Without facets + # 2017-06 19,799,459 + # 2017-07 21,077,099 + if result_facets: + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) + else: + # Without facets, show query total results over time + for index, match in enumerate(results['matches']): + output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green')) click.echo_via_pager(output) diff --git a/tests/test_shodan.py b/tests/test_shodan.py index ebe7a90..94ffc70 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -118,12 +118,19 @@ def test_exploits_count_facets(self): def test_trends_search(self): results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('total', results) self.assertIn('matches', results) self.assertIn('facets', results) - self.assertIn('total', results) self.assertTrue(results['matches']) self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + results = self.api.trends.search('apache', facets=[]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertNotIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [match['month'] for match in results['matches']]) + def test_trends_search_filters(self): results = self.api.trends.search_filters() self.assertIn('has_ipv6', results) From a08353e40b2018ec0dbf04012e9a5bfbe87f55ec Mon Sep 17 00:00:00 2001 From: John Matherly Date: Fri, 13 Oct 2023 18:15:18 -0700 Subject: [PATCH 147/148] Improved handling of downloads to prevent it from exiting prematurely if the API only returns partial or missing results for a search results page. --- setup.py | 2 +- shodan/client.py | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 2546ea7..f421898 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.30.0', + version='1.30.1', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index 7b76fdc..ab81302 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -7,6 +7,7 @@ :copyright: (c) 2014- by John Matherly """ +import math import time import requests @@ -576,15 +577,23 @@ def search_cursor(self, query, minify=True, retries=5, fields=None): :returns: A search cursor that can be used as an iterator/ generator. """ page = 1 + total_pages = 0 tries = 0 - # Placeholder results object to make the while loop below easier - results = { - 'matches': [True], - 'total': None, - } + # Grab the initial page and use the total to calculate the expected number of pages + results = self.search(query, minify=minify, page=page, fields=fields) + if results['total']: + total_pages = int(math.ceil(results['total'] / 100)) + + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 - while results['matches']: + # Keep iterating over the results from page 2 onwards + while page <= total_pages: try: results = self.search(query, minify=minify, page=page, fields=fields) for banner in results['matches']: From 87a0688d1e5b7e4bb13ae4f5fd7cb937a671cba8 Mon Sep 17 00:00:00 2001 From: John Matherly Date: Sat, 16 Dec 2023 17:29:15 -0800 Subject: [PATCH 148/148] Consolidate all Shodan methods to use the internal _request method instead of sometimes using the shodan.helpers.api_request method. New environment variable SHODAN_API_URL that can be used to overwrite the base_url used for the API requests. --- setup.py | 2 +- shodan/client.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index f421898..53bbd9a 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.30.1', + version='1.31.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff --git a/shodan/client.py b/shodan/client.py index ab81302..21c70af 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -8,13 +8,14 @@ :copyright: (c) 2014- by John Matherly """ import math +import os import time import requests import json from .exception import APIError -from .helpers import api_request, create_facet_string +from .helpers import create_facet_string from .stream import Stream @@ -314,11 +315,15 @@ def __init__(self, key, proxies=None): self._session = requests.Session() self.api_rate_limit = 1 # Requests per second self._api_query_time = None + if proxies: self._session.proxies.update(proxies) self._session.trust_env = False + + if os.environ.get('SHODAN_API_URL'): + self.base_url = os.environ.get('SHODAN_API_URL') - def _request(self, function, params, service='shodan', method='get'): + def _request(self, function, params, service='shodan', method='get', json_data=None): """General-purpose function to create web requests to SHODAN. Arguments: @@ -348,7 +353,13 @@ def _request(self, function, params, service='shodan', method='get'): try: method = method.lower() if method == 'post': - data = self._session.post(base_url + function, params) + if json_data: + data = self._session.post(base_url + function, params=params, + data=json.dumps(json_data), + headers={'content-type': 'application/json'}, + ) + else: + data = self._session.post(base_url + function, params) elif method == 'put': data = self._session.put(base_url + function, params=params) elif method == 'delete': @@ -711,8 +722,7 @@ def create_alert(self, name, ip, expires=0): 'expires': expires, } - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', - proxies=self._session.proxies) + response = self._request('/shodan/alert', params={}, json_data=data, method='post') return response @@ -732,8 +742,7 @@ def edit_alert(self, aid, ip): }, } - response = api_request(self.api_key, '/shodan/alert/{}'.format(aid), data=data, params={}, method='post', - proxies=self._session.proxies) + response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post') return response @@ -744,9 +753,9 @@ def alerts(self, aid=None, include_expired=True): else: func = '/shodan/alert/info' - response = api_request(self.api_key, func, params={ + response = self._request(func, params={ 'include_expired': include_expired, - }, proxies=self._session.proxies) + }) return response @@ -754,8 +763,7 @@ def delete_alert(self, aid): """Delete the alert with the given ID.""" func = '/shodan/alert/{}'.format(aid) - response = api_request(self.api_key, func, params={}, method='delete', - proxies=self._session.proxies) + response = self._request(func, params={}, method='delete') return response