diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b719ddb --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +build/* +*.tar.gz +*.json.gz +*.kml +*.egg +*.pyc +shodan.egg-info/* +tmp/* +MANIFEST +.vscode/ +PKG-INFO +venv/* +.idea/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8d9f9aa --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,205 @@ +CHANGELOG +========= + +1.28.0 +------ +* Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port +* Show scan ID when scanning without showing results (credit to @seadog007) +* Handle bad gateway errors (credit to @yaron-cider) + + +1.27.0 +------ +* New command: ``shodan alert export`` to save the current network monitoring configuration +* New command: ``shodan alert import`` to restore a previous network monitoring configuration +* Automatically rate limit API requests to 1 request per second (credit to @malvidin) + +1.26.1 +------ +* Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON + +1.26.0 +------ +* Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters ``) + +1.25.0 +------ +* Add new CLI command: shodan alert download + +1.24.0 +------ +* Add new CLI command: shodan alert stats + +1.23.0 +------ +* Add new CLI command: shodan alert domain + +1.22.1 +------ +* Fix bug when converting data file to CSV using Python3 + +1.22.0 +------ +* Add support for new vulnerability streaming endpoints + +1.21.3 +------ +* Fix geo.json file converter + +1.21.2 +------ +* Add support for paging through the domain information + +1.21.1 +------ +* Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command + +1.21.0 +------ +* New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. + +1.20.0 +------ +* New option "-S" for **shodan domain** to save results from the lookup +* New option "-D" for **shodan domain** to lookup open ports for IPs in the results + +1.19.0 +------ +* New method to edit the list of IPs for an existing network alert + +1.18.0 +------ +* Add library methods for the new Notifications API + +1.17.0 +------ +* Fix bug that caused unicode error when printing domain information (#106) +* Add flag to let users get their IPv6 address **shodan myip -6**(#35) + +1.16.0 +------ +* Ability to specify list of fields to include when converting to CSV/ Excel (#107) +* Filter the Shodan Firehose based on tags in the banner + +1.15.0 +------ +* New option "--skip" for download command to help users resume a download + +1.14.0 +------ +* New command **shodan version** (#104). +* Only change api_key file permissions if needed (#103) + +1.13.0 +------ +* New command **shodan domain** to lookup a domain in Shodan's DNS database +* Override environment configured settings if explicit proxy settings are supplied (@cudeso) + +1.12.1 +------ +* Fix Excel file conversion that resulted in empty .xlsx files + +1.12.0 +------ +* Add new methods to ignore/ unignore trigger notifications + +1.11.1 +------ +* Allow a single network alert to monitor multiple IP ranges (#93) + +1.11.0 +------ +* New command **shodan scan list** to list recently launched scans +* New command **shodan alert triggers** to list the available notification triggers +* New command **shodan alert enable** to enable a notification trigger +* New command **shodan alert disable** to disable a notification trigger +* New command **shodan alert info** to show details of a specific alert +* Include timestamp, vulns and tags in CSV converter (#85) +* Fixed bug that caused an exception when parsing uncompressed data files in Python3 +* Code quality improvements +* Thank you for contributions from @wagner-certat, @cclauss, @opt9, @voldmar and Antoine Neuenschwander + +1.10.4 +------ +* Fix a bug when showing old banner records that don't have the "transport" property +* Code quality improvements (bare excepts) + +1.10.3 +------ +* Change bare 'except:' statements to 'except Exception:' or more specific ones +* remove unused imports +* Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix +* List file types in **shodan convert** (#80) + +1.10.2 +------ +* Fix **shodan stats** formatting exception when faceting on **port** + +1.10.1 +------ +* Support PUT requests in the API request helper method + +1.10.0 +------ +* New command **shodan org**: manage enterprise access to Shodan for your team +* Improved unicode handling (#78) +* Remove deprecated API wrapper for shodanhq.com/api + +1.9.1 +----- +* The CHANGELOG is now part of the packages. +* Improved unicode handling in Python2 (#78) +* Add `tsv` output format for **shodan host** (#65) +* Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74) +* Improved exception handling to improve debugging **shodan init** (#77) + +1.9.0 +----- +* New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72) + +1.8.1 +----- +* Fixed bug that prevented **shodan scan submit** from finishing (#70) + +1.8.0 +----- +* Shodan CLI now installs properly on Windows (#66) +* Improved output of "shodan host" (#64, #67) +* Fixed bug that prevented an open port from being shown in "shodan host" (#63) +* No longer show an empty page if "shodan search" didn't return results (#62) +* Updated docs to make them Python3 compatible + +1.7.7 +----- +* Added "shodan data download" command to help download bulk data files + +1.7.6 +----- +* Add basic support for the Bulk Data API + +1.7.5 +----- + * Handle Cloudflare timeouts + +1.7.4 +----- + * Added "shodan radar" command + +1.7.3 +----- + * Fixed the bug #47 which was caused by the CLI using a timeout value of "0" which resulted in the "requests" library failing to connect + +1.7.2 +----- + * stream: automatically decode to unicode, fixes streaming on python3 (#45) + * Include docs in packages (#46) + * stream: handle timeout=None, None (default) can't be compared with integers (#44) + +1.7.1 +----- + * Python3 fixes for outputting images (#42) + * Add the ability to save results from host lookups via the CLI (#43) + +1.7.0 +----- + * Added "images" convert output format to let users extract images from Shodan data files (#42) diff --git a/LICENSE b/LICENSE index e344a97..0af97af 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014 John Matherly +Copyright (c) 2014- John Matherly Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/MANIFEST.in b/MANIFEST.in index 2088d21..4ba799c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,6 @@ include AUTHORS include LICENSE -include MANIFEST.in +include requirements.txt +include CHANGELOG.md +graft docs +recursive-include shodan *.py diff --git a/PKG-INFO b/PKG-INFO deleted file mode 100644 index 1fa284c..0000000 --- a/PKG-INFO +++ /dev/null @@ -1,15 +0,0 @@ -Metadata-Version: 1.1 -Name: shodan -Version: 0.7.0 -Summary: Python library for working with SHODAN -Home-page: http://github.com/achillean/shodan-python/ -Author: John Matherly -Author-email: jmath@shodanhq.com -License: UNKNOWN -Description: UNKNOWN -Platform: UNKNOWN -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Software Development :: Libraries :: Python Modules \ No newline at end of file diff --git a/README.rst b/README.rst index b53284f..14f6717 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,11 @@ -shodan: The official Python library for accessing Shodan -======================================================== +shodan: The official Python library and CLI for Shodan +====================================================== + +.. image:: https://img.shields.io/pypi/v/shodan.svg + :target: https://pypi.org/project/shodan/ + +.. image:: https://img.shields.io/github/contributors/achillean/shodan-python.svg + :target: https://github.com/achillean/shodan-python/graphs/contributors Shodan is a search engine for Internet-connected devices. Google lets you search for websites, Shodan lets you search for devices. This library provides developers easy access to all of the @@ -9,9 +15,43 @@ Features -------- - Search Shodan -- Streaming API support for real-time consumption of Shodan data +- `Fast/ bulk IP lookups `_ +- Streaming API support for real-time consumption of Shodan firehose +- `Network alerts (aka private firehose) `_ +- `Manage Email Notifications `_ - Exploit search API fully implemented +- Bulk data downloads +- Access the Shodan DNS DB to view domain information +- `Command-line interface `_ + +.. image:: https://cli.shodan.io/img/shodan-cli-preview.png + :target: https://asciinema.org/~Shodan + :width: 400px + :align: center + + +Quick Start +----------- + +.. code-block:: python + + from shodan import Shodan + + api = Shodan('MY API KEY') + + # Lookup an IP + ipinfo = api.host('8.8.8.8') + print(ipinfo) + + # Search for websites that have been "hacked" + for banner in api.search_cursor('http.title:"hacked by"'): + print(banner) + + # Get the total number of industrial control systems services on the Internet + ics_services = api.count('tag:ics') + print('Industrial Control Systems: {}'.format(ics_services['total'])) +Grab your API key from https://account.shodan.io Installation ------------ @@ -32,4 +72,4 @@ Or if you don't have pip installed (which you should seriously install): Documentation ------------- -Documentation is available at http://shodan.readthedocs.org/. +Documentation is available at https://shodan.readthedocs.org/ and https://help.shodan.io diff --git a/bin/shodan b/bin/shodan deleted file mode 100755 index d327699..0000000 --- a/bin/shodan +++ /dev/null @@ -1,464 +0,0 @@ -#!/usr/bin/env python -""" -Shodan CLI - -Note: Always run "shodan init " before trying to execute any other command! - -A simple interface to search Shodan, download data and parse compressed JSON files. -The following commands are currently supported: - - count - download - init - myip - parse - scan - search - -""" - -import click -import datetime -import gzip -import os -import os.path -import shodan -import simplejson -import time - -# Constants -SHODAN_CONFIG_DIR = '~/.shodan/' -ARRAY_SEPARATOR = ';' -COLORIZE_FIELDS = { - 'ip_str': 'green', - 'port': 'yellow', - 'data': 'white', - 'hostnames': 'magenta', - 'org': 'cyan', -} - - -# Utility methods -def get_api_key(): - shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) - keyfile = shodan_dir + '/api_key' - - # If the file doesn't yet exist let the user know that they need to - # initialize the shodan cli - if not os.path.exists(keyfile): - raise click.ClickException('Please run "shodan init " before using this command') - - # Make sure it is a read-only file - os.chmod(keyfile, 0600) - - with open(keyfile, 'r') as fin: - return fin.read().strip() - - raise click.ClickException('Please run "shodan init " before using this command') - - -def escape_data(args): - return args.encode('ascii', 'replace').replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') - -def timestr(): - return datetime.datetime.utcnow().strftime('%Y-%m-%d') - -def open_file(directory, timestr): - return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', 1) - - -@click.group() -def main(): - pass - - -@main.command() -@click.argument('key', metavar='') -def init(key): - """Initialize the Shodan command-line""" - # Create the directory if necessary - shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) - if not os.path.isdir(shodan_dir): - try: - os.mkdir(shodan_dir) - except OSError: - raise click.ClickException('Unable to create directory to store the Shodan API key (%s)' % shodan_dir) - - # Store the API key in the user's directory - keyfile = shodan_dir + '/api_key' - with open(keyfile, 'w') as fout: - fout.write(key.strip()) - click.echo(click.style('Successfully initialized', fg='green')) - - os.chmod(keyfile, 0600) - - -@main.command() -@click.argument('query', metavar='', nargs=-1) -def count(query): - """Returns the number of results for a search""" - key = get_api_key() - - # Create the query string out of the provided tuple - query = ' '.join(query).strip() - - # Make sure the user didn't supply an empty string - if query == '': - raise click.ClickException('Empty search query') - - # Perform the search - api = shodan.Shodan(key) - try: - results = api.count(query) - except shodan.APIError, e: - raise click.ClickException(e.value) - - click.echo(results['total']) - - -@main.command() -@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) -@click.argument('filename', metavar='') -@click.argument('query', metavar='', nargs=-1) -def download(limit, filename, query): - """Download search results and save them in a compressed JSON file.""" - key = get_api_key() - - # Create the query string out of the provided tuple - query = ' '.join(query).strip() - - # Make sure the user didn't supply an empty string - if query == '': - raise click.ClickException('Empty search query') - - filename = filename.strip() - if filename == '': - raise click.ClickException('Empty filename') - - # Add the appropriate extension if it's not there atm - if not filename.endswith('.json.gz'): - filename += '.json.gz' - - # Perform the search - api = shodan.Shodan(key) - - try: - total = api.count(query)['total'] - info = api.info() - except: - raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') - - # Print some summary information about the download request - click.echo('Search query:\t\t\t%s' % query) - click.echo('Total number of results:\t%s' % total) - click.echo('Query credits left:\t\t%s' % info['unlocked_left']) - click.echo('Output file:\t\t\t%s' % filename) - - if limit > total: - limit = total - - # A limit of -1 means that we should download all the data - if limit == -1: - limit = total - - with gzip.open(filename, 'w') as fout: - count = 0 - try: - cursor = api.search_cursor(query) - with click.progressbar(cursor, length=limit) as bar: - for banner in bar: - fout.write(simplejson.dumps(banner) + '\n') - count += 1 - - if count >= limit: - break - except: - pass - - # Let the user know we're done - if count < limit: - click.echo(click.style('Notice: fewer results were saved than requested', 'yellow')) - click.echo(click.style('Saved %s results into file %s' % (count, filename), 'green')) - - -@main.command() -@click.option('--color/--no-color', default=True) -@click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.argument('filename', metavar='', type=click.Path(exists=True)) -def parse(color, fields, separator, filename): - """Extract information out of compressed JSON files.""" - # Make sure it's some sort of json file - if not filename.endswith('.json.gz') and not filename.endswith('.json'): - raise click.ClickException('Invalid file, please make sure it is a valid Shodan JSON file') - - # Strip out any whitespace in the fields and turn them into an array - fields = [item.strip() for item in fields.split(',')] - - if len(fields) == 0: - raise click.ClickException('Please define at least one property to show') - - # Create a file handle depending on the filetype - if filename.endswith('.gz'): - fin = gzip.open(filename, 'r') - else: - fin = open(filename, 'r') - - for line in fin: - # Convert the JSON into a native Python object - banner = simplejson.loads(line) - row = '' - - # Loop over all the fields and print the banner as a row - for field in fields: - tmp = '' - if field in banner and banner[field]: - field_type = type(banner[field]) - - # If the field is an array then merge it together - if field_type == list: - tmp = ';'.join(banner[field]) - elif field_type in [int, float]: - tmp = str(banner[field]) - else: - tmp = escape_data(banner[field]) - - # Colorize certain fields if the user wants it - if color: - tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) - - # Add the field information to the row - row += tmp - row += separator - - click.echo(row) - - -@main.command() -def myip(): - """Print your external IP address""" - key = get_api_key() - - api = shodan.Shodan(key) - try: - click.echo(api.tools.myip()) - except shodan.APIError, e: - raise click.ClickException(e.value) - - -@main.command() -@click.argument('netblocks', metavar='', nargs=-1) -def scan(netblocks): - """Scan an IP/ netblock using Shodan.""" - key = get_api_key() - api = shodan.Shodan(key) - - # Submit the IPs for scanning - try: - results = api.scan(netblocks) - - click.echo(click.style('Success! ', fg='green') + '%s host(s) submitted (%s scan credits remaining)' % (results['count'], results['credits_left'])) - except shodan.APIError, e: - raise click.ClickException(e.value) - - -@main.command() -@click.option('--color/--no-color', default=True) -@click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data') -@click.option('--limit', help='The number of search results that should be returned. Maximum: 1000', default=100, type=int) -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.argument('query', metavar='', nargs=-1) -def search(color, fields, limit, separator, query): - """Search the Shodan database""" - key = get_api_key() - - # Create the query string out of the provided tuple - query = ' '.join(query).strip() - - # Make sure the user didn't supply an empty string - if query == '': - raise click.ClickException('Empty search query') - - # For now we only allow up to 1000 results at a time - if limit > 1000: - raise click.ClickException('Too many results requested, maximum is 1,000') - - # Strip out any whitespace in the fields and turn them into an array - fields = [item.strip() for item in fields.split(',')] - - if len(fields) == 0: - raise click.ClickException('Please define at least one property to show') - - # Perform the search - api = shodan.Shodan(key) - try: - results = api.search(query, limit=limit) - except shodan.APIError, e: - raise click.ClickException(e.value) - - # We buffer the entire output so we can use click's pager functionality - output = '' - for banner in results['matches']: - row = '' - - # Loop over all the fields and print the banner as a row - for field in fields: - tmp = '' - if field in banner and banner[field]: - field_type = type(banner[field]) - - # If the field is an array then merge it together - if field_type == list: - tmp = ';'.join(banner[field]) - elif field_type in [int, float]: - tmp = str(banner[field]) - else: - tmp = escape_data(banner[field]) - - # Colorize certain fields if the user wants it - if color: - tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) - - # Add the field information to the row - row += tmp - row += separator - - # click.echo(out + separator, nl=False) - output += row + '\n' - # click.echo('') - click.echo_via_pager(output) - - -@main.command() -@click.option('--limit', help='The number of results to return.', default=10, type=int) -@click.option('--facets', help='List of facets to get statistics for.', default='country,org') -@click.argument('query', metavar='', nargs=-1) -def stats(limit, facets, query): - # Setup Shodan - key = get_api_key() - api = shodan.Shodan(key) - - # Create the query string out of the provided tuple - query = ' '.join(query).strip() - - # Make sure the user didn't supply an empty string - if query == '': - raise click.ClickException('Empty search query') - - facets = facets.split(',') - facets = [(facet, limit) for facet in facets] - - # Perform the search - api = shodan.Shodan(key) - try: - results = api.count(query, facets=facets) - except shodan.APIError, e: - raise click.ClickException(e.value) - - # Print the stats tables - for facet in results['facets']: - print '# Top %s %s' % (limit, facet) - - for item in results['facets'][facet]: - print ' {:28s}'.format(item['value']), '{:12,d}'.format(item['count']) - - print '' - - -@main.command() -@click.option('--color/--no-color', default=True) -@click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') -@click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) -@click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) -@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) -@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) -def stream(color, fields, separator, limit, datadir, ports, quiet): - """Stream data in real-time.""" - # Setup the Shodan API - key = get_api_key() - api = shodan.Shodan(key) - - # Strip out any whitespace in the fields and turn them into an array - fields = [item.strip() for item in fields.split(',')] - - if len(fields) == 0: - raise click.ClickException('Please define at least one property to show') - - # Turn the list of ports into integers - if ports: - try: - ports = [int(item.strip()) for item in ports.split(',')] - except: - raise click.ClickException('Invalid list of ports') - - # Decide which stream to subscribe to based on whether or not ports were selected - if ports: - stream = api.stream.ports(ports) - else: - stream = api.stream.banners() - - counter = 0 - quit = False - last_time = timestr() - fout = None - - if datadir: - fout = open_file(datadir, last_time) - - while not quit: - try: - for banner in stream: - # Limit the number of results to output - if limit > 0: - counter += 1 - - if counter > limit: - quit = True - break - - # Write the data to the file - if datadir: - cur_time = timestr() - if cur_time != last_time: - last_time = cur_time - fout.close() - fout = open_file(datadir, last_time) - fout.write(simplejson.dumps(banner) + '\n') - - # Print the banner information to stdout - if not quiet: - row = '' - - # Loop over all the fields and print the banner as a row - for field in fields: - tmp = '' - if field in banner and banner[field]: - field_type = type(banner[field]) - - # If the field is an array then merge it together - if field_type == list: - tmp = ';'.join(banner[field]) - elif field_type in [int, float]: - tmp = str(banner[field]) - else: - tmp = escape_data(banner[field]) - - # Colorize certain fields if the user wants it - if color: - tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) - - # Add the field information to the row - row += tmp - row += separator - - click.echo(row) - except KeyboardInterrupt: - quit = True - except: - # For other errors lets just wait a few seconds and try to reconnect again - time.sleep(2) - - -if __name__ == '__main__': - main() diff --git a/docs/examples/basic-search.rst b/docs/examples/basic-search.rst index 58605ad..e61e38d 100644 --- a/docs/examples/basic-search.rst +++ b/docs/examples/basic-search.rst @@ -32,6 +32,6 @@ Basic Shodan Search # Loop through the matches and print each IP for service in result['matches']: print service['ip_str'] - except Exception, e: + except Exception as e: print 'Error: %s' % e sys.exit(1) \ No newline at end of file diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst index fce8a65..b01440e 100644 --- a/docs/examples/cert-stream.rst +++ b/docs/examples/cert-stream.rst @@ -20,11 +20,10 @@ information. # # WARNING: This script only works with people that have a subscription API plan! # And by default the Streaming API only returns 1% of the data that Shodan gathers. - # If you wish to have more access please contact us at support@shodan.io for pricing + # If you wish to have more access please contact us at sales@shodan.io for pricing # information. # # Author: achillean - import shodan import sys @@ -35,12 +34,12 @@ information. # Setup the api api = shodan.Shodan(API_KEY) - print 'Listening for certs...' + print('Listening for certs...') for banner in api.stream.ports([443, 8443]): - if 'opts' in banner and 'pem' in banner['opts']: - print banner['opts']['pem'] + if 'ssl' in banner: + # Print out all the SSL information that Shodan has collected + print(banner['ssl']) - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: {}'.format(e)) sys.exit(1) - diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst new file mode 100644 index 0000000..e4a43c3 --- /dev/null +++ b/docs/examples/gifcreator.rst @@ -0,0 +1,112 @@ +GIF Creator +----------- + +Shodan keeps a full history of all the information that has been gathered on an IP address. With the API, +you're able to retrieve that history and we're going to use that to create a tool that outputs GIFs made of +the screenshots that the Shodan crawlers gather. + +The below code requires the following Python packages: + + - arrow + - shodan + +The **arrow** package is used to parse the *timestamp* field of the banner into a Python `datetime` object. + +In addition to the above Python packages, you also need to have the **ImageMagick** software installed. If you're +working on Ubuntu or another distro using **apt** you can run the following command: + +.. code-block:: bash + + sudo apt-get install imagemagick + +This will provide us with the **convert** command which is needed to merge several images into an animated GIF. + +There are a few key Shodan methods/ parameters that make the script work: + +1. :py:func:`shodan.helpers.iterate_files()` to loop through the Shodan data file +2. **history** flag on the :py:func:`shodan.Shodan.host` method to get all the banners for an IP that Shodan has collected over the years + + + +.. code-block:: python + + #!/usr/bin/env python + # gifcreator.py + # + # Dependencies: + # - arrow + # - shodan + # + # Installation: + # sudo easy_install arrow shodan + # sudo apt-get install imagemagick + # + # Usage: + # 1. Download a json.gz file using the website or the Shodan command-line tool (https://cli.shodan.io). + # For example: + # shodan download screenshots.json.gz has_screenshot:true + # 2. Run the tool on the file: + # python gifcreator.py screenshots.json.gz + + import arrow + import os + import shodan + import shodan.helpers as helpers + import sys + + + # Settings + API_KEY = '' + MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF + MAX_SCREENS = 24 + + if len(sys.argv) != 2: + print('Usage: {} '.format(sys.argv[0])) + sys.exit(1) + + # GIFs are stored in the local "data" directory + os.mkdir('data') + + # We need to connect to the API to lookup the historical host information + api = shodan.Shodan(API_KEY) + + # Use the shodan.helpers.iterate_files() method to loop over the Shodan data file + for result in helpers.iterate_files(sys.argv[1]): + # Get the historic info + host = api.host(result['ip_str'], history=True) + + # Count how many screenshots this host has + screenshots = [] + for banner in host['data']: + # Extract the image from the banner data + if 'opts' in banner and 'screenshot' in banner['opts']: + # Sort the images by the time they were collected so the GIF will loop + # based on the local time regardless of which day the banner was taken. + timestamp = arrow.get(banner['timestamp']).time() + sort_key = timestamp.hour + screenshots.append(( + sort_key, + banner['opts']['screenshot']['data'] + )) + + # Ignore any further screenshots if we already have MAX_SCREENS number of images + if len(screenshots) >= MAX_SCREENS: + break + + # Extract the screenshots and turn them into a GIF if we've got the necessary + # amount of images. + if len(screenshots) >= MIN_SCREENS: + for (i, screenshot) in enumerate(sorted(screenshots, key=lambda x: x[0], reverse=True)): + open('/tmp/gif-image-{}.jpg'.format(i), 'w').write(screenshot[1].decode('base64')) + + # Create the actual GIF using the ImageMagick "convert" command + os.system('convert -layers OptimizePlus -delay 5x10 /tmp/gif-image-*.jpg -loop 0 +dither -colors 256 -depth 8 data/{}.gif'.format(result['ip_str'])) + + # Clean up the temporary files + os.system('rm -f /tmp/gif-image-*.jpg') + + # Show a progress indicator + print(result['ip_str']) + + +The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index d54a8ba..66e15fe 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -32,23 +32,23 @@ and country. 'port', 'asn', - # We only care about the top 5 countries, this is how we let Shodan know to return 5 instead of the - # default 10 for a facet. If you want to see more than 10, you could do ('country', 1000) for example + # We only care about the top 3 countries, this is how we let Shodan know to return 3 instead of the + # default 5 for a facet. If you want to see more than 5, you could do ('country', 1000) for example # to see the top 1,000 countries for a search query. - ('country', 5), + ('country', 3), ] FACET_TITLES = { - 'org': 'Top 10 Organizations', - 'domain': 'Top 10 Domains', - 'port': 'Top 10 Ports', - 'asn': 'Top 10 Autonomous Systems', - 'country': 'Top 5 Countries', + 'org': 'Top 5 Organizations', + 'domain': 'Top 5 Domains', + 'port': 'Top 5 Ports', + 'asn': 'Top 5 Autonomous Systems', + 'country': 'Top 3 Countries', } # Input validation if len(sys.argv) == 1: - print 'Usage: %s ' % sys.argv[0] + print('Usage: %s ' % sys.argv[0]) sys.exit(1) try: @@ -62,22 +62,22 @@ and country. # And it also runs faster than doing a search(). result = api.count(query, facets=FACETS) - print 'Shodan Summary Information' - print 'Query: %s' % query - print 'Total Results: %s\n' % result['total'] + print('Shodan Summary Information') + print('Query: %s' % query) + print('Total Results: %s\n' % result['total']) # Print the summary info from the facets for facet in result['facets']: - print FACET_TITLES[facet] + print(FACET_TITLES[facet]) for term in result['facets'][facet]: - print '%s: %s' % (term['value'], term['count']) + print('%s: %s' % (term['value'], term['count'])) # Print an empty line between summary info - print '' + print('') - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: %s' % e) sys.exit(1) """ @@ -89,38 +89,36 @@ and country. Query: apache Total Results: 34612043 - Top 10 Organizations + Top 5 Organizations Amazon.com: 808061 Ecommerce Corporation: 788704 Verio Web Hosting: 760112 Unified Layer: 627827 GoDaddy.com, LLC: 567004 - Top 10 Domains + Top 5 Domains secureserver.net: 562047 unifiedlayer.com: 494399 t-ipconnect.de: 385792 netart.pl: 194817 wanadoo.fr: 151925 - Top 10 Ports + Top 5 Ports 80: 24118703 443: 8330932 8080: 1479050 81: 359025 8443: 231441 - Top 10 Autonomous Systems + Top 5 Autonomous Systems as32392: 580002 as2914: 465786 as26496: 414998 as48030: 332000 as8560: 255774 - Top 5 Countries + Top 3 Countries US: 13227366 DE: 2900530 JP: 2014506 - CN: 1722048 - GB: 1209938 """ diff --git a/docs/index.rst b/docs/index.rst index dd52b20..956e335 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,10 @@ Streaming API. And as a bonus it also lets you search for exploits using the Sho If you're not sure where to start simply go through the "Getting Started" section of the documentation and work your way down through the examples. +For more information about Shodan and how to use the API please visit our official help center at: + + https://help.shodan.io + Introduction ~~~~~~~~~~~~ .. toctree:: @@ -26,10 +30,11 @@ Examples examples/basic-search examples/query-summary examples/cert-stream + examples/gifcreator API Reference ~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 - api \ No newline at end of file + api diff --git a/docs/tutorial.rst b/docs/tutorial.rst index e8efcb2..62744af 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -51,13 +51,13 @@ Now that we have our API object all good to go, we're ready to perform a search: results = api.search('apache') # Show the results - print 'Results found: %s' % results['total'] + print('Results found: {}'.format(results['total'])) for result in results['matches']: - print 'IP: %s' % result['ip_str'] - print result['data'] - print '' - except shodan.APIError, e: - print 'Error: %s' % e + print('IP: {}'.format(result['ip_str'])) + print(result['data']) + print('') + except shodan.APIError as e: + print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which returns a dictionary of result information. We then print how many results were found in total, @@ -101,16 +101,16 @@ To see what Shodan has available on a specific IP we can use the :py:func:`Shoda host = api.host('217.140.75.46') # Print general info - print """ - IP: %s - Organization: %s - Operating System: %s - """ % (host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a')) + print(""" + IP: {} + Organization: {} + Operating System: {} + """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a'))) # Print all banners for item in host['data']: - print """ - Port: %s - Banner: %s + print(""" + Port: {} + Banner: {} - """ % (item['port'], item['data']) \ No newline at end of file + """.format(item['port'], item['data'])) diff --git a/requirements.txt b/requirements.txt index 05b1ace..2692414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ click +click-plugins colorama -requests -simplejson +requests>=2.2.1 +XlsxWriter +ipaddress;python_version<='2.7' +tldextract \ No newline at end of file diff --git a/setup.py b/setup.py index b42a7d4..53bbd9a 100755 --- a/setup.py +++ b/setup.py @@ -1,22 +1,39 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup + + +DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') +README = open('README.rst', 'r').read() + setup( - name = 'shodan', - version = '1.2.6', - description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', - author = 'John Matherly', - author_email = 'jmath@shodan.io', - url = 'http://github.com/achillean/shodan-python/tree/master', - packages = ['shodan'], - scripts = ['bin/shodan'], - install_requires=["simplejson", "requests", "click", "colorama"], - classifiers = [ + name='shodan', + version='1.31.0', + description='Python library and command-line utility for Shodan (https://developer.shodan.io)', + long_description=README, + long_description_content_type='text/x-rst', + author='John Matherly', + author_email='jmath@shodan.io', + url='https://github.com/achillean/shodan-python', + packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], + entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, + install_requires=DEPENDENCIES, + keywords=['security', 'network'], + classifiers=[ + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries :: Python Modules', ], ) diff --git a/shodan/__init__.py b/shodan/__init__.py index 1c20901..bdfecaa 100644 --- a/shodan/__init__.py +++ b/shodan/__init__.py @@ -1,2 +1,2 @@ -from .api import WebAPI -from .client import APIError, Shodan +from shodan.client import Shodan +from shodan.exception import APIError diff --git a/shodan/__main__.py b/shodan/__main__.py new file mode 100644 index 0000000..4093b94 --- /dev/null +++ b/shodan/__main__.py @@ -0,0 +1,949 @@ +""" +Shodan CLI + +Note: Always run "shodan init " before trying to execute any other command! + +A simple interface to search Shodan, download data and parse compressed JSON files. +The following commands are currently supported: + + alert + convert + count + data + download + honeyscore + host + info + init + myip + parse + radar + scan + search + stats + stream + trends + +""" + +import click +import csv +import os +import os.path +import pkg_resources +import shodan +import shodan.helpers as helpers +import threading +import requests +import time +import json + +# The file converters that are used to go from .json.gz to various other formats +from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter + +# Constants +from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS + +# Helper methods +from shodan.cli.helpers import async_spinner, get_api_key, escape_data, timestr, open_streaming_file, get_banner_field, match_filters +from shodan.cli.host import HOST_PRINT + +# Allow 3rd-parties to develop custom commands +from click_plugins import with_plugins +from pkg_resources import iter_entry_points + +# Large subcommands are stored in separate modules +from shodan.cli.alert import alert +from shodan.cli.data import data +from shodan.cli.organization import org +from shodan.cli.scan import scan + + +# Make "-h" work like "--help" +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +CONVERTERS = { + 'kml': KmlConverter, + 'csv': CsvConverter, + 'geo.json': GeoJsonConverter, + 'images': ImagesConverter, + 'xlsx': ExcelConverter, +} + +# Define a basestring type if necessary for Python3 compatibility +try: + basestring +except NameError: + basestring = str + + +# Define the main entry point for all of our commands +# and expose a way for 3rd-party plugins to tie into the Shodan CLI. +@with_plugins(iter_entry_points('shodan.cli.plugins')) +@click.group(context_settings=CONTEXT_SETTINGS) +def main(): + pass + + +# Setup the large subcommands +main.add_command(alert) +main.add_command(data) +main.add_command(org) +main.add_command(scan) + + +@main.command() +@click.option('--fields', help='List of properties to output.', default=None) +@click.argument('input', metavar='', type=click.Path(exists=True)) +@click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) +def convert(fields, input, format): + """Convert the given input data file into a different format. The following file formats are supported: + + kml, csv, geo.json, images, xlsx + + Example: shodan convert data.json.gz kml + """ + file_size = 0 + # Check that the converter allows a custom list of fields + converter_class = CONVERTERS.get(format) + if fields: + if not hasattr(converter_class, 'fields'): + raise click.ClickException('File format doesnt support custom list of fields') + converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified + + # click.Path ensures that file path exists + file_size = os.path.getsize(input) + + # Get the basename for the input file + basename = input.replace('.json.gz', '').replace('.json', '') + + # Add the new file extension based on the format + filename = '{}.{}'.format(basename, format) + + # Open the output file + fout = open(filename, 'w') + + # Start a spinner + finished_event = threading.Event() + progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) + progress_bar_thread.start() + + # Initialize the file converter + converter = converter_class(fout) + + converter.process([input], file_size) + + finished_event.set() + progress_bar_thread.join() + + if format == 'images': + click.echo(click.style('\rSuccessfully extracted images to directory: {}'.format(converter.dirname), fg='green')) + else: + click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green')) + + +@main.command(name='domain') +@click.argument('domain', metavar='') +@click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) +@click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) +@click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True) +@click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None) +def domain_info(domain, details, save, history, type): + """View all available information for a domain""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.dns.domain_info(domain, history=history, type=type) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the host information for any IP records that were returned + hosts = {} + if details: + ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']] + ips = set(ips) + + fout = None + if save: + filename = u'{}-hosts.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for ip in ips: + try: + hosts[ip] = api.host(ip) + + # Store the banners if requested + if fout: + for banner in hosts[ip]['data']: + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) + except shodan.APIError: + pass # Ignore any API lookup errors as this isn't critical information + + # Save the DNS data + if save: + filename = u'{}.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for record in info['data']: + helpers.write_banner(fout, record) + + click.secho(info['domain'].upper(), fg='green') + + click.echo('') + for record in info['data']: + click.echo( + u'{:32} {:14} {}'.format( + click.style(record['subdomain'], fg='cyan'), + click.style(record['type'], fg='yellow'), + record['value'] + ), + nl=False, + ) + + if record['value'] in hosts: + host = hosts[record['value']] + click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) + + click.echo('') + + +@main.command() +@click.argument('key', metavar='') +def init(key): + """Initialize the Shodan command-line""" + # Create the directory if necessary + shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) + if not os.path.isdir(shodan_dir): + try: + os.makedirs(shodan_dir) + except OSError: + raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) + + # Make sure it's a valid API key + key = key.strip() + try: + api = shodan.Shodan(key) + api.info() + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Store the API key in the user's directory + keyfile = shodan_dir + '/api_key' + with open(keyfile, 'w') as fout: + fout.write(key.strip()) + click.echo(click.style('Successfully initialized', fg='green')) + + os.chmod(keyfile, 0o600) + + +@main.command() +@click.argument('query', metavar='', nargs=-1) +def count(query): + """Returns the number of results for a search""" + key = get_api_key() + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + + # Make sure the user didn't supply an empty string + if query == '': + raise click.ClickException('Empty search query') + + # Perform the search + api = shodan.Shodan(key) + try: + results = api.count(query) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.echo(results['total']) + + +@main.command() +@click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str) +@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) +@click.argument('filename', metavar='') +@click.argument('query', metavar='', nargs=-1) +def download(fields, limit, filename, query): + """Download search results and save them in a compressed JSON file.""" + key = get_api_key() + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + + # Make sure the user didn't supply an empty string + if query == '': + raise click.ClickException('Empty search query') + + filename = filename.strip() + if filename == '': + raise click.ClickException('Empty filename') + + # Add the appropriate extension if it's not there atm + if not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Strip out any whitespace in the fields and turn them into an array + if fields is not None: + fields = [item.strip() for item in fields.split(',')] + + # Perform the search + api = shodan.Shodan(key) + + try: + total = api.count(query)['total'] + info = api.info() + except Exception: + raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') + + # Print some summary information about the download request + click.echo('Search query:\t\t\t{}'.format(query)) + click.echo('Total number of results:\t{}'.format(total)) + click.echo('Query credits left:\t\t{}'.format(info['unlocked_left'])) + click.echo('Output file:\t\t\t{}'.format(filename)) + + if limit > total: + limit = total + + # A limit of -1 means that we should download all the data + if limit <= 0: + limit = total + + with helpers.open_file(filename, 'w') as fout: + count = 0 + try: + cursor = api.search_cursor(query, minify=False, fields=fields) + with click.progressbar(cursor, length=limit) as bar: + for banner in bar: + helpers.write_banner(fout, banner) + count += 1 + + if count >= limit: + break + except Exception: + pass + + # Let the user know we're done + if count < limit: + click.echo(click.style('Notice: fewer results were saved than requested', 'yellow')) + click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green')) + + +@main.command() +@click.option('--format', help='The output format for the host information. Possible values are: pretty, tsv.', default='pretty', type=click.Choice(['pretty', 'tsv'])) +@click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True) +@click.option('--filename', '-O', help='Save the host information in the given file (append if file exists).', default=None) +@click.option('--save', '-S', help='Save the host information in the a file named after the IP (append if file exists).', default=False, is_flag=True) +@click.argument('ip', metavar='') +def host(format, history, filename, save, ip): + """View all available information for an IP address""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + host = api.host(ip, history=history) + + # Print the host information to the terminal using the user-specified format + HOST_PRINT[format](host, history=history) + + # Store the results + if filename or save: + if save: + filename = '{}.json.gz'.format(ip) + + # Add the appropriate extension if it's not there atm + if not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Create/ append to the file + fout = helpers.open_file(filename) + + for banner in sorted(host['data'], key=lambda k: k['port']): + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@main.command() +def info(): + """Shows general information about your account""" + key = get_api_key() + api = shodan.Shodan(key) + try: + results = api.info() + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.echo("""Query credits available: {0} +Scan credits available: {1} + """.format(results['query_credits'], results['scan_credits'])) + + +@main.command() +@click.option('--color/--no-color', default=True) +@click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') +@click.option('--filters', '-f', help='Filter the results for specific values using key:value pairs.', multiple=True) +@click.option('--filename', '-O', help='Save the filtered results in the given file (append if file exists).') +@click.option('--separator', help='The separator between the properties of the search results.', default=u'\t') +@click.argument('filenames', metavar='', type=click.Path(exists=True), nargs=-1) +def parse(color, fields, filters, filename, separator, filenames): + """Extract information out of compressed JSON files.""" + # Strip out any whitespace in the fields and turn them into an array + fields = [item.strip() for item in fields.split(',')] + + if len(fields) == 0: + raise click.ClickException('Please define at least one property to show') + + has_filters = len(filters) > 0 + + # Setup the output file handle + fout = None + if filename: + # If no filters were provided raise an error since it doesn't make much sense w/out them + if not has_filters: + raise click.ClickException('Output file specified without any filters. Need to use filters with this option.') + + # Add the appropriate extension if it's not there atm + if not filename.endswith('.json.gz'): + filename += '.json.gz' + fout = helpers.open_file(filename) + + for banner in helpers.iterate_files(filenames): + row = u'' + + # Validate the banner against any provided filters + if has_filters and not match_filters(banner, filters): + continue + + # Append the data + if fout: + helpers.write_banner(fout, banner) + + # Loop over all the fields and print the banner as a row + for i, field in enumerate(fields): + tmp = u'' + value = get_banner_field(banner, field) + if value: + field_type = type(value) + + # If the field is an array then merge it together + if field_type == list: + tmp = u';'.join(value) + elif field_type in [int, float]: + tmp = u'{}'.format(value) + else: + tmp = escape_data(value) + + # Colorize certain fields if the user wants it + if color: + tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) + + # Add the field information to the row + if i > 0: + row += separator + row += tmp + + click.echo(row) + + +@main.command() +@click.option('--ipv6', '-6', is_flag=True, default=False, help='Try to use IPv6 instead of IPv4') +def myip(ipv6): + """Print your external IP address""" + key = get_api_key() + + api = shodan.Shodan(key) + + # Use the IPv6-enabled domain if requested + if ipv6: + api.base_url = 'https://apiv6.shodan.io' + + try: + click.echo(api.tools.myip()) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@main.command() +@click.option('--color/--no-color', default=True) +@click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data') +@click.option('--limit', help='The number of search results that should be returned. Maximum: 1000', default=100, type=int) +@click.option('--separator', help='The separator between the properties of the search results.', default='\t') +@click.argument('query', metavar='', nargs=-1) +def search(color, fields, limit, separator, query): + """Search the Shodan database""" + key = get_api_key() + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + + # Make sure the user didn't supply an empty string + if query == '': + raise click.ClickException('Empty search query') + + # For now we only allow up to 1000 results at a time + if limit > 1000: + raise click.ClickException('Too many results requested, maximum is 1,000') + + # Strip out any whitespace in the fields and turn them into an array + fields = [item.strip() for item in fields.split(',')] + + if len(fields) == 0: + raise click.ClickException('Please define at least one property to show') + + # Perform the search + api = shodan.Shodan(key) + try: + results = api.search(query, limit=limit, minify=False, fields=fields) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') + + # We buffer the entire output so we can use click's pager functionality + output = u'' + for banner in results['matches']: + row = u'' + + # Loop over all the fields and print the banner as a row + for field in fields: + tmp = u'' + value = get_banner_field(banner, field) + if value: + field_type = type(value) + + # If the field is an array then merge it together + if field_type == list: + tmp = u';'.join(value) + elif field_type in [int, float]: + tmp = u'{}'.format(value) + else: + tmp = escape_data(value) + + # Colorize certain fields if the user wants it + if color: + tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) + + # Add the field information to the row + row += tmp + row += separator + + # click.echo(out + separator, nl=False) + output += row + u'\n' + # click.echo('') + click.echo_via_pager(output) + + +@main.command() +@click.option('--limit', help='The number of results to return.', default=10, type=int) +@click.option('--facets', help='List of facets to get statistics for.', default='country,org') +@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) +@click.argument('query', metavar='', nargs=-1) +def stats(limit, facets, filename, query): + """Provide summary information about a search query""" + # Setup Shodan + key = get_api_key() + api = shodan.Shodan(key) + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + + # Make sure the user didn't supply an empty string + if query == '': + raise click.ClickException('Empty search query') + + facets = facets.split(',') + facets = [(facet, limit) for facet in facets] + + # Perform the search + try: + results = api.count(query, facets=facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Print the stats tables + for facet in results['facets']: + click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) + + for item in results['facets'][facet]: + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) + + click.echo('') + + # Create the output file if requested + fout = None + if filename: + if not filename.endswith('.csv'): + filename += '.csv' + fout = open(filename, 'w') + writer = csv.writer(fout, dialect=csv.excel) + + # Write the header + writer.writerow(['Query', query]) + + # Add an empty line to separate rows + writer.writerow([]) + + # Write the header that contains the facets + row = [] + for facet in results['facets']: + row.append(facet) + row.append('') + writer.writerow(row) + + # Every facet has 2 columns (key, value) + counter = 0 + has_items = True + while has_items: + # pylint: disable=W0612 + row = ['' for i in range(len(results['facets']) * 2)] + + pos = 0 + has_items = False + for facet in results['facets']: + values = results['facets'][facet] + + # Add the values for the facet into the current row + if len(values) > counter: + has_items = True + row[pos] = values[counter]['value'] + row[pos + 1] = values[counter]['count'] + + pos += 2 + + # Write out the row + if has_items: + writer.writerow(row) + + # Move to the next row of values + counter += 1 + + +@main.command() +@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) +@click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') +@click.option('--separator', help='The separator between the properties of the search results.', default='\t') +@click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) +@click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) +@click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) +@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) +@click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str) +@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) +@click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) +@click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) +@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) +@click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) +@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) +@click.option('--color/--no-color', default=True) +@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) +def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet): + """Stream data in real-time.""" + # Setup the Shodan API + key = get_api_key() + api = shodan.Shodan(key) + + # Temporarily change the baseurl + api.stream.base_url = streamer + + # Strip out any whitespace in the fields and turn them into an array + fields = [item.strip() for item in fields.split(',')] + + if len(fields) == 0: + raise click.ClickException('Please define at least one property to show') + + # The user must choose "ports", "countries", "asn" or nothing - can't select multiple + # filtered streams at once. + stream_type = [] + if ports: + stream_type.append('ports') + if countries: + stream_type.append('countries') + if asn: + stream_type.append('asn') + if alert: + stream_type.append('alert') + if tags: + stream_type.append('tags') + if vulns: + stream_type.append('vulns') + if custom_filters: + stream_type.append('custom_filters') + + if len(stream_type) > 1: + raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') + + stream_args = None + + # Turn the list of ports into integers + if ports: + try: + stream_args = [int(item.strip()) for item in ports.split(',')] + except ValueError: + raise click.ClickException('Invalid list of ports') + + if alert: + alert = alert.strip() + if alert.lower() != 'all': + stream_args = alert + + if asn: + stream_args = asn.split(',') + + if countries: + stream_args = countries.split(',') + + if tags: + stream_args = tags.split(',') + + if vulns: + stream_args = vulns.split(',') + + if custom_filters: + stream_args = custom_filters + + # Flatten the list of stream types + # Possible values are: + # - all + # - asn + # - countries + # - ports + if len(stream_type) == 1: + stream_type = stream_type[0] + else: + stream_type = 'all' + + # Decide which stream to subscribe to based on whether or not ports were selected + def _create_stream(name, args, timeout): + return { + 'all': api.stream.banners(timeout=timeout), + 'alert': api.stream.alert(args, timeout=timeout), + 'asn': api.stream.asn(args, timeout=timeout), + 'countries': api.stream.countries(args, timeout=timeout), + 'custom_filters': api.stream.custom(args, timeout=timeout), + 'ports': api.stream.ports(args, timeout=timeout), + 'tags': api.stream.tags(args, timeout=timeout), + 'vulns': api.stream.vulns(args, timeout=timeout), + }.get(name, 'all') + + stream = _create_stream(stream_type, stream_args, timeout=timeout) + + counter = 0 + quit = False + last_time = timestr() + fout = None + + if datadir: + fout = open_streaming_file(datadir, last_time, compresslevel) + + while not quit: + try: + for banner in stream: + # Limit the number of results to output + if limit > 0: + counter += 1 + + if counter > limit: + quit = True + break + + # Write the data to the file + if datadir: + cur_time = timestr() + if cur_time != last_time: + last_time = cur_time + fout.close() + fout = open_streaming_file(datadir, last_time) + helpers.write_banner(fout, banner) + + # Print the banner information to stdout + if not quiet: + row = u'' + + # Loop over all the fields and print the banner as a row + for field in fields: + tmp = u'' + value = get_banner_field(banner, field) + if value: + field_type = type(value) + + # If the field is an array then merge it together + if field_type == list: + tmp = u';'.join(value) + elif field_type in [int, float]: + tmp = u'{}'.format(value) + else: + tmp = escape_data(value) + + # Colorize certain fields if the user wants it + if color: + tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) + + # Add the field information to the row + row += tmp + row += separator + + click.echo(row) + except requests.exceptions.Timeout: + raise click.ClickException('Connection timed out') + except KeyboardInterrupt: + quit = True + except shodan.APIError as e: + raise click.ClickException(e.value) + except Exception: + # For other errors lets just wait a bit and try to reconnect again + time.sleep(1) + + # Create a new stream object to subscribe to + stream = _create_stream(stream_type, stream_args, timeout=timeout) + + +@main.command() +@click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str) +@click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) +@click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) +@click.argument('query', metavar='', nargs=-1) +def trends(filename, save, facets, query): + """Search Shodan historical database""" + key = get_api_key() + api = shodan.Shodan(key) + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + facets = facets.strip() + + # Make sure the user didn't supply an empty query or facets + if query == '': + raise click.ClickException('Empty search query') + + # Convert comma-separated facets string to list + parsed_facets = [] + for facet in facets.split(','): + if not facet: + continue + + parts = facet.strip().split(":") + if len(parts) > 1: + parsed_facets.append((parts[0], parts[1])) + else: + parsed_facets.append((parts[0])) + + # Perform the search + try: + results = api.trends.search(query, facets=parsed_facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') + + result_facets = [] + if results.get("facets"): + result_facets = list(results["facets"].keys()) + + # Save the results first to file if user request + if filename or save: + if not filename: + filename = '{}-trends.json.gz'.format(query.replace(' ', '-')) + elif not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Create/ append to the file + with helpers.open_file(filename) as fout: + for index, match in enumerate(results['matches']): + # Append facet info to make up a line + if result_facets: + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) + + click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) + + # We buffer the entire output so we can use click's pager functionality + output = u'' + + # Output examples: + # - Facet by os + # 2017-06 + # os + # Linux 3.x 146,502 + # Windows 7 or 8 2,189 + # + # - Without facets + # 2017-06 19,799,459 + # 2017-07 21,077,099 + if result_facets: + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) + else: + # Without facets, show query total results over time + for index, match in enumerate(results['matches']): + output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green')) + + click.echo_via_pager(output) + + +@main.command() +@click.argument('ip', metavar='') +def honeyscore(ip): + """Check whether the IP is a honeypot or not.""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + score = api.labs.honeyscore(ip) + + if score == 1.0: + click.echo(click.style('Honeypot detected', fg='red')) + elif score > 0.5: + click.echo(click.style('Probably a honeypot', fg='yellow')) + else: + click.echo(click.style('Not a honeypot', fg='green')) + + click.echo('Score: {}'.format(score)) + except Exception: + raise click.ClickException('Unable to calculate honeyscore') + + +@main.command() +def radar(): + """Real-Time Map of some results as Shodan finds them.""" + key = get_api_key() + api = shodan.Shodan(key) + + from shodan.cli.worldmap import launch_map + + try: + launch_map(api) + except shodan.APIError as e: + raise click.ClickException(e.value) + except Exception as e: + raise click.ClickException(u'{}'.format(e)) + + +@main.command() +def version(): + """Print version of this tool.""" + print(pkg_resources.get_distribution("shodan").version) + + +if __name__ == '__main__': + main() diff --git a/shodan/api.py b/shodan/api.py deleted file mode 100644 index 907f411..0000000 --- a/shodan/api.py +++ /dev/null @@ -1,225 +0,0 @@ -# The simplejson library has better JSON-parsing than the standard library and is more often updated -from simplejson import dumps, loads - -try: - # Python 2 - from urllib2 import urlopen - from urllib import urlencode -except: - # Python 3 - from urllib.request import urlopen - from urllib.parse import urlencode - -__all__ = ['WebAPI'] - -class WebAPIError(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - -class WebAPI: - """Wrapper around the SHODAN webservices API""" - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, sources=[], cve=None, osvdb=None, msb=None, bid=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - Arguments: - query -- exploit search query; same syntax as website - - Optional arguments: - sources -- metasploit, cve, osvdb, exploitdb - cve -- CVE identifier (ex. 2010-0432) - osvdb -- OSVDB identifier (ex. 11666) - msb -- Microsoft Security Bulletin ID (ex. MS05-030) - bid -- Bugtraq identifier (ex. 13951) - - """ - if sources: - query += ' source:' + ','.join(sources) - if cve: - query += ' cve:%s' % (str(cve).strip()) - if osvdb: - query += ' osvdb:%s' % (str(osvdb).strip()) - if msb: - query += ' msb:%s' % (str(msb).strip()) - if bid: - query += ' bid:%s' % (str(bid).strip()) - return self.parent._request('api', {'q': query}, service='exploits') - - class ExploitDb: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """DEPRECATED - Download the exploit code from the ExploitDB archive. - - Arguments: - id -- ID of the ExploitDB entry - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['exploitdb']) - - def search(self, query, **kwargs): - """Search the ExploitDB archive. - - Arguments: - query -- Search terms - - Returns: - A dictionary with 2 main items: matches (list) and total (int). - """ - return self.parent.search(query, sources=['exploitdb']) - - class Msf: - - def __init__(self, parent): - self.parent = parent - - def download(self, id): - """Download a metasploit module given the fullname (id) of it. - - Arguments: - id -- fullname of the module (ex. auxiliary/admin/backupexec/dump) - - Returns: - A dictionary with the following fields: - filename -- Name of the file - content-type -- Mimetype - data -- File content - """ - query = '_id:%s' % id - return self.parent.search(query, sources=['metasploit']) - - def search(self, query, **kwargs): - """Search for a Metasploit module. - """ - return self.parent.search(query, sources=['metasploit']) - - def __init__(self, key): - """Initializes the API object. - - Arguments: - key -- your API key - - """ - self.api_key = key - self.base_url = 'http://www.shodanhq.com/api/' - self.base_exploits_url = 'https://exploits.shodan.io/' - self.exploits = self.Exploits(self) - self.exploitdb = self.ExploitDb(self.exploits) - self.msf = self.Msf(self.exploits) - - def _request(self, function, params, service='shodan'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A JSON string containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - data = urlopen(base_url + function + '?' + urlencode(params)).read().decode('utf-8') - except: - raise WebAPIError('Unable to connect to Shodan') - - # Parse the text into JSON - data = loads(data) - - # Raise an exception if an error occurred - if data.get('error', None): - raise WebAPIError(data['error']) - - # Return the data - return data - - def count(self, query): - """Returns the total number of search results for the query. - """ - return self._request('count', {'q': query}) - - def locations(self, query): - """Return a break-down of all the countries and cities that the results for - the given search are located in. - """ - return self._request('locations', {'q': query}) - - def fingerprint(self, banner): - """Determine the software based on the banner. - - Arguments: - banner - HTTP banner - - Returns: - A list of software that matched the given banner. - """ - return self._request('fingerprint', {'banner': banner}) - - def host(self, ip): - """Get all available information on an IP. - - Arguments: - ip -- IP of the computer - - Returns: - All available information SHODAN has on the given IP, - subject to API key restrictions. - - """ - return self._request('host', {'ip': ip}) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('info', {}) - - def search(self, query, page=1, limit=None, offset=None): - """Search the SHODAN database. - - Arguments: - query -- search query; identical syntax to the website - - Optional arguments: - page -- page number of the search results - limit -- number of results to return - offset -- search offset to begin getting results from - - Returns: - A dictionary with 3 main items: matches, countries and total. - Visit the website for more detailed information. - - """ - args = { - 'q': query, - 'p': page, - } - if limit: - args['l'] = limit - if offset: - args['o'] = offset - - return self._request('search', args) diff --git a/shodan/cli/__init__.py b/shodan/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py new file mode 100644 index 0000000..1df11ea --- /dev/null +++ b/shodan/cli/alert.py @@ -0,0 +1,561 @@ +import click +import csv +import gzip +import json +import shodan +from tldextract import extract +from ipaddress import ip_address + +from collections import defaultdict +from operator import itemgetter +from shodan import APIError +from shodan.cli.helpers import get_api_key +from shodan.helpers import open_file, write_banner +from time import sleep + + +MAX_QUERY_LENGTH = 1000 + + +def aggregate_facet(api, networks, facets): + """Merge the results from multiple facet API queries into a single result object. + This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit + into a single API call. + """ + def _merge_custom_facets(lfacets, results): + for key in results['facets']: + if key not in lfacets: + lfacets[key] = defaultdict(int) + + for item in results['facets'][key]: + lfacets[key][item['value']] += item['count'] + + # We're going to create a custom facets dict where + # the key is the value of a facet. Normally the facets + # object is a list where each item has a "value" and "count" property. + tmp_facets = {} + count = 0 + + query = 'net:' + + for net in networks: + query += '{},'.format(net) + + # Start running API queries if the query length is getting long + if len(query) > MAX_QUERY_LENGTH: + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + query = 'net:' + + # Run any remaining search query + if query[-1] != ':': + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + + # Convert the internal facets structure back to the one that + # the API returns. + new_facets = {} + for facet in tmp_facets: + sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True) + new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items] + + # Make sure the facet keys exist even if there weren't any results + for facet, _ in facets: + if facet not in new_facets: + new_facets[facet] = [] + + return { + 'matches': [], + 'facets': new_facets, + 'total': count, + } + + +@click.group() +def alert(): + """Manage the network alerts for your account""" + pass + + +@alert.command(name='clear') +def alert_clear(): + """Remove all alerts""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + alerts = api.alerts() + for alert in alerts: + click.echo(u'Removing {} ({})'.format(alert['name'], alert['id'])) + api.delete_alert(alert['id']) + except shodan.APIError as e: + raise click.ClickException(e.value) + click.echo("Alerts deleted") + + +@alert.command(name='create') +@click.argument('name', metavar='') +@click.argument('netblocks', metavar='', nargs=-1) +def alert_create(name, netblocks): + """Create a network alert to monitor an external network""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + alert = api.create_alert(name, netblocks) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created network alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + +@alert.command(name='domain') +@click.argument('domain', metavar='', type=str) +@click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') +def alert_domain(domain, triggers): + """Create a network alert based on a domain name""" + key = get_api_key() + + api = shodan.Shodan(key) + try: + # Grab a list of IPs for the domain + domain = domain.lower() + domain_parse = extract(domain) + click.secho('Looking up domain information...', dim=True) + info = api.dns.domain_info(domain, type='A') + + if domain_parse.subdomain: + domain_ips = set([record['value'] for record in info['data'] + if record['subdomain'] == domain_parse.subdomain and + not ip_address(record['value']).is_private]) + else: + domain_ips = set([record['value'] for record in info['data'] + if not ip_address(record['value']).is_private]) + + if not domain_ips: + raise click.ClickException('No external IPs were found to be associated with this domain. ' + 'No alert was created.') + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + +@alert.command(name='download') +@click.argument('filename', metavar='', type=str) +@click.option('--alert-id', help='Specific alert ID to download the data of', default=None) +def alert_download(filename, alert_id): + """Download all information for monitored networks/ IPs.""" + key = get_api_key() + + api = shodan.Shodan(key) + ips = set() + networks = set() + + # Helper method to process batches of IPs + def batch(iterable, size=1): + iter_length = len(iterable) + for ndx in range(0, iter_length, size): + yield iterable[ndx:min(ndx + size, iter_length)] + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + if alert_id: + alerts = [api.alerts(aid=alert_id.strip())] + else: + alerts = api.alerts() + + click.echo('Compiling list of networks/ IPs to download...') + for alert in alerts: + for net in alert['filters']['ip']: + if '/' in net: + networks.add(net) + else: + ips.add(net) + + click.echo('Downloading...') + with open_file(filename) as fout: + # Check if the user is able to use batch IP lookups + batch_size = 1 + if len(ips) > 0: + api_info = api.info() + if api_info['plan'] in ['corp', 'stream-100']: + batch_size = 100 + + # Convert it to a list so we can index into it + ips = list(ips) + + # Grab all the IP information + for ip in batch(ips, size=batch_size): + try: + click.echo(ip) + results = api.host(ip) + if not isinstance(results, list): + results = [results] + + for host in results: + for banner in host['data']: + write_banner(fout, banner) + except APIError: + pass + sleep(1) # Slow down a bit to make sure we don't hit the rate limit + + # Grab all the network ranges + for net in networks: + try: + counter = 0 + click.echo(net) + for banner in api.search_cursor('net:{}'.format(net)): + write_banner(fout, banner) + + # Slow down a bit to make sure we don't hit the rate limit + if counter % 100 == 0: + sleep(1) + counter += 1 + except APIError: + pass + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') + + +@alert.command(name='export') +@click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str) +def alert_export(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + alerts = api.alerts() + + # Create the output file + click.echo('Writing alerts to file: {}'.format(filename)) + with gzip.open(filename, 'wt', encoding='utf-8') as fout: + json.dump(alerts, fout) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully exported monitored networks', fg='green') + + +@alert.command(name='import') +@click.argument('filename', metavar='') +def alert_import(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + # A mapping of the old notifier IDs to the new ones + notifier_map = {} + + try: + # Loading the alerts + click.echo('Loading alerts from: {}'.format(filename)) + with gzip.open(filename, 'rt', encoding='utf-8') as fin: + alerts = json.load(fin) + + for item in alerts: + # Create the alert + click.echo('Creating: {}'.format(item['name'])) + alert = api.create_alert(item['name'], item['filters']['ip']) + + # Enable any triggers + if item.get('triggers', {}): + triggers = ','.join(item['triggers'].keys()) + + api.enable_alert_trigger(alert['id'], triggers) + + # Add any whitelisted services for this trigger + for trigger, info in item['triggers'].items(): + if info.get('ignore', []): + for whitelist in info['ignore']: + api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port']) + + # Enable the notifiers + for prev_notifier in item.get('notifiers', []): + # We don't need to do anything for the default notifier as that + # uses the account's email address automatically. + if prev_notifier['id'] == 'default': + continue + + # Get the new notifier based on the ID of the old one + notifier = notifier_map.get(prev_notifier['id']) + + # Create the notifier if it doesn't yet exist + if notifier is None: + notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description']) + + # Add it to our map of old notifier IDs to new notifiers + notifier_map[prev_notifier['id']] = notifier + + api.add_alert_notifier(alert['id'], notifier['id']) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully imported monitored networks', fg='green') + + +@alert.command(name='info') +@click.argument('alert', metavar='') +def alert_info(alert): + """Show information about a specific alert""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.alerts(aid=alert) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(info['name'], fg='cyan') + click.secho('Created: ', nl=False, dim=True) + click.secho(info['created'], fg='magenta') + + click.secho('Notifications: ', nl=False, dim=True) + if 'triggers' in info and info['triggers']: + click.secho('enabled', fg='green') + else: + click.echo('disabled') + + click.echo('') + click.secho('Network Range(s):', dim=True) + + for network in info['filters']['ip']: + click.echo(u' > {}'.format(click.style(network, fg='yellow'))) + + click.echo('') + if 'triggers' in info and info['triggers']: + click.secho('Triggers:', dim=True) + for trigger in info['triggers']: + click.echo(u' > {}'.format(click.style(trigger, fg='yellow'))) + click.echo('') + + +@alert.command(name='list') +@click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) +def alert_list(expired): + """List all the active alerts""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + results = api.alerts(include_expired=expired) + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(results) > 0: + click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) + + for alert in results: + click.echo( + u'{:16} {:<30} {:<35} '.format( + click.style(alert['id'], fg='yellow'), + click.style(alert['name'], fg='cyan'), + click.style(', '.join(alert['filters']['ip']), fg='white') + ), + nl=False + ) + + if 'triggers' in alert and alert['triggers']: + click.secho('Triggers: ', fg='magenta', nl=False) + click.echo(', '.join(alert['triggers'].keys()), nl=False) + + if 'expired' in alert and alert['expired']: + click.secho('expired', fg='red') + else: + click.echo('') + else: + click.echo("You haven't created any alerts yet.") + + +@alert.command(name='stats') +@click.option('--limit', help='The number of results to return.', default=10, type=int) +@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) +@click.argument('facets', metavar='', nargs=-1) +def alert_stats(limit, filename, facets): + """Show summary information about your monitored networks""" + # Setup Shodan + key = get_api_key() + api = shodan.Shodan(key) + + # Make sure the user didn't supply an empty string + if not facets: + raise click.ClickException('No facets provided') + + facets = [(facet, limit) for facet in facets] + + # Get the list of IPs/ networks that the user is monitoring + networks = set() + try: + alerts = api.alerts() + for alert in alerts: + for tmp in alert['filters']['ip']: + networks.add(tmp) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the facets the user requested + try: + results = aggregate_facet(api, networks, facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared + # Print the stats tables + for facet in results['facets']: + click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) + + for item in results['facets'][facet]: + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) + + click.echo('') + + # Create the output file if requested + fout = None + if filename: + if not filename.endswith('.csv'): + filename += '.csv' + fout = open(filename, 'w') + writer = csv.writer(fout, dialect=csv.excel) + + # Write the header that contains the facets + row = [] + for facet in results['facets']: + row.append(facet) + row.append('') + writer.writerow(row) + + # Every facet has 2 columns (key, value) + counter = 0 + has_items = True + while has_items: + # pylint: disable=W0612 + row = ['' for i in range(len(results['facets']) * 2)] + + pos = 0 + has_items = False + for facet in results['facets']: + values = results['facets'][facet] + + # Add the values for the facet into the current row + if len(values) > counter: + has_items = True + row[pos] = values[counter]['value'] + row[pos + 1] = values[counter]['count'] + + pos += 2 + + # Write out the row + if has_items: + writer.writerow(row) + + # Move to the next row of values + counter += 1 + + +@alert.command(name='remove') +@click.argument('alert_id', metavar='') +def alert_remove(alert_id): + """Remove the specified alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.delete_alert(alert_id) + except shodan.APIError as e: + raise click.ClickException(e.value) + click.echo("Alert deleted") + + +@alert.command(name='triggers') +def alert_list_triggers(): + """List the available notification triggers""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + results = api.alert_triggers() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(results) > 0: + click.secho('The following triggers can be enabled on alerts:', dim=True) + click.echo('') + + for trigger in sorted(results, key=itemgetter('name')): + click.secho('{:<12} '.format('Name'), dim=True, nl=False) + click.secho(trigger['name'], fg='yellow') + + click.secho('{:<12} '.format('Description'), dim=True, nl=False) + click.secho(trigger['description'], fg='cyan') + + click.secho('{:<12} '.format('Rule'), dim=True, nl=False) + click.echo(trigger['rule']) + + click.echo('') + else: + click.echo("No triggers currently available.") + + +@alert.command(name='enable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_enable_trigger(alert_id, trigger): + """Enable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.enable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully enabled the trigger: {}'.format(trigger), fg='green') + + +@alert.command(name='disable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_disable_trigger(alert_id, trigger): + """Disable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.disable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully disabled the trigger: {}'.format(trigger), fg='green') diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py new file mode 100644 index 0000000..08b068a --- /dev/null +++ b/shodan/cli/converter/__init__.py @@ -0,0 +1,5 @@ +from .csvc import CsvConverter +from .excel import ExcelConverter +from .geojson import GeoJsonConverter +from .images import ImagesConverter +from .kml import KmlConverter diff --git a/shodan/cli/converter/base.py b/shodan/cli/converter/base.py new file mode 100644 index 0000000..14b5f29 --- /dev/null +++ b/shodan/cli/converter/base.py @@ -0,0 +1,8 @@ + +class Converter: + + def __init__(self, fout): + self.fout = fout + + def process(self, fout): + pass diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py new file mode 100644 index 0000000..2e4e2f2 --- /dev/null +++ b/shodan/cli/converter/csvc.py @@ -0,0 +1,101 @@ + +from .base import Converter +from ...helpers import iterate_files + +try: + # python 3.x: Import ABC from collections.abc + from collections.abc import MutableMapping +except ImportError: + # Python 2.x: Import ABC from collections + from collections import MutableMapping + +from csv import writer as csv_writer, excel + + +class CsvConverter(Converter): + + fields = [ + 'data', + 'hostnames', + 'ip', + 'ip_str', + 'ipv6', + 'org', + 'isp', + 'location.country_code', + 'location.city', + 'location.country_name', + 'location.latitude', + 'location.longitude', + 'os', + 'asn', + 'port', + 'tags', + 'timestamp', + 'transport', + 'product', + 'version', + 'vulns', + + 'ssl.cipher.version', + 'ssl.cipher.bits', + 'ssl.cipher.name', + 'ssl.alpn', + 'ssl.versions', + 'ssl.cert.serial', + 'ssl.cert.fingerprint.sha1', + 'ssl.cert.fingerprint.sha256', + + 'html', + 'title', + ] + + def process(self, files, file_size): + writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') + + # Write the header + writer.writerow(self.fields) + + for banner in iterate_files(files): + # The "vulns" property can't be nicely flattened as-is so we turn + # it into a list before processing the banner. + if 'vulns' in banner: + banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list + + try: + row = [] + for field in self.fields: + value = self.banner_field(banner, field) + row.append(value) + writer.writerow(row) + except Exception: + pass + + def banner_field(self, banner, flat_field): + # The provided field is a collapsed form of the actual field + fields = flat_field.split('.') + + try: + current_obj = banner + for field in fields: + current_obj = current_obj[field] + + # Convert a list into a concatenated string + if isinstance(current_obj, list): + current_obj = ','.join([str(i) for i in current_obj]) + + return current_obj + except Exception: + pass + + return '' + + def flatten(self, d, parent_key='', sep='.'): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + items.extend(self.flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py new file mode 100644 index 0000000..2021a33 --- /dev/null +++ b/shodan/cli/converter/excel.py @@ -0,0 +1,136 @@ + +from .base import Converter +from ...helpers import iterate_files, get_ip + +from collections import defaultdict +from xlsxwriter import Workbook + + +class ExcelConverter(Converter): + + fields = [ + 'port', + 'timestamp', + 'data', + 'hostnames', + 'org', + 'isp', + 'location.country_name', + 'location.country_code', + 'location.city', + 'os', + 'asn', + 'transport', + 'product', + 'version', + + 'http.server', + 'http.title', + ] + + field_names = { + 'org': 'Organization', + 'isp': 'ISP', + 'location.country_code': 'Country ISO Code', + 'location.country_name': 'Country', + 'location.city': 'City', + 'os': 'OS', + 'asn': 'ASN', + + 'http.server': 'Web Server', + 'http.title': 'Website Title', + } + + def process(self, files, file_size): + # Get the filename from the already-open file handle + filename = self.fout.name + + # Close the existing file as the XlsxWriter library handles that for us + self.fout.close() + + # Create the new workbook + workbook = Workbook(filename) + + # Check if Excel file is larger than 4GB + if file_size > 4e9: + workbook.use_zip64() + + # Define some common styles/ formats + bold = workbook.add_format({ + 'bold': 1, + }) + + # Create the main worksheet where all the raw data is shown + main_sheet = workbook.add_worksheet('Raw Data') + + # Write the header + main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently + main_sheet.set_column(0, 0, 20) + + row = 0 + col = 1 + for field in self.fields: + name = self.field_names.get(field, field.capitalize()) + main_sheet.write(row, col, name, bold) + col += 1 + row += 1 + + total = 0 + ports = defaultdict(int) + for banner in iterate_files(files): + try: + # Build the list that contains all the relevant values + data = [] + for field in self.fields: + value = self.banner_field(banner, field) + data.append(value) + + # Write those values to the main workbook + # Starting off w/ the special "IP" property + main_sheet.write_string(row, 0, get_ip(banner)) + col = 1 + + for value in data: + main_sheet.write(row, col, value) + col += 1 + row += 1 + except Exception: + pass + + # Aggregate summary information + total += 1 + ports[banner['port']] += 1 + + summary_sheet = workbook.add_worksheet('Summary') + summary_sheet.write(0, 0, 'Total', bold) + summary_sheet.write(0, 1, total) + + # Ports Distribution + summary_sheet.write(0, 3, 'Ports Distribution', bold) + row = 1 + col = 3 + for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])): + summary_sheet.write(row, col, key) + summary_sheet.write(row, col + 1, value) + row += 1 + + workbook.close() + + def banner_field(self, banner, flat_field): + # The provided field is a collapsed form of the actual field + fields = flat_field.split('.') + + try: + current_obj = banner + for field in fields: + current_obj = current_obj[field] + + # Convert a list into a concatenated string + if isinstance(current_obj, list): + current_obj = ','.join([str(i) for i in current_obj]) + + return current_obj + except Exception: + pass + + return '' diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py new file mode 100644 index 0000000..83fb935 --- /dev/null +++ b/shodan/cli/converter/geojson.py @@ -0,0 +1,52 @@ +from json import dumps +from .base import Converter +from ...helpers import get_ip, iterate_files + + +class GeoJsonConverter(Converter): + + def header(self): + self.fout.write("""{ + "type": "FeatureCollection", + "features": [ + """) + + def footer(self): + self.fout.write("""{ }]}""") + + def process(self, files, file_size): + # Write the header + self.header() + + # We only want to generate 1 datapoint for each IP - not per service + unique_hosts = set() + for banner in iterate_files(files): + ip = get_ip(banner) + if not ip: + continue + + if ip not in unique_hosts: + self.write(ip, banner) + unique_hosts.add(ip) + + self.footer() + + def write(self, ip, host): + try: + lat, lon = host['location']['latitude'], host['location']['longitude'] + feature = { + 'type': 'Feature', + 'id': ip, + 'properties': { + 'name': ip, + 'lat': lat, + 'lon': lon, + }, + 'geometry': { + 'type': 'Point', + 'coordinates': [lon, lat], + }, + } + self.fout.write(dumps(feature) + ',') + except Exception: + pass diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py new file mode 100644 index 0000000..fba9d11 --- /dev/null +++ b/shodan/cli/converter/images.py @@ -0,0 +1,51 @@ + +from .base import Converter +from ...helpers import iterate_files, get_ip, get_screenshot + +# Needed for decoding base64-strings in Python3 +from codecs import decode + +import os + + +class ImagesConverter(Converter): + + # The Images converter is special in that it creates a directory and there's + # special code in the Shodan CLI that relies on the "dirname" property to let + # the user know where the images have been stored. + dirname = None + + def process(self, files, file_size): + # Get the filename from the already-open file handle and use it as + # the directory name to store the images. + self.dirname = self.fout.name[:-7] + '-images' + + # Remove the original file that was created + self.fout.close() + os.unlink(self.fout.name) + + # Create the directory if it doesn't yet exist + if not os.path.exists(self.dirname): + os.mkdir(self.dirname) + + # Close the existing file as the XlsxWriter library handles that for us + self.fout.close() + + # Loop through all the banners in the data file + for banner in iterate_files(files): + screenshot = get_screenshot(banner) + if screenshot: + filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port']) + + # If a file with the name already exists then count up until we + # create a new, unique filename + counter = 0 + tmpname = filename + while os.path.exists(tmpname + '.jpg'): + tmpname = '{}-{}'.format(filename, counter) + counter += 1 + filename = tmpname + '.jpg' + + fout = open(filename, 'wb') + fout.write(decode(screenshot['data'].encode(), 'base64')) + fout.close() diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py new file mode 100644 index 0000000..9259ddf --- /dev/null +++ b/shodan/cli/converter/kml.py @@ -0,0 +1,106 @@ + +from .base import Converter +from ...helpers import iterate_files + + +class KmlConverter(Converter): + + def header(self): + self.fout.write(""" + + """) + + def footer(self): + self.fout.write("""""") + + def process(self, files, file_size): + # Write the header + self.header() + + hosts = {} + for banner in iterate_files(files): + ip = banner.get('ip_str', banner.get('ipv6', None)) + if not ip: + continue + + if ip not in hosts: + hosts[ip] = banner + hosts[ip]['ports'] = [] + + hosts[ip]['ports'].append(banner['port']) + + for ip, host in iter(hosts.items()): + self.write(host) + + self.footer() + + def write(self, host): + try: + ip = host.get('ip_str', host.get('ipv6', None)) + lat, lon = host['location']['latitude'], host['location']['longitude'] + + placemark = '{}]]>'.format(ip) + placemark += '{0}'.format(host['hostnames'][0]) + + placemark += '

Ports

    ' + + for port in host['ports']: + placemark += """ +
  • {} +
    • + """.format(port) + + placemark += '
' + + placemark += """ + +
powered by Shodan
+ """.format(ip) + + placemark += ']]>' + placemark += '{},{}'.format(lon, lat) + placemark += '' + + self.fout.write(placemark.encode('utf-8')) + except Exception: + pass diff --git a/shodan/cli/data.py b/shodan/cli/data.py new file mode 100644 index 0000000..98d7852 --- /dev/null +++ b/shodan/cli/data.py @@ -0,0 +1,95 @@ +import click +import requests +import shodan +import shodan.helpers as helpers + +from shodan.cli.helpers import get_api_key + + +@click.group() +def data(): + """Bulk data access to Shodan""" + pass + + +@data.command(name='list') +@click.option('--dataset', help='See the available files in the given dataset', default=None, type=str) +def data_list(dataset): + """List available datasets or the files within those datasets.""" + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + if dataset: + # Show the files within this dataset + files = api.data.list_files(dataset) + + for file in files: + click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) + click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + + # Show the SHA1 checksum if available + if file.get('sha1'): + click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False) + + click.echo('{}'.format(file['url'])) + else: + # If no dataset was provided then show a list of all datasets + datasets = api.data.list_datasets() + + for ds in datasets: + click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False) + click.echo('{}'.format(ds['description'])) + + +@data.command(name='download') +@click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int) +@click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.') +@click.argument('dataset', metavar='') +@click.argument('name', metavar='') +def data_download(chunksize, filename, dataset, name): + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + # Get the file object that the user requested which will contain the URL and total file size + file = None + try: + files = api.data.list_files(dataset) + for tmp in files: + if tmp['name'] == name: + file = tmp + break + except shodan.APIError as e: + raise click.ClickException(e.value) + + # The file isn't available + if not file: + raise click.ClickException('File not found') + + # Start downloading the file + response = requests.get(file['url'], stream=True) + + # Figure out the size of the file based on the headers + filesize = response.headers.get('content-length', None) + if not filesize: + # Fall back to using the filesize provided by the API + filesize = file['size'] + else: + filesize = int(filesize) + + chunk_size = 1024 + limit = filesize / chunk_size + + # Create a default filename based on the dataset and the filename within that dataset + if not filename: + filename = '{}-{}'.format(dataset, name) + + # Open the output file and start writing to it in chunks + with open(filename, 'wb') as fout: + with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar: + for chunk in bar: + if chunk: + fout.write(chunk) + + click.echo(click.style('Download completed: {}'.format(filename), 'green')) diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py new file mode 100644 index 0000000..bde2f07 --- /dev/null +++ b/shodan/cli/helpers.py @@ -0,0 +1,129 @@ +''' +Helper methods used across the CLI commands. +''' +import click +import datetime +import gzip +import itertools +import os +import sys +from ipaddress import ip_network, ip_address + +from .settings import SHODAN_CONFIG_DIR + +try: + basestring # Python 2 +except NameError: + basestring = (str, ) # Python 3 + + +def get_api_key(): + '''Returns the API key of the current logged-in user.''' + shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) + keyfile = shodan_dir + '/api_key' + + # If the file doesn't yet exist let the user know that they need to + # initialize the shodan cli + if not os.path.exists(keyfile): + raise click.ClickException('Please run "shodan init " before using this command') + + # Make sure it is a read-only file + if not oct(os.stat(keyfile).st_mode).endswith("600"): + os.chmod(keyfile, 0o600) + + with open(keyfile, 'r') as fin: + return fin.read().strip() + + +def escape_data(args): + # Make sure the string is unicode so the terminal can properly display it + # We do it using format() so it works across Python 2 and 3 + args = u'{}'.format(args) + return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + + +def timestr(): + return datetime.datetime.utcnow().strftime('%Y-%m-%d') + + +def open_streaming_file(directory, timestr, compresslevel=9): + return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel) + + +def get_banner_field(banner, flat_field): + # The provided field is a collapsed form of the actual field + fields = flat_field.split('.') + + try: + current_obj = banner + for field in fields: + current_obj = current_obj[field] + return current_obj + except Exception: + pass + + return None + + +def filter_with_netmask(banner, netmask): + # filtering based on netmask is a more abstract concept than + # a mere check for a specific field and thus needs its own mechanism + # this will enable users to use the net:10.0.0.0/8 syntax they are used to + # to find specific networks from a big shodan download. + network = ip_network(netmask) + ip_field = get_banner_field(banner, 'ip') + if not ip_field: + return False + banner_ip_address = ip_address(ip_field) + return banner_ip_address in network + + +def match_filters(banner, filters): + for args in filters: + flat_field, check = args.split(':', 1) + if flat_field == 'net': + return filter_with_netmask(banner, check) + + value = get_banner_field(banner, flat_field) + + # If the field doesn't exist on the banner then ignore the record + if not value: + return False + + # It must match all filters to be allowed + field_type = type(value) + + # For lists of strings we see whether the desired value is contained in the field + if field_type == list or isinstance(value, basestring): + if check not in value: + return False + elif field_type == int: + if int(check) != value: + return False + elif field_type == float: + if float(check) != value: + return False + else: + # Ignore unknown types + pass + + return True + + +def async_spinner(finished): + spinner = itertools.cycle(['-', '/', '|', '\\']) + while not finished.is_set(): + sys.stdout.write('\b{}'.format(next(spinner))) + sys.stdout.flush() + finished.wait(0.2) + + +def humanize_api_plan(plan): + return { + 'oss': 'Free', + 'dev': 'Membership', + 'basic': 'Freelancer API', + 'plus': 'Small Business API', + 'corp': 'Corporate API', + 'stream-100': 'Enterprise', + }[plan] diff --git a/shodan/cli/host.py b/shodan/cli/host.py new file mode 100644 index 0000000..8ffdeed --- /dev/null +++ b/shodan/cli/host.py @@ -0,0 +1,134 @@ +# Helper methods for printing `host` information to the terminal. +import click + +from shodan.helpers import get_ip + + +def host_print_pretty(host, history=False): + """Show the host information in a user-friendly way and try to include + as much relevant information as possible.""" + # General info + click.echo(click.style(get_ip(host), fg='green')) + if len(host['hostnames']) > 0: + click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) + + if 'city' in host and host['city']: + click.echo(u'{:25s}{}'.format('City:', host['city'])) + + if 'country_name' in host and host['country_name']: + click.echo(u'{:25s}{}'.format('Country:', host['country_name'])) + + if 'os' in host and host['os']: + click.echo(u'{:25s}{}'.format('Operating System:', host['os'])) + + if 'org' in host and host['org']: + click.echo(u'{:25s}{}'.format('Organization:', host['org'])) + + if 'last_update' in host and host['last_update']: + click.echo('{:25s}{}'.format('Updated:', host['last_update'])) + + click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports']))) + + # Output the vulnerabilities the host has + if 'vulns' in host and len(host['vulns']) > 0: + vulns = [] + for vuln in host['vulns']: + if vuln.startswith('!'): + continue + if vuln.upper() == 'CVE-2014-0160': + vulns.append(click.style('Heartbleed', fg='red')) + else: + vulns.append(click.style(vuln, fg='red')) + + if len(vulns) > 0: + click.echo('{:25s}'.format('Vulnerabilities:'), nl=False) + + for vuln in vulns: + click.echo(vuln + '\t', nl=False) + + click.echo('') + + click.echo('') + + # If the user doesn't have access to SSL/ Telnet results then we need + # to pad the host['data'] property with empty banners so they still see + # the port listed as open. (#63) + if len(host['ports']) != len(host['data']): + # Find the ports the user can't see the data for + ports = host['ports'] + for banner in host['data']: + if banner['port'] in ports: + ports.remove(banner['port']) + + # Add the placeholder banners + for port in ports: + banner = { + 'port': port, + 'transport': 'tcp', # All the filtered services use TCP + 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner + 'placeholder': True, # Don't store this banner when the file is saved + } + host['data'].append(banner) + + click.echo('Ports:') + for banner in sorted(host['data'], key=lambda k: k['port']): + product = '' + version = '' + if 'product' in banner and banner['product']: + product = banner['product'] + if 'version' in banner and banner['version']: + version = '({})'.format(banner['version']) + + click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) + if 'transport' in banner: + click.echo('/', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) + click.echo('{} {}'.format(product, version), nl=False) + + if history: + # Format the timestamp to only show the year-month-day + date = banner['timestamp'][:10] + click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) + click.echo('') + + # Show optional HTTP information + if 'http' in banner: + if 'title' in banner['http'] and banner['http']['title']: + click.echo('\t|-- HTTP title: {}'.format(banner['http']['title'])) + + # Show optional ssl info + if 'ssl' in banner: + if 'cert' in banner['ssl'] and banner['ssl']['cert']: + if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']: + issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()]) + click.echo('\t|-- Cert Issuer: {}'.format(issuer)) + if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']: + subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()]) + click.echo('\t|-- Cert Subject: {}'.format(subject)) + if 'versions' in banner['ssl'] and banner['ssl']['versions']: + click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) + if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: + click.echo('\t|-- Diffie-Hellman Parameters:') + click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) + if 'fingerprint' in banner['ssl']['dhparams']: + click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) + + +def host_print_tsv(host, history=False): + """Show the host information in a succinct, grep-friendly manner.""" + for banner in sorted(host['data'], key=lambda k: k['port']): + click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) + click.echo('\t', nl=False) + click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) + + if history: + # Format the timestamp to only show the year-month-day + date = banner['timestamp'][:10] + click.echo(click.style('\t({})'.format(date), fg='white', dim=True), nl=False) + click.echo('') + + +HOST_PRINT = { + 'pretty': host_print_pretty, + 'tsv': host_print_tsv, +} diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py new file mode 100644 index 0000000..5fbb764 --- /dev/null +++ b/shodan/cli/organization.py @@ -0,0 +1,80 @@ +import click +import shodan + +from shodan.cli.helpers import get_api_key, humanize_api_plan + + +@click.group() +def org(): + """Manage your organization's access to Shodan""" + pass + + +@org.command() +@click.option('--silent', help="Don't send a notification to the user", default=False, is_flag=True) +@click.argument('user', metavar='') +def add(silent, user): + """Add a new member""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + api.org.add_member(user, notify=not silent) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully added the new member', fg='green') + + +@org.command() +def info(): + """Show an overview of the organization""" + key = get_api_key() + api = shodan.Shodan(key) + try: + organization = api.org.info() + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(organization['name'], fg='cyan') + click.secho('Access Level: ', nl=False, dim=True) + click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') + + if organization['domains']: + click.secho('Authorized Domains: ', nl=False, dim=True) + click.echo(', '.join(organization['domains'])) + + click.echo('') + click.secho('Administrators:', dim=True) + + for admin in organization['admins']: + click.echo(u' > {:30}\t{:30}'.format( + click.style(admin['username'], fg='yellow'), + admin['email']) + ) + + click.echo('') + if organization['members']: + click.secho('Members:', dim=True) + for member in organization['members']: + click.echo(u' > {:30}\t{:30}'.format( + click.style(member['username'], fg='yellow'), + member['email']) + ) + else: + click.secho('No members yet', dim=True) + + +@org.command() +@click.argument('user', metavar='') +def remove(user): + """Remove and downgrade a member""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + api.org.remove_member(user) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully removed the member', fg='green') diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py new file mode 100644 index 0000000..cfc7aab --- /dev/null +++ b/shodan/cli/scan.py @@ -0,0 +1,343 @@ +import click +import collections +import datetime +import shodan +import shodan.helpers as helpers +import socket +import threading +import time + +from shodan.cli.helpers import get_api_key, async_spinner +from shodan.cli.settings import COLORIZE_FIELDS + + +@click.group() +def scan(): + """Scan an IP/ netblock using Shodan.""" + pass + + +@scan.command(name='list') +def scan_list(): + """Show recently launched scans""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + scans = api.scans() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(scans) > 0: + click.echo(u'# {} Scans Total - Showing 10 most recent scans:'.format(scans['total'])) + click.echo(u'# {:20} {:<15} {:<10} {:<15s}'.format('Scan ID', 'Status', 'Size', 'Timestamp')) + # click.echo('#' * 65) + for scan in scans['matches'][:10]: + click.echo( + u'{:31} {:<24} {:<10} {:<15s}'.format( + click.style(scan['id'], fg='yellow'), + click.style(scan['status'], fg='cyan'), + scan['size'], + scan['created'] + ) + ) + else: + click.echo("You haven't yet launched any scans.") + + +@scan.command(name='internet') +@click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) +@click.argument('port', type=int) +@click.argument('protocol', type=str) +def scan_internet(quiet, port, protocol): + """Scan the Internet for a specific port and protocol using the Shodan infrastructure.""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Submit the request to Shodan + click.echo('Submitting Internet scan to Shodan...', nl=False) + scan = api.scan_internet(port, protocol) + click.echo('Done') + + # If the requested port is part of the regular Shodan crawling, then + # we don't know when the scan is done so lets return immediately and + # let the user decide when to stop waiting for further results. + official_ports = api.ports() + if port in official_ports: + click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.') + else: + # Create the output file + filename = '{0}-{1}.json.gz'.format(port, protocol) + counter = 0 + with helpers.open_file(filename, 'w') as fout: + click.echo('Saving results to file: {0}'.format(filename)) + + # Start listening for results + done = False + + # Keep listening for results until the scan is done + click.echo('Waiting for data, please stand by...') + while not done: + try: + for banner in api.stream.ports([port], timeout=90): + counter += 1 + helpers.write_banner(fout, banner) + + if not quiet: + click.echo('{0:<40} {1:<20} {2}'.format( + click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), + click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), + ';'.join(banner['hostnames'])) + ) + except shodan.APIError: + # We stop waiting for results if the scan has been processed by the crawlers and + # there haven't been new results in a while + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + except socket.timeout: + # We stop waiting for results if the scan has been processed by the crawlers and + # there haven't been new results in a while + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + except Exception as e: + raise click.ClickException(repr(e)) + click.echo('Scan finished: {0} devices found'.format(counter)) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@scan.command(name='protocols') +def scan_protocols(): + """List the protocols that you can scan with using Shodan.""" + key = get_api_key() + api = shodan.Shodan(key) + try: + protocols = api.protocols() + + for name, description in iter(protocols.items()): + click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description) + except shodan.APIError as e: + raise click.ClickException(e.value) + + +@scan.command(name='submit') +@click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int) +@click.option('--filename', help='Save the results in the given file.', default='', type=str) +@click.option('--force', default=False, is_flag=True) +@click.option('--verbose', default=False, is_flag=True) +@click.argument('netblocks', metavar='', nargs=-1) +def scan_submit(wait, filename, force, verbose, netblocks): + """Scan an IP/ netblock using Shodan.""" + key = get_api_key() + api = shodan.Shodan(key) + alert = None + + # Submit the IPs for scanning + try: + # Submit the scan + scan = api.scan(netblocks, force=force) + + now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + + click.echo('') + click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left'])) + + if verbose: + click.echo('# Scan ID: {}'.format(scan['id'])) + + # Return immediately + if wait <= 0: + click.echo('Scan ID: {}'.format(scan['id'])) + click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') + else: + # Setup an alert to wait for responses + alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks) + + # Create the output file if necessary + filename = filename.strip() + fout = None + if filename != '': + # Add the appropriate extension if it's not there atm + if not filename.endswith('.json.gz'): + filename += '.json.gz' + fout = helpers.open_file(filename, 'w') + + # Start a spinner + finished_event = threading.Event() + progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) + progress_bar_thread.start() + + # Now wait a few seconds for items to get returned + hosts = collections.defaultdict(dict) + done = False + scan_start = time.time() + cache = {} + while not done: + try: + for banner in api.stream.alert(aid=alert['id'], timeout=wait): + ip = banner.get('ip', banner.get('ipv6', None)) + if not ip: + continue + + # Don't show duplicate banners + cache_key = '{}:{}'.format(ip, banner['port']) + if cache_key not in cache: + hosts[helpers.get_ip(banner)][banner['port']] = banner + cache[cache_key] = True + + # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on + if time.time() - scan_start >= 60: + scan = api.scan_status(scan['id']) + + if verbose: + click.echo('# Scan status: {}'.format(scan['status'])) + + if scan['status'] == 'DONE': + done = True + break + + except shodan.APIError: + # If the connection timed out before the timeout, that means the streaming server + # that the user tried to reach is down. In that case, lets wait briefly and try + # to connect again! + if (time.time() - scan_start) < wait: + time.sleep(0.5) + continue + + # Exit if the scan was flagged as done somehow + if done: + break + + scan = api.scan_status(scan['id']) + if scan['status'] == 'DONE': + done = True + + if verbose: + click.echo('# Scan status: {}'.format(scan['status'])) + except socket.timeout: + # If the connection timed out before the timeout, that means the streaming server + # that the user tried to reach is down. In that case, lets wait a second and try + # to connect again! + if (time.time() - scan_start) < wait: + continue + + done = True + except Exception as e: + finished_event.set() + progress_bar_thread.join() + raise click.ClickException(repr(e)) + + finished_event.set() + progress_bar_thread.join() + + def print_field(name, value): + click.echo(' {:25s}{}'.format(name, value)) + + def print_banner(banner): + click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False) + + if 'product' in banner: + click.echo(banner['product'], nl=False) + + if 'version' in banner: + click.echo(' ({})'.format(banner['version']), nl=False) + + click.echo('') + + # Show optional ssl info + if 'ssl' in banner: + if 'versions' in banner['ssl']: + # Only print SSL versions if they were successfully tested + versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')] + if len(versions) > 0: + click.echo(' |-- SSL Versions: {}'.format(', '.join(versions))) + if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: + click.echo(' |-- Diffie-Hellman Parameters:') + click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) + if 'fingerprint' in banner['ssl']['dhparams']: + click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) + + if hosts: + # Remove the remaining spinner character + click.echo('\b ') + + for ip in sorted(hosts): + host = next(iter(hosts[ip].items()))[1] + + click.echo(click.style(ip, fg='cyan'), nl=False) + if 'hostnames' in host and host['hostnames']: + click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False) + click.echo('') + + if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']: + print_field('Country', host['location']['country_name']) + + if 'city' in host['location'] and host['location']['city']: + print_field('City', host['location']['city']) + if 'org' in host and host['org']: + print_field('Organization', host['org']) + if 'os' in host and host['os']: + print_field('Operating System', host['os']) + click.echo('') + + # Output the vulnerabilities the host has + if 'vulns' in host and len(host['vulns']) > 0: + vulns = [] + for vuln in host['vulns']: + if vuln.startswith('!'): + continue + if vuln.upper() == 'CVE-2014-0160': + vulns.append(click.style('Heartbleed', fg='red')) + else: + vulns.append(click.style(vuln, fg='red')) + + if len(vulns) > 0: + click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False) + + for vuln in vulns: + click.echo(vuln + '\t', nl=False) + + click.echo('') + + # Print all the open ports: + click.echo(' Open Ports:') + for port in sorted(hosts[ip]): + print_banner(hosts[ip][port]) + + # Save the banner in a file if necessary + if fout: + helpers.write_banner(fout, hosts[ip][port]) + + click.echo('') + else: + # Prepend a \b to remove the spinner + click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.') + except shodan.APIError as e: + raise click.ClickException(e.value) + finally: + # Remove any alert + if alert: + api.delete_alert(alert['id']) + + +@scan.command(name='status') +@click.argument('scan_id', type=str) +def scan_status(scan_id): + """Check the status of an on-demand scan.""" + key = get_api_key() + api = shodan.Shodan(key) + try: + scan = api.scan_status(scan_id) + click.echo(scan['status']) + except shodan.APIError as e: + raise click.ClickException(e.value) diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py new file mode 100644 index 0000000..05c1b9f --- /dev/null +++ b/shodan/cli/settings.py @@ -0,0 +1,16 @@ + +from os import path + +if path.exists(path.expanduser("~/.shodan")): + SHODAN_CONFIG_DIR = '~/.shodan/' +else: + SHODAN_CONFIG_DIR = "~/.config/shodan/" + +COLORIZE_FIELDS = { + 'ip_str': 'green', + 'port': 'yellow', + 'data': 'white', + 'hostnames': 'magenta', + 'org': 'cyan', + 'vulns': 'red', +} diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py new file mode 100755 index 0000000..4e09872 --- /dev/null +++ b/shodan/cli/worldmap.py @@ -0,0 +1,261 @@ +''' +F-Secure Virus World Map console edition + +See README.md for more details + +Copyright 2012-2013 Jyrki Muukkonen + +Released under the MIT license. +See LICENSE.txt or http://www.opensource.org/licenses/mit-license.php + +ASCII map in map-world-01.txt is copyright: + "Map 1998 Matthew Thomas. Freely usable as long as this line is included" + +''' +import curses +import locale +import random +import time + +from shodan.exception import APIError +from shodan.helpers import get_ip + + +MAPS = { + 'world': { + # offset (as (y, x) for curses...) + 'corners': (1, 4, 23, 73), + # lat top, lon left, lat bottom, lon right + 'coords': [90.0, -180.0, -90.0, 180.0], + + # PyLint freaks out about the world map backslashes so ignore those warnings + 'data': r''' + . _..::__: ,-"-"._ |7 , _,.__ + _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ + .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ + \_.:--. `._ )`^-. "' , [_/( __,/-' + '"' \ " _L oD_,--' ) /. (| + | ,' _)_.\\._<> 6 _,' / ' + `. / [_/_'` `"( <'} ) + \\ .-. ) / `-'"..' `:._ _) ' + ` \ ( `( / `:\ > \ ,-^. /' ' + `._, "" | \`' \| ?_) {\ + `=.---. `._._ ,' "` |' ,- '. + | `-._ | / `:`<_|h--._ + ( > . | , `=.__.`-'\ + `. / | |{| ,-.,\ . + | ,' \ / `' ," \ + | / |_' | __ / + | | '-' `-' \. + |/ " / + \. ' + + ,/ ______._.--._ _..---.---------._ + ,-----"-..?----_/ ) _,-'" " ( + Map 1998 Matthew Thomas. Freely usable as long as this line is included +''' + } +} + + +class AsciiMap(object): + """ + Helper class for handling map drawing and coordinate calculations + """ + def __init__(self, map_name='world', map_conf=None, window=None, encoding=None): + if map_conf is None: + map_conf = MAPS[map_name] + self.map = map_conf['data'] + self.coords = map_conf['coords'] + self.corners = map_conf['corners'] + if window is None: + window = curses.newwin(0, 0) + self.window = window + + self.data = [] + self.data_timestamp = None + + # JSON contents _should_ be UTF8 (so, python internal unicode here...) + if encoding is None: + encoding = locale.getpreferredencoding() + self.encoding = encoding + + # check if we can use transparent background or not + if curses.can_change_color(): + curses.use_default_colors() + background = -1 + else: + background = curses.COLOR_BLACK + + tmp_colors = [ + ('red', curses.COLOR_RED, background), + ('blue', curses.COLOR_BLUE, background), + ('pink', curses.COLOR_MAGENTA, background) + ] + + self.colors = {} + if curses.has_colors(): + for i, (name, fgcolor, bgcolor) in enumerate(tmp_colors, 1): + curses.init_pair(i, fgcolor, bgcolor) + self.colors[name] = i + + def latlon_to_coords(self, lat, lon): + """ + Convert lat/lon coordinates to character positions. + Very naive version, assumes that we are drawing the whole world + TODO: filter out stuff that doesn't fit + TODO: make it possible to use "zoomed" maps + """ + width = (self.corners[3] - self.corners[1]) + height = (self.corners[2] - self.corners[0]) + + # change to 0-180, 0-360 + abs_lat = -lat + 90 + abs_lon = lon + 180 + x = (abs_lon / 360.0) * width + self.corners[1] + y = (abs_lat / 180.0) * height + self.corners[0] + return int(x), int(y) + + def set_data(self, data): + """ + Set / convert internal data. + For now it just selects a random set to show. + """ + entries = [] + + # Grab 5 random banners to display + for banner in random.sample(data, min(len(data), 5)): + desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code']) + if banner['location']['city']: + # Not all cities can be encoded in ASCII so ignore any errors + try: + desc += ' {}'.format(banner['location']['city']) + except Exception: + pass + + if 'tags' in banner and banner['tags']: + desc += ' / {}'.format(','.join(banner['tags'])) + + entry = ( + float(banner['location']['latitude']), + float(banner['location']['longitude']), + '*', + desc, + curses.A_BOLD, + 'red', + ) + entries.append(entry) + self.data = entries + + def draw(self, target): + """ Draw internal data to curses window """ + self.window.clear() + self.window.addstr(0, 0, self.map) + + # FIXME: position to be defined in map config? + row = self.corners[2] - 6 + items_to_show = 5 + for lat, lon, char, desc, attrs, color in self.data: + # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html + if desc: + desc = desc.encode(self.encoding, 'ignore').decode() + if items_to_show <= 0: + break + char_x, char_y = self.latlon_to_coords(lat, lon) + if self.colors and color: + attrs |= curses.color_pair(self.colors[color]) + self.window.addstr(char_y, char_x, char, attrs) + if desc: + det_show = "{} {}".format(char, desc) + else: + det_show = None + + if det_show is not None: + try: + self.window.addstr(row, 1, det_show, attrs) + row += 1 + items_to_show -= 1 + except Exception: + # FIXME: check window size before addstr() + break + self.window.overwrite(target) + self.window.leaveok(True) + + +class MapApp(object): + """ Virus World Map ncurses application """ + def __init__(self, api): + self.api = api + self.data = None + self.last_fetch = 0 + self.sleep = 10 # tenths of seconds, for curses.halfdelay() + self.polling_interval = 60 + + def fetch_data(self, epoch_now, force_refresh=False): + """ (Re)fetch data from JSON stream """ + refresh = False + if force_refresh or self.data is None: + refresh = True + else: + if self.last_fetch + self.polling_interval <= epoch_now: + refresh = True + + if refresh: + try: + # Grab 20 banners from the main stream + banners = [] + for banner in self.api.stream.banners(): + if 'location' in banner and banner['location']['latitude']: + banners.append(banner) + if len(banners) >= 20: + break + self.data = banners + self.last_fetch = epoch_now + except APIError: + raise + return refresh + + def run(self, scr): + """ Initialize and run the application """ + m = AsciiMap() + curses.halfdelay(self.sleep) + while True: + now = int(time.time()) + refresh = self.fetch_data(now) + m.set_data(self.data) + try: + m.draw(scr) + except curses.error: + raise Exception('Terminal window too small') + scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD) + scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD) + + # Key Input + # q - Quit + event = scr.getch() + if event == ord('q'): + break + + # redraw window (to fix encoding/rendering bugs and to hide other messages to same tty) + # user pressed 'r' or new data was fetched + if refresh: + m.window.redrawwin() + + +def launch_map(api): + app = MapApp(api) + return curses.wrapper(app.run) + + +def main(argv=None): + """ Main function / entry point """ + from shodan import Shodan + from shodan.cli.helpers import get_api_key + + api = Shodan(get_api_key()) + return launch_map(api) + + +if __name__ == '__main__': + import sys + sys.exit(main()) diff --git a/shodan/client.py b/shodan/client.py index 0eb8f65..21c70af 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -1,423 +1,805 @@ -# -*- coding: utf-8 -*- - -""" -shodan.client -~~~~~~~~~~~~~ - -This module implements the Shodan API. - -:copyright: (c) 2014 by John Matherly -""" - -import requests -import simplejson -import time - - -class APIError(Exception): - """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - -class Shodan: - """Wrapper around the Shodan REST and Streaming APIs - - :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) - :type key: str - :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. - :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. - """ - - class Tools: - - def __init__(self, parent): - self.parent = parent - - def myip(self): - """Get your current IP address as seen from the Internet. - - :returns: str -- your IP address - """ - return self.parent._request('/tools/myip', {}) - - class Exploits: - - def __init__(self, parent): - self.parent = parent - - def search(self, query, page=1, facets=None): - """Search the entire Shodan Exploits archive using the same query syntax - as the website. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :param page: The page number to access. - :type page: int - :returns: dict -- a dictionary containing the results of the search. - """ - query_args = { - 'query': query, - 'page': page, - } - if facets: - facet_str = self.parent._create_facet_string(facets) - query_args['facets'] = facet_str - - return self.parent._request('/api/search', query_args, service='exploits') - - def count(self, query, facets=None): - """Search the entire Shodan Exploits archive but only return the total # of results, - not the actual exploits. - - :param query: The exploit search query; same syntax as website. - :type query: str - :param facets: A list of strings or tuples to get summary information on. - :type facets: str - :returns: dict -- a dictionary containing the results of the search. - - """ - query_args = { - 'query': query, - } - if facets: - facet_str = self.parent._create_facet_string(facets) - query_args['facets'] = facet_str - - return self.parent._request('/api/count', query_args, service='exploits') - - class Stream: - - base_url = 'https://stream.shodan.io' - - def __init__(self, parent): - self.parent = parent - - def _create_stream(self, name): - try: - req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True) - except: - raise APIError('Unable to contact the Shodan Streaming API') - - if req.status_code != 200: - try: - raise APIError(data.json()['error']) - except: - pass - raise APIError('Invalid API key or you do not have access to the Streaming API') - return req - - def banners(self): - """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to - API subscription plans and for those it only returns a fraction of the data. - """ - stream = self._create_stream('/shodan/banners') - for line in stream.iter_lines(): - if line: - banner = simplejson.loads(line) - yield banner - - def ports(self, ports): - """ - A filtered version of the "banners" stream to only return banners that match the ports of interest. - - :param ports: A list of ports to return banner data on. - :type ports: int[] - """ - stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports])) - for line in stream.iter_lines(): - if line: - banner = simplejson.loads(line) - yield banner - - def geo(self): - """ - A stream of geolocation information for the banners. This is a stripped-down version of the "banners" stream - in case you only care about the geolocation information. - """ - stream = self._create_stream('/shodan/geo') - for line in stream.iter_lines(): - if line: - banner = simplejson.loads(line) - yield banner - - def __init__(self, key): - """Initializes the API object. - - :param key: The Shodan API key. - :type key: str - """ - self.api_key = key - self.base_url = 'https://api.shodan.io' - self.base_exploits_url = 'https://exploits.shodan.io' - self.exploits = self.Exploits(self) - self.tools = self.Tools(self) - self.stream = self.Stream(self) - - def _request(self, function, params, service='shodan', method='get'): - """General-purpose function to create web requests to SHODAN. - - Arguments: - function -- name of the function you want to execute - params -- dictionary of parameters for the function - - Returns - A dictionary containing the function's results. - - """ - # Add the API key parameter automatically - params['key'] = self.api_key - - # Determine the base_url based on which service we're interacting with - base_url = { - 'shodan': self.base_url, - 'exploits': self.base_exploits_url, - }.get(service, 'shodan') - - # Send the request - try: - if method.lower() == 'post': - data = requests.post(base_url + function, params) - else: - data = requests.get(base_url + function, params=params) - except: - raise APIError('Unable to connect to Shodan') - - # Check that the API key wasn't rejected - if data.status_code == 401: - try: - raise APIError(data.json()['error']) - except: - pass - raise APIError('Invalid API key') - - # Parse the text into JSON - try: - data = data.json() - except: - raise APIError('Unable to parse JSON response') - - # Raise an exception if an error occurred - if type(data) == dict and data.get('error', None): - raise APIError(data['error']) - - # Return the data - return data - - def _create_facet_string(self, facets): - """Converts a Python list of facets into a comma-separated string that can be understood by - the Shodan API. - """ - facet_str = '' - for facet in facets: - if isinstance(facet, basestring): - facet_str += facet - else: - facet_str += '%s:%s' % (facet[0], facet[1]) - facet_str += ',' - return facet_str[:-1] - - def count(self, query, facets=None): - """Returns the total number of search results for the query. - - :param query: Search query; identical syntax to the website - :type query: str - :param facets: (optional) A list of properties to get summary information on - :type facets: str - - :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - query_args = { - 'query': query, - } - if facets: - facet_str = self._create_facet_string(facets) - query_args['facets'] = facet_str - return self._request('/shodan/host/count', query_args) - - def host(self, ip, history=False): - """Get all available information on an IP. - - :param ip: IP of the computer - :type ip: str - :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. - :type history: bool - """ - params = {} - if history: - params['history'] = history - return self._request('/shodan/host/%s' % ip, params) - - def info(self): - """Returns information about the current API key, such as a list of add-ons - and other features that are enabled for the current user's API plan. - """ - return self._request('/api-info', {}) - - def scan(self, ips): - """Scan a network using Shodan - - :param ips: A list of IPs or netblocks in CIDR notation - :type ips: str - - :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. - """ - if isinstance(ips, basestring): - ips = [ips] - - params = { - 'ips': ','.join(ips), - } - - return self._request('/shodan/scan', params, method='post') - - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): - """Search the SHODAN database. - - :param query: Search query; identical syntax to the website - :type query: str - :param page: (optional) Page number of the search results - :type page: int - :param limit: (optional) Number of results to return - :type limit: int - :param offset: (optional) Search offset to begin getting results from - :type offset: int - :param facets: (optional) A list of properties to get summary information on - :type facets: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - - :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. - """ - args = { - 'query': query, - 'minify': minify, - } - if limit: - args['limit'] = limit - if offset: - args['offset'] = offset - else: - args['page'] = page - - if facets: - facet_str = self._create_facet_string(facets) - args['facets'] = facet_str - - return self._request('/shodan/host/search', args) - - def search_cursor(self, query, minify=True, retries=5): - """Search the SHODAN database. - - This method returns an iterator that can directly be in a loop. Use it when you want to loop over - all of the results of a search query. But this method doesn't return a "matches" array or the "total" - information. And it also can't be used with facets, it's only use is to iterate over results more - easily. - - :param query: Search query; identical syntax to the website - :type query: str - :param minify: (optional) Whether to minify the banner and only return the important data - :type minify: bool - :param retries: (optional) How often to retry the search in case it times out - :type minify: int - - :returns: A search cursor that can be used as an iterator/ generator. - """ - args = { - 'query': query, - 'minify': minify, - } - - page = 1 - tries = 0 - while page == 1 or results['matches']: - try: - results = self.search(query, minify=minify, page=page) - for banner in results['matches']: - try: - yield banner - except GeneratorExit: - return # exit out of the function - page += 1 - tries = 0 - except: - # We've retried several times but it keeps failing, so lets error out - if tries >= retries: - break - - tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason - - def search_tokens(self, query): - """Returns information about the search query itself (filters used etc.) - - :param query: Search query; identical syntax to the website - :type query: str - - :returns: A dictionary with 4 main properties: filters, errors, attributes and string. - """ - query_args = { - 'query': query, - } - return self._request('/shodan/host/search/tokens', query_args) - - def services(self): - """Get a list of services that Shodan crawls - - :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. - """ - return self._request('/shodan/services', {}) - - def queries(self, page=1, sort='timestamp', order='desc'): - """List the search queries that have been shared by other users. - - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - :param sort: Sort the list based on a property. Possible values are: votes, timestamp - :type sort: str - :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc - :type order: str - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'sort': sort, - 'order': order, - } - return self._request('/shodan/query', args) - - def queries_search(self, query, page=1): - """Search the directory of saved search queries in Shodan. - - :param query: The search string to look for in the search query - :type query: str - :param page: Page number to iterate over results; each page contains 10 items - :type page: int - - :returns: A list of saved search queries (dictionaries). - """ - args = { - 'page': page, - 'query': query, - } - return self._request('/shodan/query/search', args) - - def queries_tags(self, size=10): - """Search the directory of saved search queries in Shodan. - - :param query: The number of tags to return - :type page: int - - :returns: A list of tags. - """ - args = { - 'size': size, - } - return self._request('/shodan/query/tags', args) - +# -*- coding: utf-8 -*- +""" +shodan.client +~~~~~~~~~~~~~ + +This module implements the Shodan API. + +:copyright: (c) 2014- by John Matherly +""" +import math +import os +import time + +import requests +import json + +from .exception import APIError +from .helpers import create_facet_string +from .stream import Stream + + +# Try to disable the SSL warnings in urllib3 since not everybody can install +# C extensions. If you're able to install C extensions you can try to run: +# +# pip install requests[security] +# +# Which will download libraries that offer more full-featured SSL classes +# pylint: disable=E1101 +try: + requests.packages.urllib3.disable_warnings() +except Exception: + pass + +# Define a basestring type if necessary for Python3 compatibility +try: + basestring +except NameError: + basestring = str + + +class Shodan: + """Wrapper around the Shodan REST and Streaming APIs + + :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) + :type key: str + :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. + :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. + """ + + class Data: + + def __init__(self, parent): + self.parent = parent + + def list_datasets(self): + """Returns a list of datasets that the user has permission to download. + + :returns: A list of objects where every object describes a dataset + """ + return self.parent._request('/shodan/data', {}) + + def list_files(self, dataset): + """Returns a list of files that belong to the given dataset. + + :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' + """ + return self.parent._request('/shodan/data/{}'.format(dataset), {}) + + class Dns: + + def __init__(self, parent): + self.parent = parent + + def domain_info(self, domain, history=False, type=None, page=1): + """Grab the DNS information for a domain. + """ + args = { + 'page': page, + } + if history: + args['history'] = history + if type: + args['type'] = type + return self.parent._request('/dns/domain/{}'.format(domain), args) + + class Notifier: + + def __init__(self, parent): + self.parent = parent + + def create(self, provider, args, description=None): + """Get the settings for the specified notifier that a user has configured. + + :param provider: Provider name + :type provider: str + :param args: Provider arguments + :type args: dict + :param description: Human-friendly description of the notifier + :type description: str + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + args['provider'] = provider + + if description: + args['description'] = description + + return self.parent._request('/notifier', args, method='post') + + def edit(self, nid, args): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :param args: Provider arguments + :type args: dict + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + return self.parent._request('/notifier/{}'.format(nid), args, method='put') + + def get(self, nid): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- object describing the notifier settings + """ + return self.parent._request('/notifier/{}'.format(nid), {}) + + def list_notifiers(self): + """Returns a list of notifiers that the user has added. + + :returns: A list of notifierse that are available on the account + """ + return self.parent._request('/notifier', {}) + + def list_providers(self): + """Returns a list of supported notification providers. + + :returns: A list of providers where each object describes a provider + """ + return self.parent._request('/notifier/provider', {}) + + def remove(self, nid): + """Delete the provided notifier. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- 'success' set to True if action succeeded + """ + return self.parent._request('/notifier/{}'.format(nid), {}, method='delete') + + class Tools: + + def __init__(self, parent): + self.parent = parent + + def myip(self): + """Get your current IP address as seen from the Internet. + + :returns: str -- your IP address + """ + return self.parent._request('/tools/myip', {}) + + class Exploits: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, page=1, facets=None): + """Search the entire Shodan Exploits archive using the same query syntax + as the website. + + :param query: The exploit search query; same syntax as website. + :type query: str + :param facets: A list of strings or tuples to get summary information on. + :type facets: str + :param page: The page number to access. + :type page: int + :returns: dict -- a dictionary containing the results of the search. + """ + query_args = { + 'query': query, + 'page': page, + } + if facets: + query_args['facets'] = create_facet_string(facets) + + return self.parent._request('/api/search', query_args, service='exploits') + + def count(self, query, facets=None): + """Search the entire Shodan Exploits archive but only return the total # of results, + not the actual exploits. + + :param query: The exploit search query; same syntax as website. + :type query: str + :param facets: A list of strings or tuples to get summary information on. + :type facets: str + :returns: dict -- a dictionary containing the results of the search. + + """ + query_args = { + 'query': query, + } + if facets: + query_args['facets'] = create_facet_string(facets) + + return self.parent._request('/api/count', query_args, service='exploits') + + class Labs: + + def __init__(self, parent): + self.parent = parent + + def honeyscore(self, ip): + """Calculate the probability of an IP being an ICS honeypot. + + :param ip: IP address of the device + :type ip: str + + :returns: int -- honeyscore ranging from 0.0 to 1.0 + """ + return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) + + class Organization: + + def __init__(self, parent): + self.parent = parent + + def add_member(self, user, notify=True): + """Add the user to the organization. + + :param user: username or email address + :type user: str + :param notify: whether or not to send the user an email notification + :type notify: bool + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), { + 'notify': notify, + }, method='PUT')['success'] + + def info(self): + """Returns general information about the organization the current user is a member of. + """ + return self.parent._request('/org', {}) + + def remove_member(self, user): + """Remove the user from the organization. + + :param user: username or email address + :type user: str + + :returns: True if it succeeded and raises an Exception otherwise + """ + return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + + class Trends: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, facets): + """Search the Shodan historical database. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'facets': create_facet_string(facets), + } + + return self.parent._request('/api/v1/search', args, service='trends') + + def search_facets(self): + """This method returns a list of facets that can be used to get a breakdown of the top values for a property. + + :returns: A list of strings where each is a facet name + """ + return self.parent._request('/api/v1/search/facets', {}, service='trends') + + def search_filters(self): + """This method returns a list of search filters that can be used in the search query. + + :returns: A list of strings where each is a filter name + """ + return self.parent._request('/api/v1/search/filters', {}, service='trends') + + def __init__(self, key, proxies=None): + """Initializes the API object. + + :param key: The Shodan API key. + :type key: str + :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} + :type proxies: dict + """ + self.api_key = key + self.base_url = 'https://api.shodan.io' + self.base_exploits_url = 'https://exploits.shodan.io' + self.base_trends_url = 'https://trends.shodan.io' + self.data = self.Data(self) + self.dns = self.Dns(self) + self.exploits = self.Exploits(self) + self.trends = self.Trends(self) + self.labs = self.Labs(self) + self.notifier = self.Notifier(self) + self.org = self.Organization(self) + self.tools = self.Tools(self) + self.stream = Stream(key, proxies=proxies) + self._session = requests.Session() + self.api_rate_limit = 1 # Requests per second + self._api_query_time = None + + if proxies: + self._session.proxies.update(proxies) + self._session.trust_env = False + + if os.environ.get('SHODAN_API_URL'): + self.base_url = os.environ.get('SHODAN_API_URL') + + def _request(self, function, params, service='shodan', method='get', json_data=None): + """General-purpose function to create web requests to SHODAN. + + Arguments: + function -- name of the function you want to execute + params -- dictionary of parameters for the function + + Returns + A dictionary containing the function's results. + + """ + # Add the API key parameter automatically + params['key'] = self.api_key + + # Determine the base_url based on which service we're interacting with + base_url = { + 'shodan': self.base_url, + 'exploits': self.base_exploits_url, + 'trends': self.base_trends_url, + }.get(service, 'shodan') + + # Wait for API rate limit + if self._api_query_time is not None and self.api_rate_limit > 0: + while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time(): + time.sleep(0.1 / self.api_rate_limit) + + # Send the request + try: + method = method.lower() + if method == 'post': + if json_data: + data = self._session.post(base_url + function, params=params, + data=json.dumps(json_data), + headers={'content-type': 'application/json'}, + ) + else: + data = self._session.post(base_url + function, params) + elif method == 'put': + data = self._session.put(base_url + function, params=params) + elif method == 'delete': + data = self._session.delete(base_url + function, params=params) + else: + data = self._session.get(base_url + function, params=params) + self._api_query_time = time.time() + except Exception: + raise APIError('Unable to connect to Shodan') + + # Check that the API key wasn't rejected + if data.status_code == 401: + try: + # Return the actual error message if the API returned valid JSON + error = data.json()['error'] + except Exception as e: + # If the response looks like HTML then it's probably the 401 page that nginx returns + # for 401 responses by default + if data.text.startswith('<'): + error = 'Invalid API key' + else: + # Otherwise lets raise the error message + error = u'{}'.format(e) + + raise APIError(error) + elif data.status_code == 403: + raise APIError('Access denied (403 Forbidden)') + elif data.status_code == 502: + raise APIError('Bad Gateway (502)') + + # Parse the text into JSON + try: + data = data.json() + except ValueError: + raise APIError('Unable to parse JSON response') + + # Raise an exception if an error occurred + if type(data) == dict and 'error' in data: + raise APIError(data['error']) + + # Return the data + return data + + def count(self, query, facets=None): + """Returns the total number of search results for the query. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. + """ + query_args = { + 'query': query, + } + if facets: + query_args['facets'] = create_facet_string(facets) + return self._request('/shodan/host/count', query_args) + + def host(self, ips, history=False, minify=False): + """Get all available information on an IP. + + :param ip: IP of the computer + :type ip: str + :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. + :type history: bool + :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise. + :type minify: bool + """ + if isinstance(ips, basestring): + ips = [ips] + + params = {} + if history: + params['history'] = history + if minify: + params['minify'] = minify + return self._request('/shodan/host/{}'.format(','.join(ips)), params) + + def info(self): + """Returns information about the current API key, such as a list of add-ons + and other features that are enabled for the current user's API plan. + """ + return self._request('/api-info', {}) + + def ports(self): + """Get a list of ports that Shodan crawls + + :returns: An array containing the ports that Shodan crawls for. + """ + return self._request('/shodan/ports', {}) + + def protocols(self): + """Get a list of protocols that the Shodan on-demand scanning API supports. + + :returns: A dictionary containing the protocol name and description. + """ + return self._request('/shodan/protocols', {}) + + def scan(self, ips, force=False): + """Scan a network using Shodan + + :param ips: A list of IPs or netblocks in CIDR notation or an object structured like: + { + "9.9.9.9": [ + (443, "https"), + (8080, "http") + ], + "1.1.1.0/24": [ + (503, "modbus") + ] + } + :type ips: str or dict + :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users. + :type force: bool + + :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. + """ + if isinstance(ips, basestring): + ips = [ips] + + if isinstance(ips, dict): + networks = json.dumps(ips) + else: + networks = ','.join(ips) + + params = { + 'ips': networks, + 'force': force, + } + + return self._request('/shodan/scan', params, method='post') + + def scans(self, page=1): + """Get a list of scans submitted + + :param page: Page through the list of scans 100 results at a time + :type page: int + """ + return self._request('/shodan/scans', { + 'page': page, + }) + + def scan_internet(self, port, protocol): + """Scan a network using Shodan + + :param port: The port that should get scanned. + :type port: int + :param port: The name of the protocol as returned by the protocols() method. + :type port: str + + :returns: A dictionary with a unique ID to check on the scan progress. + """ + params = { + 'port': port, + 'protocol': protocol, + } + + return self._request('/shodan/scan/internet', params, method='post') + + def scan_status(self, scan_id): + """Get the status information about a previously submitted scan. + + :param id: The unique ID for the scan that was submitted + :type id: str + + :returns: A dictionary with general information about the scan, including its status in getting processed. + """ + return self._request('/shodan/scan/{}'.format(scan_id), {}) + + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None): + """Search the SHODAN database. + + :param query: Search query; identical syntax to the website + :type query: str + :param page: (optional) Page number of the search results + :type page: int + :param limit: (optional) Number of results to return + :type limit: int + :param offset: (optional) Search offset to begin getting results from + :type offset: int + :param facets: (optional) A list of properties to get summary information on + :type facets: str + :param minify: (optional) Whether to minify the banner and only return the important data + :type minify: bool + :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter + :type fields: str + + :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'minify': minify, + } + if limit: + args['limit'] = limit + if offset: + args['offset'] = offset + else: + args['page'] = page + + if facets: + args['facets'] = create_facet_string(facets) + + if fields and isinstance(fields, list): + args['fields'] = ','.join(fields) + + return self._request('/shodan/host/search', args) + + def search_cursor(self, query, minify=True, retries=5, fields=None): + """Search the SHODAN database. + + This method returns an iterator that can directly be in a loop. Use it when you want to loop over + all of the results of a search query. But this method doesn't return a "matches" array or the "total" + information. And it also can't be used with facets, it's only use is to iterate over results more + easily. + + :param query: Search query; identical syntax to the website + :type query: str + :param minify: (optional) Whether to minify the banner and only return the important data + :type minify: bool + :param retries: (optional) How often to retry the search in case it times out + :type retries: int + + :returns: A search cursor that can be used as an iterator/ generator. + """ + page = 1 + total_pages = 0 + tries = 0 + + # Grab the initial page and use the total to calculate the expected number of pages + results = self.search(query, minify=minify, page=page, fields=fields) + if results['total']: + total_pages = int(math.ceil(results['total'] / 100)) + + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 + + # Keep iterating over the results from page 2 onwards + while page <= total_pages: + try: + results = self.search(query, minify=minify, page=page, fields=fields) + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 + tries = 0 + except Exception: + # We've retried several times but it keeps failing, so lets error out + if tries >= retries: + raise APIError('Retry limit reached ({:d})'.format(retries)) + + tries += 1 + time.sleep(tries) # wait (1 second * retry number) if the search errored out for some reason + + def search_facets(self): + """Returns a list of search facets that can be used to get aggregate information about a search query. + + :returns: A list of strings where each is a facet name + """ + return self._request('/shodan/host/search/facets', {}) + + def search_filters(self): + """Returns a list of search filters that are available. + + :returns: A list of strings where each is a filter name + """ + return self._request('/shodan/host/search/filters', {}) + + def search_tokens(self, query): + """Returns information about the search query itself (filters used etc.) + + :param query: Search query; identical syntax to the website + :type query: str + + :returns: A dictionary with 4 main properties: filters, errors, attributes and string. + """ + query_args = { + 'query': query, + } + return self._request('/shodan/host/search/tokens', query_args) + + def services(self): + """Get a list of services that Shodan crawls + + :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. + """ + return self._request('/shodan/services', {}) + + def queries(self, page=1, sort='timestamp', order='desc'): + """List the search queries that have been shared by other users. + + :param page: Page number to iterate over results; each page contains 10 items + :type page: int + :param sort: Sort the list based on a property. Possible values are: votes, timestamp + :type sort: str + :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc + :type order: str + + :returns: A list of saved search queries (dictionaries). + """ + args = { + 'page': page, + 'sort': sort, + 'order': order, + } + return self._request('/shodan/query', args) + + def queries_search(self, query, page=1): + """Search the directory of saved search queries in Shodan. + + :param query: The search string to look for in the search query + :type query: str + :param page: Page number to iterate over results; each page contains 10 items + :type page: int + + :returns: A list of saved search queries (dictionaries). + """ + args = { + 'page': page, + 'query': query, + } + return self._request('/shodan/query/search', args) + + def queries_tags(self, size=10): + """Search the directory of saved search queries in Shodan. + + :param size: The number of tags to return + :type size: int + + :returns: A list of tags. + """ + args = { + 'size': size, + } + return self._request('/shodan/query/tags', args) + + def create_alert(self, name, ip, expires=0): + """Create a network alert/ private firehose for the specified IP range(s) + + :param name: Name of the alert + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str + + :returns: A dict describing the alert + """ + data = { + 'name': name, + 'filters': { + 'ip': ip, + }, + 'expires': expires, + } + + response = self._request('/shodan/alert', params={}, json_data=data, method='post') + + return response + + def edit_alert(self, aid, ip): + """Edit the IPs that should be monitored by the alert. + + :param aid: Alert ID + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str + + :returns: A dict describing the alert + """ + data = { + 'filters': { + 'ip': ip, + }, + } + + response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post') + + return response + + def alerts(self, aid=None, include_expired=True): + """List all of the active alerts that the user created.""" + if aid: + func = '/shodan/alert/{}/info'.format(aid) + else: + func = '/shodan/alert/info' + + response = self._request(func, params={ + 'include_expired': include_expired, + }) + + return response + + def delete_alert(self, aid): + """Delete the alert with the given ID.""" + func = '/shodan/alert/{}'.format(aid) + + response = self._request(func, params={}, method='delete') + + return response + + def alert_triggers(self): + """Return a list of available triggers that can be enabled for alerts. + + :returns: A list of triggers + """ + return self._request('/shodan/alert/triggers', {}) + + def enable_alert_trigger(self, aid, trigger): + """Enable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='put') + + def disable_alert_trigger(self, aid, trigger): + """Disable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') + + def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): + """Ignore trigger notifications for the provided IP and port.""" + # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities + # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use + # a different API endpoint. + if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') + + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') + + def unignore_alert_trigger_notification(self, aid, trigger, ip, port): + """Re-enable trigger notifications for the provided IP and port""" + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') + + def add_alert_notifier(self, aid, nid): + """Enable the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') + + def remove_alert_notifier(self, aid, nid): + """Remove the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') diff --git a/shodan/exception.py b/shodan/exception.py new file mode 100644 index 0000000..75b158e --- /dev/null +++ b/shodan/exception.py @@ -0,0 +1,11 @@ +class APIError(Exception): + """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value + + +class APITimeout(APIError): + pass diff --git a/shodan/helpers.py b/shodan/helpers.py new file mode 100644 index 0000000..563ecb4 --- /dev/null +++ b/shodan/helpers.py @@ -0,0 +1,178 @@ +import gzip +import requests +import json + +from .exception import APIError + +try: + basestring +except NameError: + basestring = str + + +def create_facet_string(facets): + """Converts a Python list of facets into a comma-separated string that can be understood by + the Shodan API. + """ + facet_str = '' + for facet in facets: + if isinstance(facet, basestring): + facet_str += facet + else: + facet_str += '{}:{}'.format(facet[0], facet[1]) + facet_str += ',' + return facet_str[:-1] + + +def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', + method='get', retries=1, proxies=None): + """General-purpose function to create web requests to SHODAN. + + Arguments: + function -- name of the function you want to execute + params -- dictionary of parameters for the function + proxies -- a proxies array for the requests library + + Returns + A dictionary containing the function's results. + + """ + # Add the API key parameter automatically + params['key'] = key + + # Send the request + tries = 0 + error = False + while tries <= retries: + try: + if method.lower() == 'post': + data = requests.post(base_url + function, json.dumps(data), params=params, + headers={'content-type': 'application/json'}, + proxies=proxies) + elif method.lower() == 'delete': + data = requests.delete(base_url + function, params=params, proxies=proxies) + elif method.lower() == 'put': + data = requests.put(base_url + function, params=params, proxies=proxies) + else: + data = requests.get(base_url + function, params=params, proxies=proxies) + + # Exit out of the loop + break + except Exception: + error = True + tries += 1 + + if error and tries >= retries: + raise APIError('Unable to connect to Shodan') + + # Check that the API key wasn't rejected + if data.status_code == 401: + try: + raise APIError(data.json()['error']) + except (ValueError, KeyError): + pass + raise APIError('Invalid API key') + + # Parse the text into JSON + try: + data = data.json() + except Exception: + raise APIError('Unable to parse JSON response') + + # Raise an exception if an error occurred + if type(data) == dict and data.get('error', None): + raise APIError(data['error']) + + # Return the data + return data + + +def iterate_files(files, fast=False): + """Loop over all the records of the provided Shodan output file(s).""" + loads = json.loads + if fast: + # Try to use ujson for parsing JSON if it's available and the user requested faster throughput + # It's significantly faster at encoding/ decoding JSON but it doesn't support as + # many options as the standard library. As such, we're mostly interested in using it for + # decoding since reading/ parsing files will use up the most time. + # pylint: disable=E0401 + try: + from ujson import loads + except Exception: + pass + + if isinstance(files, basestring): + files = [files] + + for filename in files: + # Create a file handle depending on the filetype + if filename.endswith('.gz'): + fin = gzip.open(filename, 'r') + else: + fin = open(filename, 'r') + + for line in fin: + # Ensure the line has been decoded into a string to prevent errors w/ Python3 + if not isinstance(line, basestring): + line = line.decode('utf-8') + + # Convert the JSON into a native Python object + banner = loads(line) + yield banner + + +def get_screenshot(banner): + if 'screenshot' in banner and banner['screenshot']: + return banner['screenshot'] + elif 'opts' in banner and 'screenshot' in banner['opts']: + return banner['opts']['screenshot'] + + return None + + +def get_ip(banner): + if 'ipv6' in banner: + return banner['ipv6'] + return banner['ip_str'] + + +def open_file(filename, mode='a', compresslevel=9): + return gzip.open(filename, mode, compresslevel) + + +def write_banner(fout, banner): + line = json.dumps(banner) + '\n' + fout.write(line.encode('utf-8')) + + +def humanize_bytes(byte_count, precision=1): + """Return a humanized string representation of a number of bytes. + >>> humanize_bytes(1) + '1 byte' + >>> humanize_bytes(1024) + '1.0 kB' + >>> humanize_bytes(1024*123) + '123.0 kB' + >>> humanize_bytes(1024*12342) + '12.1 MB' + >>> humanize_bytes(1024*12342,2) + '12.05 MB' + >>> humanize_bytes(1024*1234,2) + '1.21 MB' + >>> humanize_bytes(1024*1234*1111,2) + '1.31 GB' + >>> humanize_bytes(1024*1234*1111,1) + '1.3 GB' + """ + if byte_count == 1: + return '1 byte' + if byte_count < 1024: + return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') + + suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] + multiple = 1024.0 # .0 to force float on python 2 + for suffix in suffixes: + byte_count /= multiple + if byte_count < multiple: + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) diff --git a/shodan/stream.py b/shodan/stream.py new file mode 100644 index 0000000..9900b08 --- /dev/null +++ b/shodan/stream.py @@ -0,0 +1,162 @@ +import requests +import json +import ssl + +from .exception import APIError + + +class Stream: + + base_url = 'https://stream.shodan.io' + + def __init__(self, api_key, proxies=None): + self.api_key = api_key + self.proxies = proxies + + def _create_stream(self, name, query=None, timeout=None): + params = { + 'key': self.api_key, + } + stream_url = self.base_url + name + + # The user doesn't want to use a timeout + # If the timeout is specified as 0 then we also don't want to have a timeout + if (timeout and timeout <= 0) or (timeout == 0): + timeout = None + + # If the user requested a timeout then we need to disable heartbeat messages + # which are intended to keep stream connections alive even if there isn't any data + # flowing through. + if timeout: + params['heartbeat'] = False + + if query is not None: + params['query'] = query + + try: + while True: + req = requests.get(stream_url, params=params, stream=True, timeout=timeout, + proxies=self.proxies) + + # Status code 524 is special to Cloudflare + # It means that no data was sent from the streaming servers which caused Cloudflare + # to terminate the connection. + # + # We only want to exit if there was a timeout specified or the HTTP status code is + # not specific to Cloudflare. + if req.status_code != 524 or timeout >= 0: + break + except Exception: + raise APIError('Unable to contact the Shodan Streaming API') + + if req.status_code != 200: + try: + data = json.loads(req.text) + raise APIError(data['error']) + except APIError: + raise + except Exception: + pass + raise APIError('Invalid API key or you do not have access to the Streaming API') + if req.encoding is None: + req.encoding = 'utf-8' + return req + + def _iter_stream(self, stream, raw): + for line in stream.iter_lines(): + # The Streaming API sends out heartbeat messages that are newlines + # We want to ignore those messages since they don't contain any data + if line: + if raw: + yield line + else: + yield json.loads(line) + + def alert(self, aid=None, timeout=None, raw=False): + if aid: + stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout) + else: + stream = self._create_stream('/shodan/alert', timeout=timeout) + + try: + for line in self._iter_stream(stream, raw): + yield line + except requests.exceptions.ConnectionError: + raise APIError('Stream timed out') + except ssl.SSLError: + raise APIError('Stream timed out') + + def asn(self, asn, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the ASNs of interest. + + :param asn: A list of ASN to return banner data on. + :type asn: string[] + """ + stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def banners(self, raw=False, timeout=None): + """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to + API subscription plans and for those it only returns a fraction of the data. + """ + stream = self._create_stream('/shodan/banners', timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def countries(self, countries, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the countries of interest. + + :param countries: A list of countries to return banner data on. + :type countries: string[] + """ + stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def custom(self, query, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the query of interest. The query + can vary and mix-match with different arguments (ports, tags, vulns, etc). + + :param query: A space-separated list of key:value filters query to return banner data on. + :type query: string + """ + stream = self._create_stream('/shodan/custom', query=query, timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def ports(self, ports, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the ports of interest. + + :param ports: A list of ports to return banner data on. + :type ports: int[] + """ + stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def tags(self, tags, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the tags of interest. + + :param tags: A list of tags to return banner data on. + :type tags: string[] + """ + stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def vulns(self, vulns, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. + + :param vulns: A list of vulns to return banner data on. + :type vulns: string[] + """ + stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line diff --git a/shodan/threatnet.py b/shodan/threatnet.py index eca547f..cad9bdd 100644 --- a/shodan/threatnet.py +++ b/shodan/threatnet.py @@ -1,6 +1,8 @@ import requests -import simplejson -from .client import APIError +import json + +from .exception import APIError + class Threatnet: """Wrapper around the Threatnet REST and Streaming APIs @@ -9,24 +11,26 @@ class Threatnet: :type key: str :ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API. """ - + class Stream: base_url = 'https://stream.shodan.io' - def __init__(self, parent): + def __init__(self, parent, proxies=None): self.parent = parent + self.proxies = proxies def _create_stream(self, name): try: - req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True) - except: + req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, + stream=True, proxies=self.proxies) + except Exception: raise APIError('Unable to contact the Shodan Streaming API') if req.status_code != 200: try: - raise APIError(data.json()['error']) - except: + raise APIError(req.json()['error']) + except Exception: pass raise APIError('Invalid API key or you do not have access to the Streaming API') return req @@ -35,30 +39,29 @@ def events(self): stream = self._create_stream('/threatnet/events') for line in stream.iter_lines(): if line: - banner = simplejson.loads(line) + banner = json.loads(line) yield banner def backscatter(self): stream = self._create_stream('/threatnet/backscatter') for line in stream.iter_lines(): if line: - banner = simplejson.loads(line) + banner = json.loads(line) yield banner def activity(self): stream = self._create_stream('/threatnet/ssh') for line in stream.iter_lines(): if line: - banner = simplejson.loads(line) + banner = json.loads(line) yield banner - + def __init__(self, key): """Initializes the API object. - + :param key: The Shodan API key. :type key: str """ self.api_key = key self.base_url = 'https://api.shodan.io' self.stream = self.Stream(self) - diff --git a/shodan/wps.py b/shodan/wps.py deleted file mode 100644 index af9249e..0000000 --- a/shodan/wps.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -WiFi Positioning System - -Wrappers around the SkyHook and Google Locations APIs to resolve -wireless routers' MAC addresses (BSSID) to physical locations. -""" -try: - from json import dumps, loads -except: - from simplejson import dumps, loads - -try: - from urllib2 import Request, urlopen - from urllib import urlencode -except: - from urllib.request import Request, urlopen - from urllib.parse import urlencode - -class Skyhook: - - """Not yet ready for production, use the GoogleLocation class instead.""" - - def __init__(self, username='api', realm='shodan'): - self.username = username - self.realm = realm - self.url = 'https://api.skyhookwireless.com/wps2/location' - - def locate(self, mac): - # Remove the ':' - mac = mac.replace(':', '') - data = """ - - - - %s - %s - - - - %s - -50 - - """ % (self.username, self.realm, mac) - request = Request(url=self.url, data=data, headers={'Content-type': 'text/xml'}) - response = urlopen(request) - result = response.read() - return result - -class GoogleLocation: - - def __init__(self): - self.url = 'http://www.google.com/loc/json' - - def locate(self, mac): - data = { - 'version': '1.1.0', - 'request_address': True, - 'wifi_towers': [{ - 'mac_address': mac, - 'ssid': 'g', - 'signal_strength': -72 - }] - } - response = urlopen(self.url, dumps(data)) - data = response.read() - return loads(data) diff --git a/tests/test_shodan.py b/tests/test_shodan.py index e1b39c8..94ffc70 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -1,147 +1,183 @@ -import unittest -import shodan - -class ShodanTests(unittest.TestCase): - - api = None - FACETS = [ - 'port', - ('domain', 1) - ] - QUERIES = { - 'simple': 'cisco-ios', - 'minify': 'apache', - 'advanced': 'apache port:443', - 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', - } - - def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) - - def test_search_simple(self): - results = self.api.search(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure some values were returned - self.assertTrue(results['matches']) - self.assertTrue(results['total']) - - # A regular search shouldn't have the optional info - self.assertNotIn('opts', results['matches'][0]) - - def test_search_empty(self): - results = self.api.search(self.QUERIES['empty']) - self.assertTrue(len(results['matches']) == 0) - self.assertEqual(results['total'], 0) - - def test_search_facets(self): - results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_count_simple(self): - results = self.api.count(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure no values were returned - self.assertFalse(results['matches']) - self.assertTrue(results['total']) - - def test_count_facets(self): - results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_host_details(self): - host = self.api.host('147.228.101.7') - - self.assertEqual('147.228.101.7', host['ip_str']) - self.assertFalse(isinstance(host['ip'], basestring)) - - def test_search_minify(self): - results = self.api.search(self.QUERIES['minify'], minify=False) - self.assertIn('opts', results['matches'][0]) - - def test_exploits_search(self): - results = self.api.exploits.search('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(results['matches']) - - def test_exploits_search_paging(self): - results = self.api.exploits.search('apache', page=1) - match1 = results['matches'][0] - results = self.api.exploits.search('apache', page=2) - match2 = results['matches'][0] - - self.assertNotEqual(match1['_id'], match2['_id']) - - def test_exploits_search_facets(self): - results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - def test_exploits_count(self): - results = self.api.exploits.count('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(len(results['matches']) == 0) - - def test_exploits_count_facets(self): - results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) - self.assertEqual(len(results['matches']), 0) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - # Test error responses - def test_invalid_key(self): - api = shodan.Shodan('garbage') - raised = False - try: - api.search('something') - except shodan.APIError, e: - raised = True - - self.assertTrue(raised) - - def test_invalid_host_ip(self): - raised = False - try: - host = self.api.host('test') - except shodan.APIError, e: - raised = True - - self.assertTrue(raised) - - def test_search_empty_query(self): - raised = False - try: - self.api.search('') - except shodan.APIError, e: - raised = True - self.assertTrue(raised) - - def test_search_advanced_query(self): - # The free API plan can't use filters - raised = False - try: - self.api.search(self.QUERIES['advanced']) - except shodan.APIError, e: - raised = True - self.assertTrue(raised) +import unittest +import shodan + +try: + basestring +except NameError: + basestring = str + + +class ShodanTests(unittest.TestCase): + + api = None + FACETS = [ + 'port', + ('domain', 1) + ] + QUERIES = { + 'simple': 'cisco-ios', + 'minify': 'apache', + 'advanced': 'apache port:443', + 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', + } + + def setUp(self): + with open('SHODAN-API-KEY') as f: + self.api = shodan.Shodan(f.read().strip()) + + def test_search_simple(self): + results = self.api.search(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure some values were returned + self.assertTrue(results['matches']) + self.assertTrue(results['total']) + + # A regular search shouldn't have the optional info + self.assertNotIn('opts', results['matches'][0]) + + def test_search_empty(self): + results = self.api.search(self.QUERIES['empty']) + self.assertTrue(len(results['matches']) == 0) + self.assertEqual(results['total'], 0) + + def test_search_facets(self): + results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_count_simple(self): + results = self.api.count(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure no values were returned + self.assertFalse(results['matches']) + self.assertTrue(results['total']) + + def test_count_facets(self): + results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_host_details(self): + host = self.api.host('147.228.101.7') + + self.assertEqual('147.228.101.7', host['ip_str']) + self.assertFalse(isinstance(host['ip'], basestring)) + + def test_search_minify(self): + results = self.api.search(self.QUERIES['minify'], minify=False) + self.assertIn('opts', results['matches'][0]) + + def test_exploits_search(self): + results = self.api.exploits.search('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + + def test_exploits_search_paging(self): + results = self.api.exploits.search('apache', page=1) + match1 = results['matches'][0] + results = self.api.exploits.search('apache', page=2) + match2 = results['matches'][0] + + self.assertNotEqual(match1['_id'], match2['_id']) + + def test_exploits_search_facets(self): + results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_exploits_count(self): + results = self.api.exploits.count('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(len(results['matches']) == 0) + + def test_exploits_count_facets(self): + results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) + self.assertEqual(len(results['matches']), 0) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_trends_search(self): + results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + + results = self.api.trends.search('apache', facets=[]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertNotIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [match['month'] for match in results['matches']]) + + def test_trends_search_filters(self): + results = self.api.trends.search_filters() + self.assertIn('has_ipv6', results) + self.assertNotIn('http.html', results) + + def test_trends_search_facets(self): + results = self.api.trends.search_facets() + self.assertIn('product', results) + self.assertNotIn('cpe', results) + + # Test error responses + def test_invalid_key(self): + api = shodan.Shodan('garbage') + raised = False + try: + api.search('something') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_invalid_host_ip(self): + raised = False + try: + self.api.host('test') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_search_empty_query(self): + raised = False + try: + self.api.search('') + except shodan.APIError: + raised = True + self.assertTrue(raised) + + def test_search_advanced_query(self): + # The free API plan can't use filters + raised = False + try: + self.api.search(self.QUERIES['advanced']) + except shodan.APIError: + raised = True + self.assertTrue(raised) + + +if __name__ == '__main__': + unittest.main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1a9f632 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[flake8] +ignore = + E501 W293 + +exclude = + build, + docs, + shodan.egg-info, + tmp, + +per-file-ignores = + shodan/__init__.py:F401, + shodan/cli/converter/__init__.py:F401, + shodan/cli/worldmap.py:W291, \ No newline at end of file