diff --git a/MANIFEST.in b/MANIFEST.in index 6feedd3..90bd747 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ include LICENSE README.md -include aiodnsbrute/*.txt +include sublist3r2/aiodnsbrute/*.txt diff --git a/requirements.txt b/requirements.txt index f765236..b43d9c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ argparse dnspython requests -aiodnsbrute +asyncio +uvloop +tqdm +aiodns +click \ No newline at end of file diff --git a/setup.py b/setup.py index 20c9153..038fea0 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import os -from setuptools import setup +from setuptools import setup, find_packages def read(fname: str) -> str: @@ -9,8 +9,8 @@ def read(fname: str) -> str: setup( - name='Sublist3r2', - version='1.0.0', + name='sublist3r2', + version='1.0.1', python_requires='>=3.6', description='Subdomains enumeration tool for penetration testers', long_description=read('README.md'), @@ -18,11 +18,8 @@ setup( keywords='subdomain dns detection', url='https://github.com/RoninNakomoto/Sublist3r2', license='GPL-2.0', - py_modules=['sublist3r2'], + packages=find_packages(), include_package_data=True, - package_data={ - '': ['data/*.txt'], - }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -39,7 +36,11 @@ setup( 'argparse', 'dnspython', 'requests', - 'aiodnsbrute', + 'asyncio', + 'uvloop', + 'tqdm', + 'aiodns', + 'click', ], entry_points={ 'console_scripts': [ diff --git a/sublist3r2.py b/sublist3r2/__init__.py similarity index 99% rename from sublist3r2.py rename to sublist3r2/__init__.py index b4b0a76..f4ef78c 100755 --- a/sublist3r2.py +++ b/sublist3r2/__init__.py @@ -20,7 +20,10 @@ from collections import Counter # External imports import dns.resolver import requests -from aiodnsbrute.cli import aioDNSBrute +from sublist3r2.aiodnsbrute.cli import aioDNSBrute + +# Version info +__version__ = '1.0.1' # Check if we are running this on windows platform is_windows = sys.platform.startswith('win') @@ -58,12 +61,12 @@ def no_color(): def banner(): print("""%s ____ _ _ _ _ _____ ______ - / ___| _ _| |__ | (_)___| |_|___ / _ __\ __ | Sublist3r2 v1.0 + / ___| _ _| |__ | (_)___| |_|___ / _ __\ __ | Sublist3r2 v%s \___ \| | | | '_ \| | / __| __| |_ \| '__| / / a subdomains enum tool originally by @aboul3la ___) | |_| | |_) | | \__ \ |_ ___) | | / /_ maintained by Ronin Nakomoto |____/ \__,_|_.__/|_|_|___/\__|____/|_| /____|%s https://github.com/RoninNakomoto/Sublist3r2 - """ % (R, Y)) + """ % (R, __version__, Y)) def parser_error(errmsg): @@ -992,14 +995,14 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e if not silent: print(G + "[-] Starting bruteforce module now using aiodnsbrute.." + W) path_to_file = os.path.dirname(os.path.realpath(__file__)) - subs = os.path.join(path_to_file, 'data', 'subdomains-top1million-110000.txt') - resolvers = os.path.join(path_to_file, 'data', 'resolvers.txt') + subs = os.path.join(path_to_file, 'aiodnsbrute', 'subdomains-top1million-110000.txt') + resolvers = os.path.join(path_to_file, 'aiodnsbrute', 'resolvers.txt') wildcard = True verify = True query = True thread_count = threads bruteforce_list = aioDNSBrute.bruteforce_domain( - parsed_domain.netloc, resolvers, subs, wildcard, verify, search_list, thread_count, query + parsed_domain.netloc, resolvers, subs, wildcard, verify, thread_count, query ) subdomains = search_list.union(bruteforce_list) diff --git a/sublist3r2/aiodnsbrute/__init__.py b/sublist3r2/aiodnsbrute/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sublist3r2/aiodnsbrute/cli.py b/sublist3r2/aiodnsbrute/cli.py new file mode 100644 index 0000000..2b7a9fd --- /dev/null +++ b/sublist3r2/aiodnsbrute/cli.py @@ -0,0 +1,358 @@ +import random +import string +import asyncio +import functools +import os +import uvloop +import aiodns +import click +import socket +import sys +from tqdm import tqdm +from sublist3r2.aiodnsbrute.logger import ConsoleLogger + + +class aioDNSBrute(object): + """aiodnsbrute implements fast domain name brute forcing using Python's asyncio module.""" + + def __init__(self, verbosity=0, max_tasks=512): + """Constructor. + + Args: + verbosity: set output verbosity: 0 (default) is none, 3 is debug + max_tasks: the maximum number of tasks asyncio will queue (default 512) + """ + self.tasks = [] + self.errors = [] + self.fqdn = [] + self.ignore_hosts = [] + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + self.loop = asyncio.get_event_loop() + self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True) + self.sem = asyncio.BoundedSemaphore(max_tasks) + self.max_tasks = max_tasks + self.verbosity = verbosity + self.logger = ConsoleLogger(verbosity) + + async def _dns_lookup(self, name): + """Performs a DNS request using aiodns, self.lookup_type is set by the run function. + A query for A record returns which does not return metadata about + when a CNAME was resolved (just host and ttl attributes) however it should be faster. + The returned by gethostbyname contains name, aliases, and addresses, if + name is different in response we can surmise that the original domain was a CNAME entry. + + Args: + name: the domain name to resolve + + Returns: + object: if query, if gethostbyname + """ + if self.lookup_type == "query": + return await self.resolver.query(name, "A") + elif self.lookup_type == "gethostbyname": + return await self.resolver.gethostbyname(name, socket.AF_INET) + + def _dns_result_callback(self, name, future): + """Handles the pycares object passed by the _dns_lookup function. We expect an errror to + be present in the returned object because most lookups will be for names that don't exist. + c-ares errors are passed through directly, error types can be identified in ares_strerror.c + + Args: + name: original lookup name (because the query_result object doesn't contain it) + future: the completed future (pycares dns result) + """ + # Record processed we can now release the lock + self.sem.release() + # Handle known exceptions, barf on other ones + if future.exception() is not None: + try: + err_number = future.exception().args[0] + err_text = future.exception().args[1] + except IndexError: + self.logger.error(f"Couldn't parse exception: {future.exception()}") + # handle the DNS errors we expect to receive, show user unexpected errors + if err_number == 4: + # This is domain name not found, ignore it + pass + #elif err_number == 12: + # Timeout from DNS server + #self.logger.warn(f"Timeout for {name}") + elif err_number == 1: + # Server answered with no data + pass + #else: + #self.logger.error( + # f"{name} generated an unexpected exception: {future.exception()}" + #) + # for debugging/troubleshoooting keep a list of errors + # self.errors.append({'hostname': name, 'error': err_text}) + + # parse and output and store results. + else: + if self.lookup_type == "query": + ips = [ip.host for ip in future.result()] + cname = False + row = f"{name:<30}\t{ips}" + elif self.lookup_type == "gethostbyname": + r = future.result() + ips = [ip for ip in r.addresses] + if name == r.name: + cname = False + n = f"""{name:<30}\t{f"{'':<35}" if self.verbosity >= 2 else ""}""" + else: + cname = True + # format the name based on verbosity - this is kluge + short_cname = f"{r.name[:28]}.." if len(r.name) > 30 else r.name + n = f'{name}{"**" if self.verbosity <= 1 else ""}' + n = f'''{n:<30}\t{f"CNAME {short_cname:<30}" if self.verbosity >= 2 else ""}''' + row = f"{n:<30}\t{ips}" + # store the result + if set(ips) != set(self.ignore_hosts): + #self.logger.success(row) + dns_lookup_result = {"domain": name, "ip": ips} + if self.lookup_type == "gethostbyname" and cname: + dns_lookup_result["cname"] = r.name + dns_lookup_result["aliases"] = r.aliases + self.fqdn.append(dns_lookup_result) + self.logger.debug(future.result()) + self.tasks.remove(future) + if self.verbosity >= 1: + self.pbar.update() + + async def _queue_lookups(self, wordlist, domain): + """Takes a list of words and adds them to the async loop also passing the original + lookup domain name; then attaches the processing callback to deal with the result. + + Args: + wordlist: a list of names to perform lookups for + domain: the base domain to perform brute force against + """ + for word in wordlist: + # Wait on the semaphore before adding more tasks + await self.sem.acquire() + host = f"{word.strip()}.{domain}" + task = asyncio.ensure_future(self._dns_lookup(host)) + task.add_done_callback(functools.partial(self._dns_result_callback, host)) + self.tasks.append(task) + await asyncio.gather(*self.tasks, return_exceptions=True) + + def bruteforce_domain(target, resolvers=None, wordlist="subdomains-top1million-110000.txt", wildcard=True, verify=True, thread_count=7000, query=True): + subdomains_list = [] + names_list = [] + verbosity = 1 + if resolvers: + resolverfile = open(resolvers, 'r') + lines = resolverfile.read().splitlines() + resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))] + bf = aioDNSBrute(verbosity=verbosity, max_tasks=thread_count) + subdomains_list = bf.run(wordlist, target, resolvers, wildcard, verify, query) + resolverfile.close() + for r in range(1, len(subdomains_list)): + names_list.append(subdomains_list[r]['domain']) + + return names_list + + def run( + self, wordlist, domain, resolvers=None, wildcard=True, verify=True, query=True + ): + """ + Sets up the bruteforce job, does domain verification, sets resolvers, checks for wildcard + response to lookups, and sets the query type to be used. After all this, open the wordlist + file and start the brute force - with ^C handling to cleanup nicely. + + Args: + wordlist: a string containing a path to a filename to be used as a wordlist + domain: the base domain name to be used for lookups + resolvers: a list of DNS resolvers to be used (default None, uses system resolvers) + wildcard: bool, do wildcard dns detection (default true) + verify: bool, check if domain exists (default true) + query: bool, use query to do lookups (default true), false means gethostbyname is used. + + Returns: + dict containing result of lookups + """ + self.logger.info( + f"Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks..." + ) + if verify: + #self.logger.info(f"Using local resolver to verify {domain} exists.") + try: + socket.gethostbyname(domain) + except socket.gaierror as err: + self.logger.error( + f"Couldn't resolve {domain}, use the --no-verify switch to ignore this error." + ) + raise SystemExit( + self.logger.error(f"Error from host lookup: {err}") + ) + else: + self.logger.warn("Skipping domain verification. YOLO!") + if resolvers: + self.resolver.nameservers = resolvers + self.logger.info( + f"Using recursive DNS with {len(self.resolver.nameservers)} nameservers" + ) + + if wildcard: + # 63 chars is the max allowed segment length, there is practically no chance that it will be a legit record + random_sld = ( + lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}' + ) + try: + self.lookup_type = "query" + wc_check = self.loop.run_until_complete( + self._dns_lookup(f"{random_sld()}.{domain}") + ) + except aiodns.error.DNSError as err: + # we expect that the record will not exist and error 4 will be thrown + #self.logger.info( + # f"No wildcard response was detected for this domain." + #) + wc_check = None + finally: + if wc_check is not None: + self.ignore_hosts = [host.host for host in wc_check] + self.logger.warn( + f"Wildcard response detected, ignoring answers containing {self.ignore_hosts}" + ) + else: + self.logger.warn("Wildcard detection is disabled") + + if query: + #self.logger.info( + # "Using pycares `query` function to perform lookups, CNAMEs cannot be identified" + #) + self.lookup_type = "query" + else: + self.logger.info( + "Using pycares `gethostbyname` function to perform lookups, CNAME data will be appended to results (** denotes CNAME, show actual name with -vv)" + ) + self.lookup_type = "gethostbyname" + + with open(wordlist, encoding="utf-8", errors="ignore") as words: + w = words.read().splitlines() + self.logger.info(f"Wordlist loaded, proceeding with {len(w)} DNS requests") + try: + if self.verbosity >= 1: + self.pbar = tqdm( + total=len(w), unit="rec", maxinterval=0.1, mininterval=0 + ) + self.loop.run_until_complete(self._queue_lookups(w, domain)) + except KeyboardInterrupt: + self.logger.warn("Caught keyboard interrupt, cleaning up...") + asyncio.gather(*asyncio.Task.all_tasks()).cancel() + self.loop.stop() + finally: + self.loop.close() + if self.verbosity >= 1: + self.pbar.close() + self.logger.info(f"Bruteforcing Complete") + return self.fqdn + + +@click.command() +@click.option( + "--wordlist", + "-w", + help="Wordlist to use for brute force.", + default=f"{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000", +) +@click.option( + "--max-tasks", + "-t", + default=512, + help="Maximum number of tasks to run asynchronosly.", +) +@click.option( + "--resolver-file", + "-r", + type=click.File("r"), + default=None, + help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers", +) +@click.option( + "--verbosity", "-v", count=True, default=1, help="Increase output verbosity" +) +@click.option( + "--output", + "-o", + type=click.Choice(["csv", "json", "off"]), + default="off", + help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).", +) +@click.option( + "--outfile", + "-f", + type=click.File("w"), + help="Output filename. Use '-f -' to send file output to stdout overriding normal output.", +) +@click.option( + "--query/--gethostbyname", + default=True, + help="DNS lookup type to use query (default) should be faster, but won't return CNAME information.", +) +@click.option( + "--wildcard/--no-wildcard", + default=True, + help="Wildcard detection, enabled by default", +) +@click.option( + "--verify/--no-verify", + default=True, + help="Verify domain name is sane before beginning, enabled by default", +) +@click.version_option("0.3.2") +@click.argument("domain", required=True) +def main(**kwargs): + """aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module. + + credit: blark (@markbaseggio) + """ + output = kwargs.get("output") + verbosity = kwargs.get("verbosity") + resolvers = kwargs.get("resolver_file") + if output != "off": + outfile = kwargs.get("outfile") + # turn off output if we want JSON/CSV to stdout, hacky + if outfile.__class__.__name__ == "TextIOWrapper": + verbosity = 0 + if outfile is None: + # wasn't specified on command line + outfile = open(f'{kwargs["domain"]}.{output}', "w") + if resolvers: + lines = resolvers.read().splitlines() + resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))] + + bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get("max_tasks")) + results = bf.run( + wordlist=kwargs.get("wordlist"), + domain=kwargs.get("domain"), + resolvers=resolvers, + wildcard=kwargs.get("wildcard"), + verify=kwargs.get("verify"), + query=kwargs.get("query"), + ) + + if output in ("json"): + import json + json.dump(results, outfile) + + if output in ("csv"): + import csv + writer = csv.writer(outfile) + writer.writerow(["Hostname", "IPs", "CNAME", "Aliases"]) + [ + writer.writerow( + [ + r.get("domain"), + r.get("ip", [""])[0], + r.get("cname"), + r.get("aliases", [""])[0], + ] + ) + for r in results + ] + + +if __name__ == "__main__": + main() diff --git a/sublist3r2/aiodnsbrute/logger.py b/sublist3r2/aiodnsbrute/logger.py new file mode 100644 index 0000000..e95736a --- /dev/null +++ b/sublist3r2/aiodnsbrute/logger.py @@ -0,0 +1,31 @@ +from tqdm import tqdm +from click import style + + +class ConsoleLogger(object): + """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output.""" + + def __init__(self, verbosity): + self.verbosity = verbosity + self.msg_type = { + "info": ("[*]", "blue", 1), + "success": ("[+]", "green", 1), + "error": ("[-]", "red", 1), + "warn": ("[!]", "yellow", 1), + "debug": ("[D]", "cyan", 3), + } + + def __getattr__(self, attr): + try: + decorator = style( + f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True + ) + msg_verbosity = self.msg_type[attr][2] + except KeyError: + decorator = "" + msg_verbosity = 1 + finally: + if self.verbosity >= msg_verbosity: + return lambda msg: tqdm.write(f"{decorator}{msg}") + else: + return lambda msg: None diff --git a/sublist3r2/aiodnsbrute/logger/logger.py b/sublist3r2/aiodnsbrute/logger/logger.py new file mode 100644 index 0000000..e95736a --- /dev/null +++ b/sublist3r2/aiodnsbrute/logger/logger.py @@ -0,0 +1,31 @@ +from tqdm import tqdm +from click import style + + +class ConsoleLogger(object): + """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output.""" + + def __init__(self, verbosity): + self.verbosity = verbosity + self.msg_type = { + "info": ("[*]", "blue", 1), + "success": ("[+]", "green", 1), + "error": ("[-]", "red", 1), + "warn": ("[!]", "yellow", 1), + "debug": ("[D]", "cyan", 3), + } + + def __getattr__(self, attr): + try: + decorator = style( + f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True + ) + msg_verbosity = self.msg_type[attr][2] + except KeyError: + decorator = "" + msg_verbosity = 1 + finally: + if self.verbosity >= msg_verbosity: + return lambda msg: tqdm.write(f"{decorator}{msg}") + else: + return lambda msg: None diff --git a/data/resolvers.txt b/sublist3r2/aiodnsbrute/resolvers.txt similarity index 100% rename from data/resolvers.txt rename to sublist3r2/aiodnsbrute/resolvers.txt diff --git a/data/subdomains-top1million-110000.txt b/sublist3r2/aiodnsbrute/subdomains-top1million-110000.txt similarity index 100% rename from data/subdomains-top1million-110000.txt rename to sublist3r2/aiodnsbrute/subdomains-top1million-110000.txt