RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update dependencies

This commit is contained in:
Thomas Bouve 2021-10-24 01:23:23 +02:00
parent e35f70292e
commit f96487f695
10 changed files with 444 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MANIFEST.in
View File

@ -1,2 +1,2 @@
include LICENSE README.md
include aiodnsbrute/*.txt
include sublist3r2/aiodnsbrute/*.txt

6
requirements.txt
View File

@ -1,4 +1,8 @@
argparse
dnspython
requests
aiodnsbrute
asyncio
uvloop
tqdm
aiodns
click

17
setup.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
import os
from setuptools import setup
from setuptools import setup, find_packages
def read(fname: str) -> str:
@ -9,8 +9,8 @@ def read(fname: str) -> str:
setup(
name='Sublist3r2',
version='1.0.0',
name='sublist3r2',
version='1.0.1',
python_requires='>=3.6',
description='Subdomains enumeration tool for penetration testers',
long_description=read('README.md'),
@ -18,11 +18,8 @@ setup(
keywords='subdomain dns detection',
url='https://github.com/RoninNakomoto/Sublist3r2',
license='GPL-2.0',
py_modules=['sublist3r2'],
packages=find_packages(),
include_package_data=True,
package_data={
'': ['data/*.txt'],
},
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
@ -39,7 +36,11 @@ setup(
'argparse',
'dnspython',
'requests',
'aiodnsbrute',
'asyncio',
'uvloop',
'tqdm',
'aiodns',
'click',
],
entry_points={
'console_scripts': [

15
sublist3r2.py → sublist3r2/__init__.py
View File

@ -20,7 +20,10 @@ from collections import Counter
# External imports
import dns.resolver
import requests
from aiodnsbrute.cli import aioDNSBrute
from sublist3r2.aiodnsbrute.cli import aioDNSBrute
# Version info
__version__ = '1.0.1'
# Check if we are running this on windows platform
is_windows = sys.platform.startswith('win')
@ -58,12 +61,12 @@ def no_color():
def banner():
print("""%s
____ _ _ _ _ _____ ______
/ ___| _ _| |__ | (_)___| |_|___ / _ __\ __ | Sublist3r2 v1.0
/ ___| _ _| |__ | (_)___| |_|___ / _ __\ __ | Sublist3r2 v%s
\___ \| | | | '_ \| | / __| __| |_ \| '__| / / a subdomains enum tool originally by @aboul3la
___) | |_| | |_) | | \__ \ |_ ___) | | / /_ maintained by Ronin Nakomoto
|____/ \__,_|_.__/|_|_|___/\__|____/|_| /____|%s https://github.com/RoninNakomoto/Sublist3r2
""" % (R, Y))
""" % (R, __version__, Y))
def parser_error(errmsg):
@ -992,14 +995,14 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
if not silent:
print(G + "[-] Starting bruteforce module now using aiodnsbrute.." + W)
path_to_file = os.path.dirname(os.path.realpath(__file__))
subs = os.path.join(path_to_file, 'data', 'subdomains-top1million-110000.txt')
resolvers = os.path.join(path_to_file, 'data', 'resolvers.txt')
subs = os.path.join(path_to_file, 'aiodnsbrute', 'subdomains-top1million-110000.txt')
resolvers = os.path.join(path_to_file, 'aiodnsbrute', 'resolvers.txt')
wildcard = True
verify = True
query = True
thread_count = threads
bruteforce_list = aioDNSBrute.bruteforce_domain(
parsed_domain.netloc, resolvers, subs, wildcard, verify, search_list, thread_count, query
parsed_domain.netloc, resolvers, subs, wildcard, verify, thread_count, query
)
subdomains = search_list.union(bruteforce_list)

0
sublist3r2/aiodnsbrute/__init__.py Normal file
View File

358
sublist3r2/aiodnsbrute/cli.py Normal file
View File

@ -0,0 +1,358 @@
import random
import string
import asyncio
import functools
import os
import uvloop
import aiodns
import click
import socket
import sys
from tqdm import tqdm
from sublist3r2.aiodnsbrute.logger import ConsoleLogger
class aioDNSBrute(object):
"""aiodnsbrute implements fast domain name brute forcing using Python's asyncio module."""
def __init__(self, verbosity=0, max_tasks=512):
"""Constructor.
Args:
verbosity: set output verbosity: 0 (default) is none, 3 is debug
max_tasks: the maximum number of tasks asyncio will queue (default 512)
"""
self.tasks = []
self.errors = []
self.fqdn = []
self.ignore_hosts = []
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
self.loop = asyncio.get_event_loop()
self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
self.sem = asyncio.BoundedSemaphore(max_tasks)
self.max_tasks = max_tasks
self.verbosity = verbosity
self.logger = ConsoleLogger(verbosity)
async def _dns_lookup(self, name):
"""Performs a DNS request using aiodns, self.lookup_type is set by the run function.
A query for A record returns <ares_query_a_result> which does not return metadata about
when a CNAME was resolved (just host and ttl attributes) however it should be faster.
The <ares_host_result> returned by gethostbyname contains name, aliases, and addresses, if
name is different in response we can surmise that the original domain was a CNAME entry.
Args:
name: the domain name to resolve
Returns:
object: <ares_query_a_result> if query, <ares_host_result> if gethostbyname
"""
if self.lookup_type == "query":
return await self.resolver.query(name, "A")
elif self.lookup_type == "gethostbyname":
return await self.resolver.gethostbyname(name, socket.AF_INET)
def _dns_result_callback(self, name, future):
"""Handles the pycares object passed by the _dns_lookup function. We expect an errror to
be present in the returned object because most lookups will be for names that don't exist.
c-ares errors are passed through directly, error types can be identified in ares_strerror.c
Args:
name: original lookup name (because the query_result object doesn't contain it)
future: the completed future (pycares dns result)
"""
# Record processed we can now release the lock
self.sem.release()
# Handle known exceptions, barf on other ones
if future.exception() is not None:
try:
err_number = future.exception().args[0]
err_text = future.exception().args[1]
except IndexError:
self.logger.error(f"Couldn't parse exception: {future.exception()}")
# handle the DNS errors we expect to receive, show user unexpected errors
if err_number == 4:
# This is domain name not found, ignore it
pass
#elif err_number == 12:
# Timeout from DNS server
#self.logger.warn(f"Timeout for {name}")
elif err_number == 1:
# Server answered with no data
pass
#else:
#self.logger.error(
# f"{name} generated an unexpected exception: {future.exception()}"
#)
# for debugging/troubleshoooting keep a list of errors
# self.errors.append({'hostname': name, 'error': err_text})
# parse and output and store results.
else:
if self.lookup_type == "query":
ips = [ip.host for ip in future.result()]
cname = False
row = f"{name:<30}\t{ips}"
elif self.lookup_type == "gethostbyname":
r = future.result()
ips = [ip for ip in r.addresses]
if name == r.name:
cname = False
n = f"""{name:<30}\t{f"{'':<35}" if self.verbosity >= 2 else ""}"""
else:
cname = True
# format the name based on verbosity - this is kluge
short_cname = f"{r.name[:28]}.." if len(r.name) > 30 else r.name
n = f'{name}{"**" if self.verbosity <= 1 else ""}'
n = f'''{n:<30}\t{f"CNAME {short_cname:<30}" if self.verbosity >= 2 else ""}'''
row = f"{n:<30}\t{ips}"
# store the result
if set(ips) != set(self.ignore_hosts):
#self.logger.success(row)
dns_lookup_result = {"domain": name, "ip": ips}
if self.lookup_type == "gethostbyname" and cname:
dns_lookup_result["cname"] = r.name
dns_lookup_result["aliases"] = r.aliases
self.fqdn.append(dns_lookup_result)
self.logger.debug(future.result())
self.tasks.remove(future)
if self.verbosity >= 1:
self.pbar.update()
async def _queue_lookups(self, wordlist, domain):
"""Takes a list of words and adds them to the async loop also passing the original
lookup domain name; then attaches the processing callback to deal with the result.
Args:
wordlist: a list of names to perform lookups for
domain: the base domain to perform brute force against
"""
for word in wordlist:
# Wait on the semaphore before adding more tasks
await self.sem.acquire()
host = f"{word.strip()}.{domain}"
task = asyncio.ensure_future(self._dns_lookup(host))
task.add_done_callback(functools.partial(self._dns_result_callback, host))
self.tasks.append(task)
await asyncio.gather(*self.tasks, return_exceptions=True)
def bruteforce_domain(target, resolvers=None, wordlist="subdomains-top1million-110000.txt", wildcard=True, verify=True, thread_count=7000, query=True):
subdomains_list = []
names_list = []
verbosity = 1
if resolvers:
resolverfile = open(resolvers, 'r')
lines = resolverfile.read().splitlines()
resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
bf = aioDNSBrute(verbosity=verbosity, max_tasks=thread_count)
subdomains_list = bf.run(wordlist, target, resolvers, wildcard, verify, query)
resolverfile.close()
for r in range(1, len(subdomains_list)):
names_list.append(subdomains_list[r]['domain'])
return names_list
def run(
self, wordlist, domain, resolvers=None, wildcard=True, verify=True, query=True
):
"""
Sets up the bruteforce job, does domain verification, sets resolvers, checks for wildcard
response to lookups, and sets the query type to be used. After all this, open the wordlist
file and start the brute force - with ^C handling to cleanup nicely.
Args:
wordlist: a string containing a path to a filename to be used as a wordlist
domain: the base domain name to be used for lookups
resolvers: a list of DNS resolvers to be used (default None, uses system resolvers)
wildcard: bool, do wildcard dns detection (default true)
verify: bool, check if domain exists (default true)
query: bool, use query to do lookups (default true), false means gethostbyname is used.
Returns:
dict containing result of lookups
"""
self.logger.info(
f"Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks..."
)
if verify:
#self.logger.info(f"Using local resolver to verify {domain} exists.")
try:
socket.gethostbyname(domain)
except socket.gaierror as err:
self.logger.error(
f"Couldn't resolve {domain}, use the --no-verify switch to ignore this error."
)
raise SystemExit(
self.logger.error(f"Error from host lookup: {err}")
)
else:
self.logger.warn("Skipping domain verification. YOLO!")
if resolvers:
self.resolver.nameservers = resolvers
self.logger.info(
f"Using recursive DNS with {len(self.resolver.nameservers)} nameservers"
)
if wildcard:
# 63 chars is the max allowed segment length, there is practically no chance that it will be a legit record
random_sld = (
lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}'
)
try:
self.lookup_type = "query"
wc_check = self.loop.run_until_complete(
self._dns_lookup(f"{random_sld()}.{domain}")
)
except aiodns.error.DNSError as err:
# we expect that the record will not exist and error 4 will be thrown
#self.logger.info(
# f"No wildcard response was detected for this domain."
#)
wc_check = None
finally:
if wc_check is not None:
self.ignore_hosts = [host.host for host in wc_check]
self.logger.warn(
f"Wildcard response detected, ignoring answers containing {self.ignore_hosts}"
)
else:
self.logger.warn("Wildcard detection is disabled")
if query:
#self.logger.info(
# "Using pycares `query` function to perform lookups, CNAMEs cannot be identified"
#)
self.lookup_type = "query"
else:
self.logger.info(
"Using pycares `gethostbyname` function to perform lookups, CNAME data will be appended to results (** denotes CNAME, show actual name with -vv)"
)
self.lookup_type = "gethostbyname"
with open(wordlist, encoding="utf-8", errors="ignore") as words:
w = words.read().splitlines()
self.logger.info(f"Wordlist loaded, proceeding with {len(w)} DNS requests")
try:
if self.verbosity >= 1:
self.pbar = tqdm(
total=len(w), unit="rec", maxinterval=0.1, mininterval=0
)
self.loop.run_until_complete(self._queue_lookups(w, domain))
except KeyboardInterrupt:
self.logger.warn("Caught keyboard interrupt, cleaning up...")
asyncio.gather(*asyncio.Task.all_tasks()).cancel()
self.loop.stop()
finally:
self.loop.close()
if self.verbosity >= 1:
self.pbar.close()
self.logger.info(f"Bruteforcing Complete")
return self.fqdn
@click.command()
@click.option(
"--wordlist",
"-w",
help="Wordlist to use for brute force.",
default=f"{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000",
)
@click.option(
"--max-tasks",
"-t",
default=512,
help="Maximum number of tasks to run asynchronosly.",
)
@click.option(
"--resolver-file",
"-r",
type=click.File("r"),
default=None,
help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers",
)
@click.option(
"--verbosity", "-v", count=True, default=1, help="Increase output verbosity"
)
@click.option(
"--output",
"-o",
type=click.Choice(["csv", "json", "off"]),
default="off",
help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).",
)
@click.option(
"--outfile",
"-f",
type=click.File("w"),
help="Output filename. Use '-f -' to send file output to stdout overriding normal output.",
)
@click.option(
"--query/--gethostbyname",
default=True,
help="DNS lookup type to use query (default) should be faster, but won't return CNAME information.",
)
@click.option(
"--wildcard/--no-wildcard",
default=True,
help="Wildcard detection, enabled by default",
)
@click.option(
"--verify/--no-verify",
default=True,
help="Verify domain name is sane before beginning, enabled by default",
)
@click.version_option("0.3.2")
@click.argument("domain", required=True)
def main(**kwargs):
"""aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module.
credit: blark (@markbaseggio)
"""
output = kwargs.get("output")
verbosity = kwargs.get("verbosity")
resolvers = kwargs.get("resolver_file")
if output != "off":
outfile = kwargs.get("outfile")
# turn off output if we want JSON/CSV to stdout, hacky
if outfile.__class__.__name__ == "TextIOWrapper":
verbosity = 0
if outfile is None:
# wasn't specified on command line
outfile = open(f'{kwargs["domain"]}.{output}', "w")
if resolvers:
lines = resolvers.read().splitlines()
resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get("max_tasks"))
results = bf.run(
wordlist=kwargs.get("wordlist"),
domain=kwargs.get("domain"),
resolvers=resolvers,
wildcard=kwargs.get("wildcard"),
verify=kwargs.get("verify"),
query=kwargs.get("query"),
)
if output in ("json"):
import json
json.dump(results, outfile)
if output in ("csv"):
import csv
writer = csv.writer(outfile)
writer.writerow(["Hostname", "IPs", "CNAME", "Aliases"])
[
writer.writerow(
[
r.get("domain"),
r.get("ip", [""])[0],
r.get("cname"),
r.get("aliases", [""])[0],
]
)
for r in results
]
if __name__ == "__main__":
main()

31
sublist3r2/aiodnsbrute/logger.py Normal file
View File

@ -0,0 +1,31 @@
from tqdm import tqdm
from click import style
class ConsoleLogger(object):
"""A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
def __init__(self, verbosity):
self.verbosity = verbosity
self.msg_type = {
"info": ("[*]", "blue", 1),
"success": ("[+]", "green", 1),
"error": ("[-]", "red", 1),
"warn": ("[!]", "yellow", 1),
"debug": ("[D]", "cyan", 3),
}
def __getattr__(self, attr):
try:
decorator = style(
f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
)
msg_verbosity = self.msg_type[attr][2]
except KeyError:
decorator = ""
msg_verbosity = 1
finally:
if self.verbosity >= msg_verbosity:
return lambda msg: tqdm.write(f"{decorator}{msg}")
else:
return lambda msg: None

31
sublist3r2/aiodnsbrute/logger/logger.py Normal file
View File

@ -0,0 +1,31 @@
from tqdm import tqdm
from click import style
class ConsoleLogger(object):
"""A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
def __init__(self, verbosity):
self.verbosity = verbosity
self.msg_type = {
"info": ("[*]", "blue", 1),
"success": ("[+]", "green", 1),
"error": ("[-]", "red", 1),
"warn": ("[!]", "yellow", 1),
"debug": ("[D]", "cyan", 3),
}
def __getattr__(self, attr):
try:
decorator = style(
f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
)
msg_verbosity = self.msg_type[attr][2]
except KeyError:
decorator = ""
msg_verbosity = 1
finally:
if self.verbosity >= msg_verbosity:
return lambda msg: tqdm.write(f"{decorator}{msg}")
else:
return lambda msg: None

0
data/resolvers.txt → sublist3r2/aiodnsbrute/resolvers.txt
View File

0
data/subdomains-top1million-110000.txt → sublist3r2/aiodnsbrute/subdomains-top1million-110000.txt
View File