RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

initial commit

This commit is contained in:
Ahmed Aboul-Ela 2015-12-15 15:31:23 +02:00
commit 2446dc3877
6 changed files with 35254 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
subbrute/.directory Normal file
View File

@ -0,0 +1,4 @@
[Dolphin]
PreviewsShown=true
Timestamp=2015,7,3,13,23,20
Version=3

0
subbrute/__init__.py Normal file
View File

31910
subbrute/names.txt Normal file
View File

File diff suppressed because it is too large Load Diff

2014
subbrute/resolvers.txt Normal file
View File

File diff suppressed because it is too large Load Diff

631
subbrute/subbrute.py Normal file
View File

@ -0,0 +1,631 @@
#!/usr/bin/env python
#
#SubBrute v1.2
#A (very) fast subdomain enumeration tool.
#
#Maintained by rook
#Contributors:
#JordanMilne, KxCode, rc0r, memoryprint, ppaulojr
#
import re
import optparse
import os
import signal
import sys
import uuid
import random
import ctypes
import dns.resolver
import dns.rdatatype
import json
#Python 2.x and 3.x compatiablity
#We need the Queue library for exception handling
try:
import queue as Queue
except:
import Queue
#The 'multiprocessing' library does not rely upon a Global Interpreter Lock (GIL)
import multiprocessing
#Microsoft compatiablity
if sys.platform.startswith('win'):
#Drop-in replacement, subbrute + multiprocessing throws exceptions on windows.
import threading
multiprocessing.Process = threading.Thread
class verify_nameservers(multiprocessing.Process):
def __init__(self, target, record_type, resolver_q, resolver_list, wildcards):
multiprocessing.Process.__init__(self, target = self.run)
self.daemon = True
signal_init()
self.time_to_die = False
self.resolver_q = resolver_q
self.wildcards = wildcards
#Do we need wildcards for other types of records?
#This needs testing!
self.record_type = "A"
if record_type == "AAAA":
self.record_type = record_type
self.resolver_list = resolver_list
resolver = dns.resolver.Resolver()
#The domain provided by the user.
self.target = target
#1 website in the world, modify the following line when this status changes.
#www.google.cn, I'm looking at you ;)
self.most_popular_website = "www.google.com"
#We shouldn't need the backup_resolver, but we we can use them if need be.
#We must have a resolver, and localhost can work in some environments.
self.backup_resolver = resolver.nameservers + ['127.0.0.1', '8.8.8.8', '8.8.4.4']
#Ideally a nameserver should respond in less than 1 sec.
resolver.timeout = 1
resolver.lifetime = 1
try:
#Lets test the letancy of our connection.
#Google's DNS server should be an ideal time test.
resolver.nameservers = ['8.8.8.8']
resolver.query(self.most_popular_website, self.record_type)
except:
#Our connection is slower than a junebug in molasses
resolver = dns.resolver.Resolver()
self.resolver = resolver
def end(self):
self.time_to_die = True
#This process cannot block forever, it needs to check if its time to die.
def add_nameserver(self, nameserver):
keep_trying = True
while not self.time_to_die and keep_trying:
try:
self.resolver_q.put(nameserver, timeout = 1)
trace("Added nameserver:", nameserver)
keep_trying = False
except Exception as e:
if type(e) == Queue.Full or str(type(e)) == "<class 'queue.Full'>":
keep_trying = True
def verify(self, nameserver_list):
added_resolver = False
for server in nameserver_list:
if self.time_to_die:
#We are done here.
break
server = server.strip()
if server:
self.resolver.nameservers = [server]
try:
#test_result = self.resolver.query(self.most_popular_website, "A")
#should throw an exception before this line.
if True:#test_result:
#Only add the nameserver to the queue if we can detect wildcards.
if(self.find_wildcards(self.target)):# and self.find_wildcards(".com")
#wildcards have been added to the set, it is now safe to be added to the queue.
#blocking queue, this process will halt on put() when the queue is full:
self.add_nameserver(server)
added_resolver = True
else:
trace("Rejected nameserver - wildcard:", server)
except Exception as e:
#Rejected server :(
trace("Rejected nameserver - unreliable:", server, type(e))
return added_resolver
def run(self):
#Every user will get a different set of resovlers, this helps redistribute traffic.
random.shuffle(self.resolver_list)
if not self.verify(self.resolver_list):
#This should never happen, inform the user.
sys.stderr.write('Warning: No nameservers found, trying fallback list.\n')
#Try and fix it for the user:
self.verify(self.backup_resolver)
#End of the resolvers list.
try:
self.resolver_q.put(False, timeout = 1)
except:
pass
#Only add the nameserver to the queue if we can detect wildcards.
#Returns False on error.
def find_wildcards(self, host):
#We want sovle the following three problems:
#1)The target might have a wildcard DNS record.
#2)The target maybe using geolocaiton-aware DNS.
#3)The DNS server we are testing may respond to non-exsistant 'A' records with advertizements.
#I have seen a CloudFlare Enterprise customer with the first two conditions.
try:
#This is case #3, these spam nameservers seem to be more trouble then they are worth.
wildtest = self.resolver.query(uuid.uuid4().hex + ".com", "A")
if len(wildtest):
trace("Spam DNS detected:", host)
return False
except:
pass
test_counter = 8
looking_for_wildcards = True
while looking_for_wildcards and test_counter >= 0 :
looking_for_wildcards = False
#Don't get lost, this nameserver could be playing tricks.
test_counter -= 1
try:
testdomain = "%s.%s" % (uuid.uuid4().hex, host)
wildtest = self.resolver.query(testdomain, self.record_type)
#This 'A' record may contain a list of wildcards.
if wildtest:
for w in wildtest:
w = str(w)
if w not in self.wildcards:
#wildcards were detected.
self.wildcards[w] = None
#We found atleast one wildcard, look for more.
looking_for_wildcards = True
except Exception as e:
if type(e) == dns.resolver.NXDOMAIN or type(e) == dns.name.EmptyLabel:
#not found
return True
else:
#This resolver maybe flakey, we don't want it for our tests.
trace("wildcard exception:", self.resolver.nameservers, type(e))
return False
#If we hit the end of our depth counter and,
#there are still wildcards, then reject this nameserver because it smells bad.
return (test_counter >= 0)
class lookup(multiprocessing.Process):
def __init__(self, in_q, out_q, resolver_q, domain, wildcards, spider_blacklist):
multiprocessing.Process.__init__(self, target = self.run)
signal_init()
self.required_nameservers = 16
self.in_q = in_q
self.out_q = out_q
self.resolver_q = resolver_q
self.domain = domain
self.wildcards = wildcards
self.spider_blacklist = spider_blacklist
self.resolver = dns.resolver.Resolver()
#Force pydns to use our nameservers
self.resolver.nameservers = []
def get_ns(self):
ret = []
try:
ret = [self.resolver_q.get_nowait()]
if ret == False:
#Queue is empty, inform the rest.
self.resolver_q.put(False)
ret = []
except:
pass
return ret
def get_ns_blocking(self):
ret = []
ret = [self.resolver_q.get()]
if ret == False:
trace("get_ns_blocking - Resolver list is empty.")
#Queue is empty, inform the rest.
self.resolver_q.put(False)
ret = []
return ret
def check(self, host, record_type = "A", retries = 0):
trace("Checking:", host)
cname_record = []
retries = 0
if len(self.resolver.nameservers) <= self.required_nameservers:
#This process needs more nameservers, lets see if we have one avaible
self.resolver.nameservers += self.get_ns()
#Ok we should be good to go.
while True:
try:
#Query the nameserver, this is not simple...
if not record_type or record_type == "A":
resp = self.resolver.query(host)
#Crawl the response
hosts = extract_hosts(str(resp.response), self.domain)
for h in hosts:
if h not in self.spider_blacklist:
self.spider_blacklist[h]=None
trace("Found host with spider:", h)
self.in_q.put((h, record_type, 0))
return resp
if record_type == "CNAME":
#A max 20 lookups
for x in range(20):
try:
resp = self.resolver.query(host, record_type)
except dns.resolver.NoAnswer:
resp = False
pass
if resp and resp[0]:
host = str(resp[0]).rstrip(".")
cname_record.append(host)
else:
return cname_record
else:
#All other records:
return self.resolver.query(host, record_type)
except Exception as e:
if type(e) == dns.resolver.NoNameservers:
#We should never be here.
#We must block, another process should try this host.
#do we need a limit?
self.in_q.put((host, record_type, 0))
self.resolver.nameservers += self.get_ns_blocking()
return False
elif type(e) == dns.resolver.NXDOMAIN:
#"Non-existent domain name."
return False
elif type(e) == dns.resolver.NoAnswer:
#"The response did not contain an answer."
if retries >= 1:
trace("NoAnswer retry")
return False
retries += 1
elif type(e) == dns.resolver.Timeout:
trace("lookup failure:", host, retries)
#Check if it is time to give up.
if retries >= 3:
if retries > 3:
#Sometimes 'internal use' subdomains will timeout for every request.
#As far as I'm concerned, the authorative name server has told us this domain exists,
#we just can't know the address value using this method.
return ['Mutiple Query Timeout - External address resolution was restricted']
else:
#Maybe another process can take a crack at it.
self.in_q.put((host, record_type, retries + 1))
return False
retries += 1
#retry...
elif type(e) == IndexError:
#Some old versions of dnspython throw this error,
#doesn't seem to affect the results, and it was fixed in later versions.
pass
elif type(e) == TypeError:
# We'll get here if the number procs > number of resolvers.
# This is an internal error do we need a limit?
self.in_q.put((host, record_type, 0))
return False
elif type(e) == dns.rdatatype.UnknownRdatatype:
error("DNS record type not supported:", record_type)
else:
trace("Problem processing host:", host)
#dnspython threw some strange exception...
raise e
def run(self):
#This process needs one resolver before it can start looking.
self.resolver.nameservers += self.get_ns_blocking()
while True:
found_addresses = []
work = self.in_q.get()
#Check if we have hit the end marker
while not work:
#Look for a re-queued lookup
try:
work = self.in_q.get(blocking = False)
#if we took the end marker of the queue we need to put it back
if work:
self.in_q.put(False)
except:#Queue.Empty
trace('End of work queue')
#There isn't an item behind the end marker
work = False
break
#Is this the end all work that needs to be done?
if not work:
#Perpetuate the end marker for all threads to see
self.in_q.put(False)
#Notify the parent that we have died of natural causes
self.out_q.put(False)
break
else:
if len(work) == 3:
#keep track of how many times this lookup has timedout.
(hostname, record_type, timeout_retries) = work
response = self.check(hostname, record_type, timeout_retries)
else:
(hostname, record_type) = work
response = self.check(hostname, record_type)
sys.stdout.flush()
trace(response)
#self.wildcards is populated by the verify_nameservers() thread.
#This variable doesn't need a muetex, because it has a queue.
#A queue ensure nameserver cannot be used before it's wildcard entries are found.
reject = False
if response:
for a in response:
a = str(a)
if a in self.wildcards:
trace("resovled wildcard:", hostname)
reject= True
#reject this domain.
break;
else:
found_addresses.append(a)
if not reject:
#This request is filled, send the results back
result = (hostname, record_type, found_addresses)
self.out_q.put(result)
#Extract relevant hosts
#The dot at the end of a domain signifies the root,
#and all TLDs are subs of the root.
host_match = re.compile(r"((?<=[\s])[a-zA-Z0-9_-]+\.(?:[a-zA-Z0-9_-]+\.?)+(?=[\s]))")
def extract_hosts(data, hostname):
#made a global to avoid re-compilation
global host_match
ret = []
hosts = re.findall(host_match, data)
for fh in hosts:
host = fh.rstrip(".")
#Is this host in scope?
if host.endswith(hostname):
ret.append(host)
return ret
#Return a list of unique sub domains, sorted by frequency.
#Only match domains that have 3 or more sections subdomain.domain.tld
domain_match = re.compile("([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+")
def extract_subdomains(file_name):
#Avoid re-compilation
global domain_match
subs = {}
sub_file = open(file_name).read()
f_all = re.findall(domain_match, sub_file)
del sub_file
for i in f_all:
if i.find(".") >= 0:
p = i.split(".")[0:-1]
#gobble everything that might be a TLD
while p and len(p[-1]) <= 3:
p = p[0:-1]
#remove the domain name
p = p[0:-1]
#do we have a subdomain.domain left?
if len(p) >= 1:
trace(str(p), " : ", i)
for q in p:
if q :
#domain names can only be lower case.
q = q.lower()
if q in subs:
subs[q] += 1
else:
subs[q] = 1
#Free some memory before the sort...
del f_all
#Sort by freq in desc order
subs_sorted = sorted(subs.keys(), key = lambda x: subs[x], reverse = True)
return subs_sorted
def print_target(target, record_type = None, subdomains = "names.txt", resolve_list = "resolvers.txt", process_count = 16, output = False, json_output = False, found_subdomains=[],verbose=False):
subdomains_list = []
results_temp = []
run(target, record_type, subdomains, resolve_list, process_count)
for result in run(target, record_type, subdomains, resolve_list, process_count):
(hostname, record_type, response) = result
if not record_type:
result = hostname
else:
result = "%s,%s" % (hostname, ",".join(response).strip(","))
if result not in found_subdomains:
if verbose:
print result
subdomains_list.append(result)
return set(subdomains_list)
def run(target, record_type = None, subdomains = "names.txt", resolve_list = "resolvers.txt", process_count = 16):
subdomains = check_open(subdomains)
resolve_list = check_open(resolve_list)
if (len(resolve_list) / 16) < process_count:
sys.stderr.write('Warning: Fewer than 16 resovlers per thread, consider adding more nameservers to resolvers.txt.\n')
wildcards = multiprocessing.Manager().dict()
spider_blacklist = multiprocessing.Manager().dict()
in_q = multiprocessing.Queue()
out_q = multiprocessing.Queue()
#have a buffer of at most two new nameservers that lookup processes can draw from.
resolve_q = multiprocessing.Queue(maxsize = 2)
#Make a source of fast nameservers avaiable for other processes.
verify_nameservers_proc = verify_nameservers(target, record_type, resolve_q, resolve_list, wildcards)
verify_nameservers_proc.start()
#The empty string
in_q.put((target, record_type))
spider_blacklist[target]=None
#A list of subdomains is the input
for s in subdomains:
s = str(s).strip()
if s:
if s.find(","):
#SubBrute should be forgiving, a comma will never be in a url
#but the user might try an use a CSV file as input.
s=s.split(",")[0]
if not s.endswith(target):
hostname = "%s.%s" % (s, target)
else:
#A user might feed an output list as a subdomain list.
hostname = s
if hostname not in spider_blacklist:
spider_blacklist[hostname]=None
work = (hostname, record_type)
in_q.put(work)
#Terminate the queue
in_q.put(False)
for i in range(process_count):
worker = lookup(in_q, out_q, resolve_q, target, wildcards, spider_blacklist)
worker.start()
threads_remaining = process_count
while True:
try:
#The output is valid hostnames
result = out_q.get(True, 10)
#we will get an empty exception before this runs.
if not result:
threads_remaining -= 1
else:
#run() is a generator, and yields results from the work queue
yield result
except Exception as e:
#The cx_freeze version uses queue.Empty instead of Queue.Empty :(
if type(e) == Queue.Empty or str(type(e)) == "<class 'queue.Empty'>":
pass
else:
raise(e)
#make sure everyone is complete
if threads_remaining <= 0:
break
trace("killing nameserver process")
#We no longer require name servers.
try:
killproc(pid = verify_nameservers_proc.pid)
except:
#Windows threading.tread
verify_nameservers_proc.end()
trace("End")
#exit handler for signals. So ctrl+c will work.
#The 'multiprocessing' library each process is it's own process which side-steps the GIL
#If the user wants to exit prematurely, each process must be killed.
def killproc(signum = 0, frame = 0, pid = False):
if not pid:
pid = os.getpid()
if sys.platform.startswith('win'):
try:
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(1, 0, pid)
kernel32.TerminateProcess(handle, 0)
except:
#Oah windows.
pass
else:
os.kill(pid, 9)
#Toggle debug output
verbose = False
def trace(*args, **kwargs):
if verbose:
for a in args:
sys.stderr.write(str(a))
sys.stderr.write(" ")
sys.stderr.write("\n")
def error(*args, **kwargs):
for a in args:
sys.stderr.write(str(a))
sys.stderr.write(" ")
sys.stderr.write("\n")
sys.exit(1)
def check_open(input_file):
ret = []
#If we can't find a resolver from an input file, then we need to improvise.
try:
ret = open(input_file).readlines()
except:
error("File not found:", input_file)
if not len(ret):
error("File is empty:", input_file)
return ret
#Every 'multiprocessing' process needs a signal handler.
#All processes need to die, we don't want to leave zombies.
def signal_init():
#Escliate signal to prevent zombies.
signal.signal(signal.SIGINT, killproc)
try:
signal.signal(signal.SIGTSTP, killproc)
signal.signal(signal.SIGQUIT, killproc)
except:
#Windows
pass
if __name__ == "__main__":
if getattr(sys, 'frozen', False):
# cx_freeze windows:
base_path = os.path.dirname(sys.executable)
multiprocessing.freeze_support()
else:
#everything else:
base_path = os.path.dirname(os.path.realpath(__file__))
parser = optparse.OptionParser("usage: %prog [options] target")
parser.add_option("-s", "--subs", dest = "subs", default = os.path.join(base_path, "names.txt"),
type = "string", help = "(optional) list of subdomains, default = 'names.txt'")
parser.add_option("-r", "--resolvers", dest = "resolvers", default = os.path.join(base_path, "resolvers.txt"),
type = "string", help = "(optional) A list of DNS resolvers, if this list is empty it will OS's internal resolver default = 'resolvers.txt'")
parser.add_option("-t", "--targets_file", dest = "targets", default = "",
type = "string", help = "(optional) A file containing a newline delimited list of domains to brute force.")
parser.add_option("-o", "--output", dest = "output", default = False, help = "(optional) Output to file (Greppable Format)")
parser.add_option("-j", "--json", dest="json", default = False, help="(optional) Output to file (JSON Format)")
parser.add_option("-a", "-A", action = 'store_true', dest = "ipv4", default = False,
help = "(optional) Print all IPv4 addresses for sub domains (default = off).")
parser.add_option("--type", dest = "type", default = False,
type = "string", help = "(optional) Print all reponses for an arbitrary DNS record type (CNAME, AAAA, TXT, SOA, MX...)")
parser.add_option("-c", "--process_count", dest = "process_count",
default = 16, type = "int",
help = "(optional) Number of lookup theads to run. default = 16")
parser.add_option("-f", "--filter_subs", dest = "filter", default = "",
type = "string", help = "(optional) A file containing unorganized domain names which will be filtered into a list of subdomains sorted by frequency. This was used to build names.txt.")
parser.add_option("-v", "--verbose", action = 'store_true', dest = "verbose", default = False,
help = "(optional) Print debug information.")
(options, args) = parser.parse_args()
verbose = options.verbose
if len(args) < 1 and options.filter == "" and options.targets == "":
parser.error("You must provie a target. Use -h for help.")
if options.filter != "":
#cleanup this file and print it out
for d in extract_subdomains(options.filter):
print(d)
sys.exit()
if options.targets != "":
targets = check_open(options.targets) #the domains
else:
targets = args #multiple arguments on the cli: ./subbrute.py google.com gmail.com yahoo.com if (len(resolver_list) / 16) < options.process_count:
output = False
if options.output:
try:
output = open(options.output, "w")
except:
error("Failed writing to file:", options.output)
json_output = False
if options.json:
try:
json_output = open(options.json, "w")
except:
error("Failed writing to file:", options.json)
record_type = False
if options.ipv4:
record_type="A"
if options.type:
record_type = str(options.type).upper()
threads = []
for target in targets:
target = target.strip()
if target:
#target => domain
#record_type =>
#options.subs => file the contain the subdomains list
#options.process_count => process count default = 16
#options.resolvers => the resolvers file
#options.output
#options.json
print target, record_type, options.subs, options.resolvers, options.process_count, output, json_output
print_target(target, record_type, options.subs, options.resolvers, options.process_count, output, json_output)

695
sublist3r.py Normal file
View File

@ -0,0 +1,695 @@
#!/usr/bin/env python
# SubList3r v0.1
# By Ahmed Aboul-Ela - twitter.com/aboul3la
import re
import sys
import os
import argparse
import time
import requests
import urlparse
import urllib
import hashlib
import multiprocessing
import threading
import dns.resolver
from subbrute import subbrute
from collections import Counter
#Console Colors
G = '\033[92m' #green
Y = '\033[93m' #yellow
B = '\033[94m' #blue
R = '\033[91m' #red
W = '\033[0m' #white
def banner():
print """%s
____ _ _ _ _ _____
/ ___| _ _| |__ | (_)___| |_|___ / _ __
\___ \| | | | '_ \| | / __| __| |_ \| '__|
___) | |_| | |_) | | \__ \ |_ ___) | |
|____/ \__,_|_.__/|_|_|___/\__|____/|_|%s%s
# Fast Subomains Enumeration tool using Search Engines and BruteForce
# Coded By Ahmed Aboul-Ela - @aboul3la
# Special Thanks to Ibrahim Mosaad - @ibrahim_mosaad for his contributions%s
"""%(R,W,Y,W)
def parser_error(errmsg):
banner()
print "Usage: python "+sys.argv[0]+" [Options] use -h for help"
print R+"Error: "+errmsg+W
sys.exit()
def parse_args():
#parse the arguments
parser = argparse.ArgumentParser(epilog='\tExample: \r\npython '+sys.argv[0]+" -d google.com")
parser.error = parser_error
parser._optionals.title = "OPTIONS"
parser.add_argument('-d','--domain', help='Domain name to enumrate it\'s subdomains', required=True)
parser.add_argument('-b','--bruteforce', help='Enable the subbrute bruteforce module', nargs='?', default=False)
parser.add_argument('-v','--verbose', help='Enable Verbosity and display results in realtime', nargs='?', default=False)
parser.add_argument('-t','--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=10)
parser.add_argument('-o','--output', help='Save the results to text file')
return parser.parse_args()
def write_file(filename,subdomains):
#saving subdomains results to output file
print "%s[-] Saving results to file: %s%s%s%s"%(Y,W,R,filename,W)
f = open(str(filename),'wb')
for subdomain in subdomains:
f.write(subdomain+"\r\n")
f.close()
class enumratorBase(object):
def __init__(self, base_url, engine_name, domain , subdomains=[]):
self.domain = urlparse.urlparse(domain).netloc
self.session=requests.Session()
self.subdomains = []
self.timeout = 10
self.base_url = base_url
self.engine_name = engine_name
self.print_banner()
def print_banner(self):
""" subclass can override this if they want a fancy banner :)"""
print G+"[-] Searching now in %s.." %(self.engine_name)+W
return
def send_req(self, query, page_no=1):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive'
}
url = self.base_url.format(query=query, page_no=page_no)
try:
resp = self.session.get(url, headers=headers, timeout=self.timeout)
except Exception as e:
print e
raise
return resp.text
def check_max_subdomains(self,count):
if self.MAX_DOMAINS == 0:
return False
if count >= self.MAX_DOMAINS:
return True
else:
return False
def check_max_pages(self,num):
if self.MAX_PAGES == 0:
return False
if num >= self.MAX_PAGES:
return True
else:
return False
#Override
def extract_domains(self, resp):
""" chlid class should override this function """
return
#override
def check_response_errors(self,resp):
""" chlid class should override this function
The function should return True if there are no errors and False otherwise
"""
return True
def should_sleep(self):
"""Some enumrators require sleeping to avoid bot detections like Google enumerator"""
return
def generate_query(self):
""" chlid class should override this function """
return
def get_page(self,num):
""" chlid class that user different pagnation counter should override this function """
return num+10
def enumerate(self,altquery=False):
flag = True
page_no = 0
prev_links =[]
prev_subdomains = []
retries = 0
while flag:
query = self.generate_query()
count = query.count(self.domain) #finding the number of subdomains found so far
#if they we reached the maximum number of subdomains in search query then we should go over the pages
if self.check_max_subdomains(count):
page_no= self.get_page(page_no)
if self.check_max_pages(page_no): #maximum pages for Google to avoid getting blocked
return self.subdomains
resp = self.send_req(query, page_no)
#check if there is any error occured
if not self.check_response_errors(resp):
return self.subdomains
links = self.extract_domains(resp)
#if the previous page hyperlinks was the similar to the current one, then maybe we have reached the last page
if links == prev_links:
retries+=1
page_no= self.get_page(page_no)
#make another retry maybe it isn't the last page
if retries >= 3:
return self.subdomains
prev_links = links
self.should_sleep()
return self.subdomains
class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
def __init__(self, base_url, engine_name, domain , subdomains=[], q=None,lock=threading.Lock()):
enumratorBase.__init__(self, base_url, engine_name, domain, subdomains)
multiprocessing.Process.__init__(self)
self.lock = lock
self.q=q
return
def run(self):
domain_list = self.enumerate()
self.q.put(domain_list)
class GoogleEnum(enumratorBaseThreaded):
def __init__(self, domain , subdomains=[], q=None):
base_url = "https://google.com/search?q={query}&btnG=Search&hl=en-US&biw=&bih=&gbv=1&start={page_no}&filter=0"
self.engine_name="Google"
self.MAX_DOMAINS = 11
self.MAX_PAGES = 200
super(GoogleEnum, self).__init__(base_url, self.engine_name,domain, subdomains,q=q)
self.q=q
return
def extract_domains(self, resp):
link_regx = re.compile('<cite.*?>(.*?)<\/cite>')
try:
links_list = link_regx.findall(resp)
for link in links_list:
link = re.sub('<span.*>','',link)
if not link.startswith('http'):
link="http://"+link
subdomain = urlparse.urlparse(link).netloc
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
return links_list
def check_response_errors(self, resp):
if 'Our systems have detected unusual traffic' in resp:
print R+"Error: Google now probably is blocking our requests"+W
print R+"Exiting..."+W
return False
return True
def should_sleep(self):
time.sleep(5)
return
def generate_query(self):
if len(self.subdomains) > 0:
query = "site:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS-2]))
else:
query = "site:{domain} -www.{domain}".format(domain=self.domain)
return query
class YahooEnum(enumratorBaseThreaded):
def __init__(self, domain , subdomains=[], q=None):
base_url = "https://search.yahoo.com/search?p={query}&b={page_no}"
self.engine_name="Yahoo"
self.MAX_DOMAINS = 10
self.MAX_PAGES = 0
super(YahooEnum, self).__init__(base_url, self.engine_name,domain, subdomains, q=q)
self.q=q
return
def extract_domains(self, resp):
link_regx2 = re.compile('<span class=" fz-15px fw-m fc-12th wr-bw">(.*?)</span>')
link_regx = re.compile('<span class="txt"><span class=" cite fw-xl fz-15px">(.*?)</span>')
try:
links = link_regx.findall(resp)
links2 = link_regx2.findall(resp)
links_list = links+links2
for link in links_list:
link = re.sub("<(\/)?b>","",link)
if not link.startswith('http'):
link="http://"+link
subdomain = urlparse.urlparse(link).netloc
if not subdomain.endswith(self.domain):
continue
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != self.domain and subdomain != '':
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
return links_list
def should_sleep(self):
return
def get_page(self,num):
return num+10
def generate_query(self):
if len(self.subdomains) > 0:
query = "site:{domain} -domain:www.{domain} -domain:{found}".format(domain=self.domain, found=' -domain:'.join(self.subdomains[:77]))
else:
query = "site:{domain}".format(domain=self.domain)
return query
class AskEnum(enumratorBaseThreaded):
def __init__(self, domain , subdomains=[], q=None):
base_url = 'http://www.ask.com/web?q={query}&page={page_no}&qid=8D6EE6BF52E0C04527E51F64F22C4534&o=0&l=dir&qsrc=998&qo=pagination'
self.engine_name="Ask"
self.MAX_DOMAINS = 11
self.MAX_PAGES = 0
enumratorBaseThreaded.__init__(self, base_url, self.engine_name,domain, subdomains, q=q)
self.q=q
return
def extract_domains(self, resp):
link_regx = re.compile('<p class="web-result-url">(.*?)</p>')
try:
links_list = link_regx.findall(resp)
for link in links_list:
if not link.startswith('http'):
link="http://"+link
subdomain = urlparse.urlparse(link).netloc
if subdomain not in self.subdomains and subdomain != self.domain:
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
return links_list
def get_page(self,num):
return num+1
def generate_query(self):
if len(self.subdomains) > 0:
query = "site:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS]))
else:
query = "site:{domain} -www.{domain}".format(domain=self.domain)
return query
class BingEnum(enumratorBaseThreaded):
def __init__(self, domain , subdomains=[], q=None):
base_url = 'https://www.bing.com/search?q={query}&go=Submit&first={page_no}'
self.engine_name="Bing"
self.MAX_DOMAINS = 30
self.MAX_PAGES = 0
enumratorBaseThreaded.__init__(self, base_url, self.engine_name,domain, subdomains,q=q)
self.q=q
return
def extract_domains(self, resp):
link_regx = re.compile('<li class="b_algo"><h2><a href="(.*?)"')
link_regx2 = re.compile('<div class="b_title"><h2><a href="(.*?)"')
try:
links = link_regx.findall(resp)
links2 = link_regx2.findall(resp)
links_list = links+links2
for link in links_list:
link = re.sub('<(\/)?strong>|<span.*?>|<|>','',link)
if not link.startswith('http'):
link="http://"+link
subdomain = urlparse.urlparse(link).netloc
if subdomain not in self.subdomains and subdomain != self.domain:
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
return links_list
def generate_query(self):
if len(self.subdomains) > 0:
query = "domain:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS]))
else:
query = "domain:{domain} -www.{domain}".format(domain=self.domain)
return query
class BaiduEnum(enumratorBaseThreaded):
def __init__(self, domain , subdomains=[], q=None):
base_url = 'http://www.baidu.com/s?pn={page_no}&wd={query}'
self.engine_name="Baidu"
self.MAX_DOMAINS = 2
self.MAX_PAGES = 760
enumratorBaseThreaded.__init__(self, base_url, self.engine_name,domain, subdomains, q=q)
self.querydomain = self.domain
self.q=q
return
def extract_domains(self, resp):
found_newdomain = False
subdomain_list = []
link_regx = re.compile('<a.*?class="c-showurl".*?>(.*?)</a>')
try:
links = link_regx.findall(resp)
for link in links:
link = re.sub('<.*?>|>|<|&nbsp;','',link)
if not link.startswith('http'):
link="http://"+link
subdomain = urlparse.urlparse(link).netloc
if subdomain.endswith(self.domain):
subdomain_list.append(subdomain)
if subdomain not in self.subdomains and subdomain != self.domain:
found_newdomain = True
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
if not found_newdomain and len(subdomain_list) != 0:
self.querydomain = self.findsubs(subdomain_list)
return links
def findsubs(self,subdomains):
count = Counter(subdomains)
subdomain1 = max(count, key=count.get)
count.pop(subdomain1,"None")
if len(count) > 0:
subdomain2 = max(count, key=count.get)
else:
subdomain2 = ''
return (subdomain1,subdomain2)
def check_response_errors(self, resp):
return True
def should_sleep(self):
return
def generate_query(self):
if len(self.subdomains) > 0 and self.querydomain != self.domain:
query = "site:{domain} -site:{found} ".format(domain=self.domain, found=' -site:'.join(self.querydomain))
else:
query = "site:{domain}".format(domain=self.domain)
return query
class NetcraftEnum(multiprocessing.Process):
def __init__(self, domain , subdomains=[], q=None,lock=threading.Lock()):
self.base_url = 'http://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session=requests.Session()
self.engine_name="Netcraft"
multiprocessing.Process.__init__(self)
self.lock = lock
self.q=q
self.timeout = 10
self.print_banner()
return
def run(self):
domain_list = self.enumerate()
self.q.put(domain_list)
return
def print_banner(self):
print G+"[-] Searching now in %s.." %(self.engine_name)+W
return
def req(self,url,cookies=dict()):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
}
try:
resp = self.session.get(url, headers=headers, timeout=self.timeout,cookies=cookies)
except Exception as e:
print e
raise
return resp
def get_next(self,resp):
link_regx = re.compile('<A href="(.*?)"><b>Next page</b></a>')
link = link_regx.findall(resp)
link = re.sub('host=.*?%s'%self.domain,'host=%s'%self.domain,link[0])
url = 'http://searchdns.netcraft.com'+link
return url
def create_cookies(self,cookie):
cookies = dict()
cookies_list = cookie[0:cookie.find(';')].split("=")
cookies[cookies_list[0]] = cookies_list[1]
cookies['netcraft_js_verification_response'] = hashlib.sha1(urllib.unquote(cookies_list[1])).hexdigest()
return cookies
def enumerate(self):
start_url = self.base_url.format(domain='example.com')
resp = self.req(start_url)
cookies = self.create_cookies(resp.headers['set-cookie'])
url = self.base_url.format(domain=self.domain)
while True:
resp = self.req(url,cookies).text
self.extract_domains(resp)
if not 'Next page' in resp:
return self.subdomains
break
url = self.get_next(resp)
def extract_domains(self, resp):
link_regx = re.compile('<a href="http://toolbar.netcraft.com/site_report\?url=(.*)">')
try:
links_list = link_regx.findall(resp)
for link in links_list:
subdomain = urlparse.urlparse(link).netloc
if not subdomain.endswith(self.domain):
continue
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
except Exception as e:
pass
return links_list
class DNSdumpster(multiprocessing.Process):
def __init__(self, domain , subdomains=[], q=None,lock=threading.Lock()):
self.base_url = 'https://dnsdumpster.com/'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session=requests.Session()
self.engine_name="DNSdumpster"
multiprocessing.Process.__init__(self)
self.lock = lock
self.q=q
self.timeout = 10
self.print_banner()
return
def run(self):
domain_list = self.enumerate()
self.q.put(domain_list)
return
def print_banner(self):
print G+"[-] Searching now in %s.." %(self.engine_name)+W
return
def check_host(self,host):
is_valid = False
Resolver = dns.resolver.Resolver()
Resolver.nameservers = ['8.8.8.8', '8.8.4.4']
try:
ip = Resolver.query(host, 'A')[0].to_text()
if ip != '' and ip is not None:
is_valid = True
except:
pass
return is_valid
def req(self,req_method,url,params=dict()):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Referer':'https://dnsdumpster.com'
}
try:
if req_method == 'GET':
resp = self.session.get(url, headers=headers, timeout=self.timeout)
else:
resp = self.session.post(url,data=params,headers=headers,timeout=self.timeout)
except Exception as e:
print e
raise
return resp.text
def get_csrftoken(self,resp):
csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />",re.S)
token = csrf_regex.findall(resp)[0]
return token.strip()
def enumerate(self):
resp = self.req('GET',self.base_url)
token = self.get_csrftoken(resp)
params = {'csrfmiddlewaretoken':token,'targetip':self.domain}
post_resp = self.req('POST',self.base_url,params)
self.extract_domains(post_resp)
return self.subdomains
def extract_domains(self, resp):
tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>',re.S)
link_regex = re.compile('<td class="col-md-4">(.*?)<br>',re.S)
links = []
results_tbl = tbl_regex.findall(resp)[0]
links_list = link_regex.findall(results_tbl)
links = list(set(links_list))
for link in links:
subdomain = link.strip()
if not subdomain.endswith(self.domain):
continue
if self.check_host(subdomain) and subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
if verbose:
print "%s%s: %s%s"%(R,self.engine_name,W,subdomain)
self.subdomains.append(subdomain)
return links
def main():
args = parse_args()
domain = args.domain
threads = args.threads
savefile = args.output
google_list = []
bing_list = []
baidu_list = []
bruteforce_list = set()
subdomains_queue = multiprocessing.Queue()
#Check Verbosity
global verbose
verbose = args.verbose
if verbose or verbose is None:
verbose = True
#Check Bruteforce Status
enable_bruteforce = args.bruteforce
if enable_bruteforce or enable_bruteforce is None:
enable_bruteforce = True
#Validate domain
domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,5}$")
if not domain_check.match(domain):
print R+"Error: Please enter a valid domain"+W
sys.exit()
if not domain.startswith('http://') or not domain.startswith('https://'):
domain = 'http://'+domain
#Print the Banner
banner()
parsed_domain = urlparse.urlparse(domain)
print B+"[-] Enumerating subdomains now for %s"%parsed_domain.netloc+W
if verbose:
print Y+"[-] verbosity is enabled, will show the subdomains results in realtime"+W
#Start the engines enumeration
enum_baidu = BaiduEnum(domain, verbose,q=subdomains_queue)
enum_yahoo = YahooEnum(domain, verbose,q=subdomains_queue)
enum_google = GoogleEnum(domain, verbose, q=subdomains_queue)
enum_bing = BingEnum(domain, verbose, q=subdomains_queue)
enum_ask = AskEnum(domain, verbose, q=subdomains_queue)
enum_netcraft = NetcraftEnum(domain, verbose, q=subdomains_queue)
enum_dnsdumpester = DNSdumpster(domain, verbose, q=subdomains_queue)
enum_baidu.start()
enum_yahoo.start()
enum_google.start()
enum_bing.start()
enum_ask.start()
enum_netcraft.start()
enum_dnsdumpester.start()
enum_baidu.join()
enum_yahoo.join()
enum_google.join()
enum_bing.join()
enum_ask.join()
enum_netcraft.join()
enum_dnsdumpester.join()
search_list = set()
while not subdomains_queue.empty():
search_list= search_list.union(subdomains_queue.get())
if enable_bruteforce:
print G+"[-] Starting bruteforce module now using subbrute.."+W
record_type = False
subs = os.path.join(os.path.dirname(os.path.realpath(__file__)), "subbrute/names.txt")
resolvers=os.path.join(os.path.dirname(os.path.realpath(__file__)) ,"subbrute/resolvers.txt")
process_count=threads
output = False
json_output=False
bruteforce_list = subbrute.print_target(parsed_domain.netloc, record_type, subs, resolvers, process_count, output, json_output, search_list,verbose)
subdomains = search_list.union(bruteforce_list)
if len(subdomains) > 0:
if savefile is not None:
write_file(savefile,subdomains)
print Y+"[-] Total Unique Subdomains Found: %s"%len(subdomains)+W
for subdomain in subdomains:
print G+subdomain+W
if __name__=="__main__":
main()