Merge pull request #2 from cclauss/patch-1
Avoid [] or {} as default value for function params
This commit is contained in:
Ahmed Aboul-Ela
commit
13feabda83
164
sublist3r.py
164
sublist3r.py
|
|
@ -1,6 +1,8 @@
|
|||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
# SubList3r v0.1
|
||||
# By Ahmed Aboul-Ela - twitter.com/aboul3la
|
||||
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
|
|
@ -50,7 +52,7 @@ def parse_args():
|
|||
parser = argparse.ArgumentParser(epilog = '\tExample: \r\npython '+sys.argv[0]+" -d google.com")
|
||||
parser.error = parser_error
|
||||
parser._optionals.title = "OPTIONS"
|
||||
parser.add_argument('-d','--domain', help='Domain name to enumrate it\'s subdomains', required=True)
|
||||
parser.add_argument('-d', '--domain', help="Domain name to enumrate it's subdomains", required=True)
|
||||
parser.add_argument('-b', '--bruteforce', help='Enable the subbrute bruteforce module', nargs='?', default=False)
|
||||
parser.add_argument('-v', '--verbose', help='Enable Verbosity and display results in realtime', nargs='?', default=False)
|
||||
parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=10)
|
||||
|
|
@ -60,13 +62,13 @@ def parse_args():
|
|||
def write_file(filename, subdomains):
|
||||
#saving subdomains results to output file
|
||||
print "%s[-] Saving results to file: %s%s%s%s"%(Y,W,R,filename,W)
|
||||
f = open(str(filename),'wb')
|
||||
with open(str(filename), 'wb') as f:
|
||||
for subdomain in subdomains:
|
||||
f.write(subdomain+"\r\n")
|
||||
f.close()
|
||||
|
||||
class enumratorBase(object):
|
||||
def __init__(self, base_url, engine_name, domain , subdomains=[]):
|
||||
def __init__(self, base_url, engine_name, domain, subdomains=None):
|
||||
subdomains = subdomains or []
|
||||
self.domain = urlparse.urlparse(domain).netloc
|
||||
self.session = requests.Session()
|
||||
self.subdomains = []
|
||||
|
|
@ -81,7 +83,6 @@ class enumratorBase(object):
|
|||
return
|
||||
|
||||
def send_req(self, query, page_no=1):
|
||||
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-GB,en;q=0.5',
|
||||
|
|
@ -100,18 +101,12 @@ class enumratorBase(object):
|
|||
def check_max_subdomains(self,count):
|
||||
if self.MAX_DOMAINS == 0:
|
||||
return False
|
||||
if count >= self.MAX_DOMAINS:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return count >= self.MAX_DOMAINS
|
||||
|
||||
def check_max_pages(self, num):
|
||||
if self.MAX_PAGES == 0:
|
||||
return False
|
||||
if num >= self.MAX_PAGES:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return num >= self.MAX_PAGES
|
||||
|
||||
#Override
|
||||
def extract_domains(self, resp):
|
||||
|
|
@ -133,7 +128,6 @@ class enumratorBase(object):
|
|||
""" chlid class should override this function """
|
||||
return
|
||||
|
||||
|
||||
def get_page(self, num):
|
||||
""" chlid class that user different pagnation counter should override this function """
|
||||
return num + 10
|
||||
|
|
@ -149,20 +143,18 @@ class enumratorBase(object):
|
|||
query = self.generate_query()
|
||||
count = query.count(self.domain) #finding the number of subdomains found so far
|
||||
|
||||
#if they we reached the maximum number of subdomains in search query then we should go over the pages
|
||||
#if they we reached the maximum number of subdomains in search query
|
||||
#then we should go over the pages
|
||||
if self.check_max_subdomains(count):
|
||||
page_no = self.get_page(page_no)
|
||||
|
||||
|
||||
if self.check_max_pages(page_no): #maximum pages for Google to avoid getting blocked
|
||||
return self.subdomains
|
||||
|
||||
resp = self.send_req(query, page_no)
|
||||
|
||||
#check if there is any error occured
|
||||
if not self.check_response_errors(resp):
|
||||
return self.subdomains
|
||||
|
||||
links = self.extract_domains(resp)
|
||||
|
||||
#if the previous page hyperlinks was the similar to the current one, then maybe we have reached the last page
|
||||
|
|
@ -181,7 +173,8 @@ class enumratorBase(object):
|
|||
|
||||
|
||||
class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
|
||||
def __init__(self, base_url, engine_name, domain , subdomains=[], q=None,lock=threading.Lock()):
|
||||
def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, lock=threading.Lock()):
|
||||
subdomains = subdomains or []
|
||||
enumratorBase.__init__(self, base_url, engine_name, domain, subdomains)
|
||||
multiprocessing.Process.__init__(self)
|
||||
self.lock = lock
|
||||
|
|
@ -194,7 +187,8 @@ class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
|
|||
|
||||
|
||||
class GoogleEnum(enumratorBaseThreaded):
|
||||
def __init__(self, domain , subdomains=[], q=None):
|
||||
def __init__(self, domain, subdomains=None, q=None):
|
||||
subdomains = subdomains or []
|
||||
base_url = "https://google.com/search?q={query}&btnG=Search&hl=en-US&biw=&bih=&gbv=1&start={page_no}&filter=0"
|
||||
self.engine_name = "Google"
|
||||
self.MAX_DOMAINS = 11
|
||||
|
|
@ -212,8 +206,7 @@ class GoogleEnum(enumratorBaseThreaded):
|
|||
if not link.startswith('http'):
|
||||
link="http://"+link
|
||||
subdomain = urlparse.urlparse(link).netloc
|
||||
|
||||
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
|
||||
if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
|
||||
if verbose:
|
||||
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
|
||||
self.subdomains.append(subdomain)
|
||||
|
|
@ -228,20 +221,22 @@ class GoogleEnum(enumratorBaseThreaded):
|
|||
return False
|
||||
return True
|
||||
|
||||
|
||||
def should_sleep(self):
|
||||
time.sleep(5)
|
||||
return
|
||||
|
||||
def generate_query(self):
|
||||
if len(self.subdomains) > 0:
|
||||
query = "site:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS-2]))
|
||||
if self.subdomains:
|
||||
fmt = 'site:{domain} -www.{domain} -{found}'
|
||||
found = ' -'.join(self.subdomains[:self.MAX_DOMAINS-2])
|
||||
query = fmt.format(domain=self.domain, found=found)
|
||||
else:
|
||||
query = "site:{domain} -www.{domain}".format(domain=self.domain)
|
||||
return query
|
||||
|
||||
class YahooEnum(enumratorBaseThreaded):
|
||||
def __init__(self, domain , subdomains=[], q=None):
|
||||
def __init__(self, domain, subdomains=None, q=None):
|
||||
subdomains = subdomains or []
|
||||
base_url = "https://search.yahoo.com/search?p={query}&b={page_no}"
|
||||
self.engine_name = "Yahoo"
|
||||
self.MAX_DOMAINS = 10
|
||||
|
|
@ -259,16 +254,12 @@ class YahooEnum(enumratorBaseThreaded):
|
|||
links_list = links+links2
|
||||
for link in links_list:
|
||||
link = re.sub("<(\/)?b>","", link)
|
||||
|
||||
if not link.startswith('http'):
|
||||
link="http://"+link
|
||||
|
||||
subdomain = urlparse.urlparse(link).netloc
|
||||
|
||||
if not subdomain.endswith(self.domain):
|
||||
continue
|
||||
|
||||
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != self.domain and subdomain != '':
|
||||
if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
|
||||
if verbose:
|
||||
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
|
||||
self.subdomains.append(subdomain)
|
||||
|
|
@ -284,14 +275,17 @@ class YahooEnum(enumratorBaseThreaded):
|
|||
return num + 10
|
||||
|
||||
def generate_query(self):
|
||||
if len(self.subdomains) > 0:
|
||||
query = "site:{domain} -domain:www.{domain} -domain:{found}".format(domain=self.domain, found=' -domain:'.join(self.subdomains[:77]))
|
||||
if self.subdomains:
|
||||
fmt = 'site:{domain} -domain:www.{domain} -domain:{found}'
|
||||
found = ' -domain:'.join(self.subdomains[:77])
|
||||
query = fmt.format(domain=self.domain, found=found)
|
||||
else:
|
||||
query = "site:{domain}".format(domain=self.domain)
|
||||
return query
|
||||
|
||||
class AskEnum(enumratorBaseThreaded):
|
||||
def __init__(self, domain , subdomains=[], q=None):
|
||||
def __init__(self, domain, subdomains=None, q=None):
|
||||
subdomains = subdomains or []
|
||||
base_url = 'http://www.ask.com/web?q={query}&page={page_no}&qid=8D6EE6BF52E0C04527E51F64F22C4534&o=0&l=dir&qsrc=998&qo=pagination'
|
||||
self.engine_name = "Ask"
|
||||
self.MAX_DOMAINS = 11
|
||||
|
|
@ -321,15 +315,18 @@ class AskEnum(enumratorBaseThreaded):
|
|||
return num + 1
|
||||
|
||||
def generate_query(self):
|
||||
if len(self.subdomains) > 0:
|
||||
query = "site:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS]))
|
||||
if self.subdomains:
|
||||
fmt = 'site:{domain} -www.{domain} -{found}'
|
||||
found = ' -'.join(self.subdomains[:self.MAX_DOMAINS])
|
||||
query = fmt.format(domain=self.domain, found=found)
|
||||
else:
|
||||
query = "site:{domain} -www.{domain}".format(domain=self.domain)
|
||||
|
||||
return query
|
||||
|
||||
class BingEnum(enumratorBaseThreaded):
|
||||
def __init__(self, domain , subdomains=[], q=None):
|
||||
def __init__(self, domain, subdomains=None, q=None):
|
||||
subdomains = subdomains or []
|
||||
base_url = 'https://www.bing.com/search?q={query}&go=Submit&first={page_no}'
|
||||
self.engine_name = "Bing"
|
||||
self.MAX_DOMAINS = 30
|
||||
|
|
@ -355,22 +352,24 @@ class BingEnum(enumratorBaseThreaded):
|
|||
if verbose:
|
||||
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
|
||||
self.subdomains.append(subdomain)
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return links_list
|
||||
|
||||
def generate_query(self):
|
||||
if len(self.subdomains) > 0:
|
||||
query = "domain:{domain} -www.{domain} -{found}".format(domain=self.domain, found=' -'.join(self.subdomains[:self.MAX_DOMAINS]))
|
||||
if self.subdomains:
|
||||
fmt = 'domain:{domain} -www.{domain} -{found}'
|
||||
found = ' -'.join(self.subdomains[:self.MAX_DOMAINS])
|
||||
query = fmt.format(domain=self.domain, found=found)
|
||||
else:
|
||||
query = "domain:{domain} -www.{domain}".format(domain=self.domain)
|
||||
return query
|
||||
|
||||
|
||||
class BaiduEnum(enumratorBaseThreaded):
|
||||
def __init__(self, domain , subdomains=[], q=None):
|
||||
def __init__(self, domain, subdomains=None, q=None):
|
||||
subdomains = subdomains or []
|
||||
base_url = 'http://www.baidu.com/s?pn={page_no}&wd={query}'
|
||||
self.engine_name = "Baidu"
|
||||
self.MAX_DOMAINS = 2
|
||||
|
|
@ -400,8 +399,7 @@ class BaiduEnum(enumratorBaseThreaded):
|
|||
self.subdomains.append(subdomain)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
if not found_newdomain and len(subdomain_list) != 0:
|
||||
if not found_newdomain and subdomain_list:
|
||||
self.querydomain = self.findsubs(subdomain_list)
|
||||
return links
|
||||
|
||||
|
|
@ -409,15 +407,9 @@ class BaiduEnum(enumratorBaseThreaded):
|
|||
count = Counter(subdomains)
|
||||
subdomain1 = max(count, key=count.get)
|
||||
count.pop(subdomain1, "None")
|
||||
|
||||
if len(count) > 0:
|
||||
subdomain2 = max(count, key=count.get)
|
||||
else:
|
||||
subdomain2 = ''
|
||||
|
||||
subdomain2 = max(count, key=count.get) if count else ''
|
||||
return (subdomain1, subdomain2)
|
||||
|
||||
|
||||
def check_response_errors(self, resp):
|
||||
return True
|
||||
|
||||
|
|
@ -425,15 +417,16 @@ class BaiduEnum(enumratorBaseThreaded):
|
|||
return
|
||||
|
||||
def generate_query(self):
|
||||
if len(self.subdomains) > 0 and self.querydomain != self.domain:
|
||||
query = "site:{domain} -site:{found} ".format(domain=self.domain, found=' -site:'.join(self.querydomain))
|
||||
if self.subdomains and self.querydomain != self.domain:
|
||||
found = ' -site:'.join(self.querydomain)
|
||||
query = "site:{domain} -site:{found} ".format(domain=self.domain, found=found)
|
||||
else:
|
||||
query = "site:{domain}".format(domain=self.domain)
|
||||
|
||||
return query
|
||||
|
||||
class NetcraftEnum(multiprocessing.Process):
|
||||
def __init__(self, domain , subdomains=[], q=None,lock=threading.Lock()):
|
||||
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock()):
|
||||
subdomains = subdomains or []
|
||||
self.base_url = 'http://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}'
|
||||
self.domain = urlparse.urlparse(domain).netloc
|
||||
self.subdomains = []
|
||||
|
|
@ -455,7 +448,8 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
print G+"[-] Searching now in %s.." %(self.engine_name)+W
|
||||
return
|
||||
|
||||
def req(self,url,cookies=dict()):
|
||||
def req(self, url, cookies=None)
|
||||
cookies = cookies or {}
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-GB,en;q=0.5',
|
||||
|
|
@ -468,7 +462,6 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
raise
|
||||
return resp
|
||||
|
||||
|
||||
def get_next(self, resp):
|
||||
link_regx = re.compile('<A href="(.*?)"><b>Next page</b></a>')
|
||||
link = link_regx.findall(resp)
|
||||
|
|
@ -476,7 +469,6 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
url = 'http://searchdns.netcraft.com'+link
|
||||
return url
|
||||
|
||||
|
||||
def create_cookies(self, cookie):
|
||||
cookies = dict()
|
||||
cookies_list = cookie[0:cookie.find(';')].split("=")
|
||||
|
|
@ -484,7 +476,6 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
cookies['netcraft_js_verification_response'] = hashlib.sha1(urllib.unquote(cookies_list[1])).hexdigest()
|
||||
return cookies
|
||||
|
||||
|
||||
def enumerate(self):
|
||||
start_url = self.base_url.format(domain='example.com')
|
||||
resp = self.req(start_url)
|
||||
|
|
@ -506,7 +497,7 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
subdomain = urlparse.urlparse(link).netloc
|
||||
if not subdomain.endswith(self.domain):
|
||||
continue
|
||||
if subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
|
||||
if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
|
||||
if verbose:
|
||||
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
|
||||
self.subdomains.append(subdomain)
|
||||
|
|
@ -515,10 +506,9 @@ class NetcraftEnum(multiprocessing.Process):
|
|||
return links_list
|
||||
|
||||
|
||||
|
||||
|
||||
class DNSdumpster(multiprocessing.Process):
|
||||
def __init__(self, domain , subdomains=[], q=None,lock=threading.Lock()):
|
||||
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock()):
|
||||
subdomains = subdomains or []
|
||||
self.base_url = 'https://dnsdumpster.com/'
|
||||
self.domain = urlparse.urlparse(domain).netloc
|
||||
self.subdomains = []
|
||||
|
|
@ -546,14 +536,14 @@ class DNSdumpster(multiprocessing.Process):
|
|||
Resolver.nameservers = ['8.8.8.8', '8.8.4.4']
|
||||
try:
|
||||
ip = Resolver.query(host, 'A')[0].to_text()
|
||||
if ip != '' and ip is not None:
|
||||
if ip:
|
||||
is_valid = True
|
||||
except:
|
||||
pass
|
||||
return is_valid
|
||||
|
||||
def req(self,req_method,url,params=dict()):
|
||||
|
||||
def req(self, req_method, url, params=None):
|
||||
params = params or {}
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-GB,en;q=0.5',
|
||||
|
|
@ -566,13 +556,11 @@ class DNSdumpster(multiprocessing.Process):
|
|||
resp = self.session.get(url, headers=headers, timeout=self.timeout)
|
||||
else:
|
||||
resp = self.session.post(url, data=params, headers=headers, timeout=self.timeout)
|
||||
|
||||
except Exception as e:
|
||||
print e
|
||||
raise
|
||||
return resp.text
|
||||
|
||||
|
||||
def get_csrftoken(self, resp):
|
||||
csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />",re.S)
|
||||
token = csrf_regex.findall(resp)[0]
|
||||
|
|
@ -597,14 +585,13 @@ class DNSdumpster(multiprocessing.Process):
|
|||
subdomain = link.strip()
|
||||
if not subdomain.endswith(self.domain):
|
||||
continue
|
||||
if self.check_host(subdomain) and subdomain not in self.subdomains and subdomain != self.domain and subdomain != '':
|
||||
if self.check_host(subdomain) and subdomain and subdomain not in self.subdomains and subdomain != self.domain:
|
||||
if verbose:
|
||||
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
|
||||
self.subdomains.append(subdomain)
|
||||
return links
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
domain = args.domain
|
||||
|
|
@ -646,40 +633,23 @@ def main():
|
|||
print Y+"[-] verbosity is enabled, will show the subdomains results in realtime"+W
|
||||
|
||||
#Start the engines enumeration
|
||||
enum_baidu = BaiduEnum(domain, verbose,q=subdomains_queue)
|
||||
enum_yahoo = YahooEnum(domain, verbose,q=subdomains_queue)
|
||||
enum_google = GoogleEnum(domain, verbose, q=subdomains_queue)
|
||||
enum_bing = BingEnum(domain, verbose, q=subdomains_queue)
|
||||
enum_ask = AskEnum(domain, verbose, q=subdomains_queue)
|
||||
enum_netcraft = NetcraftEnum(domain, verbose, q=subdomains_queue)
|
||||
enum_dnsdumpester = DNSdumpster(domain, verbose, q=subdomains_queue)
|
||||
|
||||
enum_baidu.start()
|
||||
enum_yahoo.start()
|
||||
enum_google.start()
|
||||
enum_bing.start()
|
||||
enum_ask.start()
|
||||
enum_netcraft.start()
|
||||
enum_dnsdumpester.start()
|
||||
|
||||
enum_baidu.join()
|
||||
enum_yahoo.join()
|
||||
enum_google.join()
|
||||
enum_bing.join()
|
||||
enum_ask.join()
|
||||
enum_netcraft.join()
|
||||
enum_dnsdumpester.join()
|
||||
enums = [enum(domain, verbose, q=subdomains_queue) for enum in AskEnum,
|
||||
BaiduEnum, BingEnum, DNSdumpster, GoogleEnum, NetcraftEnum, YahooEnum]
|
||||
for enum in enums:
|
||||
enum.start()
|
||||
for enum in enums:
|
||||
enum.join()
|
||||
|
||||
search_list = set()
|
||||
|
||||
while not subdomains_queue.empty():
|
||||
search_list= search_list.union(subdomains_queue.get())
|
||||
|
||||
if enable_bruteforce:
|
||||
print G+"[-] Starting bruteforce module now using subbrute.."+W
|
||||
record_type = False
|
||||
subs = os.path.join(os.path.dirname(os.path.realpath(__file__)), "subbrute/names.txt")
|
||||
resolvers=os.path.join(os.path.dirname(os.path.realpath(__file__)) ,"subbrute/resolvers.txt")
|
||||
path_to_file = os.path.dirname(os.path.realpath(__file__))
|
||||
subs = os.path.join(path_to_file, 'subbrute', 'names.txt')
|
||||
resolvers = os.path.join(path_to_file, 'subbrute', 'resolvers.txt')
|
||||
process_count = threads
|
||||
output = False
|
||||
json_output = False
|
||||
|
|
@ -687,8 +657,8 @@ def main():
|
|||
|
||||
subdomains = search_list.union(bruteforce_list)
|
||||
|
||||
if len(subdomains) > 0:
|
||||
if savefile is not None:
|
||||
if subdomains:
|
||||
if savefile:
|
||||
write_file(savefile, subdomains)
|
||||
print Y+"[-] Total Unique Subdomains Found: %s"%len(subdomains)+W
|
||||
for subdomain in subdomains:
|
||||
|
|
|
|||
Loading…
Reference in New Issue