RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhancing OOP, fixing major bugs, implemente sublister as a module where you can call it from other programs

This commit is contained in:
Ibrahim 2016-10-27 10:46:25 +01:00
parent 7b1657e3df
commit 5823d9ee83
1 changed files with 33 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

233
sublist3r.py
View File

@ -432,7 +432,6 @@ class BingEnum(enumratorBaseThreaded):
query = "domain:{domain} -www.{domain}".format(domain=self.domain) query = "domain:{domain} -www.{domain}".format(domain=self.domain)
return query return query
class BaiduEnum(enumratorBaseThreaded): class BaiduEnum(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
@ -491,35 +490,14 @@ class BaiduEnum(enumratorBaseThreaded):
query = "site:{domain} -site:www.{domain}".format(domain=self.domain) query = "site:{domain} -site:www.{domain}".format(domain=self.domain)
return query return query
class NetcraftEnum(multiprocessing.Process): class NetcraftEnum(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'http://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}' self.base_url = 'http://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session = requests.Session()
self.engine_name = "Netcraft" self.engine_name = "Netcraft"
multiprocessing.Process.__init__(self) self.lock = threading.Lock()
self.lock = lock super(NetcraftEnum, self).__init__(self.base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.q = q self.q = q
self.timeout = 10
self.silent = silent
self.verbose=verbose
self.print_banner()
return
def print_(self, text):
if not self.silent:
print text
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain)
return
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return return
def req(self, url, cookies=None): def req(self, url, cookies=None):
@ -536,14 +514,6 @@ class NetcraftEnum(multiprocessing.Process):
resp = None resp = None
return resp return resp
def get_response(self,response):
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def get_next(self, resp): def get_next(self, resp):
link_regx = re.compile('<A href="(.*?)"><b>Next page</b></a>') link_regx = re.compile('<A href="(.*?)"><b>Next page</b></a>')
link = link_regx.findall(resp) link = link_regx.findall(resp)
@ -594,38 +564,17 @@ class NetcraftEnum(multiprocessing.Process):
pass pass
return links_list return links_list
class DNSdumpster(enumratorBaseThreaded):
class DNSdumpster(multiprocessing.Process): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'https://dnsdumpster.com/' base_url = 'https://dnsdumpster.com/'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.live_subdomains = [] self.live_subdomains = []
self.session = requests.Session()
self.engine_name = "DNSdumpster" self.engine_name = "DNSdumpster"
multiprocessing.Process.__init__(self)
self.threads = 70 self.threads = 70
self.lock = threading.BoundedSemaphore(value=self.threads) self.lock = threading.BoundedSemaphore(value=self.threads)
self.q = q self.q = q
self.timeout = 25 self.timeout = 25
self.silent = silent super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.verbose = verbose
self.print_banner()
return
def print_(self, text):
if not self.silent:
print text
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain)
return
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return return
def check_host(self,host): def check_host(self,host):
@ -664,14 +613,6 @@ class DNSdumpster(multiprocessing.Process):
resp = None resp = None
return self.get_response(resp) return self.get_response(resp)
def get_response(self,response):
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def get_csrftoken(self, resp): def get_csrftoken(self, resp):
csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />",re.S) csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />",re.S)
token = csrf_regex.findall(resp)[0] token = csrf_regex.findall(resp)[0]
@ -689,7 +630,6 @@ class DNSdumpster(multiprocessing.Process):
t.join() t.join()
return self.live_subdomains return self.live_subdomains
def extract_domains(self, resp): def extract_domains(self, resp):
tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>',re.S) tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>',re.S)
link_regex = re.compile('<td class="col-md-4">(.*?)<br>',re.S) link_regex = re.compile('<td class="col-md-4">(.*?)<br>',re.S)
@ -708,38 +648,19 @@ class DNSdumpster(multiprocessing.Process):
self.subdomains.append(subdomain.strip()) self.subdomains.append(subdomain.strip())
return links return links
class Virustotal(multiprocessing.Process): class Virustotal(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'https://www.virustotal.com/en/domain/{domain}/information/' base_url = 'https://www.virustotal.com/en/domain/{domain}/information/'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session = requests.Session()
self.engine_name = "Virustotal" self.engine_name = "Virustotal"
self.silent = silent self.lock = threading.Lock()
multiprocessing.Process.__init__(self)
self.lock = lock
self.q = q self.q = q
self.timeout = 10 self.timeout = 10
self.verbose = verbose super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.print_banner()
return return
def run(self): #the main send_req need to be rewritten
domain_list = self.enumerate() def send_req(self, url):
for domain in domain_list:
self.q.append(domain)
return
def print_(self, text):
if not self.silent:
print text
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return
def req(self, url):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-GB,en;q=0.5', 'Accept-Language': 'en-GB,en;q=0.5',
@ -754,17 +675,10 @@ class Virustotal(multiprocessing.Process):
return self.get_response(resp) return self.get_response(resp)
def get_response(self,response): #once the send_req is rewritten we don't need to call this function, the stock one should be ok
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def enumerate(self): def enumerate(self):
url = self.base_url.format(domain=self.domain) url = self.base_url.format(domain=self.domain)
resp = self.req(url) resp = self.send_req(url)
self.extract_domains(resp) self.extract_domains(resp)
return self.subdomains return self.subdomains
@ -784,34 +698,15 @@ class Virustotal(multiprocessing.Process):
pass pass
class ThreatCrowd(multiprocessing.Process): class ThreatCrowd(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}' base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session = requests.Session()
self.engine_name = "ThreatCrowd" self.engine_name = "ThreatCrowd"
multiprocessing.Process.__init__(self) self.lock = threading.Lock()
self.lock = lock
self.q = q self.q = q
self.timeout = 20 self.timeout = 20
self.silent = silent super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.verbose = verbose
self.print_banner()
return
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain)
return
def print_(self, text):
if not self.silent:
print text
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return return
def req(self, url): def req(self, url):
@ -828,14 +723,6 @@ class ThreatCrowd(multiprocessing.Process):
return self.get_response(resp) return self.get_response(resp)
def get_response(self,response):
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def enumerate(self): def enumerate(self):
url = self.base_url.format(domain=self.domain) url = self.base_url.format(domain=self.domain)
resp = self.req(url) resp = self.req(url)
@ -863,35 +750,15 @@ class ThreatCrowd(multiprocessing.Process):
except Exception as e: except Exception as e:
pass pass
class CrtSearch(multiprocessing.Process): class CrtSearch(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'https://crt.sh/?q=%25.{domain}' base_url = 'https://crt.sh/?q=%25.{domain}'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session = requests.Session()
self.engine_name = "SSL Certificates" self.engine_name = "SSL Certificates"
multiprocessing.Process.__init__(self) self.lock = threading.Lock()
self.lock = lock
self.q = q self.q = q
self.timeout = 25 self.timeout = 25
self.silent = silent super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.verbose = verbose
self.print_banner()
return
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain)
return
def print_(self, text):
if not self.silent:
print text
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return return
def req(self, url): def req(self, url):
@ -908,14 +775,6 @@ class CrtSearch(multiprocessing.Process):
return self.get_response(resp) return self.get_response(resp)
def get_response(self,response):
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def enumerate(self): def enumerate(self):
url = self.base_url.format(domain=self.domain) url = self.base_url.format(domain=self.domain)
resp = self.req(url) resp = self.req(url)
@ -938,36 +797,17 @@ class CrtSearch(multiprocessing.Process):
except Exception as e: except Exception as e:
pass pass
class PassiveDNS(multiprocessing.Process): class PassiveDNS(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
self.base_url = 'http://ptrarchive.com/tools/search.htm?label={domain}' base_url = 'http://ptrarchive.com/tools/search.htm?label={domain}'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.session = requests.Session()
self.engine_name = "PassiveDNS" self.engine_name = "PassiveDNS"
multiprocessing.Process.__init__(self) self.lock = threading.Lock()
self.lock = lock
self.q = q self.q = q
self.timeout = 25 self.timeout = 25
self.silent = silent super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
self.verbose = verbose
self.print_banner()
return return
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain)
return
def print_(self, text):
if not self.silent:
print text
def print_banner(self):
self.print_(G+"[-] Searching now in %s.." %(self.engine_name)+W)
return
def req(self, url): def req(self, url):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0',
@ -984,13 +824,6 @@ class PassiveDNS(multiprocessing.Process):
return self.get_response(resp) return self.get_response(resp)
def get_response(self,response):
if response is None:
return 0
if hasattr(response, "text"):
return response.text
else:
return response.content
def enumerate(self): def enumerate(self):
url = self.base_url.format(domain=self.domain) url = self.base_url.format(domain=self.domain)
@ -1133,4 +966,4 @@ if __name__=="__main__":
banner() banner()
res = main(domain, threads, savefile, ports, silent=True, verbose=verbose, enable_bruteforce=enable_bruteforce) res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce)