RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Apply Patrowl patches

This commit is contained in:
Nicolas Mattiocco 2020-09-18 14:20:41 +02:00
parent 729d649ec5
commit b567c5e92a
3 changed files with 54 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +1,2 @@
*.pyc *.pyc
env

1
requirements.txt
View File

@ -1,3 +1,4 @@
argparse argparse
dnspython dnspython
requests requests
certifi

92
sublist3r.py
View File

@ -2,6 +2,7 @@
# coding: utf-8 # coding: utf-8
# Sublist3r v1.0 # Sublist3r v1.0
# By Ahmed Aboul-Ela - twitter.com/aboul3la # By Ahmed Aboul-Ela - twitter.com/aboul3la
# Updated by Nicolas Mattiocco <nicolas@patrowl.io>
# modules in standard library # modules in standard library
import re import re
@ -22,7 +23,7 @@ from subbrute import subbrute
import dns.resolver import dns.resolver
import requests import requests
# Python 2.x and 3.x compatiablity # Python 2.x and 3.x compatibility
if sys.version > '3': if sys.version > '3':
import urllib.parse as urlparse import urllib.parse as urlparse
import urllib.parse as urllib import urllib.parse as urllib
@ -35,7 +36,7 @@ else:
try: try:
import requests.packages.urllib3 import requests.packages.urllib3
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
except: except Exception:
pass pass
# Check if we are running this on windows platform # Check if we are running this on windows platform
@ -50,15 +51,15 @@ if is_windows:
R = '\033[91m' # red R = '\033[91m' # red
W = '\033[0m' # white W = '\033[0m' # white
try: try:
import win_unicode_console , colorama import win_unicode_console
import colorama
win_unicode_console.enable() win_unicode_console.enable()
colorama.init() colorama.init()
#Now the unicode will work ^_^ # Now the unicode will work ^_^
except: except Exception:
print("[!] Error: Coloring libraries not installed, no coloring will be used [Check the readme]") print("[!] Error: Coloring libraries not installed, no coloring will be used [Check the readme]")
G = Y = B = R = W = G = Y = B = R = W = '' G = Y = B = R = W = G = Y = B = R = W = ''
else: else:
G = '\033[92m' # green G = '\033[92m' # green
Y = '\033[93m' # yellow Y = '\033[93m' # yellow
@ -66,6 +67,7 @@ else:
R = '\033[91m' # red R = '\033[91m' # red
W = '\033[0m' # white W = '\033[0m' # white
def no_color(): def no_color():
global G, Y, B, R, W global G, Y, B, R, W
G = Y = B = R = W = '' G = Y = B = R = W = ''
@ -195,12 +197,12 @@ class enumratorBase(object):
# override # override
def extract_domains(self, resp): def extract_domains(self, resp):
""" chlid class should override this function """ """ child class should override this function """
return return
# override # override
def check_response_errors(self, resp): def check_response_errors(self, resp):
""" chlid class should override this function """ child class should override this function
The function should return True if there are no errors and False otherwise The function should return True if there are no errors and False otherwise
""" """
return True return True
@ -210,11 +212,11 @@ class enumratorBase(object):
return return
def generate_query(self): def generate_query(self):
""" chlid class should override this function """ """ child class should override this function """
return return
def get_page(self, num): def get_page(self, num):
""" chlid class that user different pagnation counter should override this function """ """ child class that user different pagnation counter should override this function """
return num + 10 return num + 10
def enumerate(self, altquery=False): def enumerate(self, altquery=False):
@ -236,7 +238,7 @@ class enumratorBase(object):
return self.subdomains return self.subdomains
resp = self.send_req(query, page_no) resp = self.send_req(query, page_no)
# check if there is any error occured # check if there is any error occurred
if not self.check_response_errors(resp): if not self.check_response_errors(resp):
return self.subdomains return self.subdomains
links = self.extract_domains(resp) links = self.extract_domains(resp)
@ -266,8 +268,9 @@ class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
def run(self): def run(self):
domain_list = self.enumerate() domain_list = self.enumerate()
for domain in domain_list: if domain_list is not None:
self.q.append(domain) for domain in domain_list:
self.q.append(domain)
class GoogleEnum(enumratorBaseThreaded): class GoogleEnum(enumratorBaseThreaded):
@ -565,16 +568,16 @@ class NetcraftEnum(enumratorBaseThreaded):
def enumerate(self): def enumerate(self):
start_url = self.base_url.format(domain='example.com') start_url = self.base_url.format(domain='example.com')
resp = self.req(start_url) resp = self.req(start_url)
cookies = self.get_cookies(resp.headers) if 'headers' in dir(resp):
url = self.base_url.format(domain=self.domain) cookies = self.get_cookies(resp.headers)
while True: url = self.base_url.format(domain=self.domain)
resp = self.get_response(self.req(url, cookies)) while True:
self.extract_domains(resp) resp = self.get_response(self.req(url, cookies))
if 'Next Page' not in resp: self.extract_domains(resp)
return self.subdomains if 'Next Page' not in resp:
break return self.subdomains
url = self.get_next(resp) url = self.get_next(resp)
self.should_sleep() self.should_sleep()
def extract_domains(self, resp): def extract_domains(self, resp):
links_list = list() links_list = list()
@ -611,13 +614,14 @@ class DNSdumpster(enumratorBaseThreaded):
Resolver.nameservers = ['8.8.8.8', '8.8.4.4'] Resolver.nameservers = ['8.8.8.8', '8.8.4.4']
self.lock.acquire() self.lock.acquire()
try: try:
ip = Resolver.query(host, 'A')[0].to_text() # ip = Resolver.query(host, 'A')[0].to_text()
ip = dns.resolver.Resolver.resolve()
if ip: if ip:
if self.verbose: if self.verbose:
self.print_("%s%s: %s%s" % (R, self.engine_name, W, host)) self.print_("%s%s: %s%s" % (R, self.engine_name, W, host))
is_valid = True is_valid = True
self.live_subdomains.append(host) self.live_subdomains.append(host)
except: except Exception:
pass pass
self.lock.release() self.lock.release()
return is_valid return is_valid
@ -625,7 +629,7 @@ class DNSdumpster(enumratorBaseThreaded):
def req(self, req_method, url, params=None): def req(self, req_method, url, params=None):
params = params or {} params = params or {}
headers = dict(self.headers) headers = dict(self.headers)
headers['Referer'] = 'https://dnsdumpster.com' headers['Referrer'] = 'https://dnsdumpster.com'
try: try:
if req_method == 'GET': if req_method == 'GET':
resp = self.session.get(url, headers=headers, timeout=self.timeout) resp = self.session.get(url, headers=headers, timeout=self.timeout)
@ -644,14 +648,15 @@ class DNSdumpster(enumratorBaseThreaded):
def enumerate(self): def enumerate(self):
self.lock = threading.BoundedSemaphore(value=70) self.lock = threading.BoundedSemaphore(value=70)
resp = self.req('GET', self.base_url) resp = self.req('GET', self.base_url)
token = self.get_csrftoken(resp) if isinstance(resp, str):
params = {'csrfmiddlewaretoken': token, 'targetip': self.domain} token = self.get_csrftoken(resp)
post_resp = self.req('POST', self.base_url, params) params = {'csrfmiddlewaretoken': token, 'targetip': self.domain}
self.extract_domains(post_resp) post_resp = self.req('POST', self.base_url, params)
for subdomain in self.subdomains: self.extract_domains(post_resp)
t = threading.Thread(target=self.check_host, args=(subdomain,)) for subdomain in self.subdomains:
t.start() t = threading.Thread(target=self.check_host, args=(subdomain,))
t.join() t.start()
t.join()
return self.live_subdomains return self.live_subdomains
def extract_domains(self, resp): def extract_domains(self, resp):
@ -697,7 +702,11 @@ class Virustotal(enumratorBaseThreaded):
def enumerate(self): def enumerate(self):
while self.url != '': while self.url != '':
resp = self.send_req(self.url) resp = self.send_req(self.url)
resp = json.loads(resp) try:
resp = json.loads(resp)
except Exception:
self.print_(R + "[!] Error: Virustotal returns 'int' values" + W)
break
if 'error' in resp: if 'error' in resp:
self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W) self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
break break
@ -709,7 +718,7 @@ class Virustotal(enumratorBaseThreaded):
return self.subdomains return self.subdomains
def extract_domains(self, resp): def extract_domains(self, resp):
#resp is already parsed as json # resp is already parsed as json
try: try:
for i in resp['data']: for i in resp['data']:
if i['type'] == 'domain': if i['type'] == 'domain':
@ -765,7 +774,8 @@ class ThreatCrowd(enumratorBaseThreaded):
class CrtSearch(enumratorBaseThreaded): class CrtSearch(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
base_url = 'https://crt.sh/?q=%25.{domain}' # base_url = 'https://crt.sh/?q=%25.{domain}'
base_url = 'https://crt.sh/?q=.{domain}'
self.engine_name = "SSL Certificates" self.engine_name = "SSL Certificates"
self.q = q self.q = q
super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
@ -813,6 +823,7 @@ class CrtSearch(enumratorBaseThreaded):
print(e) print(e)
pass pass
class PassiveDNS(enumratorBaseThreaded): class PassiveDNS(enumratorBaseThreaded):
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or [] subdomains = subdomains or []
@ -825,7 +836,7 @@ class PassiveDNS(enumratorBaseThreaded):
def req(self, url): def req(self, url):
try: try:
resp = self.session.get(url, headers=self.headers, timeout=self.timeout) resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
except Exception as e: except Exception:
resp = None resp = None
return self.get_response(resp) return self.get_response(resp)
@ -847,7 +858,7 @@ class PassiveDNS(enumratorBaseThreaded):
if self.verbose: if self.verbose:
self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain)) self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
self.subdomains.append(subdomain.strip()) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception:
pass pass
@ -1000,7 +1011,8 @@ def interactive():
if args.no_color: if args.no_color:
no_color() no_color()
banner() banner()
res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines) main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines)
if __name__ == "__main__": if __name__ == "__main__":
interactive() interactive()