diff --git a/turbolist3r.py b/turbolist3r.py index 5aaf7c6..746db81 100644 --- a/turbolist3r.py +++ b/turbolist3r.py @@ -26,7 +26,7 @@ import socket from collections import Counter # external modules -#from subbrute import subbrute +# from subbrute import subbrute import dns.resolver import requests # import dnslib, which provides better features compared to dns.resolver for finding subdomains @@ -46,6 +46,7 @@ else: # there's also an option to disable the SSL warning: try: import requests.packages.urllib3 + requests.packages.urllib3.disable_warnings() except: pass @@ -55,7 +56,6 @@ is_windows = sys.platform.startswith('win') global debug - # Console Colors if is_windows: # Windows deserve coloring too :D @@ -63,23 +63,24 @@ if is_windows: Y = '\033[93m' # yellow B = '\033[94m' # blue R = '\033[91m' # red - W = '\033[0m' # white + W = '\033[0m' # white try: - import win_unicode_console , colorama + import win_unicode_console, colorama + win_unicode_console.enable() colorama.init() - #Now the unicode will work ^_^ + # Now the unicode will work ^_^ except: print("[!] Error: Coloring libraries not installed ,no coloring will be used [Check the readme]") G = Y = B = R = W = G = Y = B = R = W = '' - + else: G = '\033[92m' # green Y = '\033[93m' # yellow B = '\033[94m' # blue R = '\033[91m' # red - W = '\033[0m' # white + W = '\033[0m' # white def banner(): @@ -94,6 +95,7 @@ def banner(): # Forked by Carl Pearson - github.com/fleetcaptain """ % (R, W, Y)) + def parser_error(errmsg): banner() print("Usage: python " + sys.argv[0] + " [Options] use -h for help") @@ -109,8 +111,10 @@ def parse_args(): parser.add_argument('-d', '--domain', help="Domain name to enumerate it's subdomains", required=True) parser.add_argument('-b', '--bruteforce', help='Enable the subbrute bruteforce module', nargs='?', default=False) parser.add_argument('-p', '--ports', help='Scan the found subdomains against specified tcp ports') - parser.add_argument('-v', '--verbose', help='Enable Verbosity and display results in realtime', nargs='?', default=False) - parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=30) + parser.add_argument('-v', '--verbose', help='Enable Verbosity and display results in realtime', nargs='?', + default=False) + parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, + default=30) parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines') parser.add_argument('-o', '--output', help='Save just domain names to specified text file') parser.add_argument('-a', '--analysis', help='Do analysis of the results and save to specified text file') @@ -163,6 +167,12 @@ class enumratorBase(object): self.engine_name = engine_name self.silent = silent self.verbose = verbose + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.8', + 'Accept-Encoding': 'gzip', + } self.print_banner() def print_(self, text): @@ -176,17 +186,10 @@ class enumratorBase(object): return def send_req(self, query, page_no=1): - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'Connection': 'keep-alive', - } url = self.base_url.format(query=query, page_no=page_no) try: - resp = self.session.get(url, headers=headers, timeout=self.timeout) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) except Exception: resp = None return self.get_response(resp) @@ -259,7 +262,7 @@ class enumratorBase(object): retries += 1 page_no = self.get_page(page_no) - # make another retry maybe it isn't the last page + # make another retry maybe it isn't the last page if retries >= 3: return self.subdomains @@ -270,7 +273,8 @@ class enumratorBase(object): class enumratorBaseThreaded(multiprocessing.Process, enumratorBase): - def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): + def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, + verbose=True): subdomains = subdomains or [] enumratorBase.__init__(self, base_url, engine_name, domain, subdomains, silent=silent, verbose=verbose) multiprocessing.Process.__init__(self) @@ -291,7 +295,8 @@ class GoogleEnum(enumratorBaseThreaded): self.engine_name = "Google" self.MAX_DOMAINS = 11 self.MAX_PAGES = 200 - super(GoogleEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(GoogleEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) self.q = q return @@ -313,7 +318,7 @@ class GoogleEnum(enumratorBaseThreaded): return links_list def check_response_errors(self, resp): - if 'Our systems have detected unusual traffic' in resp: + if (type(resp) is str or type(resp) is unicode) and 'Our systems have detected unusual traffic' in resp: self.print_(R + "[!] Error: Google probably now is blocking our requests" + W) self.print_(R + "[~] Finished now the Google Enumeration ..." + W) return False @@ -340,7 +345,8 @@ class YahooEnum(enumratorBaseThreaded): self.engine_name = "Yahoo" self.MAX_DOMAINS = 10 self.MAX_PAGES = 0 - super(YahooEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(YahooEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) self.q = q return @@ -391,12 +397,13 @@ class AskEnum(enumratorBaseThreaded): self.engine_name = "Ask" self.MAX_DOMAINS = 11 self.MAX_PAGES = 0 - enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) self.q = q return def extract_domains(self, resp): - link_regx = re.compile('

(.*?)

') + link_regx = re.compile('(.*?)', re.IGNORECASE) try: links_list = link_regx.findall(resp) for link in links_list: @@ -477,7 +484,8 @@ class BaiduEnum(enumratorBaseThreaded): self.engine_name = "Baidu" self.MAX_DOMAINS = 2 self.MAX_PAGES = 760 - enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) self.querydomain = self.domain self.q = q return @@ -535,21 +543,16 @@ class NetcraftEnum(enumratorBaseThreaded): self.base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}' self.engine_name = "Netcraft" self.lock = threading.Lock() - super(NetcraftEnum, self).__init__(self.base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(NetcraftEnum, self).__init__(self.base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) self.q = q return def req(self, url, cookies=None): cookies = cookies or {} - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - } try: - resp = self.session.get(url, headers=headers, timeout=self.timeout, cookies=cookies) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout, cookies=cookies) except Exception as e: self.print_(e) resp = None @@ -559,7 +562,7 @@ class NetcraftEnum(enumratorBaseThreaded): link_regx = re.compile('Next page') link = link_regx.findall(resp) link = re.sub('host=.*?%s' % self.domain, 'host=%s' % self.domain, link[0]) - url = 'http://searchdns.netcraft.com' + link + url = 'https://searchdns.netcraft.com' + link return url def create_cookies(self, cookie): @@ -615,7 +618,8 @@ class DNSdumpster(enumratorBaseThreaded): self.threads = 70 self.lock = threading.BoundedSemaphore(value=self.threads) self.q = q - super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) return def check_host(self, host): @@ -637,14 +641,8 @@ class DNSdumpster(enumratorBaseThreaded): def req(self, req_method, url, params=None): params = params or {} - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'Referer': 'https://dnsdumpster.com' - } - + headers = dict(self.headers) + headers['Referer'] = 'https://dnsdumpster.com' try: if req_method == 'GET': resp = self.session.get(url, headers=headers, timeout=self.timeout) @@ -698,20 +696,15 @@ class Virustotal(enumratorBaseThreaded): self.engine_name = "Virustotal" self.lock = threading.Lock() self.q = q - super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) return # the main send_req need to be rewritten def send_req(self, url): - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - } try: - resp = self.session.get(url, headers=headers, timeout=self.timeout) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) except Exception as e: self.print_(e) resp = None @@ -748,19 +741,14 @@ class ThreatCrowd(enumratorBaseThreaded): self.engine_name = "ThreatCrowd" self.lock = threading.Lock() self.q = q - super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) return def req(self, url): - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - } try: - resp = self.session.get(url, headers=headers, timeout=self.timeout) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) except Exception: resp = None @@ -800,19 +788,14 @@ class CrtSearch(enumratorBaseThreaded): self.engine_name = "SSL Certificates" self.lock = threading.Lock() self.q = q - super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) return def req(self, url): - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/40.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-GB,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - } try: - resp = self.session.get(url, headers=headers, timeout=self.timeout) + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) except Exception: resp = None @@ -835,7 +818,7 @@ class CrtSearch(enumratorBaseThreaded): continue if '@' in subdomain: - subdomain = subdomain[subdomain.find('@')+1:] + subdomain = subdomain[subdomain.find('@') + 1:] if subdomain not in self.subdomains and subdomain != self.domain: if self.verbose: @@ -852,19 +835,32 @@ class PassiveDNS(enumratorBaseThreaded): self.engine_name = "PassiveDNS" self.lock = threading.Lock() self.q = q - super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, + verbose=verbose) return + def get_agent(self, ua=None): + + agents_url = 'http://www.webuseragents.com/recent' + try: + resp = session.get(agents_url, headers=self.headers, timeout=self.timeout) + agents_list = self.get_response(resp) + agents_regex = re.compile('-- " + record) - elif rtype == "A": - ahosts.append(name + " -->-- " + record) - # round robin the resolvers - server = server + 1 - server = server % len(resolvers) - - # update user on our progress - every 30 hosts - count = count + 1 - if (count % 30) == 0: - print str(count) + '/' + total - except KeyboardInterrupt: - print(R + '\n[-] User exit' + W) - exit() - except: - # Generally unknown error. Keep going - # Known errors: subdomain sample starting with a dot, ex .domain.com - continue + # res is the list of subdomains e.g. www.example.com, mail.example.com, etc + resolvers = ['8.8.8.8', '8.8.4.4', '9.9.9.9', '75.75.75.75'] + server = 0 + count = 0 + total = str(len(res)) + print("") + print(B + "[-] Beginning analysis of " + total + " subdomains..." + W) + for subdomain in res: + try: + name = subdomain.replace('\n', '').replace('\r', '') + (rtype, record) = lookup(name, resolvers[server]) + # if the query did not return an error, then add result to appropriate array + if rtype != "ERROR": + if rtype == "CNAME": + cnames.append(name + " -->-- " + record) + elif rtype == "A": + ahosts.append(name + " -->-- " + record) + # round robin the resolvers + server = server + 1 + server = server % len(resolvers) - ahosts.sort() - cnames.sort() + # update user on our progress - every 30 hosts + count = count + 1 + if (count % 30) == 0: + print(str(count) + '/' + total) + except KeyboardInterrupt: + print(R + '\n[-] User exit' + W) + exit() + except: + # Generally unknown error. Keep going + # Known errors: subdomain sample starting with a dot, ex .domain.com + continue - # output analysis results to console - for x in range(0, len(ahosts)): - print(G + ahosts[x] + W) - print "\n" - for x in range(0, len(cnames)): - print(G + cnames[x] + W) + ahosts.sort() + cnames.sort() - #print "" - # save the analysis to a file. Merge the arrays into one list for easier reading - write_file(analysis, ahosts + ["\n"] + cnames) + # output analysis results to console + for x in range(0, len(ahosts)): + print(G + ahosts[x] + W) + print("\n") + for x in range(0, len(cnames)): + print(G + cnames[x] + W) + # print "" + # save the analysis to a file. Merge the arrays into one list for easier reading + write_file(analysis, ahosts + ["\n"] + cnames)