diff --git a/README.md b/README.md index c860b29..5604534 100644 --- a/README.md +++ b/README.md @@ -17,27 +17,10 @@ git clone https://github.com/aboul3la/Sublist3r.git ## Recommended Python Version: -Sublist3r currently supports **Python 2** and **Python 3**. +Sublist3r currently supports**Python 3.8+ -* The recommended version for Python 2 is **2.7.x** -* The recommended version for Python 3 is **3.4.x** +* The recommended version for Python 3 is **3.8+** -## Dependencies: - -Sublist3r depends on the `requests`, `dnspython` and `argparse` python modules. - -These dependencies can be installed using the requirements file: - -- Installation on Windows: -``` -c:\python27\python.exe -m pip install -r requirements.txt -``` -- Installation on Linux -``` -sudo pip install -r requirements.txt -``` - -Alternatively, each module can be installed independently as shown below. #### Requests Module (http://docs.python-requests.org/en/latest/) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b61373e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 498ea9d..5c7639c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -argparse dnspython requests diff --git a/setup.py b/setup.py index eb2383c..a297348 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,13 @@ from setuptools import setup, find_packages setup( name='Sublist3r', - version='1.0', - python_requires='>=2.7', - install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], - packages=find_packages()+['.'], + version='2.0', + python_requires='>=3.8', + install_requires=['dnspython', 'requests'], + packages=find_packages(), + py_modules=['sublist3r'], include_package_data=True, + package_data={'subbrute': ['*.txt']}, url='https://github.com/aboul3la/Sublist3r', license='GPL-2.0', description='Subdomains enumeration tool for penetration testers', @@ -19,12 +21,15 @@ setup( 'License :: OSI Approved :: GNU General Public License v2', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Topic :: Security', ], keywords='subdomain dns detection', diff --git a/subbrute/subbrute.py b/subbrute/subbrute.py index c7b1ab6..f0873be 100644 --- a/subbrute/subbrute.py +++ b/subbrute/subbrute.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # #SubBrute v1.2 #A (very) fast subdomain enumeration tool. @@ -67,7 +67,7 @@ class verify_nameservers(multiprocessing.Process): #Lets test the letancy of our connection. #Google's DNS server should be an ideal time test. resolver.nameservers = ['8.8.8.8'] - resolver.query(self.most_popular_website, self.record_type) + resolver.resolve(self.most_popular_website, self.record_type) except: #Our connection is slower than a junebug in molasses resolver = dns.resolver.Resolver() @@ -138,7 +138,7 @@ class verify_nameservers(multiprocessing.Process): #I have seen a CloudFlare Enterprise customer with the first two conditions. try: #This is case #3, these spam nameservers seem to be more trouble then they are worth. - wildtest = self.resolver.query(uuid.uuid4().hex + ".com", "A") + wildtest = self.resolver.resolve(uuid.uuid4().hex + ".com", "A") if len(wildtest): trace("Spam DNS detected:", host) return False @@ -152,7 +152,7 @@ class verify_nameservers(multiprocessing.Process): test_counter -= 1 try: testdomain = "%s.%s" % (uuid.uuid4().hex, host) - wildtest = self.resolver.query(testdomain, self.record_type) + wildtest = self.resolver.resolve(testdomain, self.record_type) #This 'A' record may contain a list of wildcards. if wildtest: for w in wildtest: @@ -204,7 +204,10 @@ class lookup(multiprocessing.Process): def get_ns_blocking(self): ret = [] - ret = [self.resolver_q.get()] + try: + ret = [self.resolver_q.get()] + except KeyboardInterrupt: + return [] if ret == False: trace("get_ns_blocking - Resolver list is empty.") #Queue is empty, inform the rest. @@ -224,7 +227,7 @@ class lookup(multiprocessing.Process): try: #Query the nameserver, this is not simple... if not record_type or record_type == "A": - resp = self.resolver.query(host) + resp = self.resolver.resolve(host) #Crawl the response hosts = extract_hosts(str(resp.response), self.domain) for h in hosts: @@ -237,7 +240,7 @@ class lookup(multiprocessing.Process): #A max 20 lookups for x in range(20): try: - resp = self.resolver.query(host, record_type) + resp = self.resolver.resolve(host, record_type) except dns.resolver.NoAnswer: resp = False pass @@ -248,7 +251,7 @@ class lookup(multiprocessing.Process): return cname_record else: #All other records: - return self.resolver.query(host, record_type) + return self.resolver.resolve(host, record_type) except Exception as e: if type(e) == dns.resolver.NoNameservers: @@ -299,59 +302,62 @@ class lookup(multiprocessing.Process): raise e def run(self): - #This process needs one resolver before it can start looking. - self.resolver.nameservers += self.get_ns_blocking() - while True: - found_addresses = [] - work = self.in_q.get() - #Check if we have hit the end marker - while not work: - #Look for a re-queued lookup - try: - work = self.in_q.get(blocking = False) - #if we took the end marker of the queue we need to put it back - if work: - self.in_q.put(False) - except:#Queue.Empty - trace('End of work queue') - #There isn't an item behind the end marker - work = False + try: + #This process needs one resolver before it can start looking. + self.resolver.nameservers += self.get_ns_blocking() + while True: + found_addresses = [] + work = self.in_q.get() + #Check if we have hit the end marker + while not work: + #Look for a re-queued lookup + try: + work = self.in_q.get(block=False) + #if we took the end marker of the queue we need to put it back + if work: + self.in_q.put(False) + except:#Queue.Empty + trace('End of work queue') + #There isn't an item behind the end marker + work = False + break + #Is this the end all work that needs to be done? + if not work: + #Perpetuate the end marker for all threads to see + self.in_q.put(False) + #Notify the parent that we have died of natural causes + self.out_q.put(False) break - #Is this the end all work that needs to be done? - if not work: - #Perpetuate the end marker for all threads to see - self.in_q.put(False) - #Notify the parent that we have died of natural causes - self.out_q.put(False) - break - else: - if len(work) == 3: - #keep track of how many times this lookup has timedout. - (hostname, record_type, timeout_retries) = work - response = self.check(hostname, record_type, timeout_retries) else: - (hostname, record_type) = work - response = self.check(hostname, record_type) - sys.stdout.flush() - trace(response) - #self.wildcards is populated by the verify_nameservers() thread. - #This variable doesn't need a muetex, because it has a queue. - #A queue ensure nameserver cannot be used before it's wildcard entries are found. - reject = False - if response: - for a in response: - a = str(a) - if a in self.wildcards: - trace("resovled wildcard:", hostname) - reject= True - #reject this domain. - break; - else: - found_addresses.append(a) - if not reject: - #This request is filled, send the results back - result = (hostname, record_type, found_addresses) - self.out_q.put(result) + if len(work) == 3: + #keep track of how many times this lookup has timedout. + (hostname, record_type, timeout_retries) = work + response = self.check(hostname, record_type, timeout_retries) + else: + (hostname, record_type) = work + response = self.check(hostname, record_type) + sys.stdout.flush() + trace(response) + #self.wildcards is populated by the verify_nameservers() thread. + #This variable doesn't need a muetex, because it has a queue. + #A queue ensure nameserver cannot be used before it's wildcard entries are found. + reject = False + if response: + for a in response: + a = str(a) + if a in self.wildcards: + trace("resovled wildcard:", hostname) + reject= True + #reject this domain. + break; + else: + found_addresses.append(a) + if not reject: + #This request is filled, send the results back + result = (hostname, record_type, found_addresses) + self.out_q.put(result) + except KeyboardInterrupt: + return #Extract relevant hosts #The dot at the end of a domain signifies the root, @@ -371,7 +377,7 @@ def extract_hosts(data, hostname): #Return a list of unique sub domains, sorted by frequency. #Only match domains that have 3 or more sections subdomain.domain.tld -domain_match = re.compile("([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+") +domain_match = re.compile(r"([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+") def extract_subdomains(file_name): #Avoid re-compilation global domain_match @@ -462,9 +468,11 @@ def run(target, record_type = None, subdomains = "names.txt", resolve_list = "re in_q.put(work) #Terminate the queue in_q.put(False) + workers = [] for i in range(process_count): worker = lookup(in_q, out_q, resolve_q, target, wildcards, spider_blacklist) worker.start() + workers.append(worker) threads_remaining = process_count while True: try: @@ -476,6 +484,20 @@ def run(target, record_type = None, subdomains = "names.txt", resolve_list = "re else: #run() is a generator, and yields results from the work queue yield result + except KeyboardInterrupt: + try: + killproc(pid = verify_nameservers_proc.pid) + except: + try: + verify_nameservers_proc.end() + except: + pass + for w in workers: + try: + killproc(pid = w.pid) + except: + pass + return except Exception as e: #The cx_freeze version uses queue.Empty instead of Queue.Empty :( if type(e) == Queue.Empty or str(type(e)) == "": @@ -533,7 +555,14 @@ def check_open(input_file): try: ret = open(input_file).readlines() except: - error("File not found:", input_file) + if not os.path.isabs(input_file): + try: + base_path = os.path.dirname(os.path.realpath(__file__)) + ret = open(os.path.join(base_path, input_file)).readlines() + except: + error("File not found:", input_file) + else: + error("File not found:", input_file) if not len(ret): error("File is empty:", input_file) return ret diff --git a/sublist3r.py b/sublist3r.py index 760e5ce..1e736e9 100755 --- a/sublist3r.py +++ b/sublist3r.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding: utf-8 # Sublist3r v1.0 # By Ahmed Aboul-Ela - twitter.com/aboul3la @@ -23,12 +23,8 @@ import dns.resolver import requests # Python 2.x and 3.x compatiablity -if sys.version > '3': - import urllib.parse as urlparse - import urllib.parse as urllib -else: - import urlparse - import urllib +import urllib.parse as urlparse +import urllib.parse as urllib # In case you cannot install some of the required development packages # there's also an option to disable the SSL warning: @@ -72,7 +68,7 @@ def no_color(): def banner(): - print("""%s + print(r"""%s ____ _ _ _ _ _____ / ___| _ _| |__ | (_)___| |_|___ / _ __ \___ \| | | | '_ \| | / __| __| |_ \| '__| @@ -283,7 +279,7 @@ class GoogleEnum(enumratorBaseThreaded): def extract_domains(self, resp): links_list = list() - link_regx = re.compile('(.*?)<\/cite>') + link_regx = re.compile(r'(.*?)') try: links_list = link_regx.findall(resp) for link in links_list: @@ -300,7 +296,7 @@ class GoogleEnum(enumratorBaseThreaded): return links_list def check_response_errors(self, resp): - if (type(resp) is str or type(resp) is unicode) and 'Our systems have detected unusual traffic' in resp: + if isinstance(resp, str) and 'Our systems have detected unusual traffic' in resp: self.print_(R + "[!] Error: Google probably now is blocking our requests" + W) self.print_(R + "[~] Finished now the Google Enumeration ..." + W) return False @@ -340,7 +336,7 @@ class YahooEnum(enumratorBaseThreaded): links2 = link_regx2.findall(resp) links_list = links + links2 for link in links_list: - link = re.sub("<(\/)?b>", "", link) + link = re.sub(r"<(/)?b>", "", link) if not link.startswith('http'): link = "http://" + link subdomain = urlparse.urlparse(link).netloc @@ -436,7 +432,7 @@ class BingEnum(enumratorBaseThreaded): links_list = links + links2 for link in links_list: - link = re.sub('<(\/)?strong>||<|>', '', link) + link = re.sub(r'<(/)?strong>||<|>', '', link) if not link.startswith('http'): link = "http://" + link subdomain = urlparse.urlparse(link).netloc @@ -611,7 +607,7 @@ class DNSdumpster(enumratorBaseThreaded): Resolver.nameservers = ['8.8.8.8', '8.8.4.4'] self.lock.acquire() try: - ip = Resolver.query(host, 'A')[0].to_text() + ip = Resolver.resolve(host, 'A')[0].to_text() if ip: if self.verbose: self.print_("%s%s: %s%s" % (R, self.engine_name, W, host)) @@ -638,13 +634,19 @@ class DNSdumpster(enumratorBaseThreaded): def get_csrftoken(self, resp): csrf_regex = re.compile('', re.S) - token = csrf_regex.findall(resp)[0] - return token.strip() + if not resp: + return None + tokens = csrf_regex.findall(resp) + if not tokens: + return None + return tokens[0].strip() def enumerate(self): self.lock = threading.BoundedSemaphore(value=70) resp = self.req('GET', self.base_url) token = self.get_csrftoken(resp) + if not token: + return self.live_subdomains params = {'csrfmiddlewaretoken': token, 'targetip': self.domain} post_resp = self.req('POST', self.base_url, params) self.extract_domains(post_resp) @@ -655,7 +657,7 @@ class DNSdumpster(enumratorBaseThreaded): return self.live_subdomains def extract_domains(self, resp): - tbl_regex = re.compile('<\/a>Host Records.*?(.*?)', re.S) + tbl_regex = re.compile(r'Host Records.*?(.*?)', re.S) link_regex = re.compile('(.*?)
', re.S) links = [] try: @@ -685,21 +687,41 @@ class Virustotal(enumratorBaseThreaded): # the main send_req need to be rewritten def send_req(self, url): + headers = dict(self.headers) + headers['Accept'] = 'application/json' + headers['Referer'] = 'https://www.virustotal.com/' + headers['User-Agent'] = random.choice([ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ]) try: - resp = self.session.get(url, headers=self.headers, timeout=self.timeout) + resp = self.session.get(url, headers=headers, timeout=self.timeout) except Exception as e: self.print_(e) resp = None + if resp is None: + return None + + if getattr(resp, 'status_code', None) in (403, 429): + return None + return self.get_response(resp) # once the send_req is rewritten we don't need to call this function, the stock one should be ok def enumerate(self): while self.url != '': resp = self.send_req(self.url) - resp = json.loads(resp) + if not resp: + break + try: + resp = json.loads(resp) + except Exception: + break if 'error' in resp: - self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W) + if self.verbose: + self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W) break if 'links' in resp and 'next' in resp['links']: self.url = resp['links']['next'] @@ -895,13 +917,13 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e enable_bruteforce = True # Validate domain - domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$") + domain_check = re.compile(r"^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$") if not domain_check.match(domain): if not silent: print(R + "Error: Please enter a valid domain" + W) return [] - if not domain.startswith('http://') or not domain.startswith('https://'): + if not domain.startswith('http://') and not domain.startswith('https://'): domain = 'http://' + domain parsed_domain = urlparse.urlparse(domain)