RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge b6ae52a4ac into 729d649ec5

This commit is contained in:
idenroad 2026-01-29 17:53:10 -05:00 committed by GitHub
parent 729d649ec5 b6ae52a4ac
commit ba3122fedb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 152 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -17,27 +17,10 @@ git clone https://github.com/aboul3la/Sublist3r.git
## Recommended Python Version:
Sublist3r currently supports **Python 2** and **Python 3**.
Sublist3r currently supports**Python 3.8+
* The recommended version for Python 2 is **2.7.x**
* The recommended version for Python 3 is **3.4.x**
* The recommended version for Python 3 is **3.8+**
## Dependencies:
Sublist3r depends on the `requests`, `dnspython` and `argparse` python modules.
These dependencies can be installed using the requirements file:
- Installation on Windows:
```
c:\python27\python.exe -m pip install -r requirements.txt
```
- Installation on Linux
```
sudo pip install -r requirements.txt
```
Alternatively, each module can be installed independently as shown below.
#### Requests Module (http://docs.python-requests.org/en/latest/)

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

1
requirements.txt
View File

@ -1,3 +1,2 @@
argparse
dnspython
requests

23
setup.py
View File

@ -2,11 +2,13 @@ from setuptools import setup, find_packages
setup(
name='Sublist3r',
version='1.0',
python_requires='>=2.7',
install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''],
packages=find_packages()+['.'],
version='2.0',
python_requires='>=3.8',
install_requires=['dnspython', 'requests'],
packages=find_packages(),
py_modules=['sublist3r'],
include_package_data=True,
package_data={'subbrute': ['*.txt']},
url='https://github.com/aboul3la/Sublist3r',
license='GPL-2.0',
description='Subdomains enumeration tool for penetration testers',
@ -19,12 +21,15 @@ setup(
'License :: OSI Approved :: GNU General Public License v2',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
'Programming Language :: Python :: 3.13',
'Programming Language :: Python :: 3.14',
'Topic :: Security',
],
keywords='subdomain dns detection',

151
subbrute/subbrute.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
#SubBrute v1.2
#A (very) fast subdomain enumeration tool.
@ -67,7 +67,7 @@ class verify_nameservers(multiprocessing.Process):
#Lets test the letancy of our connection.
#Google's DNS server should be an ideal time test.
resolver.nameservers = ['8.8.8.8']
resolver.query(self.most_popular_website, self.record_type)
resolver.resolve(self.most_popular_website, self.record_type)
except:
#Our connection is slower than a junebug in molasses
resolver = dns.resolver.Resolver()
@ -138,7 +138,7 @@ class verify_nameservers(multiprocessing.Process):
#I have seen a CloudFlare Enterprise customer with the first two conditions.
try:
#This is case #3, these spam nameservers seem to be more trouble then they are worth.
wildtest = self.resolver.query(uuid.uuid4().hex + ".com", "A")
wildtest = self.resolver.resolve(uuid.uuid4().hex + ".com", "A")
if len(wildtest):
trace("Spam DNS detected:", host)
return False
@ -152,7 +152,7 @@ class verify_nameservers(multiprocessing.Process):
test_counter -= 1
try:
testdomain = "%s.%s" % (uuid.uuid4().hex, host)
wildtest = self.resolver.query(testdomain, self.record_type)
wildtest = self.resolver.resolve(testdomain, self.record_type)
#This 'A' record may contain a list of wildcards.
if wildtest:
for w in wildtest:
@ -204,7 +204,10 @@ class lookup(multiprocessing.Process):
def get_ns_blocking(self):
ret = []
ret = [self.resolver_q.get()]
try:
ret = [self.resolver_q.get()]
except KeyboardInterrupt:
return []
if ret == False:
trace("get_ns_blocking - Resolver list is empty.")
#Queue is empty, inform the rest.
@ -224,7 +227,7 @@ class lookup(multiprocessing.Process):
try:
#Query the nameserver, this is not simple...
if not record_type or record_type == "A":
resp = self.resolver.query(host)
resp = self.resolver.resolve(host)
#Crawl the response
hosts = extract_hosts(str(resp.response), self.domain)
for h in hosts:
@ -237,7 +240,7 @@ class lookup(multiprocessing.Process):
#A max 20 lookups
for x in range(20):
try:
resp = self.resolver.query(host, record_type)
resp = self.resolver.resolve(host, record_type)
except dns.resolver.NoAnswer:
resp = False
pass
@ -248,7 +251,7 @@ class lookup(multiprocessing.Process):
return cname_record
else:
#All other records:
return self.resolver.query(host, record_type)
return self.resolver.resolve(host, record_type)
except Exception as e:
if type(e) == dns.resolver.NoNameservers:
@ -299,59 +302,62 @@ class lookup(multiprocessing.Process):
raise e
def run(self):
#This process needs one resolver before it can start looking.
self.resolver.nameservers += self.get_ns_blocking()
while True:
found_addresses = []
work = self.in_q.get()
#Check if we have hit the end marker
while not work:
#Look for a re-queued lookup
try:
work = self.in_q.get(blocking = False)
#if we took the end marker of the queue we need to put it back
if work:
self.in_q.put(False)
except:#Queue.Empty
trace('End of work queue')
#There isn't an item behind the end marker
work = False
try:
#This process needs one resolver before it can start looking.
self.resolver.nameservers += self.get_ns_blocking()
while True:
found_addresses = []
work = self.in_q.get()
#Check if we have hit the end marker
while not work:
#Look for a re-queued lookup
try:
work = self.in_q.get(block=False)
#if we took the end marker of the queue we need to put it back
if work:
self.in_q.put(False)
except:#Queue.Empty
trace('End of work queue')
#There isn't an item behind the end marker
work = False
break
#Is this the end all work that needs to be done?
if not work:
#Perpetuate the end marker for all threads to see
self.in_q.put(False)
#Notify the parent that we have died of natural causes
self.out_q.put(False)
break
#Is this the end all work that needs to be done?
if not work:
#Perpetuate the end marker for all threads to see
self.in_q.put(False)
#Notify the parent that we have died of natural causes
self.out_q.put(False)
break
else:
if len(work) == 3:
#keep track of how many times this lookup has timedout.
(hostname, record_type, timeout_retries) = work
response = self.check(hostname, record_type, timeout_retries)
else:
(hostname, record_type) = work
response = self.check(hostname, record_type)
sys.stdout.flush()
trace(response)
#self.wildcards is populated by the verify_nameservers() thread.
#This variable doesn't need a muetex, because it has a queue.
#A queue ensure nameserver cannot be used before it's wildcard entries are found.
reject = False
if response:
for a in response:
a = str(a)
if a in self.wildcards:
trace("resovled wildcard:", hostname)
reject= True
#reject this domain.
break;
else:
found_addresses.append(a)
if not reject:
#This request is filled, send the results back
result = (hostname, record_type, found_addresses)
self.out_q.put(result)
if len(work) == 3:
#keep track of how many times this lookup has timedout.
(hostname, record_type, timeout_retries) = work
response = self.check(hostname, record_type, timeout_retries)
else:
(hostname, record_type) = work
response = self.check(hostname, record_type)
sys.stdout.flush()
trace(response)
#self.wildcards is populated by the verify_nameservers() thread.
#This variable doesn't need a muetex, because it has a queue.
#A queue ensure nameserver cannot be used before it's wildcard entries are found.
reject = False
if response:
for a in response:
a = str(a)
if a in self.wildcards:
trace("resovled wildcard:", hostname)
reject= True
#reject this domain.
break;
else:
found_addresses.append(a)
if not reject:
#This request is filled, send the results back
result = (hostname, record_type, found_addresses)
self.out_q.put(result)
except KeyboardInterrupt:
return
#Extract relevant hosts
#The dot at the end of a domain signifies the root,
@ -371,7 +377,7 @@ def extract_hosts(data, hostname):
#Return a list of unique sub domains, sorted by frequency.
#Only match domains that have 3 or more sections subdomain.domain.tld
domain_match = re.compile("([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+")
domain_match = re.compile(r"([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+")
def extract_subdomains(file_name):
#Avoid re-compilation
global domain_match
@ -462,9 +468,11 @@ def run(target, record_type = None, subdomains = "names.txt", resolve_list = "re
in_q.put(work)
#Terminate the queue
in_q.put(False)
workers = []
for i in range(process_count):
worker = lookup(in_q, out_q, resolve_q, target, wildcards, spider_blacklist)
worker.start()
workers.append(worker)
threads_remaining = process_count
while True:
try:
@ -476,6 +484,20 @@ def run(target, record_type = None, subdomains = "names.txt", resolve_list = "re
else:
#run() is a generator, and yields results from the work queue
yield result
except KeyboardInterrupt:
try:
killproc(pid = verify_nameservers_proc.pid)
except:
try:
verify_nameservers_proc.end()
except:
pass
for w in workers:
try:
killproc(pid = w.pid)
except:
pass
return
except Exception as e:
#The cx_freeze version uses queue.Empty instead of Queue.Empty :(
if type(e) == Queue.Empty or str(type(e)) == "<class 'queue.Empty'>":
@ -533,7 +555,14 @@ def check_open(input_file):
try:
ret = open(input_file).readlines()
except:
error("File not found:", input_file)
if not os.path.isabs(input_file):
try:
base_path = os.path.dirname(os.path.realpath(__file__))
ret = open(os.path.join(base_path, input_file)).readlines()
except:
error("File not found:", input_file)
else:
error("File not found:", input_file)
if not len(ret):
error("File is empty:", input_file)
return ret

64
sublist3r.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# coding: utf-8
# Sublist3r v1.0
# By Ahmed Aboul-Ela - twitter.com/aboul3la
@ -23,12 +23,8 @@ import dns.resolver
import requests
# Python 2.x and 3.x compatiablity
if sys.version > '3':
import urllib.parse as urlparse
import urllib.parse as urllib
else:
import urlparse
import urllib
import urllib.parse as urlparse
import urllib.parse as urllib
# In case you cannot install some of the required development packages
# there's also an option to disable the SSL warning:
@ -72,7 +68,7 @@ def no_color():
def banner():
print("""%s
print(r"""%s
____ _ _ _ _ _____
/ ___| _ _| |__ | (_)___| |_|___ / _ __
\___ \| | | | '_ \| | / __| __| |_ \| '__|
@ -283,7 +279,7 @@ class GoogleEnum(enumratorBaseThreaded):
def extract_domains(self, resp):
links_list = list()
link_regx = re.compile('<cite.*?>(.*?)<\/cite>')
link_regx = re.compile(r'<cite.*?>(.*?)</cite>')
try:
links_list = link_regx.findall(resp)
for link in links_list:
@ -300,7 +296,7 @@ class GoogleEnum(enumratorBaseThreaded):
return links_list
def check_response_errors(self, resp):
if (type(resp) is str or type(resp) is unicode) and 'Our systems have detected unusual traffic' in resp:
if isinstance(resp, str) and 'Our systems have detected unusual traffic' in resp:
self.print_(R + "[!] Error: Google probably now is blocking our requests" + W)
self.print_(R + "[~] Finished now the Google Enumeration ..." + W)
return False
@ -340,7 +336,7 @@ class YahooEnum(enumratorBaseThreaded):
links2 = link_regx2.findall(resp)
links_list = links + links2
for link in links_list:
link = re.sub("<(\/)?b>", "", link)
link = re.sub(r"<(/)?b>", "", link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
@ -436,7 +432,7 @@ class BingEnum(enumratorBaseThreaded):
links_list = links + links2
for link in links_list:
link = re.sub('<(\/)?strong>|<span.*?>|<|>', '', link)
link = re.sub(r'<(/)?strong>|<span.*?>|<|>', '', link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
@ -611,7 +607,7 @@ class DNSdumpster(enumratorBaseThreaded):
Resolver.nameservers = ['8.8.8.8', '8.8.4.4']
self.lock.acquire()
try:
ip = Resolver.query(host, 'A')[0].to_text()
ip = Resolver.resolve(host, 'A')[0].to_text()
if ip:
if self.verbose:
self.print_("%s%s: %s%s" % (R, self.engine_name, W, host))
@ -638,13 +634,19 @@ class DNSdumpster(enumratorBaseThreaded):
def get_csrftoken(self, resp):
csrf_regex = re.compile('<input type="hidden" name="csrfmiddlewaretoken" value="(.*?)">', re.S)
token = csrf_regex.findall(resp)[0]
return token.strip()
if not resp:
return None
tokens = csrf_regex.findall(resp)
if not tokens:
return None
return tokens[0].strip()
def enumerate(self):
self.lock = threading.BoundedSemaphore(value=70)
resp = self.req('GET', self.base_url)
token = self.get_csrftoken(resp)
if not token:
return self.live_subdomains
params = {'csrfmiddlewaretoken': token, 'targetip': self.domain}
post_resp = self.req('POST', self.base_url, params)
self.extract_domains(post_resp)
@ -655,7 +657,7 @@ class DNSdumpster(enumratorBaseThreaded):
return self.live_subdomains
def extract_domains(self, resp):
tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>', re.S)
tbl_regex = re.compile(r'<a name="hostanchor"></a>Host Records.*?<table.*?>(.*?)</table>', re.S)
link_regex = re.compile('<td class="col-md-4">(.*?)<br>', re.S)
links = []
try:
@ -685,21 +687,41 @@ class Virustotal(enumratorBaseThreaded):
# the main send_req need to be rewritten
def send_req(self, url):
headers = dict(self.headers)
headers['Accept'] = 'application/json'
headers['Referer'] = 'https://www.virustotal.com/'
headers['User-Agent'] = random.choice([
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
])
try:
resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
resp = self.session.get(url, headers=headers, timeout=self.timeout)
except Exception as e:
self.print_(e)
resp = None
if resp is None:
return None
if getattr(resp, 'status_code', None) in (403, 429):
return None
return self.get_response(resp)
# once the send_req is rewritten we don't need to call this function, the stock one should be ok
def enumerate(self):
while self.url != '':
resp = self.send_req(self.url)
resp = json.loads(resp)
if not resp:
break
try:
resp = json.loads(resp)
except Exception:
break
if 'error' in resp:
self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
if self.verbose:
self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
break
if 'links' in resp and 'next' in resp['links']:
self.url = resp['links']['next']
@ -895,13 +917,13 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
enable_bruteforce = True
# Validate domain
domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
domain_check = re.compile(r"^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
if not domain_check.match(domain):
if not silent:
print(R + "Error: Please enter a valid domain" + W)
return []
if not domain.startswith('http://') or not domain.startswith('https://'):
if not domain.startswith('http://') and not domain.startswith('https://'):
domain = 'http://' + domain
parsed_domain = urlparse.urlparse(domain)