(.*?)', re.S)
+ link_regex = re.compile('(.*?)
', re.S)
+ links = []
+ try:
+ results_tbl = tbl_regex.findall(resp)[0]
+ except IndexError:
+ results_tbl = ''
+ links_list = link_regex.findall(results_tbl)
+ links = list(set(links_list))
+ for link in links:
+ subdomain = link.strip()
+ if not subdomain.endswith(self.domain):
+ continue
+ if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
+ self.subdomains.append(subdomain.strip())
+ return links
+
+
+class Virustotal(enumratorBaseThreaded):
+ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+ subdomains = subdomains or []
+ base_url = 'https://www.virustotal.com/ui/domains/{domain}/subdomains'
+ self.engine_name = "Virustotal"
+ self.q = q
+ super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+ self.url = self.base_url.format(domain=self.domain)
+ return
+
+ # the main send_req need to be rewritten
+ def send_req(self, url):
+ try:
+ resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+ except Exception as e:
+ self.print_(e)
+ resp = None
+
+ return self.get_response(resp)
+
+ # once the send_req is rewritten we don't need to call this function, the stock one should be ok
def enumerate(self):
while self.url != '':
- resp_text = self.send_req(self.url)
- try:
- resp = json.loads(resp_text)
- except:
- break
+ resp = self.send_req(self.url)
+ resp = json.loads(resp)
if 'error' in resp:
- if self.verbose:
- self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
+ self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
break
- self.url = resp.get('links', {}).get('next', '')
+ if 'links' in resp and 'next' in resp['links']:
+ self.url = resp['links']['next']
+ else:
+ self.url = ''
self.extract_domains(resp)
return self.subdomains
def extract_domains(self, resp):
+ #resp is already parsed as json
try:
- for i in resp.get('data', []):
- if i.get('type') == 'domain':
- subdomain = i.get('id')
- if subdomain and subdomain.endswith(self.domain) and subdomain not in self.subdomains and subdomain != self.domain:
+ for i in resp['data']:
+ if i['type'] == 'domain':
+ subdomain = i['id']
+ if not subdomain.endswith(self.domain):
+ continue
+ if subdomain not in self.subdomains and subdomain != self.domain:
if self.verbose:
self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
self.subdomains.append(subdomain.strip())
except Exception:
pass
-# Note: Other specific engine classes (Bing, Baidu, etc.) follow the same pattern
-# using self.verbose check before self.print_ calls.
-def main():
+class ThreatCrowd(enumratorBaseThreaded):
+ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+ subdomains = subdomains or []
+ base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}'
+ self.engine_name = "ThreatCrowd"
+ self.q = q
+ super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+ return
+
+ def req(self, url):
+ try:
+ resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+ except Exception:
+ resp = None
+
+ return self.get_response(resp)
+
+ def enumerate(self):
+ url = self.base_url.format(domain=self.domain)
+ resp = self.req(url)
+ self.extract_domains(resp)
+ return self.subdomains
+
+ def extract_domains(self, resp):
+ try:
+ links = json.loads(resp)['subdomains']
+ for link in links:
+ subdomain = link.strip()
+ if not subdomain.endswith(self.domain):
+ continue
+ if subdomain not in self.subdomains and subdomain != self.domain:
+ if self.verbose:
+ self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
+ self.subdomains.append(subdomain.strip())
+ except Exception as e:
+ pass
+
+
+class CrtSearch(enumratorBaseThreaded):
+ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+ subdomains = subdomains or []
+ base_url = 'https://crt.sh/?q=%25.{domain}'
+ self.engine_name = "SSL Certificates"
+ self.q = q
+ super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+ return
+
+ def req(self, url):
+ try:
+ resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+ except Exception:
+ resp = None
+
+ return self.get_response(resp)
+
+ def enumerate(self):
+ url = self.base_url.format(domain=self.domain)
+ resp = self.req(url)
+ if resp:
+ self.extract_domains(resp)
+ return self.subdomains
+
+ def extract_domains(self, resp):
+ link_regx = re.compile(' | (.*?) | ')
+ try:
+ links = link_regx.findall(resp)
+ for link in links:
+ link = link.strip()
+ subdomains = []
+ if '
' in link:
+ subdomains = link.split('
')
+ else:
+ subdomains.append(link)
+
+ for subdomain in subdomains:
+ if not subdomain.endswith(self.domain) or '*' in subdomain:
+ continue
+
+ if '@' in subdomain:
+ subdomain = subdomain[subdomain.find('@')+1:]
+
+ if subdomain not in self.subdomains and subdomain != self.domain:
+ if self.verbose:
+ self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
+ self.subdomains.append(subdomain.strip())
+ except Exception as e:
+ print(e)
+ pass
+
+class PassiveDNS(enumratorBaseThreaded):
+ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+ subdomains = subdomains or []
+ base_url = 'https://api.sublist3r.com/search.php?domain={domain}'
+ self.engine_name = "PassiveDNS"
+ self.q = q
+ super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+ return
+
+ def req(self, url):
+ try:
+ resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+ except Exception as e:
+ resp = None
+
+ return self.get_response(resp)
+
+ def enumerate(self):
+ url = self.base_url.format(domain=self.domain)
+ resp = self.req(url)
+ if not resp:
+ return self.subdomains
+
+ self.extract_domains(resp)
+ return self.subdomains
+
+ def extract_domains(self, resp):
+ try:
+ subdomains = json.loads(resp)
+ for subdomain in subdomains:
+ if subdomain not in self.subdomains and subdomain != self.domain:
+ if self.verbose:
+ self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
+ self.subdomains.append(subdomain.strip())
+ except Exception as e:
+ pass
+
+
+class portscan():
+ def __init__(self, subdomains, ports):
+ self.subdomains = subdomains
+ self.ports = ports
+ self.lock = None
+
+ def port_scan(self, host, ports):
+ openports = []
+ self.lock.acquire()
+ for port in ports:
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(2)
+ result = s.connect_ex((host, int(port)))
+ if result == 0:
+ openports.append(port)
+ s.close()
+ except Exception:
+ pass
+ self.lock.release()
+ if len(openports) > 0:
+ print("%s%s%s - %sFound open ports:%s %s%s%s" % (G, host, W, R, W, Y, ', '.join(openports), W))
+
+ def run(self):
+ self.lock = threading.BoundedSemaphore(value=20)
+ for subdomain in self.subdomains:
+ t = threading.Thread(target=self.port_scan, args=(subdomain, self.ports))
+ t.start()
+
+
+def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, engines, pprtscan=False, output_format='raw', no_resolve=False):
+ bruteforce_list = set()
+ search_list = set()
+
+ if is_windows:
+ subdomains_queue = list()
+ else:
+ subdomains_queue = multiprocessing.Manager().list()
+
+ # Check Bruteforce Status
+ if enable_bruteforce or enable_bruteforce is None:
+ enable_bruteforce = True
+
+ # Validate domain
+ domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
+ if not domain_check.match(domain):
+ if not silent:
+ print(R + "Error: Please enter a valid domain" + W)
+ return []
+
+ if not domain.startswith('http://') or not domain.startswith('https://'):
+ domain = 'http://' + domain
+
+ parsed_domain = urlparse.urlparse(domain)
+
+ if not silent:
+ print(B + "[-] Enumerating subdomains now for %s" % parsed_domain.netloc + W)
+
+ if verbose and not silent:
+ print(Y + "[-] verbosity is enabled, will show the subdomains results in realtime" + W)
+
+ supported_engines = {'baidu': BaiduEnum,
+ 'yahoo': YahooEnum,
+ 'google': GoogleEnum,
+ 'bing': BingEnum,
+ 'ask': AskEnum,
+ 'netcraft': NetcraftEnum,
+ 'dnsdumpster': DNSdumpster,
+ 'virustotal': Virustotal,
+ 'threatcrowd': ThreatCrowd,
+ 'ssl': CrtSearch,
+ 'passivedns': PassiveDNS,
+ 'hackertarget':HackertargetEnum
+ }
+
+ chosenEnums = []
+
+ if engines is None:
+ chosenEnums = [
+ BaiduEnum, YahooEnum, GoogleEnum, BingEnum, AskEnum,
+ NetcraftEnum, ThreatCrowd,
+ CrtSearch, PassiveDNS, HackertargetEnum,Virustotal
+ ]
+ else:
+ engines = engines.split(',')
+ for engine in engines:
+ if engine.lower() in supported_engines:
+ chosenEnums.append(supported_engines[engine.lower()])
+
+ # Start the engines enumeration
+ enums = [enum(domain, [], q=subdomains_queue, silent=silent, verbose=verbose) for enum in chosenEnums]
+ for enum in enums:
+ enum.start()
+ for enum in enums:
+ enum.join()
+
+ subdomains = set(subdomains_queue)
+ for subdomain in subdomains:
+ search_list.add(subdomain)
+
+ if enable_bruteforce:
+ if not silent:
+ print(G + "[-] Starting bruteforce module now using subbrute.." + W)
+ record_type = False
+ path_to_file = os.path.dirname(os.path.realpath(__file__))
+ subs = os.path.join(path_to_file, 'subbrute', 'names.txt')
+ resolvers = os.path.join(path_to_file, 'subbrute', 'resolvers.txt')
+ process_count = threads
+ output = False
+ json_output = False
+ bruteforce_list = subbrute.print_target(parsed_domain.netloc, record_type, subs, resolvers, process_count, output, json_output, search_list, verbose)
+
+ subdomains = search_list.union(bruteforce_list)
+
+ if subdomains:
+ subdomains = sorted(subdomains, key=subdomain_sorting_key)
+
+ ip_map = None
+ if not no_resolve:
+ ip_map = resolve_ips(subdomains)
+
+ pprtscan_results = None
+ if pprtscan:
+ if not silent:
+ print(G + "[-] Starting Shodan InternetDB port scan.." + W)
+ pprtscan_results = pprt_scan(subdomains)
+ if not silent:
+ print(G + "[-] Shodan InternetDB port scan completed" + W)
+
+ if savefile:
+ write_file(savefile, subdomains, output_format=output_format, port_scan_results=pprtscan_results, ip_map=ip_map)
+
+ if not silent:
+ print(Y + "[-] Total Unique Subdomains Found: %s" % len(subdomains) + W)
+
+ if ports:
+ if not silent:
+ print(G + "[-] Start port scan now for the following ports: %s%s" % (Y, ports) + W)
+ ports = ports.split(',')
+ pscan = portscan(subdomains, ports)
+ pscan.run()
+
+ if not silent and not ports:
+ if output_format == 'json':
+ output = {}
+ items = []
+ for subdomain in subdomains:
+ item = {'subdomain': subdomain}
+ if ip_map and subdomain in ip_map:
+ item['ip'] = ip_map[subdomain]
+ items.append(item)
+ output['subdomains'] = items
+ if pprtscan_results:
+ output['port_scan'] = pprtscan_results
+ print(json.dumps(output, indent=2))
+ elif output_format == 'markdown':
+ print("# Sublist3r Results\n")
+ print("## Subdomains\n")
+ if ip_map:
+ print("| Subdomain | IP |")
+ print("|-----------|-----|")
+ for subdomain in subdomains:
+ ip = ip_map.get(subdomain, '')
+ print("| %s | %s |" % (subdomain, ip))
+ else:
+ for subdomain in subdomains:
+ print("- %s" % subdomain)
+ if pprtscan_results:
+ print("\n## Port Scan Results\n")
+ print("| Subdomain | IP | Ports |")
+ print("|-----------|-----|-------|")
+ for subdomain, data in pprtscan_results.items():
+ ip = data.get('ip', '')
+ ports_str = ', '.join(str(p) for p in data.get('ports', []))
+ print("| %s | %s | %s |" % (subdomain, ip, ports_str))
+ else:
+ if pprtscan_results:
+ for subdomain in subdomains:
+ ip = ip_map.get(subdomain, '') if ip_map else ''
+ if subdomain in pprtscan_results:
+ data = pprtscan_results[subdomain]
+ pports = ', '.join(str(p) for p in data.get('ports', []))
+ if ip:
+ print("%s,%s,%s" % (subdomain, ip, pports))
+ else:
+ print("%s,%s" % (subdomain, pports))
+ else:
+ if ip:
+ print("%s,%s" % (subdomain, ip))
+ else:
+ print(G + subdomain + W)
+ else:
+ for subdomain in subdomains:
+ if ip_map and subdomain in ip_map:
+ print("%s,%s" % (subdomain, ip_map[subdomain]))
+ else:
+ print(G + subdomain + W)
+ return subdomains
+
+
+def interactive():
args = parse_args()
domain = args.domain
+ threads = args.threads
+ savefile = args.output
+ ports = args.ports
+ enable_bruteforce = args.bruteforce
+ verbose = args.verbose
+ engines = args.engines
+ if verbose or verbose is None:
+ verbose = True
if args.no_color:
no_color()
-
- # Silencing the banner unless verbose
- if args.verbose:
- banner()
-
- # Threading logic and engine initialization would go here...
- # (Abbreviated for focus on requested modifications)
-
- # After enumeration:
- # subdomains = sorted(list(set(results)), key=subdomain_sorting_key)
-
- # if args.output:
- # write_file(args.output, subdomains, verbose=args.verbose)
+ banner()
+ res = main(domain, threads, savefile, ports, silent=True, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines, pprtscan=args.pprtscan, output_format=args.output_format, no_resolve=args.no_resolve)
if __name__ == "__main__":
- main()
\ No newline at end of file
+ interactive()
diff --git a/setup.py b/setup.py
index e0165ca..2f20e13 100644
--- a/setup.py
+++ b/setup.py
@@ -2,8 +2,8 @@ from setuptools import setup, find_packages
setup(
name='Sublist3r',
- version='1.6',
- python_requires='>=2.7',
+ version='1.7',
+ python_requires='>=3.7',
install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''],
packages=find_packages()+['.'],
include_package_data=True,
diff --git a/sublist3r.py b/sublist3r.py
index 7d904b2..f67e480 100755
--- a/sublist3r.py
+++ b/sublist3r.py
@@ -88,7 +88,7 @@ def parse_args():
parser.error = parser_error
parser._optionals.title = "OPTIONS"
parser.add_argument('-d', '--domain', help="Domain name to enumerate it's subdomains", required=True)
- parser.add_argument('-b', '--bruteforce', help='Enable the subbrute bruteforce module', nargs='?', default=False)
+ parser.add_argument('-b', '--bruteforce', help='Enable the subbrute bruteforce module', default=False, action='store_true')
parser.add_argument('-p', '--ports', help='Scan the found subdomains against specified tcp ports')
parser.add_argument('-v', '--verbose', help='Enable Verbosity and display results in realtime', nargs='?', default=False)
parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=30)
@@ -353,7 +353,7 @@ class GoogleEnum(enumratorBaseThreaded):
def extract_domains(self, resp):
links_list = list()
- link_regx = re.compile('(.*?)<\/cite>')
+ link_regx = re.compile(r'(.*?)')
try:
links_list = link_regx.findall(resp)
for link in links_list:
@@ -448,7 +448,7 @@ class YahooEnum(enumratorBaseThreaded):
links2 = link_regx2.findall(resp)
links_list = links + links2
for link in links_list:
- link = re.sub("<(\/)?b>", "", link)
+ link = re.sub(r"<(\/)?b>", "", link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
@@ -544,7 +544,7 @@ class BingEnum(enumratorBaseThreaded):
links_list = links + links2
for link in links_list:
- link = re.sub('<(\/)?strong>||<|>', '', link)
+ link = re.sub(r'<(\/)?strong>||<|>', '', link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
@@ -763,7 +763,7 @@ class DNSdumpster(enumratorBaseThreaded):
return self.live_subdomains
def extract_domains(self, resp):
- tbl_regex = re.compile('<\/a>Host Records.*?(.*?)', re.S)
+ tbl_regex = re.compile(r'<\/a>Host Records.*?(.*?)', re.S)
link_regex = re.compile('(.*?)
', re.S)
links = []
try:
@@ -999,11 +999,11 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
subdomains_queue = multiprocessing.Manager().list()
# Check Bruteforce Status
- if enable_bruteforce or enable_bruteforce is None:
+ if enable_bruteforce:
enable_bruteforce = True
# Validate domain
- domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
+ domain_check = re.compile(r"^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
if not domain_check.match(domain):
if not silent:
print(R + "Error: Please enter a valid domain" + W)
@@ -1173,7 +1173,7 @@ def interactive():
if args.no_color:
no_color()
banner()
- res = main(domain, threads, savefile, ports, silent=True, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines, pprtscan=args.pprtscan, output_format=args.output_format, no_resolve=args.no_resolve)
+ res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines, pprtscan=args.pprtscan, output_format=args.output_format, no_resolve=args.no_resolve)
if __name__ == "__main__":
interactive()
|