Merge branch 'aboul3la:master' into master

README.md

@@ -1,6 +1,6 @@
 ## About Sublist3r 
 
-Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT. It helps penetration testers and bug hunters collect and gather subdomains for the domain they are targeting. Sublist3r enumerates subdomains using many search engines such as Google, Yahoo, Bing, Baidu, and Ask. Sublist3r also enumerates subdomains using Netcraft, Virustotal, ThreatCrowd, DNSdumpster, and ReverseDNS.
+Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT. It helps penetration testers and bug hunters collect and gather subdomains for the domain they are targeting. Sublist3r enumerates subdomains using many search engines such as Google, Yahoo, Bing, Baidu and Ask. Sublist3r also enumerates subdomains using Netcraft, Virustotal, ThreatCrowd, DNSdumpster and ReverseDNS.
 
 [subbrute](https://github.com/TheRook/subbrute) was integrated with Sublist3r to increase the possibility of finding more subdomains using bruteforce with an improved wordlist. The credit goes to TheRook who is the author of subbrute.
 
@@ -26,11 +26,11 @@ sublist3r -b -d yourdomain.com -t 37 -v -o /root/sublist3r_yourdomain_com
 Sublist3r currently supports **Python 2** and **Python 3**.
 
 * The recommended version for Python 2 is **2.7.x**
-* The recommened version for Python 3 is **3.4.x**
+* The recommended version for Python 3 is **3.4.x**
 
 ## Dependencies:
 
-Sublist3r depends on the `requests`, `dnspython`, and `argparse` python modules.
+Sublist3r depends on the `requests`, `dnspython` and `argparse` python modules.
 
 These dependencies can be installed using the requirements file:
 

sublist3r.py

@@ -66,6 +66,10 @@ else:
     R = '\033[91m'  # red
     W = '\033[0m'   # white
 
+def no_color():
+    global G, Y, B, R, W
+    G = Y = B = R = W = ''
+
 
 def banner():
     print("""%s
@@ -98,6 +102,7 @@ def parse_args():
     parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=30)
     parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines')
     parser.add_argument('-o', '--output', help='Save the results to text file')
+    parser.add_argument('-n', '--no-color', help='Output without color', default=False, action='store_true')
     return parser.parse_args()
 
 
@@ -252,11 +257,10 @@ class enumratorBase(object):
 
 
 class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
-    def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True):
+    def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, silent=False, verbose=True):
         subdomains = subdomains or []
         enumratorBase.__init__(self, base_url, engine_name, domain, subdomains, silent=silent, verbose=verbose)
         multiprocessing.Process.__init__(self)
-        self.lock = lock
         self.q = q
         return
 
@@ -520,7 +524,6 @@ class NetcraftEnum(enumratorBaseThreaded):
         subdomains = subdomains or []
         self.base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}'
         self.engine_name = "Netcraft"
-        self.lock = threading.Lock()
         super(NetcraftEnum, self).__init__(self.base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
         self.q = q
         return
@@ -534,11 +537,14 @@ class NetcraftEnum(enumratorBaseThreaded):
             resp = None
         return resp
 
+    def should_sleep(self):
+        time.sleep(random.randint(1, 2))
+        return
+
     def get_next(self, resp):
-        link_regx = re.compile('<A href="(.*?)"><b>Next page</b></a>')
+        link_regx = re.compile('<a.*?href="(.*?)">Next Page')
         link = link_regx.findall(resp)
-        link = re.sub('host=.*?%s' % self.domain, 'host=%s' % self.domain, link[0])
+        url = 'http://searchdns.netcraft.com' + link[0]
-        url = 'http://searchdns.netcraft.com' + link
         return url
 
     def create_cookies(self, cookie):
@@ -564,14 +570,15 @@ class NetcraftEnum(enumratorBaseThreaded):
         while True:
             resp = self.get_response(self.req(url, cookies))
             self.extract_domains(resp)
-            if 'Next page' not in resp:
+            if 'Next Page' not in resp:
                 return self.subdomains
                 break
             url = self.get_next(resp)
+            self.should_sleep()
 
     def extract_domains(self, resp):
         links_list = list()
-        link_regx = re.compile('<a href="http://toolbar.netcraft.com/site_report\?url=(.*)">')
+        link_regx = re.compile('<a class="results-table__host" href="(.*?)"')
         try:
             links_list = link_regx.findall(resp)
             for link in links_list:
@@ -593,9 +600,8 @@ class DNSdumpster(enumratorBaseThreaded):
         base_url = 'https://dnsdumpster.com/'
         self.live_subdomains = []
         self.engine_name = "DNSdumpster"
-        self.threads = 70
-        self.lock = threading.BoundedSemaphore(value=self.threads)
         self.q = q
+        self.lock = None
         super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
         return
 
@@ -631,11 +637,12 @@ class DNSdumpster(enumratorBaseThreaded):
         return self.get_response(resp)
 
     def get_csrftoken(self, resp):
-        csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />", re.S)
+        csrf_regex = re.compile('<input type="hidden" name="csrfmiddlewaretoken" value="(.*?)">', re.S)
         token = csrf_regex.findall(resp)[0]
         return token.strip()
 
     def enumerate(self):
+        self.lock = threading.BoundedSemaphore(value=70)
         resp = self.req('GET', self.base_url)
         token = self.get_csrftoken(resp)
         params = {'csrfmiddlewaretoken': token, 'targetip': self.domain}
@@ -669,11 +676,11 @@ class DNSdumpster(enumratorBaseThreaded):
 class Virustotal(enumratorBaseThreaded):
     def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
         subdomains = subdomains or []
-        base_url = 'https://www.virustotal.com/en/domain/{domain}/information/'
+        base_url = 'https://www.virustotal.com/ui/domains/{domain}/subdomains'
         self.engine_name = "Virustotal"
-        self.lock = threading.Lock()
         self.q = q
         super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        self.url = self.base_url.format(domain=self.domain)
         return
 
     # the main send_req need to be rewritten
@@ -688,17 +695,25 @@ class Virustotal(enumratorBaseThreaded):
 
     # once the send_req is rewritten we don't need to call this function, the stock one should be ok
     def enumerate(self):
-        url = self.base_url.format(domain=self.domain)
+        while self.url != '':
-        resp = self.send_req(url)
+            resp = self.send_req(self.url)
+            resp = json.loads(resp)
+            if 'error' in resp:
+                self.print_(R + "[!] Error: Virustotal probably now is blocking our requests" + W)
+                break
+            if 'links' in resp and 'next' in resp['links']:
+                self.url = resp['links']['next']
+            else:
+                self.url = ''
             self.extract_domains(resp)
         return self.subdomains
 
     def extract_domains(self, resp):
-        link_regx = re.compile('<div class="enum.*?">.*?<a target="_blank" href=".*?">(.*?)</a>', re.S)
+        #resp is already parsed as json
         try:
-            links = link_regx.findall(resp)
+            for i in resp['data']:
-            for link in links:
+                if i['type'] == 'domain':
-                subdomain = link.strip()
+                    subdomain = i['id']
                     if not subdomain.endswith(self.domain):
                         continue
                     if subdomain not in self.subdomains and subdomain != self.domain:
@@ -714,7 +729,6 @@ class ThreatCrowd(enumratorBaseThreaded):
         subdomains = subdomains or []
         base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}'
         self.engine_name = "ThreatCrowd"
-        self.lock = threading.Lock()
         self.q = q
         super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
         return
@@ -753,7 +767,6 @@ class CrtSearch(enumratorBaseThreaded):
         subdomains = subdomains or []
         base_url = 'https://crt.sh/?q=%25.{domain}'
         self.engine_name = "SSL Certificates"
-        self.lock = threading.Lock()
         self.q = q
         super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
         return
@@ -778,7 +791,14 @@ class CrtSearch(enumratorBaseThreaded):
         try:
             links = link_regx.findall(resp)
             for link in links:
-                subdomain = link.strip()
+                link = link.strip()
+                subdomains = []
+                if '<BR>' in link:
+                    subdomains = link.split('<BR>')
+                else:
+                    subdomains.append(link)
+
+                for subdomain in subdomains:
                     if not subdomain.endswith(self.domain) or '*' in subdomain:
                         continue
 
@@ -790,15 +810,14 @@ class CrtSearch(enumratorBaseThreaded):
                             self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
                         self.subdomains.append(subdomain.strip())
         except Exception as e:
+            print(e)
             pass
 
-
 class PassiveDNS(enumratorBaseThreaded):
     def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
         subdomains = subdomains or []
         base_url = 'https://api.sublist3r.com/search.php?domain={domain}'
         self.engine_name = "PassiveDNS"
-        self.lock = threading.Lock()
         self.q = q
         super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
         return
@@ -836,8 +855,7 @@ class portscan():
     def __init__(self, subdomains, ports):
         self.subdomains = subdomains
         self.ports = ports
-        self.threads = 20
+        self.lock = None
-        self.lock = threading.BoundedSemaphore(value=self.threads)
 
     def port_scan(self, host, ports):
         openports = []
@@ -857,6 +875,7 @@ class portscan():
             print("%s%s%s - %sFound open ports:%s %s%s%s" % (G, host, W, R, W, Y, ', '.join(openports), W))
 
     def run(self):
+        self.lock = threading.BoundedSemaphore(value=20)
         for subdomain in self.subdomains:
             t = threading.Thread(target=self.port_scan, args=(subdomain, self.ports))
             t.start()
@@ -978,6 +997,8 @@ def interactive():
     engines = args.engines
     if verbose or verbose is None:
         verbose = True
+    if args.no_color:
+        no_color()
     banner()
     res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines)