Merge branch 'dev'

Github didn't like web PR so following manual instructions

README.md

@@ -73,7 +73,7 @@ Short Form    | Long Form     | Description
 
 Note that turbolist3r has not been tested on Windows.
 
-Turbolist3r depends on the `dnslib`, `requests`, `dnspython`, and `argparse` python modules.
+Turbolist3r depends on the `dnslib`, `requests`, and `argparse` python modules.
 
 #### dnslib Module
 
@@ -99,18 +99,6 @@ sudo yum install python-requests
 sudo pip install requests
 ```
 
-#### dnspython Module (http://www.dnspython.org/)
-
-- Install for Ubuntu/Debian:
-```
-sudo apt-get install python-dnspython
-```
-
-- Install using pip:
-```
-sudo pip install dnspython
-```
-
 #### argparse Module
 
 - Install for Ubuntu/Debian:

requirements.txt

@@ -1,4 +1,3 @@
 dnslib
 argparse
-dnspython
 requests

turbolist3r.py

@@ -4,8 +4,6 @@
 # By Carl Pearson - github.com/fleetcaptain
 # Based on Sublist3r code created by Ahmed Aboul-Ela - twitter.com/aboul3la
 #
-# TODO - merge Sublist3r dns requests with dnslib to avoid duplication of dns libraries
-#
 
 
 # modules in standard library
@@ -26,9 +24,11 @@ from collections import Counter
 try:
     from subbrute import subbrute
 except:
-	print "Failed to import subbrute, you will not be able to bruteforce"
+    print("Failed to import subbrute, you will not be able to bruteforce")
-import dns.resolver
+
+#import dns.resolver
 import requests
+
 # import dnslib, which provides better features compared to dns.resolver for finding subdomains
 # for example, a return status of NXDOMAIN causes an exception with dns.resolver, but dnslib allows
 # us to easily capture the reply, which could indicate the precsence of subdomain takeover
@@ -55,6 +55,8 @@ except:
 is_windows = sys.platform.startswith('win')
 
 global debug
+global resolvers # set resolvers as global so we can use it in multiple places
+global RESOLVER_COUNT # number of resolvers we are using
 
 # Console Colors
 if is_windows:
@@ -643,20 +645,36 @@ class DNSdumpster(enumratorBaseThreaded):
                                           verbose=verbose)
         return
 
-	def check_host(self, host):
+    def check_host(self, host, resolver):
         is_valid = False
-		Resolver = dns.resolver.Resolver()
+        response = None
-		Resolver.nameservers = ['8.8.8.8', '8.8.4.4', '1.1.1.1', '1.0.0.1']
+        use_tcp=False
         self.lock.acquire()
+        query = dnslib.DNSRecord.question(host, 'A')
         try:
-			ip = Resolver.query(host, 'A')[0].to_text()
+            response_q = query.send(resolver, 53, use_tcp, timeout=3)
-			if ip:
+            if response_q:
+                response = dnslib.DNSRecord.parse(response_q)
+        except Exception as e:
+            # probably socket timed out
+            if self.verbose or debug:
+                print("Error: rcode for " + host + " in DNSDumpster = " + rcode)
+                print(str(e))
+            pass
+        if debug:
+                print(str(response))
+                print()
+
+        if response:
+            rcode = dnslib.RCODE[response.header.rcode]
             if self.verbose:
-					self.print_("%s%s: %s%s" % (R, self.engine_name, W, host))
+                print("Rcode for " + host + " in DNSDumpster = " + rcode)
+            # including NXDOMAIN here will result in some false positives, since some subdomains won't exist
+            # but since Turbolist3r is focused on subdomain takeover, we do want to know about these as NXDOMAIN 
+            # is an indicator the subdomain might be vulnerable to takeover
+            if rcode == 'NOERROR' or rcode == 'NXDOMAIN':
                  is_valid = True
                  self.live_subdomains.append(host)
-		except:
-			pass
         self.lock.release()
         return is_valid
 
@@ -685,8 +703,14 @@ class DNSdumpster(enumratorBaseThreaded):
         params = {'csrfmiddlewaretoken': token, 'targetip': self.domain}
         post_resp = self.req('POST', self.base_url, params)
         self.extract_domains(post_resp)
+        server = 0
         for subdomain in self.subdomains:
-			t = threading.Thread(target=self.check_host, args=(subdomain,))
+            # round robin the resolvers
+            server = server + 1
+            server = server % RESOLVER_COUNT
+            resolver = resolvers[server]
+            # start thread with current subdomain and resolver as arguments
+            t = threading.Thread(target=self.check_host, args=(subdomain,resolver,))
             t.start()
             t.join()
         return self.live_subdomains
@@ -1158,7 +1182,7 @@ if __name__ == "__main__":
     # Did the user specifiy a custom resolver file?
     # If so, try to read it here, so if there is an error we don't waste
     # running the rest of the script before erroring out
-	resolvers = []
+    
     if (server_file != None):
         try:
             f = open(server_file, 'r')
@@ -1178,6 +1202,8 @@ if __name__ == "__main__":
         # use a default list of resolvers
         resolvers = ['8.8.8.8', '8.8.4.4', '9.9.9.9', '1.1.1.1', '1.0.0.1']    
        
+    RESOLVER_COUNT = len(resolvers)
+ 
     if (inputfile != None):
         print(B + "[-] Reading subdomains from " + inputfile + W)
         f = open(inputfile, 'r')
@@ -1209,7 +1235,7 @@ if __name__ == "__main__":
                         ahosts.append(name + " -->-- " + record)
                 # round robin the resolvers
                 server = server + 1
-				server = server % len(resolvers)
+                server = server % RESOLVER_COUNT
 
                 # update user on our progress - every 30 hosts
                 count = count + 1