From 3cfa9426f3574de410c56b9009950d1924508d6f Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sat, 23 Oct 2021 23:45:57 +0200
Subject: [PATCH 1/7] Add setup.py, clean long lines, strip python2 code

---
 setup.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 setup.py

diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..20c9153
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+import os
+from setuptools import setup
+
+
+def read(fname: str) -> str:
+    """Open files relative to package."""
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+setup(
+    name='Sublist3r2',
+    version='1.0.0',
+    python_requires='>=3.6',
+    description='Subdomains enumeration tool for penetration testers',
+    long_description=read('README.md'),
+    long_description_content_type='text/markdown',
+    keywords='subdomain dns detection',
+    url='https://github.com/RoninNakomoto/Sublist3r2',
+    license='GPL-2.0',
+    py_modules=['sublist3r2'],
+    include_package_data=True,
+    package_data={
+        '': ['data/*.txt'],
+    },
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'Intended Audience :: Telecommunications Industry',
+        'License :: OSI Approved :: GNU General Public License v2',
+        'Operating System :: POSIX :: Linux',
+        'Operating System :: MacOS',
+        'Programming Language :: Python :: 3',
+        'Topic :: Security',
+    ],
+    install_requires=[
+        'argparse',
+        'dnspython',
+        'requests',
+        'aiodnsbrute',
+    ],
+    entry_points={
+        'console_scripts': [
+            'sublist3r2 = sublist3r2:interactive',
+        ],
+    },
+)

From e35f70292ee145498a6176a4a5b687e64564b1da Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sat, 23 Oct 2021 23:50:20 +0200
Subject: [PATCH 2/7] Add setup.py, clean long lines, strip python2 code

---
 aiodnsbrute/__init__.py                       |   0
 aiodnsbrute/cli.py                            | 359 ------------------
 aiodnsbrute/logger.py                         |  31 --
 aiodnsbrute/logger/logger.py                  |  31 --
 {aiodnsbrute => data}/resolvers.txt           |   0
 .../subdomains-top1million-110000.txt         |   0
 sublist3r2.py                                 | 226 ++++++-----
 7 files changed, 127 insertions(+), 520 deletions(-)
 delete mode 100644 aiodnsbrute/__init__.py
 delete mode 100644 aiodnsbrute/cli.py
 delete mode 100644 aiodnsbrute/logger.py
 delete mode 100644 aiodnsbrute/logger/logger.py
 rename {aiodnsbrute => data}/resolvers.txt (100%)
 rename {aiodnsbrute => data}/subdomains-top1million-110000.txt (100%)

diff --git a/aiodnsbrute/__init__.py b/aiodnsbrute/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/aiodnsbrute/cli.py b/aiodnsbrute/cli.py
deleted file mode 100644
index 0c4ecbc..0000000
--- a/aiodnsbrute/cli.py
+++ /dev/null
@@ -1,359 +0,0 @@
-import random
-import string
-import asyncio
-import functools
-import os
-import uvloop
-import aiodns
-import click
-import socket
-import sys
-from tqdm import tqdm
-from aiodnsbrute.logger import ConsoleLogger
-
-
-class aioDNSBrute(object):
-    """aiodnsbrute implements fast domain name brute forcing using Python's asyncio module."""
-
-    def __init__(self, verbosity=0, max_tasks=512):
-        """Constructor.
-
-        Args:
-            verbosity: set output verbosity: 0 (default) is none, 3 is debug
-            max_tasks: the maximum number of tasks asyncio will queue (default 512)
-        """
-        self.tasks = []
-        self.errors = []
-        self.fqdn = []
-        self.ignore_hosts = []
-        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
-        self.loop = asyncio.get_event_loop()
-        self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
-        self.sem = asyncio.BoundedSemaphore(max_tasks)
-        self.max_tasks = max_tasks
-        self.verbosity = verbosity
-        self.logger = ConsoleLogger(verbosity)
-
-    async def _dns_lookup(self, name):
-        """Performs a DNS request using aiodns, self.lookup_type is set by the run function.
-        A query for A record returns <ares_query_a_result> which does not return metadata about
-        when a CNAME was resolved (just host and ttl attributes) however it should be faster.
-        The <ares_host_result> returned by gethostbyname contains name, aliases, and addresses, if
-        name is different in response we can surmise that the original domain was a CNAME entry.
-
-        Args:
-            name: the domain name to resolve
-
-        Returns:
-            object: <ares_query_a_result> if query, <ares_host_result> if gethostbyname
-        """
-        if self.lookup_type == "query":
-            return await self.resolver.query(name, "A")
-        elif self.lookup_type == "gethostbyname":
-            return await self.resolver.gethostbyname(name, socket.AF_INET)
-
-    def _dns_result_callback(self, name, future):
-        """Handles the pycares object passed by the _dns_lookup function. We expect an errror to
-        be present in the returned object because most lookups will be for names that don't exist.
-        c-ares errors are passed through directly, error types can be identified in ares_strerror.c
-
-        Args:
-            name: original lookup name (because the query_result object doesn't contain it)
-            future: the completed future (pycares dns result)
-        """
-        # Record processed we can now release the lock
-        self.sem.release()
-        # Handle known exceptions, barf on other ones
-        if future.exception() is not None:
-            try:
-                err_number = future.exception().args[0]
-                err_text = future.exception().args[1]
-            except IndexError:
-                self.logger.error(f"Couldn't parse exception: {future.exception()}")
-            # handle the DNS errors we expect to receive, show user unexpected errors
-            if err_number == 4:
-                # This is domain name not found, ignore it
-                pass
-           # elif err_number == 12:
-                # Timeout from DNS server
-                #self.logger.warn(f"Timeout for {name}")
-            elif err_number == 1:
-                # Server answered with no data
-                pass
-            #else:
-                #self.logger.error(
-                #    f"{name} generated an unexpected exception: {future.exception()}"
-                #)
-            # for debugging/troubleshoooting keep a list of errors
-            # self.errors.append({'hostname': name, 'error': err_text})
-
-        # parse and output and store results.
-        else:
-            if self.lookup_type == "query":
-                ips = [ip.host for ip in future.result()]
-                cname = False
-                row = f"{name:<30}\t{ips}"
-            elif self.lookup_type == "gethostbyname":
-                r = future.result()
-                ips = [ip for ip in r.addresses]
-                if name == r.name:
-                    cname = False
-                    n = f"""{name:<30}\t{f"{'':<35}" if self.verbosity >= 2 else ""}"""
-                else:
-                    cname = True
-                    # format the name based on verbosity - this is kluge
-                    short_cname = f"{r.name[:28]}.." if len(r.name) > 30 else r.name
-                    n = f'{name}{"**" if self.verbosity <= 1 else ""}'
-                    n = f'''{n:<30}\t{f"CNAME {short_cname:<30}" if self.verbosity >= 2 else ""}'''
-                row = f"{n:<30}\t{ips}"
-            # store the result
-            if set(ips) != set(self.ignore_hosts):
-                #self.logger.success(row)
-                dns_lookup_result = {"domain": name, "ip": ips}
-                if self.lookup_type == "gethostbyname" and cname:
-                    dns_lookup_result["cname"] = r.name
-                    dns_lookup_result["aliases"] = r.aliases
-                self.fqdn.append(dns_lookup_result)
-            self.logger.debug(future.result())
-        self.tasks.remove(future)
-        if self.verbosity >= 1:
-            self.pbar.update()
-       
-
-    async def _queue_lookups(self, wordlist, domain):
-        """Takes a list of words and adds them to the async loop also passing the original
-        lookup domain name; then attaches the processing callback to deal with the result.
-
-        Args:
-            wordlist: a list of names to perform lookups for
-            domain: the base domain to perform brute force against
-        """
-        for word in wordlist:
-            # Wait on the semaphore before adding more tasks
-            await self.sem.acquire()
-            host = f"{word.strip()}.{domain}"
-            task = asyncio.ensure_future(self._dns_lookup(host))
-            task.add_done_callback(functools.partial(self._dns_result_callback, host))
-            self.tasks.append(task)
-        await asyncio.gather(*self.tasks, return_exceptions=True)
-
-    def bruteforce_domain(target, resolvers=None, wordlist="subdomains-top1million-110000.txt", wildcard=True, verify=True, found_subdomains=[], thread_count=7000, query=True):
-        subdomains_list = []
-        names_list = []
-        verbosity=1
-        if resolvers:
-              resolverfile = open(resolvers,'r')
-              lines = resolverfile.read().splitlines()
-              resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
-        bf = aioDNSBrute(verbosity=verbosity, max_tasks=thread_count)
-        subdomains_list = bf.run(wordlist, target, resolvers, wildcard, verify, query)
-        resolverfile.close()
-        for r in range(1, len(subdomains_list)):  
-              names_list.append(subdomains_list[r]['domain'])
-
-        return names_list
-
-    def run(
-        self, wordlist, domain, resolvers=None, wildcard=True, verify=True, query=True
-    ):
-        """
-        Sets up the bruteforce job, does domain verification, sets resolvers, checks for wildcard
-        response to lookups, and sets the query type to be used. After all this, open the wordlist
-        file and start the brute force - with ^C handling to cleanup nicely.
-
-        Args:
-            wordlist: a string containing a path to a filename to be used as a wordlist
-            domain: the base domain name to be used for lookups
-            resolvers: a list of DNS resolvers to be used (default None, uses system resolvers)
-            wildcard: bool, do wildcard dns detection (default true)
-            verify: bool, check if domain exists (default true)
-            query: bool, use query to do lookups (default true), false means gethostbyname is used.
-
-        Returns:
-            dict containing result of lookups
-        """
-        self.logger.info(
-            f"Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks..."
-        )
-        if verify:
-            #self.logger.info(f"Using local resolver to verify {domain} exists.")
-            try:
-                socket.gethostbyname(domain)
-            except socket.gaierror as err:
-                self.logger.error(
-                    f"Couldn't resolve {domain}, use the --no-verify switch to ignore this error."
-                )
-                raise SystemExit(
-                    self.logger.error(f"Error from host lookup: {err}")
-                )
-        else:
-            self.logger.warn("Skipping domain verification. YOLO!")
-        if resolvers:
-            self.resolver.nameservers = resolvers
-        self.logger.info(
-            f"Using recursive DNS with {len(self.resolver.nameservers)} nameservers"
-        )
-
-        if wildcard:
-            # 63 chars is the max allowed segment length, there is practically no chance that it will be a legit record
-            random_sld = (
-                lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}'
-            )
-            try:
-                self.lookup_type = "query"
-                wc_check = self.loop.run_until_complete(
-                    self._dns_lookup(f"{random_sld()}.{domain}")
-                )
-            except aiodns.error.DNSError as err:
-                # we expect that the record will not exist and error 4 will be thrown
-                #self.logger.info(
-                #    f"No wildcard response was detected for this domain."
-                #)
-                wc_check = None
-            finally:
-                if wc_check is not None:
-                    self.ignore_hosts = [host.host for host in wc_check]
-                    self.logger.warn(
-                        f"Wildcard response detected, ignoring answers containing {self.ignore_hosts}"
-                    )
-        else:
-            self.logger.warn("Wildcard detection is disabled")
-
-        if query:
-            #self.logger.info(
-            #    "Using pycares `query` function to perform lookups, CNAMEs cannot be identified"
-            #)
-            self.lookup_type = "query"
-        else:
-            self.logger.info(
-                "Using pycares `gethostbyname` function to perform lookups, CNAME data will be appended to results (** denotes CNAME, show actual name with -vv)"
-            )
-            self.lookup_type = "gethostbyname"
-
-        with open(wordlist, encoding="utf-8", errors="ignore") as words:
-            w = words.read().splitlines()
-        self.logger.info(f"Wordlist loaded, proceeding with {len(w)} DNS requests")
-        try:
-            if self.verbosity >= 1:
-                self.pbar = tqdm(
-                    total=len(w), unit="rec", maxinterval=0.1, mininterval=0
-                )
-            self.loop.run_until_complete(self._queue_lookups(w, domain))
-        except KeyboardInterrupt:
-            self.logger.warn("Caught keyboard interrupt, cleaning up...")
-            asyncio.gather(*asyncio.Task.all_tasks()).cancel()
-            self.loop.stop()
-        finally:
-            self.loop.close()
-            if self.verbosity >= 1:
-                self.pbar.close()
-            self.logger.info(f"Bruteforcing Complete")
-        return self.fqdn
-
-
-@click.command()
-@click.option(
-    "--wordlist",
-    "-w",
-    help="Wordlist to use for brute force.",
-    default=f"{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000",
-)
-@click.option(
-    "--max-tasks",
-    "-t",
-    default=512,
-    help="Maximum number of tasks to run asynchronosly.",
-)
-@click.option(
-    "--resolver-file",
-    "-r",
-    type=click.File("r"),
-    default=None,
-    help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers",
-)
-@click.option(
-    "--verbosity", "-v", count=True, default=1, help="Increase output verbosity"
-)
-@click.option(
-    "--output",
-    "-o",
-    type=click.Choice(["csv", "json", "off"]),
-    default="off",
-    help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).",
-)
-@click.option(
-    "--outfile",
-    "-f",
-    type=click.File("w"),
-    help="Output filename. Use '-f -' to send file output to stdout overriding normal output.",
-)
-@click.option(
-    "--query/--gethostbyname",
-    default=True,
-    help="DNS lookup type to use query (default) should be faster, but won't return CNAME information.",
-)
-@click.option(
-    "--wildcard/--no-wildcard",
-    default=True,
-    help="Wildcard detection, enabled by default",
-)
-@click.option(
-    "--verify/--no-verify",
-    default=True,
-    help="Verify domain name is sane before beginning, enabled by default",
-)
-@click.version_option("0.3.2")
-@click.argument("domain", required=True)
-def main(**kwargs):
-    """aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module.
-
-    credit: blark (@markbaseggio)
-    """
-    output = kwargs.get("output")
-    verbosity = kwargs.get("verbosity")
-    resolvers = kwargs.get("resolver_file")
-    if output != "off":
-        outfile = kwargs.get("outfile")
-        # turn off output if we want JSON/CSV to stdout, hacky
-        if outfile.__class__.__name__ == "TextIOWrapper":
-            verbosity = 0
-        if outfile is None:
-            # wasn't specified on command line
-            outfile = open(f'{kwargs["domain"]}.{output}', "w")
-    if resolvers:
-        lines = resolvers.read().splitlines()
-        resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
-
-    bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get("max_tasks"))
-    results = bf.run(
-        wordlist=kwargs.get("wordlist"),
-        domain=kwargs.get("domain"),
-        resolvers=resolvers,
-        wildcard=kwargs.get("wildcard"),
-        verify=kwargs.get("verify"),
-        query=kwargs.get("query"),
-    )
-
-    if output in ("json"):
-        import json
-        json.dump(results, outfile)
-
-    if output in ("csv"):
-        import csv
-        writer = csv.writer(outfile)
-        writer.writerow(["Hostname", "IPs", "CNAME", "Aliases"])
-        [
-            writer.writerow(
-                [
-                    r.get("domain"),
-                    r.get("ip", [""])[0],
-                    r.get("cname"),
-                    r.get("aliases", [""])[0],
-                ]
-            )
-            for r in results
-        ]
-
-
-if __name__ == "__main__":
-    main()
diff --git a/aiodnsbrute/logger.py b/aiodnsbrute/logger.py
deleted file mode 100644
index e95736a..0000000
--- a/aiodnsbrute/logger.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from tqdm import tqdm
-from click import style
-
-
-class ConsoleLogger(object):
-    """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
-
-    def __init__(self, verbosity):
-        self.verbosity = verbosity
-        self.msg_type = {
-            "info": ("[*]", "blue", 1),
-            "success": ("[+]", "green", 1),
-            "error": ("[-]", "red", 1),
-            "warn": ("[!]", "yellow", 1),
-            "debug": ("[D]", "cyan", 3),
-        }
-
-    def __getattr__(self, attr):
-        try:
-            decorator = style(
-                f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
-            )
-            msg_verbosity = self.msg_type[attr][2]
-        except KeyError:
-            decorator = ""
-            msg_verbosity = 1
-        finally:
-            if self.verbosity >= msg_verbosity:
-                return lambda msg: tqdm.write(f"{decorator}{msg}")
-            else:
-                return lambda msg: None
diff --git a/aiodnsbrute/logger/logger.py b/aiodnsbrute/logger/logger.py
deleted file mode 100644
index e95736a..0000000
--- a/aiodnsbrute/logger/logger.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from tqdm import tqdm
-from click import style
-
-
-class ConsoleLogger(object):
-    """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
-
-    def __init__(self, verbosity):
-        self.verbosity = verbosity
-        self.msg_type = {
-            "info": ("[*]", "blue", 1),
-            "success": ("[+]", "green", 1),
-            "error": ("[-]", "red", 1),
-            "warn": ("[!]", "yellow", 1),
-            "debug": ("[D]", "cyan", 3),
-        }
-
-    def __getattr__(self, attr):
-        try:
-            decorator = style(
-                f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
-            )
-            msg_verbosity = self.msg_type[attr][2]
-        except KeyError:
-            decorator = ""
-            msg_verbosity = 1
-        finally:
-            if self.verbosity >= msg_verbosity:
-                return lambda msg: tqdm.write(f"{decorator}{msg}")
-            else:
-                return lambda msg: None
diff --git a/aiodnsbrute/resolvers.txt b/data/resolvers.txt
similarity index 100%
rename from aiodnsbrute/resolvers.txt
rename to data/resolvers.txt
diff --git a/aiodnsbrute/subdomains-top1million-110000.txt b/data/subdomains-top1million-110000.txt
similarity index 100%
rename from aiodnsbrute/subdomains-top1million-110000.txt
rename to data/subdomains-top1million-110000.txt
diff --git a/sublist3r2.py b/sublist3r2.py
index f66517f..b4b0a76 100755
--- a/sublist3r2.py
+++ b/sublist3r2.py
@@ -2,41 +2,25 @@
 # coding: utf-8
 # Sublist3r2 v1.0
 
-
-# modules in standard library
-import re
-import sys
-import os
+# Builtin imports
 import argparse
-import time
 import hashlib
-import random
-import multiprocessing
-import threading
-import socket
 import json
+import multiprocessing
+import os
+import random
+import re
+import socket
+import sys
+import threading
+import time
+import urllib.parse as urlparse
 from collections import Counter
 
-# external modules
+# External imports
 import dns.resolver
 import requests
-from aiodnsbrute.cli import aioDNSBrute 
-
-# Python 2.x and 3.x compatiablity
-if sys.version > '3':
-    import urllib.parse as urlparse
-    import urllib.parse as urllib
-else:
-    import urlparse
-    import urllib
-
-# In case you cannot install some of the required development packages
-# there's also an option to disable the SSL warning:
-try:
-    import requests.packages.urllib3
-    requests.packages.urllib3.disable_warnings()
-except:
-    pass
+from aiodnsbrute.cli import aioDNSBrute
 
 # Check if we are running this on windows platform
 is_windows = sys.platform.startswith('win')
@@ -50,15 +34,14 @@ if is_windows:
     R = '\033[91m'  # red
     W = '\033[0m'   # white
     try:
-        import win_unicode_console , colorama
+        import colorama
+        import win_unicode_console
         win_unicode_console.enable()
         colorama.init()
-        #Now the unicode will work ^_^
-    except:
+        # Now the unicode will work ^_^
+    except Exception:
         print("[!] Error: Coloring libraries not installed, no coloring will be used [Check the readme]")
-        G = Y = B = R = W = G = Y = B = R = W = ''
-
-
+        G = Y = B = R = W = ''
 else:
     G = '\033[92m'  # green
     Y = '\033[93m'  # yellow
@@ -66,6 +49,7 @@ else:
     R = '\033[91m'  # red
     W = '\033[0m'   # white
 
+
 def no_color():
     global G, Y, B, R, W
     G = Y = B = R = W = ''
@@ -73,13 +57,13 @@ def no_color():
 
 def banner():
     print("""%s
-         ____        _     _ _     _   _____      _ ____
+         ____        _     _ _     _   _____     ______
         / ___| _   _| |__ | (_)___| |_|___ / _ __\ __  |    Sublist3r2 v1.0
         \___ \| | | | '_ \| | / __| __| |_ \| '__|  / /     a subdomains enum tool originally by @aboul3la
          ___) | |_| | |_) | | \__ \ |_ ___) | |    / /_     maintained by Ronin Nakomoto
-        |____/ \__,_|_.__/|_|_|___/\__|____/|_|   /____|%s%s    https://github.com/RoninNakomoto/Sublist3r2
+        |____/ \__,_|_.__/|_|_|___/\__|____/|_|   /____|%s    https://github.com/RoninNakomoto/Sublist3r2
 
-    """ % (R, W, Y))
+    """ % (R, Y))
 
 
 def parser_error(errmsg):
@@ -258,7 +242,10 @@ class enumratorBase(object):
 class enumratorBaseThreaded(multiprocessing.Process, enumratorBase):
     def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, silent=False, verbose=True):
         subdomains = subdomains or []
-        enumratorBase.__init__(self, base_url, engine_name, domain, subdomains, silent=silent, verbose=verbose)
+        enumratorBase.__init__(
+            self, base_url, engine_name, domain, subdomains,
+            silent=silent, verbose=verbose
+        )
         multiprocessing.Process.__init__(self)
         self.q = q
         return
@@ -276,13 +263,16 @@ class GoogleEnum(enumratorBaseThreaded):
         self.engine_name = "Google"
         self.MAX_DOMAINS = 11
         self.MAX_PAGES = 200
-        super(GoogleEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(GoogleEnum, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.q = q
         return
 
     def extract_domains(self, resp):
         links_list = list()
-        link_regx = re.compile('<cite.*?>(.*?)<\/cite>')
+        link_regx = re.compile(r'<cite.*?>(.*?)<\/cite>')
         try:
             links_list = link_regx.findall(resp)
             for link in links_list:
@@ -299,7 +289,7 @@ class GoogleEnum(enumratorBaseThreaded):
         return links_list
 
     def check_response_errors(self, resp):
-        if (type(resp) is str or type(resp) is unicode) and 'Our systems have detected unusual traffic' in resp:
+        if (type(resp) is str) and 'Our systems have detected unusual traffic' in resp:
             self.print_(R + "[!] Error: Google probably now is blocking our requests" + W)
             self.print_(R + "[~] Finished now the Google Enumeration ..." + W)
             return False
@@ -326,20 +316,23 @@ class YahooEnum(enumratorBaseThreaded):
         self.engine_name = "Yahoo"
         self.MAX_DOMAINS = 10
         self.MAX_PAGES = 0
-        super(YahooEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(YahooEnum, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.q = q
         return
 
     def extract_domains(self, resp):
-        link_regx2 = re.compile('<span class=" fz-.*? fw-m fc-12th wr-bw.*?">(.*?)</span>')
-        link_regx = re.compile('<span class="txt"><span class=" cite fw-xl fz-15px">(.*?)</span>')
+        link_regx2 = re.compile(r'<span class=" fz-.*? fw-m fc-12th wr-bw.*?">(.*?)</span>')
+        link_regx = re.compile(r'<span class="txt"><span class=" cite fw-xl fz-15px">(.*?)</span>')
         links_list = []
         try:
             links = link_regx.findall(resp)
             links2 = link_regx2.findall(resp)
             links_list = links + links2
             for link in links_list:
-                link = re.sub("<(\/)?b>", "", link)
+                link = re.sub(r'<(\/)?b>', '', link)
                 if not link.startswith('http'):
                     link = "http://" + link
                 subdomain = urlparse.urlparse(link).netloc
@@ -377,13 +370,16 @@ class AskEnum(enumratorBaseThreaded):
         self.engine_name = "Ask"
         self.MAX_DOMAINS = 11
         self.MAX_PAGES = 0
-        enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        enumratorBaseThreaded.__init__(
+            self, base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.q = q
         return
 
     def extract_domains(self, resp):
         links_list = list()
-        link_regx = re.compile('<p class="web-result-url">(.*?)</p>')
+        link_regx = re.compile(r'<p class="web-result-url">(.*?)</p>')
         try:
             links_list = link_regx.findall(resp)
             for link in links_list:
@@ -420,22 +416,25 @@ class BingEnum(enumratorBaseThreaded):
         self.engine_name = "Bing"
         self.MAX_DOMAINS = 30
         self.MAX_PAGES = 0
-        enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent)
+        enumratorBaseThreaded.__init__(
+            self, base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent
+        )
         self.q = q
         self.verbose = verbose
         return
 
     def extract_domains(self, resp):
         links_list = list()
-        link_regx = re.compile('<li class="b_algo"><h2><a href="(.*?)"')
-        link_regx2 = re.compile('<div class="b_title"><h2><a href="(.*?)"')
+        link_regx = re.compile(r'<li class="b_algo"><h2><a href="(.*?)"')
+        link_regx2 = re.compile(r'<div class="b_title"><h2><a href="(.*?)"')
         try:
             links = link_regx.findall(resp)
             links2 = link_regx2.findall(resp)
             links_list = links + links2
 
             for link in links_list:
-                link = re.sub('<(\/)?strong>|<span.*?>|<|>', '', link)
+                link = re.sub(r'<(\/)?strong>|<span.*?>|<|>', '', link)
                 if not link.startswith('http'):
                     link = "http://" + link
                 subdomain = urlparse.urlparse(link).netloc
@@ -465,7 +464,10 @@ class BaiduEnum(enumratorBaseThreaded):
         self.engine_name = "Baidu"
         self.MAX_DOMAINS = 2
         self.MAX_PAGES = 760
-        enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        enumratorBaseThreaded.__init__(
+            self, base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.querydomain = self.domain
         self.q = q
         return
@@ -474,11 +476,11 @@ class BaiduEnum(enumratorBaseThreaded):
         links = list()
         found_newdomain = False
         subdomain_list = []
-        link_regx = re.compile('<a.*?class="c-showurl".*?>(.*?)</a>')
+        link_regx = re.compile(r'<a.*?class="c-showurl".*?>(.*?)</a>')
         try:
             links = link_regx.findall(resp)
             for link in links:
-                link = re.sub('<.*?>|>|<|&nbsp;', '', link)
+                link = re.sub(r'<.*?>|>|<|&nbsp;', '', link)
                 if not link.startswith('http'):
                     link = "http://" + link
                 subdomain = urlparse.urlparse(link).netloc
@@ -523,7 +525,10 @@ class NetcraftEnum(enumratorBaseThreaded):
         subdomains = subdomains or []
         self.base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}'
         self.engine_name = "Netcraft"
-        super(NetcraftEnum, self).__init__(self.base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(NetcraftEnum, self).__init__(
+            self.base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.q = q
         return
 
@@ -541,7 +546,7 @@ class NetcraftEnum(enumratorBaseThreaded):
         return
 
     def get_next(self, resp):
-        link_regx = re.compile('<a.*?href="(.*?)">Next Page')
+        link_regx = re.compile(r'<a.*?href="(.*?)">Next Page')
         link = link_regx.findall(resp)
         url = 'http://searchdns.netcraft.com' + link[0]
         return url
@@ -551,7 +556,9 @@ class NetcraftEnum(enumratorBaseThreaded):
         cookies_list = cookie[0:cookie.find(';')].split("=")
         cookies[cookies_list[0]] = cookies_list[1]
         # hashlib.sha1 requires utf-8 encoded str
-        cookies['netcraft_js_verification_response'] = hashlib.sha1(urllib.unquote(cookies_list[1]).encode('utf-8')).hexdigest()
+        cookies['netcraft_js_verification_response'] = hashlib.sha1(
+            urlparse.unquote(cookies_list[1]).encode('utf-8')
+        ).hexdigest()
         return cookies
 
     def get_cookies(self, headers):
@@ -577,7 +584,7 @@ class NetcraftEnum(enumratorBaseThreaded):
 
     def extract_domains(self, resp):
         links_list = list()
-        link_regx = re.compile('<a class="results-table__host" href="(.*?)"')
+        link_regx = re.compile(r'<a class="results-table__host" href="(.*?)"')
         try:
             links_list = link_regx.findall(resp)
             for link in links_list:
@@ -601,7 +608,10 @@ class DNSdumpster(enumratorBaseThreaded):
         self.engine_name = "DNSdumpster"
         self.q = q
         self.lock = None
-        super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(DNSdumpster, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         return
 
     def check_host(self, host):
@@ -616,7 +626,7 @@ class DNSdumpster(enumratorBaseThreaded):
                     self.print_("%s%s: %s%s" % (R, self.engine_name, W, host))
                 is_valid = True
                 self.live_subdomains.append(host)
-        except:
+        except Exception:
             pass
         self.lock.release()
         return is_valid
@@ -636,7 +646,7 @@ class DNSdumpster(enumratorBaseThreaded):
         return self.get_response(resp)
 
     def get_csrftoken(self, resp):
-        csrf_regex = re.compile('<input type="hidden" name="csrfmiddlewaretoken" value="(.*?)">', re.S)
+        csrf_regex = re.compile(r'<input type="hidden" name="csrfmiddlewaretoken" value="(.*?)">', re.S)
         token = csrf_regex.findall(resp)[0]
         return token.strip()
 
@@ -654,8 +664,8 @@ class DNSdumpster(enumratorBaseThreaded):
         return self.live_subdomains
 
     def extract_domains(self, resp):
-        tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>', re.S)
-        link_regex = re.compile('<td class="col-md-4">(.*?)<br>', re.S)
+        tbl_regex = re.compile(r'<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>', re.S)
+        link_regex = re.compile(r'<td class="col-md-4">(.*?)<br>', re.S)
         links = []
         try:
             results_tbl = tbl_regex.findall(resp)[0]
@@ -678,23 +688,26 @@ class Virustotal(enumratorBaseThreaded):
         base_url = 'https://www.virustotal.com/api/v3/domains/{domain}/subdomains'
         self.engine_name = "Virustotal"
         if os.getenv("VT_APIKEY") is None:
-            VT_APIKEY=input(B + "[+] Enter VirusTotal API key, press Enter for none: " + W)
-            VT_APIKEY=VT_APIKEY.strip()
+            VT_APIKEY = input(B + "[+] Enter VirusTotal API key, press Enter for none: " + W)
+            VT_APIKEY = VT_APIKEY.strip()
             if VT_APIKEY != "":
-                os.environ["VT_APIKEY"]=(VT_APIKEY)
+                os.environ["VT_APIKEY"] = (VT_APIKEY)
         else:
             VT_APIKEY = os.getenv("VT_APIKEY")
-        os.environ["VT_APIKEY"]=(VT_APIKEY)
+        os.environ["VT_APIKEY"] = (VT_APIKEY)
         self.apikey = os.getenv('VT_APIKEY', None)
         self.q = q
-        super(Virustotal, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(Virustotal, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         self.url = self.base_url.format(domain=self.domain)
         return
 
     # the main send_req need to be rewritten
     def send_req(self, url):
         try:
-            self.headers.update({'X-ApiKey':self.apikey})
+            self.headers.update({'X-ApiKey': self.apikey})
             resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
         except Exception as e:
             self.print_(e)
@@ -705,11 +718,10 @@ class Virustotal(enumratorBaseThreaded):
     def enumerate(self):
         if self.apikey:
             while self.url != '':
-                #try:
                 resp = self.send_req(self.url)
                 resp = json.loads(resp)
                 if 'error' in resp:
-                    self.print_(R + "Error Code: {}".format(resp['error']["code"]) +W)
+                    self.print_(R + "Error Code: {}".format(resp['error']["code"]) + W)
                     self.print_(R + "Virus Total Server Message: {}".format(resp['error']["message"]) + W)
                     break
                 if 'links' in resp and 'next' in resp['links']:
@@ -719,8 +731,8 @@ class Virustotal(enumratorBaseThreaded):
                 self.extract_domains(resp)
         else:
             self.print_(R + "[!] Error: VirusTotal API key environment variable not found. Skipping" + W)
-            self.print_(R + "[!] set VT_APIKEY to your virus total API key using:   export VT_APIKEY=Your_VT_API_KEY_VALUE" + W)
-            self.print_(B + "[!] To get a VT APIKEY, register at https://www.virustotal.com/gui/join-us" +W)
+            self.print_(R + "[!] set VT_APIKEY to your virus total API key using: `export VT_APIKEY=Your_VT_API_KEY_VALUE`" + W)
+            self.print_(B + "[!] To get a VT APIKEY, register at https://www.virustotal.com/gui/join-us" + W)
         return self.subdomains
 
     def extract_domains(self, resp):
@@ -745,7 +757,10 @@ class ThreatCrowd(enumratorBaseThreaded):
         base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}'
         self.engine_name = "ThreatCrowd"
         self.q = q
-        super(ThreatCrowd, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(ThreatCrowd, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         return
 
     def req(self, url):
@@ -783,7 +798,10 @@ class CrtSearch(enumratorBaseThreaded):
         base_url = 'https://crt.sh/?q=%25.{domain}'
         self.engine_name = "SSL Certificates"
         self.q = q
-        super(CrtSearch, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(CrtSearch, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         return
 
     def req(self, url):
@@ -802,7 +820,7 @@ class CrtSearch(enumratorBaseThreaded):
         return self.subdomains
 
     def extract_domains(self, resp):
-        link_regx = re.compile('<TD>(.*?)</TD>')
+        link_regx = re.compile(r'<TD>(.*?)</TD>')
         try:
             links = link_regx.findall(resp)
             for link in links:
@@ -818,7 +836,7 @@ class CrtSearch(enumratorBaseThreaded):
                         continue
 
                     if '@' in subdomain:
-                        subdomain = subdomain[subdomain.find('@')+1:]
+                        subdomain = subdomain[subdomain.find('@') + 1:]
 
                     if subdomain not in self.subdomains and subdomain != self.domain:
                         if self.verbose:
@@ -828,13 +846,17 @@ class CrtSearch(enumratorBaseThreaded):
             print(e)
             pass
 
+
 class PassiveDNS(enumratorBaseThreaded):
     def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
         subdomains = subdomains or []
         base_url = 'https://api.sublist3r.com/search.php?domain={domain}'
         self.engine_name = "PassiveDNS"
         self.q = q
-        super(PassiveDNS, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        super(PassiveDNS, self).__init__(
+            base_url, self.engine_name, domain, subdomains,
+            q=q, silent=silent, verbose=verbose
+        )
         return
 
     def req(self, url):
@@ -910,7 +932,7 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
         enable_bruteforce = True
 
     # Validate domain
-    domain_check = re.compile("^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$")
+    domain_check = re.compile(r'^(http|https)?[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,}$')
     if not domain_check.match(domain):
         if not silent:
             print(R + "Error: Please enter a valid domain" + W)
@@ -927,18 +949,19 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
     if verbose and not silent:
         print(Y + "[-] verbosity is enabled, will show the subdomains results in realtime" + W)
 
-    supported_engines = {'baidu': BaiduEnum,
-                         'yahoo': YahooEnum,
-                         'google': GoogleEnum,
-                         'bing': BingEnum,
-                         'ask': AskEnum,
-                         'netcraft': NetcraftEnum,
-                         'dnsdumpster': DNSdumpster,
-                         'virustotal': Virustotal,
-                         'threatcrowd': ThreatCrowd,
-                         'ssl': CrtSearch,
-                         'passivedns': PassiveDNS
-                         }
+    supported_engines = {
+        'baidu': BaiduEnum,
+        'yahoo': YahooEnum,
+        'google': GoogleEnum,
+        'bing': BingEnum,
+        'ask': AskEnum,
+        'netcraft': NetcraftEnum,
+        'dnsdumpster': DNSdumpster,
+        'virustotal': Virustotal,
+        'threatcrowd': ThreatCrowd,
+        'ssl': CrtSearch,
+        'passivedns': PassiveDNS
+    }
 
     chosenEnums = []
 
@@ -968,15 +991,16 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
     if enable_bruteforce:
         if not silent:
             print(G + "[-] Starting bruteforce module now using aiodnsbrute.." + W)
-        record_type = False
         path_to_file = os.path.dirname(os.path.realpath(__file__))
-        subs = os.path.join(path_to_file, 'aiodnsbrute', 'subdomains-top1million-110000.txt')
-        resolvers = os.path.join(path_to_file, 'aiodnsbrute', 'resolvers.txt')
-        wildcard=True
-        verify=True
-        query=True
+        subs = os.path.join(path_to_file, 'data', 'subdomains-top1million-110000.txt')
+        resolvers = os.path.join(path_to_file, 'data', 'resolvers.txt')
+        wildcard = True
+        verify = True
+        query = True
         thread_count = threads
-        bruteforce_list = aioDNSBrute.bruteforce_domain(parsed_domain.netloc, resolvers, subs, wildcard, verify, search_list, thread_count, query)
+        bruteforce_list = aioDNSBrute.bruteforce_domain(
+            parsed_domain.netloc, resolvers, subs, wildcard, verify, search_list, thread_count, query
+        )
     subdomains = search_list.union(bruteforce_list)
 
     if subdomains:
@@ -1015,7 +1039,11 @@ def interactive():
     if args.no_color:
         no_color()
     banner()
-    res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines)
+    main(
+        domain, threads, savefile, ports,
+        silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines
+    )
+
 
 if __name__ == "__main__":
     interactive()

From f96487f695873cc29bb24ae6554310a8a281c05f Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sun, 24 Oct 2021 01:23:23 +0200
Subject: [PATCH 3/7] Update dependencies

---
 MANIFEST.in                                   |   2 +-
 requirements.txt                              |   6 +-
 setup.py                                      |  17 +-
 sublist3r2.py => sublist3r2/__init__.py       |  15 +-
 sublist3r2/aiodnsbrute/__init__.py            |   0
 sublist3r2/aiodnsbrute/cli.py                 | 358 ++++++++++++++++++
 sublist3r2/aiodnsbrute/logger.py              |  31 ++
 sublist3r2/aiodnsbrute/logger/logger.py       |  31 ++
 .../aiodnsbrute}/resolvers.txt                |   0
 .../subdomains-top1million-110000.txt         |   0
 10 files changed, 444 insertions(+), 16 deletions(-)
 rename sublist3r2.py => sublist3r2/__init__.py (99%)
 create mode 100644 sublist3r2/aiodnsbrute/__init__.py
 create mode 100644 sublist3r2/aiodnsbrute/cli.py
 create mode 100644 sublist3r2/aiodnsbrute/logger.py
 create mode 100644 sublist3r2/aiodnsbrute/logger/logger.py
 rename {data => sublist3r2/aiodnsbrute}/resolvers.txt (100%)
 rename {data => sublist3r2/aiodnsbrute}/subdomains-top1million-110000.txt (100%)

diff --git a/MANIFEST.in b/MANIFEST.in
index 6feedd3..90bd747 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,2 @@
 include LICENSE README.md
-include aiodnsbrute/*.txt
+include sublist3r2/aiodnsbrute/*.txt
diff --git a/requirements.txt b/requirements.txt
index f765236..b43d9c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,8 @@
 argparse
 dnspython
 requests
-aiodnsbrute
+asyncio
+uvloop
+tqdm
+aiodns
+click
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 20c9153..038fea0 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import os
-from setuptools import setup
+from setuptools import setup, find_packages
 
 
 def read(fname: str) -> str:
@@ -9,8 +9,8 @@ def read(fname: str) -> str:
 
 
 setup(
-    name='Sublist3r2',
-    version='1.0.0',
+    name='sublist3r2',
+    version='1.0.1',
     python_requires='>=3.6',
     description='Subdomains enumeration tool for penetration testers',
     long_description=read('README.md'),
@@ -18,11 +18,8 @@ setup(
     keywords='subdomain dns detection',
     url='https://github.com/RoninNakomoto/Sublist3r2',
     license='GPL-2.0',
-    py_modules=['sublist3r2'],
+    packages=find_packages(),
     include_package_data=True,
-    package_data={
-        '': ['data/*.txt'],
-    },
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
@@ -39,7 +36,11 @@ setup(
         'argparse',
         'dnspython',
         'requests',
-        'aiodnsbrute',
+        'asyncio',
+        'uvloop',
+        'tqdm',
+        'aiodns',
+        'click',
     ],
     entry_points={
         'console_scripts': [
diff --git a/sublist3r2.py b/sublist3r2/__init__.py
similarity index 99%
rename from sublist3r2.py
rename to sublist3r2/__init__.py
index b4b0a76..f4ef78c 100755
--- a/sublist3r2.py
+++ b/sublist3r2/__init__.py
@@ -20,7 +20,10 @@ from collections import Counter
 # External imports
 import dns.resolver
 import requests
-from aiodnsbrute.cli import aioDNSBrute
+from sublist3r2.aiodnsbrute.cli import aioDNSBrute
+
+# Version info
+__version__ = '1.0.1'
 
 # Check if we are running this on windows platform
 is_windows = sys.platform.startswith('win')
@@ -58,12 +61,12 @@ def no_color():
 def banner():
     print("""%s
          ____        _     _ _     _   _____     ______
-        / ___| _   _| |__ | (_)___| |_|___ / _ __\ __  |    Sublist3r2 v1.0
+        / ___| _   _| |__ | (_)___| |_|___ / _ __\ __  |    Sublist3r2 v%s
         \___ \| | | | '_ \| | / __| __| |_ \| '__|  / /     a subdomains enum tool originally by @aboul3la
          ___) | |_| | |_) | | \__ \ |_ ___) | |    / /_     maintained by Ronin Nakomoto
         |____/ \__,_|_.__/|_|_|___/\__|____/|_|   /____|%s    https://github.com/RoninNakomoto/Sublist3r2
 
-    """ % (R, Y))
+    """ % (R, __version__, Y))
 
 
 def parser_error(errmsg):
@@ -992,14 +995,14 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
         if not silent:
             print(G + "[-] Starting bruteforce module now using aiodnsbrute.." + W)
         path_to_file = os.path.dirname(os.path.realpath(__file__))
-        subs = os.path.join(path_to_file, 'data', 'subdomains-top1million-110000.txt')
-        resolvers = os.path.join(path_to_file, 'data', 'resolvers.txt')
+        subs = os.path.join(path_to_file, 'aiodnsbrute', 'subdomains-top1million-110000.txt')
+        resolvers = os.path.join(path_to_file, 'aiodnsbrute', 'resolvers.txt')
         wildcard = True
         verify = True
         query = True
         thread_count = threads
         bruteforce_list = aioDNSBrute.bruteforce_domain(
-            parsed_domain.netloc, resolvers, subs, wildcard, verify, search_list, thread_count, query
+            parsed_domain.netloc, resolvers, subs, wildcard, verify, thread_count, query
         )
     subdomains = search_list.union(bruteforce_list)
 
diff --git a/sublist3r2/aiodnsbrute/__init__.py b/sublist3r2/aiodnsbrute/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/sublist3r2/aiodnsbrute/cli.py b/sublist3r2/aiodnsbrute/cli.py
new file mode 100644
index 0000000..2b7a9fd
--- /dev/null
+++ b/sublist3r2/aiodnsbrute/cli.py
@@ -0,0 +1,358 @@
+import random
+import string
+import asyncio
+import functools
+import os
+import uvloop
+import aiodns
+import click
+import socket
+import sys
+from tqdm import tqdm
+from sublist3r2.aiodnsbrute.logger import ConsoleLogger
+
+
+class aioDNSBrute(object):
+    """aiodnsbrute implements fast domain name brute forcing using Python's asyncio module."""
+
+    def __init__(self, verbosity=0, max_tasks=512):
+        """Constructor.
+
+        Args:
+            verbosity: set output verbosity: 0 (default) is none, 3 is debug
+            max_tasks: the maximum number of tasks asyncio will queue (default 512)
+        """
+        self.tasks = []
+        self.errors = []
+        self.fqdn = []
+        self.ignore_hosts = []
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+        self.loop = asyncio.get_event_loop()
+        self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
+        self.sem = asyncio.BoundedSemaphore(max_tasks)
+        self.max_tasks = max_tasks
+        self.verbosity = verbosity
+        self.logger = ConsoleLogger(verbosity)
+
+    async def _dns_lookup(self, name):
+        """Performs a DNS request using aiodns, self.lookup_type is set by the run function.
+        A query for A record returns <ares_query_a_result> which does not return metadata about
+        when a CNAME was resolved (just host and ttl attributes) however it should be faster.
+        The <ares_host_result> returned by gethostbyname contains name, aliases, and addresses, if
+        name is different in response we can surmise that the original domain was a CNAME entry.
+
+        Args:
+            name: the domain name to resolve
+
+        Returns:
+            object: <ares_query_a_result> if query, <ares_host_result> if gethostbyname
+        """
+        if self.lookup_type == "query":
+            return await self.resolver.query(name, "A")
+        elif self.lookup_type == "gethostbyname":
+            return await self.resolver.gethostbyname(name, socket.AF_INET)
+
+    def _dns_result_callback(self, name, future):
+        """Handles the pycares object passed by the _dns_lookup function. We expect an errror to
+        be present in the returned object because most lookups will be for names that don't exist.
+        c-ares errors are passed through directly, error types can be identified in ares_strerror.c
+
+        Args:
+            name: original lookup name (because the query_result object doesn't contain it)
+            future: the completed future (pycares dns result)
+        """
+        # Record processed we can now release the lock
+        self.sem.release()
+        # Handle known exceptions, barf on other ones
+        if future.exception() is not None:
+            try:
+                err_number = future.exception().args[0]
+                err_text = future.exception().args[1]
+            except IndexError:
+                self.logger.error(f"Couldn't parse exception: {future.exception()}")
+            # handle the DNS errors we expect to receive, show user unexpected errors
+            if err_number == 4:
+                # This is domain name not found, ignore it
+                pass
+            #elif err_number == 12:
+                # Timeout from DNS server
+                #self.logger.warn(f"Timeout for {name}")
+            elif err_number == 1:
+                # Server answered with no data
+                pass
+            #else:
+                #self.logger.error(
+                #    f"{name} generated an unexpected exception: {future.exception()}"
+                #)
+            # for debugging/troubleshoooting keep a list of errors
+            # self.errors.append({'hostname': name, 'error': err_text})
+
+        # parse and output and store results.
+        else:
+            if self.lookup_type == "query":
+                ips = [ip.host for ip in future.result()]
+                cname = False
+                row = f"{name:<30}\t{ips}"
+            elif self.lookup_type == "gethostbyname":
+                r = future.result()
+                ips = [ip for ip in r.addresses]
+                if name == r.name:
+                    cname = False
+                    n = f"""{name:<30}\t{f"{'':<35}" if self.verbosity >= 2 else ""}"""
+                else:
+                    cname = True
+                    # format the name based on verbosity - this is kluge
+                    short_cname = f"{r.name[:28]}.." if len(r.name) > 30 else r.name
+                    n = f'{name}{"**" if self.verbosity <= 1 else ""}'
+                    n = f'''{n:<30}\t{f"CNAME {short_cname:<30}" if self.verbosity >= 2 else ""}'''
+                row = f"{n:<30}\t{ips}"
+            # store the result
+            if set(ips) != set(self.ignore_hosts):
+                #self.logger.success(row)
+                dns_lookup_result = {"domain": name, "ip": ips}
+                if self.lookup_type == "gethostbyname" and cname:
+                    dns_lookup_result["cname"] = r.name
+                    dns_lookup_result["aliases"] = r.aliases
+                self.fqdn.append(dns_lookup_result)
+            self.logger.debug(future.result())
+        self.tasks.remove(future)
+        if self.verbosity >= 1:
+            self.pbar.update()
+
+    async def _queue_lookups(self, wordlist, domain):
+        """Takes a list of words and adds them to the async loop also passing the original
+        lookup domain name; then attaches the processing callback to deal with the result.
+
+        Args:
+            wordlist: a list of names to perform lookups for
+            domain: the base domain to perform brute force against
+        """
+        for word in wordlist:
+            # Wait on the semaphore before adding more tasks
+            await self.sem.acquire()
+            host = f"{word.strip()}.{domain}"
+            task = asyncio.ensure_future(self._dns_lookup(host))
+            task.add_done_callback(functools.partial(self._dns_result_callback, host))
+            self.tasks.append(task)
+        await asyncio.gather(*self.tasks, return_exceptions=True)
+
+    def bruteforce_domain(target, resolvers=None, wordlist="subdomains-top1million-110000.txt", wildcard=True, verify=True, thread_count=7000, query=True):
+        subdomains_list = []
+        names_list = []
+        verbosity = 1
+        if resolvers:
+            resolverfile = open(resolvers, 'r')
+            lines = resolverfile.read().splitlines()
+            resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
+        bf = aioDNSBrute(verbosity=verbosity, max_tasks=thread_count)
+        subdomains_list = bf.run(wordlist, target, resolvers, wildcard, verify, query)
+        resolverfile.close()
+        for r in range(1, len(subdomains_list)):  
+            names_list.append(subdomains_list[r]['domain'])
+
+        return names_list
+
+    def run(
+        self, wordlist, domain, resolvers=None, wildcard=True, verify=True, query=True
+    ):
+        """
+        Sets up the bruteforce job, does domain verification, sets resolvers, checks for wildcard
+        response to lookups, and sets the query type to be used. After all this, open the wordlist
+        file and start the brute force - with ^C handling to cleanup nicely.
+
+        Args:
+            wordlist: a string containing a path to a filename to be used as a wordlist
+            domain: the base domain name to be used for lookups
+            resolvers: a list of DNS resolvers to be used (default None, uses system resolvers)
+            wildcard: bool, do wildcard dns detection (default true)
+            verify: bool, check if domain exists (default true)
+            query: bool, use query to do lookups (default true), false means gethostbyname is used.
+
+        Returns:
+            dict containing result of lookups
+        """
+        self.logger.info(
+            f"Brute forcing {domain} with a maximum of {self.max_tasks} concurrent tasks..."
+        )
+        if verify:
+            #self.logger.info(f"Using local resolver to verify {domain} exists.")
+            try:
+                socket.gethostbyname(domain)
+            except socket.gaierror as err:
+                self.logger.error(
+                    f"Couldn't resolve {domain}, use the --no-verify switch to ignore this error."
+                )
+                raise SystemExit(
+                    self.logger.error(f"Error from host lookup: {err}")
+                )
+        else:
+            self.logger.warn("Skipping domain verification. YOLO!")
+        if resolvers:
+            self.resolver.nameservers = resolvers
+        self.logger.info(
+            f"Using recursive DNS with {len(self.resolver.nameservers)} nameservers"
+        )
+
+        if wildcard:
+            # 63 chars is the max allowed segment length, there is practically no chance that it will be a legit record
+            random_sld = (
+                lambda: f'{"".join(random.choice(string.ascii_lowercase + string.digits) for i in range(63))}'
+            )
+            try:
+                self.lookup_type = "query"
+                wc_check = self.loop.run_until_complete(
+                    self._dns_lookup(f"{random_sld()}.{domain}")
+                )
+            except aiodns.error.DNSError as err:
+                # we expect that the record will not exist and error 4 will be thrown
+                #self.logger.info(
+                #    f"No wildcard response was detected for this domain."
+                #)
+                wc_check = None
+            finally:
+                if wc_check is not None:
+                    self.ignore_hosts = [host.host for host in wc_check]
+                    self.logger.warn(
+                        f"Wildcard response detected, ignoring answers containing {self.ignore_hosts}"
+                    )
+        else:
+            self.logger.warn("Wildcard detection is disabled")
+
+        if query:
+            #self.logger.info(
+            #    "Using pycares `query` function to perform lookups, CNAMEs cannot be identified"
+            #)
+            self.lookup_type = "query"
+        else:
+            self.logger.info(
+                "Using pycares `gethostbyname` function to perform lookups, CNAME data will be appended to results (** denotes CNAME, show actual name with -vv)"
+            )
+            self.lookup_type = "gethostbyname"
+
+        with open(wordlist, encoding="utf-8", errors="ignore") as words:
+            w = words.read().splitlines()
+        self.logger.info(f"Wordlist loaded, proceeding with {len(w)} DNS requests")
+        try:
+            if self.verbosity >= 1:
+                self.pbar = tqdm(
+                    total=len(w), unit="rec", maxinterval=0.1, mininterval=0
+                )
+            self.loop.run_until_complete(self._queue_lookups(w, domain))
+        except KeyboardInterrupt:
+            self.logger.warn("Caught keyboard interrupt, cleaning up...")
+            asyncio.gather(*asyncio.Task.all_tasks()).cancel()
+            self.loop.stop()
+        finally:
+            self.loop.close()
+            if self.verbosity >= 1:
+                self.pbar.close()
+            self.logger.info(f"Bruteforcing Complete")
+        return self.fqdn
+
+
+@click.command()
+@click.option(
+    "--wordlist",
+    "-w",
+    help="Wordlist to use for brute force.",
+    default=f"{os.path.dirname(os.path.realpath(__file__))}/wordlists/bitquark_20160227_subdomains_popular_1000",
+)
+@click.option(
+    "--max-tasks",
+    "-t",
+    default=512,
+    help="Maximum number of tasks to run asynchronosly.",
+)
+@click.option(
+    "--resolver-file",
+    "-r",
+    type=click.File("r"),
+    default=None,
+    help="A text file containing a list of DNS resolvers to use, one per line, comments start with #. Default: use system resolvers",
+)
+@click.option(
+    "--verbosity", "-v", count=True, default=1, help="Increase output verbosity"
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Choice(["csv", "json", "off"]),
+    default="off",
+    help="Output results to DOMAIN.csv/json (extension automatically appended when not using -f).",
+)
+@click.option(
+    "--outfile",
+    "-f",
+    type=click.File("w"),
+    help="Output filename. Use '-f -' to send file output to stdout overriding normal output.",
+)
+@click.option(
+    "--query/--gethostbyname",
+    default=True,
+    help="DNS lookup type to use query (default) should be faster, but won't return CNAME information.",
+)
+@click.option(
+    "--wildcard/--no-wildcard",
+    default=True,
+    help="Wildcard detection, enabled by default",
+)
+@click.option(
+    "--verify/--no-verify",
+    default=True,
+    help="Verify domain name is sane before beginning, enabled by default",
+)
+@click.version_option("0.3.2")
+@click.argument("domain", required=True)
+def main(**kwargs):
+    """aiodnsbrute is a command line tool for brute forcing domain names utilizing Python's asyncio module.
+
+    credit: blark (@markbaseggio)
+    """
+    output = kwargs.get("output")
+    verbosity = kwargs.get("verbosity")
+    resolvers = kwargs.get("resolver_file")
+    if output != "off":
+        outfile = kwargs.get("outfile")
+        # turn off output if we want JSON/CSV to stdout, hacky
+        if outfile.__class__.__name__ == "TextIOWrapper":
+            verbosity = 0
+        if outfile is None:
+            # wasn't specified on command line
+            outfile = open(f'{kwargs["domain"]}.{output}', "w")
+    if resolvers:
+        lines = resolvers.read().splitlines()
+        resolvers = [x.strip() for x in lines if (x and not x.startswith("#"))]
+
+    bf = aioDNSBrute(verbosity=verbosity, max_tasks=kwargs.get("max_tasks"))
+    results = bf.run(
+        wordlist=kwargs.get("wordlist"),
+        domain=kwargs.get("domain"),
+        resolvers=resolvers,
+        wildcard=kwargs.get("wildcard"),
+        verify=kwargs.get("verify"),
+        query=kwargs.get("query"),
+    )
+
+    if output in ("json"):
+        import json
+        json.dump(results, outfile)
+
+    if output in ("csv"):
+        import csv
+        writer = csv.writer(outfile)
+        writer.writerow(["Hostname", "IPs", "CNAME", "Aliases"])
+        [
+            writer.writerow(
+                [
+                    r.get("domain"),
+                    r.get("ip", [""])[0],
+                    r.get("cname"),
+                    r.get("aliases", [""])[0],
+                ]
+            )
+            for r in results
+        ]
+
+
+if __name__ == "__main__":
+    main()
diff --git a/sublist3r2/aiodnsbrute/logger.py b/sublist3r2/aiodnsbrute/logger.py
new file mode 100644
index 0000000..e95736a
--- /dev/null
+++ b/sublist3r2/aiodnsbrute/logger.py
@@ -0,0 +1,31 @@
+from tqdm import tqdm
+from click import style
+
+
+class ConsoleLogger(object):
+    """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
+
+    def __init__(self, verbosity):
+        self.verbosity = verbosity
+        self.msg_type = {
+            "info": ("[*]", "blue", 1),
+            "success": ("[+]", "green", 1),
+            "error": ("[-]", "red", 1),
+            "warn": ("[!]", "yellow", 1),
+            "debug": ("[D]", "cyan", 3),
+        }
+
+    def __getattr__(self, attr):
+        try:
+            decorator = style(
+                f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
+            )
+            msg_verbosity = self.msg_type[attr][2]
+        except KeyError:
+            decorator = ""
+            msg_verbosity = 1
+        finally:
+            if self.verbosity >= msg_verbosity:
+                return lambda msg: tqdm.write(f"{decorator}{msg}")
+            else:
+                return lambda msg: None
diff --git a/sublist3r2/aiodnsbrute/logger/logger.py b/sublist3r2/aiodnsbrute/logger/logger.py
new file mode 100644
index 0000000..e95736a
--- /dev/null
+++ b/sublist3r2/aiodnsbrute/logger/logger.py
@@ -0,0 +1,31 @@
+from tqdm import tqdm
+from click import style
+
+
+class ConsoleLogger(object):
+    """A quick and dirty metasploit style console output logger that doesn't mess up tqdm output."""
+
+    def __init__(self, verbosity):
+        self.verbosity = verbosity
+        self.msg_type = {
+            "info": ("[*]", "blue", 1),
+            "success": ("[+]", "green", 1),
+            "error": ("[-]", "red", 1),
+            "warn": ("[!]", "yellow", 1),
+            "debug": ("[D]", "cyan", 3),
+        }
+
+    def __getattr__(self, attr):
+        try:
+            decorator = style(
+                f"{self.msg_type[attr][0]} ", fg=self.msg_type[attr][1], bold=True
+            )
+            msg_verbosity = self.msg_type[attr][2]
+        except KeyError:
+            decorator = ""
+            msg_verbosity = 1
+        finally:
+            if self.verbosity >= msg_verbosity:
+                return lambda msg: tqdm.write(f"{decorator}{msg}")
+            else:
+                return lambda msg: None
diff --git a/data/resolvers.txt b/sublist3r2/aiodnsbrute/resolvers.txt
similarity index 100%
rename from data/resolvers.txt
rename to sublist3r2/aiodnsbrute/resolvers.txt
diff --git a/data/subdomains-top1million-110000.txt b/sublist3r2/aiodnsbrute/subdomains-top1million-110000.txt
similarity index 100%
rename from data/subdomains-top1million-110000.txt
rename to sublist3r2/aiodnsbrute/subdomains-top1million-110000.txt

From 06ae67a7bbdb314e0eb4357ced52a4f67537b00f Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sun, 24 Oct 2021 01:38:20 +0200
Subject: [PATCH 4/7] Update naming

---
 sublist3r2/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sublist3r2/__init__.py b/sublist3r2/__init__.py
index f4ef78c..f2e0d0f 100755
--- a/sublist3r2/__init__.py
+++ b/sublist3r2/__init__.py
@@ -2,7 +2,7 @@
 # coding: utf-8
 # Sublist3r2 v1.0
 
-# Builtin imports
+# Builtin modules
 import argparse
 import hashlib
 import json
@@ -17,7 +17,7 @@ import time
 import urllib.parse as urlparse
 from collections import Counter
 
-# External imports
+# External modules
 import dns.resolver
 import requests
 from sublist3r2.aiodnsbrute.cli import aioDNSBrute

From 5157ac5454095124eb0984b23231764c1b990c10 Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sun, 24 Oct 2021 01:46:48 +0200
Subject: [PATCH 5/7] Update dockerfile

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c0f1769..a8b4fd0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,7 @@ ENV PATH=${PATH}:/Sublist3r2
 
 RUN apt-get update && \
 	apt-get install -y build-essential libffi-dev libgit2-dev && \
-	pip install -r /Sublist3r2/requirements.txt && \
+	pip install /Sublist3r2 && \
 	addgroup Sublist3r2 --force-badname && \
 	useradd -g Sublist3r2 -d /Sublist3r2 -s /bin/sh Sublist3r2 && \
 	chown -R Sublist3r2:Sublist3r2 /Sublist3r2 && \
@@ -24,6 +24,6 @@ RUN apt-get update && \
 
 USER Sublist3r2
 
-ENTRYPOINT ["sublist3r2.py"] 
+ENTRYPOINT ["sublist3r2"] 
 
 CMD ["-h"]

From 691766d8271e2ed6822c4094b790e8ee2e511b0c Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sun, 24 Oct 2021 20:29:53 +0200
Subject: [PATCH 6/7] Clean VT_APIKEY code

---
 sublist3r2/__init__.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/sublist3r2/__init__.py b/sublist3r2/__init__.py
index f2e0d0f..1ebce6a 100755
--- a/sublist3r2/__init__.py
+++ b/sublist3r2/__init__.py
@@ -690,15 +690,13 @@ class Virustotal(enumratorBaseThreaded):
         subdomains = subdomains or []
         base_url = 'https://www.virustotal.com/api/v3/domains/{domain}/subdomains'
         self.engine_name = "Virustotal"
-        if os.getenv("VT_APIKEY") is None:
-            VT_APIKEY = input(B + "[+] Enter VirusTotal API key, press Enter for none: " + W)
-            VT_APIKEY = VT_APIKEY.strip()
-            if VT_APIKEY != "":
-                os.environ["VT_APIKEY"] = (VT_APIKEY)
-        else:
-            VT_APIKEY = os.getenv("VT_APIKEY")
-        os.environ["VT_APIKEY"] = (VT_APIKEY)
-        self.apikey = os.getenv('VT_APIKEY', None)
+        self.apikey = os.getenv("VT_APIKEY")
+
+        if self.apikey is None:
+            vt_apikey = input(B + "[+] Enter VirusTotal API key, press Enter for none: " + W).strip()
+            if vt_apikey != "":
+                self.apikey = vt_apikey
+
         self.q = q
         super(Virustotal, self).__init__(
             base_url, self.engine_name, domain, subdomains,

From b21a7bcbc42bb4b278cc8b330d99be636e5ca735 Mon Sep 17 00:00:00 2001
From: Thomas Bouve <crahan@n00.be>
Date: Sun, 24 Oct 2021 22:34:57 +0200
Subject: [PATCH 7/7] Ignore escape sequence linting errors in banner

---
 sublist3r2/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sublist3r2/__init__.py b/sublist3r2/__init__.py
index 1ebce6a..9ce1e32 100755
--- a/sublist3r2/__init__.py
+++ b/sublist3r2/__init__.py
@@ -66,7 +66,7 @@ def banner():
          ___) | |_| | |_) | | \__ \ |_ ___) | |    / /_     maintained by Ronin Nakomoto
         |____/ \__,_|_.__/|_|_|___/\__|____/|_|   /____|%s    https://github.com/RoninNakomoto/Sublist3r2
 
-    """ % (R, __version__, Y))
+    """ % (R, __version__, Y))  # noqa
 
 
 def parser_error(errmsg):