Matched patterns now get saved to scans/patterns.log

Added lock to prevent multiple asynchronous functions from writing to the same file(s) at the same time.
Moved HTTP Server pattern to nmap-http service scan.
Updated several service scans to use tee so that output is also written to stdout and can be captured.
Moved one gobuster manual command to a scan.

autorecon.py

@@ -139,30 +139,47 @@ if 'password_wordlist' in service_scans_config:
     if isinstance(service_scans_config['password_wordlist'], str):
         password_wordlist = service_scans_config['password_wordlist']
 
-async def read_stream(stream, address, tag='?', patterns=None, color=Fore.BLUE):
+async def read_stream(stream, target, tag='?', patterns=None, color=Fore.BLUE):
+    address = target.address
     while True:
         line = await stream.readline()
         if line:
             line = str(line.rstrip(), 'utf8', 'ignore')
             debug(color + '[' + Style.BRIGHT + address + ' ' + tag + Style.NORMAL + '] ' + Fore.RESET + '{line}', color=color)
-            if verbose >= 1:
+
             for p in global_patterns:
                 matches = re.findall(p['pattern'], line)
                 if 'description' in p:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}' + p['description'].replace('{match}', '{bblue}{match}{crst}{bmagenta}') + '{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - ' + p['description'] + '\n\n'))
                 else:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}Matched Pattern: {bblue}{match}{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - Matched Pattern: {match}\n\n'))
 
             for p in patterns:
                 matches = re.findall(p['pattern'], line)
                 if 'description' in p:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}' + p['description'].replace('{match}', '{bblue}{match}{crst}{bmagenta}') + '{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - ' + p['description'] + '\n\n'))
                 else:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}Matched Pattern: {bblue}{match}{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - Matched Pattern: {match}\n\n'))
         else:
             break
 
@@ -173,20 +190,22 @@ async def run_cmd(semaphore, cmd, target, tag='?', patterns=None):
 
         info('Running task {bgreen}{tag}{rst} on {byellow}{address}{rst}' + (' with {bblue}{cmd}{rst}' if verbose >= 1 else ''))
 
+        async with target.lock:
             with open(os.path.join(scandir, '_commands.log'), 'a') as file:
                 file.writelines(e('{cmd}\n\n'))
 
         process = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
 
         await asyncio.wait([
-            read_stream(process.stdout, address, tag=tag, patterns=patterns),
+            read_stream(process.stdout, target, tag=tag, patterns=patterns),
-            read_stream(process.stderr, address, tag=tag, patterns=patterns, color=Fore.RED)
+            read_stream(process.stderr, target, tag=tag, patterns=patterns, color=Fore.RED)
         ])
 
         await process.wait()
 
     if process.returncode != 0:
         error('Task {bred}{tag}{rst} on {byellow}{address}{rst} returned non-zero exit code: {process.returncode}')
+        async with target.lock:
             with open(os.path.join(scandir, '_errors.log'), 'a') as file:
                 file.writelines(e('[*] Task {tag} returned non-zero exit code: {process.returncode}. Command: {cmd}\n'))
     else:
@@ -207,15 +226,23 @@ async def parse_port_scan(stream, tag, address, pattern):
             if parse_match:
                 ports.append(parse_match.group('port'))
 
-            if verbose >= 1:
+
             for p in global_patterns:
                 matches = re.findall(p['pattern'], line)
                 if 'description' in p:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}' + p['description'].replace('{match}', '{bblue}{match}{crst}{bmagenta}') + '{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - ' + p['description'] + '\n\n'))
                 else:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}Matched Pattern: {bblue}{match}{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - Matched Pattern: {match}\n\n'))
         else:
             break
 
@@ -234,15 +261,22 @@ async def parse_service_detection(stream, tag, address, pattern):
             if parse_match:
                 services.append((parse_match.group('protocol').lower(), int(parse_match.group('port')), parse_match.group('service')))
 
-            if verbose >= 1:
             for p in global_patterns:
                 matches = re.findall(p['pattern'], line)
                 if 'description' in p:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}' + p['description'].replace('{match}', '{bblue}{match}{crst}{bmagenta}') + '{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - ' + p['description'] + '\n\n'))
                 else:
                     for match in matches:
+                        if verbose >= 1:
                             info('Task {bgreen}{tag}{rst} on {byellow}{address}{rst} - {bmagenta}Matched Pattern: {bblue}{match}{rst}')
+                        async with target.lock:
+                            with open(os.path.join(target.scandir, 'patterns.log'), 'a') as file:
+                                file.writelines(e('{tag} - Matched Pattern: {match}\n\n'))
         else:
             break
 
@@ -262,6 +296,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
             info('Running port scan {bgreen}{tag}{rst} on {byellow}{address}{rst}' + (' with {bblue}{command}{rst}' if verbose >= 1 else ''))
 
+            async with target.lock:
                 with open(os.path.join(scandir, '_commands.log'), 'a') as file:
                     file.writelines(e('{command}\n\n'))
 
@@ -269,7 +304,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
             output = [
                 parse_port_scan(process.stdout, tag, address, pattern),
-                read_stream(process.stderr, address, tag=tag, color=Fore.RED)
+                read_stream(process.stderr, target, tag=tag, color=Fore.RED)
             ]
 
             results = await asyncio.gather(*output)
@@ -278,6 +313,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
             if process.returncode != 0:
                 error('Port scan {bred}{tag}{rst} on {byellow}{address}{rst} returned non-zero exit code: {process.returncode}')
+                async with target.lock:
                     with open(os.path.join(scandir, '_errors.log'), 'a') as file:
                         file.writelines(e('[*] Port scan {tag} returned non-zero exit code: {process.returncode}. Command: {command}\n'))
                 return {'returncode': process.returncode}
@@ -295,6 +331,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
         info('Running service detection {bgreen}{tag}{rst} on {byellow}{address}{rst}' + (' with {bblue}{command}{rst}' if verbose >= 1 else ''))
 
+        async with target.lock:
             with open(os.path.join(scandir, '_commands.log'), 'a') as file:
                 file.writelines(e('{command}\n\n'))
 
@@ -302,7 +339,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
         output = [
             parse_service_detection(process.stdout, tag, address, pattern),
-            read_stream(process.stderr, address, tag=tag, color=Fore.RED)
+            read_stream(process.stderr, target, tag=tag, color=Fore.RED)
         ]
 
         results = await asyncio.gather(*output)
@@ -311,6 +348,7 @@ async def run_portscan(semaphore, tag, target, service_detection, port_scan=None
 
         if process.returncode != 0:
             error('Service detection {bred}{tag}{rst} on {byellow}{address}{rst} returned non-zero exit code: {process.returncode}')
+            async with target.lock:
                 with open(os.path.join(scandir, '_errors.log'), 'a') as file:
                     file.writelines(e('[*] Service detection {tag} returned non-zero exit code: {process.returncode}. Command: {command}\n'))
         else:
@@ -498,10 +536,14 @@ def scan_host(target, concurrent_scans):
     os.makedirs(scandir, exist_ok=True)
 
     os.makedirs(os.path.abspath(os.path.join(scandir, 'xml')), exist_ok=True)
+    open(os.path.abspath(os.path.join(scandir, 'patterns.log')), 'a').close()
 
     open(os.path.abspath(os.path.join(reportdir, 'local.txt')), 'a').close()
     open(os.path.abspath(os.path.join(reportdir, 'proof.txt')), 'a').close()
 
+    # Use a lock when writing to specific files that may be written to by other asynchronous functions.
+    target.lock = asyncio.Lock()
+
     # Get event loop for current process.
     loop = asyncio.get_event_loop()
 
@@ -521,6 +563,7 @@ class Target:
         self.reportdir = ''
         self.scandir = ''
         self.scans = []
+        self.lock = None
 
 if __name__ == '__main__':
 

patterns.toml

@@ -1,7 +1,3 @@
 [[pattern]]
 description = 'Nmap script found a potential vulnerability. ({match})'
 pattern = 'State: (?:(?:LIKELY\_?)?VULNERABLE)'
-
-[[pattern]]
-description = 'Identified HTTP Server: {match}'
-pattern = 'Server: ([^\n]+)'

service-scans.toml

@@ -68,13 +68,17 @@ ignore-service-names = [
     name = 'nmap-http'
     command = 'nmap {nmap_extra} -sV -p {port} --script="(http* or ssl*) and not (broadcast or dos or external or http-slowloris* or fuzzer)" -oN "{scandir}/{protocol}_{port}_http_nmap.txt" -oX "{scandir}/xml/{protocol}_{port}_{scheme}_nmap.xml" {address}'
 
+        [[http.scan.pattern]]
+        description = 'Identified HTTP Server: {match}'
+        pattern = 'Server: ([^\n]+)'
+
     [[http.scan]]
     name = 'curl-index'
-    command = 'curl -sSik {scheme}://{address}:{port}/ -m 10 -o "{scandir}/{protocol}_{port}_{scheme}_index.html"'
+    command = 'curl -sSik {scheme}://{address}:{port}/ -m 10 2>&1 | tee "{scandir}/{protocol}_{port}_{scheme}_index.html"'
 
     [[http.scan]]
     name = 'curl-robots'
-    command = 'curl -sSik {scheme}://{address}:{port}/robots.txt -m 10 -o "{scandir}/{protocol}_{port}_{scheme}_robots.txt"'
+    command = 'curl -sSik {scheme}://{address}:{port}/robots.txt -m 10 2>&1 | tee "{scandir}/{protocol}_{port}_{scheme}_robots.txt"'
 
     [[http.scan]]
     name = 'whatweb'
@@ -84,10 +88,13 @@ ignore-service-names = [
     name = 'nikto'
     command = 'nikto -ask=no -h {scheme}://{address}:{port} 2>&1 | tee "{scandir}/{protocol}_{port}_{scheme}_nikto.txt"'
 
+    [[http.scan]]
+    name = 'gobuster'
+    command = 'gobuster -u {scheme}://{address}:{port}/ -w /usr/share/seclists/Discovery/Web-Content/big.txt -e -k -l -s "200,204,301,302,307,403" -x "txt,html,php,asp,aspx" 2>&1 | tee "{scandir}/{protocol}_{port}_{scheme}_gobuster.txt"'
+
     [[http.manual]]
     description = '(dirsearch) Multi-threaded recursive directory/file enumeration for web servers using various wordlists:'
     commands = [
-        'dirsearch -u {scheme}://{address}:{port}/ --plain-text-report="{scandir}/{protocol}_{port}_{scheme}_dirsearch_big.txt" -t 16 -r -e html,php,asp,aspx -f -w /usr/share/seclists/Discovery/Web-Content/big.txt',
         'dirsearch -u {scheme}://{address}:{port}/ --plain-text-report="{scandir}/{protocol}_{port}_{scheme}_dirsearch_common.txt" -t 16 -r -e html,php,asp,aspx -f -w /usr/share/seclists/Discovery/Web-Content/common.txt',
         'dirsearch -u {scheme}://{address}:{port}/ --plain-text-report="{scandir}/{protocol}_{port}_{scheme}_dirsearch_dirbuster.txt" -t 16 -r -e html,php,asp,aspx -f -w /usr/share/wordlists/dirbuster/directory-list-2.3-medium.txt'
     ]
@@ -95,7 +102,6 @@ ignore-service-names = [
     [[http.manual]]
     description = '(dirb) Recursive directory/file enumeration for web servers using various wordlists (same as dirsearch above):'
     commands = [
-        'dirb {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_dirb_big.txt" /usr/share/seclists/Discovery/Web-Content/big.txt',
         'dirb {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_dirb_common.txt" /usr/share/seclists/Discovery/Web-Content/common.txt',
         'dirb {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_dirb_dirbuster.txt" /usr/share/wordlists/dirbuster/directory-list-2.3-medium.txt'
     ]
@@ -103,9 +109,8 @@ ignore-service-names = [
     [[http.manual]]
     description = '(gobuster) Directory/file enumeration for web servers using various wordlists (same as dirb above):'
     commands = [
-        'gobuster -u {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_gobuster_big.txt" -w /usr/share/seclists/Discovery/Web-Content/big.txt -s "200,204,301,302,307,403,500" -e',
+        'gobuster -u {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_gobuster_common.txt" -w /usr/share/seclists/Discovery/Web-Content/common.txt -s "200,204,301,302,307,403" -e',
-        'gobuster -u {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_gobuster_common.txt" -w /usr/share/seclists/Discovery/Web-Content/common.txt -s "200,204,301,302,307,403,500" -e',
+        'gobuster -u {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_gobuster_dirbuster.txt" -w /usr/share/wordlists/dirbuster/directory-list-2.3-medium.txt -s "200,204,301,302,307,403" -e'
-        'gobuster -u {scheme}://{address}:{port}/ -o "{scandir}/{protocol}_{port}_{scheme}_gobuster_dirbuster.txt" -w /usr/share/wordlists/dirbuster/directory-list-2.3-medium.txt -s "200,204,301,302,307,403,500" -e'
     ]
 
     [[http.manual]]
@@ -351,7 +356,7 @@ service-names = [
 
     [[snmp.scan]]
     name = 'onesixtyone'
-    command = 'onesixtyone -c /usr/share/seclists/Discovery/SNMP/common-snmp-community-strings.txt -dd -o "{scandir}/{protocol}_{port}_snmp_onesixtyone.txt" {address}'
+    command = 'onesixtyone -c /usr/share/seclists/Discovery/SNMP/common-snmp-community-strings.txt -dd {address} 2>&1 | tee "{scandir}/{protocol}_{port}_snmp_onesixtyone.txt"'
     run_once = true
     ports.udp = [161]