From 6e9339328ea0626b554c54695f53a674044f8789 Mon Sep 17 00:00:00 2001
From: Tib3rius <48113936+Tib3rius@users.noreply.github.com>
Date: Tue, 24 Aug 2021 20:30:13 -0400
Subject: [PATCH] New functionality.

Added a stream readlines() function to read all lines into a list.
Added fformat() function, giving plugin authors more access to variables.
Fixed "Curl Robots" plugin (suggestion by Alh4zr3d) so it only saves the robots.txt file if it finds one.
---
 autorecon.py    | 31 +++++++++++++++++++++++++++++++
 plugins/http.py | 12 ++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/autorecon.py b/autorecon.py
index 40b27a7..bcbba97 100644
--- a/autorecon.py
+++ b/autorecon.py
@@ -227,6 +227,16 @@ class CommandStreamReader(object):
 				else:
 					await asyncio.sleep(0.1)
 
+	async def readlines(self):
+		lines = []
+		while True:
+			line = await self.readline()
+			if line is not None:
+				lines.append(line)
+			else:
+				break
+		return lines
+
 class Plugin(object):
 
 	def __init__(self):
@@ -577,6 +587,9 @@ def e(*args, frame_index=1, **kvargs):
 
 	return string.Formatter().vformat(' '.join(args), args, vals)
 
+def fformat(s):
+	return e(s, frame_index=3)
+
 def cprint(*args, color=Fore.RESET, char='*', sep=' ', end='\n', frame_index=1, file=sys.stdout, printmsg=True, **kvargs):
 	frame = sys._getframe(frame_index)
 
@@ -808,7 +821,25 @@ async def service_scan(plugin, service):
 			break
 
 	async with semaphore:
+		# Create variables for fformat references.
+		address = service.target.address
+		scandir = service.target.scandir
+		protocol = service.protocol
+		port = service.port
+		name = service.name
+
+		# Special cases for HTTP.
+		http_scheme = 'https' if 'https' in service.name or service.secure is True else 'http'
+
+		nmap_extra = service.target.autorecon.args.nmap
+		if service.target.autorecon.args.nmap_append:
+			nmap_extra += ' ' + service.target.autorecon.args.nmap_append
+
+		if protocol == 'udp':
+			nmap_extra += ' -sU'
+
 		tag = service.tag() + '/' + plugin.slug
+
 		info('Service scan {bblue}' + plugin.name + ' (' + tag + '){rst} running against {byellow}' + service.target.address + '{rst}')
 
 		async with service.target.lock:
diff --git a/plugins/http.py b/plugins/http.py
index 052e1be..a86b4b8 100644
--- a/plugins/http.py
+++ b/plugins/http.py
@@ -1,4 +1,4 @@
-from autorecon import ServiceScan, error
+from autorecon import ServiceScan, error, info, fformat
 from shutil import which
 import os
 
@@ -67,7 +67,15 @@ class CurlRobots(ServiceScan):
 
 	async def run(self, service):
 		if service.protocol == 'tcp':
-			await service.execute('curl -sSik {http_scheme}://{address}:{port}/robots.txt', outfile='{protocol}_{port}_{http_scheme}_curl-robots.txt')
+			_, stdout, _ = await service.execute('curl -sSikf {http_scheme}://{address}:{port}/robots.txt')
+			lines = await stdout.readlines()
+
+			if lines:
+				filename = fformat('{scandir}/{protocol}_{port}_{http_scheme}_curl-robots.txt')
+				with open(filename, mode='wt', encoding='utf8') as robots:
+					robots.write('\n'.join(lines))
+			else:
+				info('{bblue}[' + fformat('{tag}') + ']{rst} There did not appear to be a robots.txt file in the webroot (/).')
 
 class DirBuster(ServiceScan):