Sn1per Community Edition by @xer0dayz - https://xerosecurity.com

bin/github-subdomains.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python3.5
+
+# I don't believe in license.
+# You can do whatever you want with this program.
+
+import os
+import sys
+import re
+import time
+import requests
+import random
+import argparse
+from functools import partial
+from colored import fg, bg, attr
+from multiprocessing.dummy import Pool
+
+
+TOKENS_FILE = os.path.dirname(os.path.realpath(__file__))+'/.tokens'
+
+
+def githubApiSearchCode( search, page ):
+    headers = {"Authorization":"token "+random.choice(t_tokens)}
+    url = 'https://api.github.com/search/code?s=indexed&type=Code&o=desc&q=' + search + '&page=' + str(page)
+    # print(url)
+
+    try:
+        r = requests.get( url, headers=headers, timeout=5 )
+        json = r.json()
+        return json
+    except Exception as e:
+        print( "%s[-] error occurred: %s%s" % (fg('red'),e,attr(0)) )
+        return False
+
+
+def getRawUrl( result ):
+    raw_url = result['html_url'];
+    raw_url = raw_url.replace( 'https://github.com/', 'https://raw.githubusercontent.com/' )
+    raw_url = raw_url.replace( '/blob/', '/' )
+    return raw_url;
+
+
+def readCode( regexp, source, result ):
+    url = getRawUrl( result )
+    code = doGetCode( url )
+    # print(code)
+
+    if code:
+        matches = re.findall( regexp, code )
+        if matches:
+            for sub in  matches:
+                # print(sub)
+                sub = sub[0].replace('2F','').lower().strip()
+                if len(sub) and not sub in t_history:
+                    t_history.append( sub )
+                    sys.stdout.write( "%s" % sub )
+                    if source:
+                        sys.stdout.write( "\t-> %s" % result['html_url'] )
+                    sys.stdout.write( "\n" )
+
+
+def doGetCode( url ):
+    # print( url )
+    try:
+        r = requests.get( url, timeout=5 )
+    except Exception as e:
+        sys.stdout.write( "%s[-] error occurred: %s%s\n" % (fg('red'),e,attr(0)) )
+        return False
+
+    return r.text
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument( "-t","--token",help="auth token (required)" )
+parser.add_argument( "-d","--domain",help="domain you are looking for (required)" )
+parser.add_argument( "-e","--extend",help="also look for <dummy>example.com", action="store_true" )
+parser.add_argument( "-s","--source",help="display first url where subdomains are found", action="store_true" )
+parser.parse_args()
+args = parser.parse_args()
+
+t_tokens = []
+if args.token:
+    t_tokens = args.token.split(',')
+else:
+    if os.path.isfile(TOKENS_FILE):
+        fp = open(TOKENS_FILE,'r')
+        t_tokens = fp.read().split("\n")
+        fp.close()
+
+if not len(t_tokens):
+    parser.error( 'auth token is missing' )
+
+if args.source:
+    _source = True
+else:
+    _source = False
+
+if args.domain:
+    _domain = args.domain
+else:
+    parser.error( 'domain is missing' )
+
+t_history = []
+page = 1
+_search = '"' + _domain + '"'
+
+### this is a test, looks like we got more result that way
+import tldextract
+t_host_parse = tldextract.extract( _domain )
+_search = '"' + t_host_parse.domain + '"'
+# print( t_host_parse )
+# exit()
+###
+
+# egrep -io "[0-9a-z_\-\.]+\.([0-9a-z_\-]+)?`echo $h|awk -F '.' '{print $(NF-1)}'`([0-9a-z_\-\.]+)?\.[a-z]{1,5}"
+
+
+if args.extend:
+    # _regexp = r'[0-9a-zA-Z_\-\.]+' + _domain.replace('.','\.')
+    _regexp = r'([0-9a-z_\-\.]+\.([0-9a-z_\-]+)?'+t_host_parse.domain+'([0-9a-z_\-\.]+)?\.[a-z]{1,5})'
+else:
+    _regexp = r'(([0-9a-zA-Z_\-\.]+)\.' + _domain.replace('.','\.')+')'
+# print(_regexp)
+
+# for page in range(1,10):
+while True:
+    time.sleep( 1 )
+    t_json = githubApiSearchCode( _search, page )
+    # print(t_json)
+    page = page + 1
+
+    if not t_json or 'documentation_url' in t_json or not 'items' in t_json or not len(t_json['items']):
+        break
+
+    pool = Pool( 30 )
+    pool.map( partial(readCode,_regexp,_source), t_json['items'] )
+    pool.close()
+    pool.join()

modes/javascript-analysis.sh

@@ -0,0 +1,33 @@
+      mkdir -p $LOOT_DIR/web/javascript/$TARGET 2> /dev/null
+      cd $LOOT_DIR/web/javascript/$TARGET
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DOWNLOADING ALL JAVASCRIPT FILES $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      for a in `grep "\.js" $LOOT_DIR/web/spider-$TARGET.txt | head -n $MAX_JAVASCRIPT_FILES | cut -d\? -f1 | sort -u`; do echo "Downloading - $a" && FILENAME=$(echo "$a" | awk -F/ '{print $(NF-0)}') && curl --connect-timeout 10 --max-time 10 -s -R -L --insecure $a | js-beautify - > $FILENAME 2> /dev/null; done;
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DISPLAYING ALL JAVASCRIPT COMMENTS $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      cat $LOOT_DIR/web/javascript/$TARGET/*.js 2> /dev/null | egrep "\/\/|\/\*" | sort -u | tee $LOOT_DIR/web/javascript-$TARGET-comments.txt 
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DISPLAYING ALL JAVASCRIPT LINKS $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      cat $LOOT_DIR/web/javascript/$TARGET/*.js 2> /dev/null | grep -Eo "(http|https)://[a-zA-Z0-9./?=_-]*" | sort -u | tee $LOOT_DIR/web/javascript-$TARGET-urls.txt
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED RUNNING LINKFINDER $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      cd $PLUGINS_DIR/LinkFinder/
+      for a in `ls $LOOT_DIR/web/javascript/$TARGET/*.js 2> /dev/null`; do echo "Analyzing - $a" && FILENAME=$(echo "$a" | awk -F/ '{print $(NF-0)}') && python linkfinder.py -d -i $a -o cli | egrep -v "application\/|SSL error" > $LOOT_DIR/web/javascript-linkfinder-$TARGET-$FILENAME.txt 2> /dev/null; done;
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DISPLAYING PATH RELATIVE LINKS $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      grep -h -v $LOOT_DIR/web/javascript-linkfinder-$TARGET-*.txt 2> /dev/null | grep -v "Running " | awk '{print $1}' | sort -u | tee $LOOT_DIR/web/javascript-$TARGET-path-relative.txt
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DISPLAYING JAVASCRIPT URLS $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      grep -h http $LOOT_DIR/web/javascript-linkfinder-$TARGET-*.txt 2> /dev/null | grep -v "Running " | awk '{print $1}' | egrep "http\:\/\/|https\:\/\/" | sort -u | tee $LOOT_DIR/web/javascript-$TARGET-linkfinder-urls.txt
+      sort -u $LOOT_DIR/web/javascript-$TARGET-urls.txt $LOOT_DIR/web/javascript-$TARGET-linkfinder-urls.txt 2> /dev/null > $LOOT_DIR/web/javascript-$TARGET-urls-sorted.txt
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      echo -e "$OKRED DISPLAYING JAVASCRIPT DOMAINS $RESET"
+      echo -e "${OKGREEN}====================================================================================${RESET}•x${OKGREEN}[`date +"%Y-%m-%d](%H:%M)"`${RESET}x•"
+      grep -h http $LOOT_DIR/web/javascript-linkfinder-$TARGET-*.txt 2> /dev/null | grep -v "Running " | awk '{print $1}' | egrep "http\:\/\/|https\:\/\/" | cut -d\/ -f3 | sort -u | tee $LOOT_DIR/web/javascript-$TARGET-domains.txt
+      WEB_JAVASCRIPT_ANALYSIS="0"