RomanNum3ral
/
Sublist3r
mirror of https://github.com/aboul3la/Sublist3r.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #24 from robinlennox/patch/fix_duplicates 

Avoid any duplicates in subdomains
This commit is contained in:
Ahmed Aboul-Ela 2016-07-17 04:27:15 +03:00 committed by GitHub
parent b685aaf37b ff6b78a869
commit cf0c734996
1 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
sublist3r.py
View File

@ -230,7 +230,7 @@ class GoogleEnum(enumratorBaseThreaded):
if subdomain and subdomain not in self.subdomains and subdomain != self.domain: if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
return links_list return links_list
@ -283,7 +283,7 @@ class YahooEnum(enumratorBaseThreaded):
if subdomain and subdomain not in self.subdomains and subdomain != self.domain: if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -326,7 +326,7 @@ class AskEnum(enumratorBaseThreaded):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -372,7 +372,7 @@ class BingEnum(enumratorBaseThreaded):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -417,7 +417,7 @@ class BaiduEnum(enumratorBaseThreaded):
found_newdomain = True found_newdomain = True
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
if not found_newdomain and subdomain_list: if not found_newdomain and subdomain_list:
@ -538,7 +538,7 @@ class NetcraftEnum(multiprocessing.Process):
if subdomain and subdomain not in self.subdomains and subdomain != self.domain: if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
return links_list return links_list
@ -648,7 +648,7 @@ class DNSdumpster(multiprocessing.Process):
if not subdomain.endswith(self.domain): if not subdomain.endswith(self.domain):
continue continue
if subdomain and subdomain not in self.subdomains and subdomain != self.domain: if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
return links return links
class Virustotal(multiprocessing.Process): class Virustotal(multiprocessing.Process):
@ -716,7 +716,7 @@ class Virustotal(multiprocessing.Process):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -791,7 +791,7 @@ class ThreatCrowd(multiprocessing.Process):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -860,7 +860,7 @@ class CrtSearch(multiprocessing.Process):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass
@ -929,7 +929,7 @@ class PassiveDNS(multiprocessing.Process):
if subdomain not in self.subdomains and subdomain != self.domain: if subdomain not in self.subdomains and subdomain != self.domain:
if verbose: if verbose:
print "%s%s: %s%s"%(R, self.engine_name, W, subdomain) print "%s%s: %s%s"%(R, self.engine_name, W, subdomain)
self.subdomains.append(subdomain) self.subdomains.append(subdomain.strip())
except Exception as e: except Exception as e:
pass pass