private List <string> LoadSubdomains(string url)
{
string testUrl = url.Trim(new char[] { '*', '.' });
var subDomains = new List <string>();
//This will have to get moved to a central db
List <string> knownSubdomains = new List <string>();
string existingFile = Path.Combine(Settings.ExistingDir, testUrl.Replace(".", "") + ".txt");
if (File.Exists(existingFile))
{
knownSubdomains = File.ReadAllLines(existingFile).ToList();
}
HunterRequest request = new HunterRequest();
request.Domain = testUrl;
request.KnownSubdomains = knownSubdomains;
request.SecurityTrailsAPIKey = Settings.SecurityTrailsAPI;
request.VirusTotalAPIKey = Settings.VirusTotalAPI;
subDomains = Hunter.GatherAll(request);
Log("Subdomains found: " + subDomains.Count);
TextFileLogger.WriteOverwriteFile(Settings.ExistingDir, testUrl.Replace(".", "") + ".txt", subDomains);
return(subDomains);
}
private void _btnStart_Click(object sender, EventArgs e)
{
//string domain = _txtDomain.Text;
_btnStart.Text = "Service Started...";
_btnStart.Enabled = false;
List <string> domains = File.ReadAllLines(Settings.SourceFile).ToList();
//DomainController domainController = new DomainController();
//SubdomainController subdomainController = new SubdomainController();
//DomainFindResult findResult = domainController.FindAll();
// foreach (Clark.Domain.Data.Domain domainItem in findResult.Items)
foreach (string domain in domains)
{
//string domain = domainItem.DomainName;
//_chkCrawlSites
Log("========================");
Log("Starting " + domain + " at " + DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToShortTimeString());
Log("========================");
List <string> knownSubdomains = new List <string>();
string existingFile = Path.Combine(Settings.ExistingDir, domain.Replace(".", "") + ".txt");
if (File.Exists(existingFile))
{
knownSubdomains = File.ReadAllLines(existingFile).ToList();
}
List <string> subDomains = new List <string>();
if (System.IO.File.GetLastWriteTime(existingFile) > DateTime.Now.AddDays(-60) && knownSubdomains.Count != 0)
{
subDomains = knownSubdomains;
Log("Subdomains loaded from file: " + subDomains.Count);
}
else
{
HunterRequest request = new HunterRequest();
request.Domain = domain;
request.KnownSubdomains = knownSubdomains;
request.SecurityTrailsAPIKey = Settings.SecurityTrailsAPI;
request.VirusTotalAPIKey = Settings.VirusTotalAPI;
subDomains = Hunter.GatherAll(request);
Log("Subdomains found: " + subDomains.Count);
TextFileLogger.WriteOverwriteFile(Settings.ExistingDir, domain.Replace(".", "") + ".txt", subDomains);
}
subDomains.Insert(0, domain);
List <string> chunked = new List <string>();
foreach (List <string> subDomainChunk in subDomains.ChunkBy(_threadCount))
{
_countdown = new CountdownEvent(Math.Min(_threadCount, subDomainChunk.Count));
List <Thread> lstThreads = new List <Thread>();
foreach (string chunck in subDomainChunk)
{
//Subdomain sub = new Subdomain();
//sub.DateFound = DateTime.Now;
//sub.DomainId = domainItem.DomainId;
//sub.SubdomainName = chunck.ToString().Replace(domain, "").Trim('.');
//subdomainController.Insert(sub);
Thread th = new Thread(() => { TestDomain(chunck.ToString(), true); });
lstThreads.Add(th);
}
foreach (Thread th in lstThreads)
{
th.Start();
}
_countdown.Wait();
}
}
}