public Rapid7Results(string name, string file, HostList hosts)
{
using (StreamWriter writer = new StreamWriter("data/output/services-" + name, true))
using (StreamReader reader = new StreamReader("data/" + file))
{
while (!reader.EndOfStream)
{
try
{
string jsonString = reader.ReadLine();
int hostStringStart = jsonString.LastIndexOf("\"ip\":") + 7;
int hostStringEnd = jsonString.IndexOf('"', hostStringStart);
string hostString = jsonString.Substring(hostStringStart, hostStringEnd - hostStringStart);
IPAddress host = IPAddress.Parse(hostString);
lock (hosts.Hosts)
{
if (!hosts.Hosts.Keys.Contains(host))
{
continue;
}
}
int portStringStart = jsonString.LastIndexOf("\"port\":") + 8;
int portStringEnd = jsonString.IndexOf(',', portStringStart);
string portString = jsonString.Substring(portStringStart, portStringEnd - portStringStart);
writer.Write(host.ToString());
writer.Write(";");
writer.Write(name.Replace(';', ','));
writer.Write(";");
writer.Write(portString);
writer.Write(";");
writer.Write(jsonString.Replace(';', ','));
writer.WriteLine();
}
catch
{
}
}
}
}
public Rapid7Results(string name, string file, HostList hosts)
{
using (StreamWriter writer = new StreamWriter("data/output/services-" + name, true))
using (StreamReader reader = new StreamReader("data/" + file))
{
while (!reader.EndOfStream)
{
try
{
string jsonString = reader.ReadLine();
int hostStringStart = jsonString.LastIndexOf("\"ip\":") + 7;
int hostStringEnd = jsonString.IndexOf('"', hostStringStart);
string hostString = jsonString.Substring(hostStringStart, hostStringEnd - hostStringStart);
IPAddress host = IPAddress.Parse(hostString);
lock (hosts.Hosts)
{
if (!hosts.Hosts.Keys.Contains(host))
continue;
}
int portStringStart = jsonString.LastIndexOf("\"port\":") + 8;
int portStringEnd = jsonString.IndexOf(',', portStringStart);
string portString = jsonString.Substring(portStringStart, portStringEnd - portStringStart);
writer.Write(host.ToString());
writer.Write(";");
writer.Write(name.Replace(';', ','));
writer.Write(";");
writer.Write(portString);
writer.Write(";");
writer.Write(jsonString.Replace(';', ','));
writer.WriteLine();
}
catch
{
}
}
}
}
public static void Main(string[] args)
{
if (Directory.Exists("data/output"))
Directory.Delete("data/output", true);
Directory.CreateDirectory("data/output");
ThreadPool.SetMaxThreads(2, 1);
ThreadPool.SetMinThreads(1, 1);
Console.WriteLine("Loading list of dutch hosts...");
HostList = new HostList("nl.csv");
Console.WriteLine("Found {0} dutch hosts.", HostList.Hosts.Count);
FindServiceDescriptor[] services = new FindServiceDescriptor[]
{
new FindServiceDescriptor(143, "IMAP", FindZmapFile("143-imap-starttls-full_ipv4", true), FindZmapFile("143-imap-starttls-full_ipv4", false)),
new FindServiceDescriptor(21, "FTP", FindZmapFile("21-ftp-banner-full_ipv4", true), FindZmapFile("21-ftp-banner-full_ipv4", false)),
new FindServiceDescriptor(995, "POP3S", FindZmapFile("995-pop3s-tls-full_ipv4", true), FindZmapFile("995-pop3s-tls-full_ipv4", false)),
new FindServiceDescriptor(443, "Heartbleed", FindZmapFile("443-https-heartbleed-full_ipv4", true), FindZmapFile("443-https-heartbleed-full_ipv4", false)),
new FindServiceDescriptor(25, "SMTP", FindZmapFile("25-smtp-starttls-full_ipv4", true), FindZmapFile("25-smtp-starttls-full_ipv4", false)),
new FindServiceDescriptor(993, "IMAPS", FindZmapFile("993-imaps-tls-full_ipv4", true), FindZmapFile("993-imaps-tls-full_ipv4", false)),
new FindServiceDescriptor(443, "HTTPS", FindZmapFile("443-https-tls-full_ipv4", true), FindZmapFile("443-https-tls-full_ipv4", false)),
new FindServiceDescriptor(110, "POP3", FindZmapFile("110-pop3-starttls-full_ipv4", true), FindZmapFile("110-pop3-starttls-full_ipv4", false)),
new FindServiceDescriptor("HTTP", FindRapid7File("http"))
};
foreach (var service in services)
ThreadPool.QueueUserWorkItem(FindServices, service);
Console.WriteLine("Synchronizing threads...");
foreach (var service in services)
service.WaitHandle.WaitOne();
Console.WriteLine("Done.");
CveDocument = new CveDocument();
if (!Directory.Exists("output"))
Directory.CreateDirectory("output");
string resultPath = "/media/koen/2.3.2-22-amd641/output";
ResultProcessor results = new ResultProcessor(resultPath);
List<string> uniqueHttpAddressList;
if (!File.Exists("UniqueAddressList"))
{
Console.WriteLine("Generating unique address list...");
uniqueHttpAddressList = results.GetUniqueAddressList(true);
Utilities.SaveObject("UniqueAddressList", uniqueHttpAddressList);
}
else
{
Console.WriteLine("Loading unique address list...");
uniqueHttpAddressList = (List<string>)Utilities.LoadObject("UniqueAddressList");
}
Console.WriteLine("Found {0} unique HTTP servers.", uniqueHttpAddressList.Count);
int randomHostCount = Utilities.CalculateSampleSize(uniqueHttpAddressList.Count);
Console.WriteLine("Fetching {0} random hosts.", randomHostCount);
IPAddress[] randomHosts = results.GetRandomHosts(randomHostCount, uniqueHttpAddressList);
Console.WriteLine("{0} random hosts fetched.", randomHosts.Length);
List<Host> hostList;
if (!File.Exists("HostInformation"))
{
hostList = new List<Host>(randomHosts.Length);
Console.WriteLine("Generating host information...");
for (int i = 0; i < randomHosts.Length; i++)
hostList.Add(new Host(randomHosts[i]));
results.FillHostInformation(hostList);
Console.WriteLine("Generated host information.");
Utilities.SaveObject("HostInformation", hostList);
}
else
{
Console.WriteLine("Loading host information...");
hostList = (List<Host>)Utilities.LoadObject("HostInformation");
}
List<string> hostIPs = new List<string>();
foreach (var host in hostList)
hostIPs.Add(host.AddressString);
File.WriteAllLines("hosts.txt", hostIPs);
Console.WriteLine("Searching for software banners/versions and CVE's...");
FindAndDumpSoftwareBannersAndCves(hostList);
Console.WriteLine("Searching for Heartbleed...");
FindAndDumpHeartbleed(resultPath, hostList);
Console.WriteLine("Done!");
List<Host> shodanHostList;
if (!File.Exists("ShodanHostInformation"))
{
Console.WriteLine("Loading Shodan host list...");
shodanHostList = GetHostListFromShodan(hostList.Select(h => h.AddressString).ToList());
Console.WriteLine("Found {0} Shodan hosts.", shodanHostList.Count);
Utilities.SaveObject("ShodanHostInformation", shodanHostList);
}
else
{
Console.WriteLine("Loading Shodan host information...");
shodanHostList = (List<Host>)Utilities.LoadObject("ShodanHostInformation");
}
Console.WriteLine("Searching for Shodan software banners/versions and CVE's...");
FindAndDumpSoftwareBannersAndCves(shodanHostList, "shodan-");
Console.WriteLine("Searching Shodan for comments on websites...");
FindAndProcessWebsiteComments(shodanHostList);
}
public static void Main(string[] args)
{
if (Directory.Exists("data/output"))
Directory.Delete("data/output", true);
Directory.CreateDirectory("data/output");
ThreadPool.SetMaxThreads(2, 1);
ThreadPool.SetMinThreads(1, 1);
Console.WriteLine("Loading list of dutch hosts...");
HostList = new HostList("nl.csv");
Console.WriteLine("Found {0} dutch hosts.", HostList.Hosts.Count);
FindServiceDescriptor[] services = new FindServiceDescriptor[]
{
new FindServiceDescriptor(143, "IMAP", FindZmapFile("143-imap-starttls-full_ipv4", true), FindZmapFile("143-imap-starttls-full_ipv4", false)),
new FindServiceDescriptor(21, "FTP", FindZmapFile("21-ftp-banner-full_ipv4", true), FindZmapFile("21-ftp-banner-full_ipv4", false)),
new FindServiceDescriptor(995, "POP3S", FindZmapFile("995-pop3s-tls-full_ipv4", true), FindZmapFile("995-pop3s-tls-full_ipv4", false)),
new FindServiceDescriptor(443, "Heartbleed", FindZmapFile("443-https-heartbleed-full_ipv4", true), FindZmapFile("443-https-heartbleed-full_ipv4", false)),
new FindServiceDescriptor(25, "SMTP", FindZmapFile("25-smtp-starttls-full_ipv4", true), FindZmapFile("25-smtp-starttls-full_ipv4", false)),
new FindServiceDescriptor(993, "IMAPS", FindZmapFile("993-imaps-tls-full_ipv4", true), FindZmapFile("993-imaps-tls-full_ipv4", false)),
new FindServiceDescriptor(443, "HTTPS", FindZmapFile("443-https-tls-full_ipv4", true), FindZmapFile("443-https-tls-full_ipv4", false)),
new FindServiceDescriptor(110, "POP3", FindZmapFile("110-pop3-starttls-full_ipv4", true), FindZmapFile("110-pop3-starttls-full_ipv4", false)),
new FindServiceDescriptor("HTTP", FindRapid7File("http"))
};
foreach (var service in services)
ThreadPool.QueueUserWorkItem(FindServices, service);
Console.WriteLine("Synchronizing threads...");
foreach (var service in services)
service.WaitHandle.WaitOne();
Console.WriteLine("Done.");
CveDocument = new CveDocument();
if (!Directory.Exists("output"))
Directory.CreateDirectory("output");
string resultPath = "/media/koen/2.3.2-22-amd641/output";
ResultProcessor results = new ResultProcessor(resultPath);
List<string> uniqueHttpAddressList;
if (!File.Exists("UniqueAddressList"))
{
Console.WriteLine("Generating unique address list...");
uniqueHttpAddressList = results.GetUniqueAddressList(true);
Utilities.SaveObject("UniqueAddressList", uniqueHttpAddressList);
}
else
{
Console.WriteLine("Loading unique address list...");
uniqueHttpAddressList = (List<string>)Utilities.LoadObject("UniqueAddressList");
}
Console.WriteLine("Found {0} unique HTTP servers.", uniqueHttpAddressList.Count);
int randomHostCount = Utilities.CalculateSampleSize(uniqueHttpAddressList.Count);
Console.WriteLine("Fetching {0} random hosts.", randomHostCount);
IPAddress[] randomHosts = results.GetRandomHosts(randomHostCount, uniqueHttpAddressList);
Console.WriteLine("{0} random hosts fetched.", randomHosts.Length);
List<Host> hostList;
if (!File.Exists("HostInformation"))
{
hostList = new List<Host>(randomHosts.Length);
Console.WriteLine("Generating host information...");
for (int i = 0; i < randomHosts.Length; i++)
hostList.Add(new Host(randomHosts[i]));
results.FillHostInformation(hostList);
Console.WriteLine("Generated host information.");
Utilities.SaveObject("HostInformation", hostList);
}
else
{
Console.WriteLine("Loading host information...");
hostList = (List<Host>)Utilities.LoadObject("HostInformation");
}
List<string> hostIPs = new List<string>();
foreach (var host in hostList)
hostIPs.Add(host.AddressString);
File.WriteAllLines("hosts.txt", hostIPs);
Console.WriteLine("Searching for software banners/versions and CVE's...");
FindAndDumpSoftwareBannersAndCves(hostList);
Console.WriteLine("Searching for Heartbleed...");
FindAndDumpHeartbleed(resultPath, hostList);
Console.WriteLine("Done!");
List<Host> shodanHostList;
if (!File.Exists("ShodanHostInformation"))
{
Console.WriteLine("Loading Shodan host list...");
shodanHostList = GetHostListFromShodan(hostList.Select(h => h.AddressString).ToList());
Console.WriteLine("Found {0} Shodan hosts.", shodanHostList.Count);
Utilities.SaveObject("ShodanHostInformation", shodanHostList);
}
else
{
Console.WriteLine("Loading Shodan host information...");
shodanHostList = (List<Host>)Utilities.LoadObject("ShodanHostInformation");
}
Console.WriteLine("Searching for Shodan software banners/versions and CVE's...");
FindAndDumpSoftwareBannersAndCves(shodanHostList, "shodan-");
Console.WriteLine("Searching Shodan for comments on websites...");
FindAndProcessWebsiteComments(shodanHostList);
}
