public static void Main(string[] args)
        {
            if (Directory.Exists("data/output"))
                Directory.Delete("data/output", true);
            Directory.CreateDirectory("data/output");

            ThreadPool.SetMaxThreads(2, 1);
            ThreadPool.SetMinThreads(1, 1);

            Console.WriteLine("Loading list of dutch hosts...");

            HostList = new HostList("nl.csv");
            Console.WriteLine("Found {0} dutch hosts.", HostList.Hosts.Count);

            FindServiceDescriptor[] services = new FindServiceDescriptor[]
            {
                new FindServiceDescriptor(143, "IMAP", FindZmapFile("143-imap-starttls-full_ipv4", true), FindZmapFile("143-imap-starttls-full_ipv4", false)),
                new FindServiceDescriptor(21, "FTP", FindZmapFile("21-ftp-banner-full_ipv4", true), FindZmapFile("21-ftp-banner-full_ipv4", false)),
                new FindServiceDescriptor(995, "POP3S", FindZmapFile("995-pop3s-tls-full_ipv4", true), FindZmapFile("995-pop3s-tls-full_ipv4", false)),
                new FindServiceDescriptor(443, "Heartbleed", FindZmapFile("443-https-heartbleed-full_ipv4", true), FindZmapFile("443-https-heartbleed-full_ipv4", false)),
                new FindServiceDescriptor(25, "SMTP", FindZmapFile("25-smtp-starttls-full_ipv4", true), FindZmapFile("25-smtp-starttls-full_ipv4", false)),
                new FindServiceDescriptor(993, "IMAPS", FindZmapFile("993-imaps-tls-full_ipv4", true), FindZmapFile("993-imaps-tls-full_ipv4", false)),
                new FindServiceDescriptor(443, "HTTPS", FindZmapFile("443-https-tls-full_ipv4", true), FindZmapFile("443-https-tls-full_ipv4", false)),
                new FindServiceDescriptor(110, "POP3", FindZmapFile("110-pop3-starttls-full_ipv4", true), FindZmapFile("110-pop3-starttls-full_ipv4", false)),
                new FindServiceDescriptor("HTTP", FindRapid7File("http"))
            };

            foreach (var service in services)
                ThreadPool.QueueUserWorkItem(FindServices, service);

            Console.WriteLine("Synchronizing threads...");
            foreach (var service in services)
                service.WaitHandle.WaitOne();

            Console.WriteLine("Done.");

            CveDocument = new CveDocument();

            if (!Directory.Exists("output"))
                Directory.CreateDirectory("output");

            string resultPath = "/media/koen/2.3.2-22-amd641/output";
            ResultProcessor results = new ResultProcessor(resultPath);

            List<string> uniqueHttpAddressList;
            if (!File.Exists("UniqueAddressList"))
            {
                Console.WriteLine("Generating unique address list...");
                uniqueHttpAddressList = results.GetUniqueAddressList(true);
                Utilities.SaveObject("UniqueAddressList", uniqueHttpAddressList);
            }
            else
            {
                Console.WriteLine("Loading unique address list...");
                uniqueHttpAddressList = (List<string>)Utilities.LoadObject("UniqueAddressList");
            }

            Console.WriteLine("Found {0} unique HTTP servers.", uniqueHttpAddressList.Count);

            int randomHostCount = Utilities.CalculateSampleSize(uniqueHttpAddressList.Count);

            Console.WriteLine("Fetching {0} random hosts.", randomHostCount);
            IPAddress[] randomHosts = results.GetRandomHosts(randomHostCount, uniqueHttpAddressList);

            Console.WriteLine("{0} random hosts fetched.", randomHosts.Length);

            List<Host> hostList;
            if (!File.Exists("HostInformation"))
            {
                hostList = new List<Host>(randomHosts.Length);

                Console.WriteLine("Generating host information...");

                for (int i = 0; i < randomHosts.Length; i++)
                    hostList.Add(new Host(randomHosts[i]));

                results.FillHostInformation(hostList);

                Console.WriteLine("Generated host information.");

                Utilities.SaveObject("HostInformation", hostList);
            }
            else
            {
                Console.WriteLine("Loading host information...");
                hostList = (List<Host>)Utilities.LoadObject("HostInformation");
            }

            List<string> hostIPs = new List<string>();
            foreach (var host in hostList)
                hostIPs.Add(host.AddressString);

            File.WriteAllLines("hosts.txt", hostIPs);

            Console.WriteLine("Searching for software banners/versions and CVE's...");
            FindAndDumpSoftwareBannersAndCves(hostList);
            Console.WriteLine("Searching for Heartbleed...");
            FindAndDumpHeartbleed(resultPath, hostList);
            Console.WriteLine("Done!");

            List<Host> shodanHostList;
            if (!File.Exists("ShodanHostInformation"))
            {
                Console.WriteLine("Loading Shodan host list...");
                shodanHostList = GetHostListFromShodan(hostList.Select(h => h.AddressString).ToList());
                Console.WriteLine("Found {0} Shodan hosts.", shodanHostList.Count);

                Utilities.SaveObject("ShodanHostInformation", shodanHostList);
            }
            else
            {
                Console.WriteLine("Loading Shodan host information...");
                shodanHostList = (List<Host>)Utilities.LoadObject("ShodanHostInformation");
            }

            Console.WriteLine("Searching for Shodan software banners/versions and CVE's...");
            FindAndDumpSoftwareBannersAndCves(shodanHostList, "shodan-");

            Console.WriteLine("Searching Shodan for comments on websites...");
            FindAndProcessWebsiteComments(shodanHostList);
        }
Example #2
0
        public static void Main(string[] args)
        {
            /*if (Directory.Exists("data/output"))
                Directory.Delete("data/output", true);
            Directory.CreateDirectory("data/output");

            ThreadPool.SetMaxThreads(2, 1);
            ThreadPool.SetMinThreads(1, 1);

            Console.WriteLine("Loading list of dutch hosts...");

            HostList = new HostList("nl.csv");
            Console.WriteLine("Found {0} dutch hosts.", HostList.Hosts.Count);

            FindServiceDescriptor[] services = new FindServiceDescriptor[]
            { 
                new FindServiceDescriptor(143, "IMAP", "5elhwfrqv15nq5px-143-imap-starttls-full_ipv4-20150617T163103-zgrab-results.json", "5elhwfrqv15nq5px-143-imap-starttls-full_ipv4-20150617T163103-zmap-results.csv"),
                new FindServiceDescriptor(21, "FTP", "7ngdfqqrhmqdce38-21-ftp-banner-full_ipv4-20150801T233003-zgrab-results.json", "7ngdfqqrhmqdce38-21-ftp-banner-full_ipv4-20150801T233003-zmap-results.csv"),
                new FindServiceDescriptor(995, "POP3S", "gf1z452301hyhs3w-995-pop3s-tls-full_ipv4-20150802T140000-zgrab-results.json", "gf1z452301hyhs3w-995-pop3s-tls-full_ipv4-20150802T140000-zmap-results.csv"),
                new FindServiceDescriptor(443, "Heartbleed", "ju8g62b9picx0i3i-443-https-heartbleed-full_ipv4-20150706T000000-zgrab-results.json", "ju8g62b9picx0i3i-443-https-heartbleed-full_ipv4-20150706T000000-zmap-results.csv"),
                new FindServiceDescriptor(25, "SMTP", "klnqp1y00vooeonh-25-smtp-starttls-full_ipv4-20150803T040000-zgrab-results.json", "klnqp1y00vooeonh-25-smtp-starttls-full_ipv4-20150803T040000-zmap-results.csv"),
                new FindServiceDescriptor(993, "IMAPS", "pt15h1gy6uic493j-993-imaps-tls-full_ipv4-20150721T120000-zgrab-results.json", "pt15h1gy6uic493j-993-imaps-tls-full_ipv4-20150721T120000-zmap-results.csv"),
                new FindServiceDescriptor(443, "HTTPS", "ydns0pmlsiu0996u-443-https-tls-full_ipv4-20150804T010006-zgrab-results.json", "ydns0pmlsiu0996u-443-https-tls-full_ipv4-20150804T010006-zmap-results.csv"),
                new FindServiceDescriptor(110, "POP3", "z2nk2bbxgipkjl9k-110-pop3-starttls-full_ipv4-20150729T221221-zgrab-results.json", "z2nk2bbxgipkjl9k-110-pop3-starttls-full_ipv4-20150729T221221-zmap-results.csv"),
                new FindServiceDescriptor("HTTP", "20150721-http")
            };

            foreach (var service in services)
                ThreadPool.QueueUserWorkItem(FindServices, service);

            Console.WriteLine("Synchronizing threads...");
            foreach (var service in services)
                service.WaitHandle.WaitOne();

            Console.WriteLine("Done.");*/

            CveDocument = new CveDocument();

            if (!Directory.Exists("output"))
                Directory.CreateDirectory("output");

            string resultPath = "/media/koen/2.3.2-22-amd641/output";
            ResultProcessor results = new ResultProcessor(resultPath);

            List<string> uniqueHttpAddressList;
            if (!File.Exists("UniqueAddressList"))
            {
                Console.WriteLine("Generating unique address list...");
                uniqueHttpAddressList = results.GetUniqueAddressList(true);
                Utilities.SaveObject("UniqueAddressList", uniqueHttpAddressList);
            }
            else
            {
                Console.WriteLine("Loading unique address list...");
                uniqueHttpAddressList = (List<string>)Utilities.LoadObject("UniqueAddressList");
            }

            Console.WriteLine("Found {0} unique HTTP servers.", uniqueHttpAddressList.Count);

            int randomHostCount = Utilities.CalculateSampleSize(uniqueHttpAddressList.Count);

            Console.WriteLine("Fetching {0} random hosts.", randomHostCount);
            IPAddress[] randomHosts = results.GetRandomHosts(randomHostCount, uniqueHttpAddressList);

            Console.WriteLine("{0} random hosts fetched.", randomHosts.Length);

            List<Host> hostList;
            if (!File.Exists("HostInformation"))
            {
                hostList = new List<Host>(randomHosts.Length);

                Console.WriteLine("Generating host information...");

                for (int i = 0; i < randomHosts.Length; i++)
                    hostList.Add(new Host(randomHosts[i]));

                results.FillHostInformation(hostList);

                Console.WriteLine("Generated host information.");

                Utilities.SaveObject("HostInformation", hostList);
            }
            else
            {
                Console.WriteLine("Loading host information...");
                hostList = (List<Host>)Utilities.LoadObject("HostInformation");
            }

            List<string> hostIPs = new List<string>();
            foreach (var host in hostList)
                hostIPs.Add(host.AddressString);

            File.WriteAllLines("hosts.txt", hostIPs);

            Console.WriteLine("Searching for software banners/versions and CVE's...");
            FindAndDumpSoftwareBannersAndCves(hostList);
            Console.WriteLine("Searching for Heartbleed...");
            FindAndDumpHeartbleed(resultPath, hostList);
            Console.WriteLine("Done!");

            List<Host> shodanHostList;
            if (!File.Exists("ShodanHostInformation"))
            {
                Console.WriteLine("Loading Shodan host list...");
                shodanHostList = GetHostListFromShodan(hostList.Select(h => h.AddressString).ToList());
                Console.WriteLine("Found {0} Shodan hosts.", shodanHostList.Count);

                Utilities.SaveObject("ShodanHostInformation", shodanHostList);
            }
            else
            {
                Console.WriteLine("Loading Shodan host information...");
                shodanHostList = (List<Host>)Utilities.LoadObject("ShodanHostInformation");
            }

            Console.WriteLine("Searching for Shodan software banners/versions and CVE's...");
            FindAndDumpSoftwareBannersAndCves(shodanHostList, "shodan-");

            Console.WriteLine("Searching Shodan for comments on websites...");
            FindAndProcessWebsiteComments(shodanHostList);
        }