/// <summary> /// Load the imports /// </summary> /// <param name="options"></param> internal static List <string> PopulateTargets(ref CommandOptions options) { List <string> errors = new List <string>(); string lineOfText; // Transfer each hostname in to the process one from the user defined list, cleaning and removing dupes as we go foreach (string hostname in options.Hostnames) { Match targetMatcher = UriCleaner.Match(hostname); string cleanTarget = targetMatcher.Groups[1].Value; if (cleanTarget == null || cleanTarget.Length < 4 || cleanTarget.Length > 255) { errors.Add(String.Format("Invalid target: {0} which was cleaned as {1}", hostname, cleanTarget)); } else if (!options.Targets.Contains(cleanTarget)) { options.Targets.Add(cleanTarget); } } // Load hostnames from the import file(s) foreach (var fileName in options.ImportFilenames) { if (!File.Exists(fileName)) { errors.Add("Invalid filename: " + fileName); continue; } try { using (var filestream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var file = new StreamReader(filestream, System.Text.Encoding.Default, true, 4096)) { while ((lineOfText = file.ReadLine()) != null) { string cleanLine = lineOfText.Trim(); if (cleanLine.StartsWith("#")) { continue; // Comment line } Match targetMatcher = UriCleaner.Match(cleanLine); string cleanTarget = targetMatcher.Groups[1].Value; if (cleanTarget == null || cleanTarget.Length < 4 || cleanTarget.Length > 255) { errors.Add(String.Format("Invalid target: {0} which was cleaned as {1}", cleanLine, cleanTarget)); } else if (!options.Targets.Contains(cleanTarget)) { options.Targets.Add(cleanTarget); } } } } catch (Exception) { errors.Add("Unable to read file: " + fileName); } } return(errors); }
/// <summary> /// Obtain the results of a host /// </summary> /// <param name="options"></param> /// <param name="hostname"></param> /// <returns></returns> internal static ScanResult ScanHost(ref CommandOptions options, string hostname) { ScanResult result = new ScanResult() { Hostname = hostname, StartTime = DateTime.UtcNow }; string apiResult = null; if (options.Verbosity > 0) { Console.WriteLine("[.] Scanning: {0}", hostname); } // server can report overload, so have a limited number of tests int overLoadRetry = 0; while (overLoadRetry < MaxOverloadRetries) { using (WebClient client = new WebClient()) { result.Tries = 1; Uri endpoint = BuildQueryUri(options, hostname, true); if (options.MinVerbosity(Verbosity.Standard)) { Console.WriteLine("[.] Requesting: " + endpoint.ToString()); } // Iterate for up to the permitted number of tries while (result.Tries <= options.MaxTries) { // Get the revised one - it can change after the first request due to the caching mode if (result.Tries == 2) { endpoint = endpoint = BuildQueryUri(options, hostname, false); if (options.MinVerbosity(Verbosity.Detailed)) { Console.WriteLine("[.] Request string is now: " + endpoint.ToString()); } } // Obtain the response try { apiResult = client.DownloadString(endpoint); } catch (WebException ex) { if (ex.Status == WebExceptionStatus.Timeout) { result.Status = QueryStatus.Timeout; } else if (ex.Response != null) { switch (((HttpWebResponse)ex.Response).StatusCode) { case (HttpStatusCode)429: // TooManyRequests - Not a const in .NET Framework target result.Status = QueryStatus.RateLimited; // Adapt to manage this if it's the first time seeing it if (options.MaxParallel > 1 && overLoadRetry == 0) { options.MaxParallel--; } // Sleep for a random time as there may well be other threads also hitting this Random sleepRandomiser = new Random(); #pragma warning disable SCS0005 // Weak random generator - this is not intended to be a strong form of random int rateLimitedSleepDelay = sleepRandomiser.Next(1000, 10000); #pragma warning restore SCS0005 // Weak random generator if (options.MinVerbosity(Verbosity.Detailed)) { Console.WriteLine("[.] Server overloaded. Pausing {0} for {1}", hostname, rateLimitedSleepDelay); } System.Threading.Thread.Sleep(rateLimitedSleepDelay); continue; case HttpStatusCode.ServiceUnavailable: // 503 result.Status = QueryStatus.Maintenance; break; case (HttpStatusCode)529: // 529 result.Status = QueryStatus.Overloaded; break; case (HttpStatusCode)441: // Unknown. Received with host of 127.0.0.1 result.Status = QueryStatus.WebError; break; default: result.Status = QueryStatus.WebError; if (options.Verbosity == Verbosity.Responses) { Console.WriteLine("[!] Error with the Web Server... Techie details follow:\n" + ex.ToString()); } else if (options.MinVerbosity(Verbosity.Errors)) { Console.WriteLine("[!] Error with the Web Server"); } break; } } if (result.Status != QueryStatus.RateLimited) { return(result); } } if (options.Verbosity == Verbosity.Responses) { Console.WriteLine("[.] Raw server response: " + apiResult); } // Convert to a parsed JSON object try { result.Content = JObject.Parse(apiResult); } catch (Exception) { result.Status = QueryStatus.ResponseError; return(result); } // Check it string responseStatus = (string)result.Content["status"]; if (responseStatus == "READY") { // This means it's finished, not that the test was correct. e.g. checks for a host that responds to ping but has no HTTPS endpoint (test case: amazon.ie) bool validScan = false; foreach (var item in result.Content["endpoints"]) { if ((string)item["statusMessage"] == "Ready") { validScan = true; } } if (validScan) { result.Status = QueryStatus.Success; } else { result.Status = QueryStatus.HostError; // e.g. "Unable to connect to the server" } return(result); } if (responseStatus == "ERROR") { if ((string)result.Content["statusMessage"] == "Unable to resolve domain name") { result.Status = QueryStatus.InvalidHostname; } else { result.Status = QueryStatus.ServiceError; } return(result); } result.Tries++; int pauseMs = CalculatePauseTime(options, result); if (options.MinVerbosity(Verbosity.Detailed)) { Console.WriteLine("[.] Pausing {1} for {0} ms", pauseMs, hostname); } // Pause for a little mo System.Threading.Thread.Sleep(pauseMs); } } overLoadRetry++; } if (result.Status != QueryStatus.RateLimited) { result.Status = QueryStatus.Timeout; } return(result); }