/// <summary>
/// Load the imports
/// </summary>
/// <param name="options"></param>
internal static List <string> PopulateTargets(ref CommandOptions options)
{
List <string> errors = new List <string>();
string lineOfText;
// Transfer each hostname in to the process one from the user defined list, cleaning and removing dupes as we go
foreach (string hostname in options.Hostnames)
{
Match targetMatcher = UriCleaner.Match(hostname);
string cleanTarget = targetMatcher.Groups[1].Value;
if (cleanTarget == null || cleanTarget.Length < 4 || cleanTarget.Length > 255)
{
errors.Add(String.Format("Invalid target: {0} which was cleaned as {1}", hostname, cleanTarget));
}
else if (!options.Targets.Contains(cleanTarget))
{
options.Targets.Add(cleanTarget);
}
}
// Load hostnames from the import file(s)
foreach (var fileName in options.ImportFilenames)
{
if (!File.Exists(fileName))
{
errors.Add("Invalid filename: " + fileName);
continue;
}
try
{
using (var filestream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var file = new StreamReader(filestream, System.Text.Encoding.Default, true, 4096))
{
while ((lineOfText = file.ReadLine()) != null)
{
string cleanLine = lineOfText.Trim();
if (cleanLine.StartsWith("#"))
{
continue; // Comment line
}
Match targetMatcher = UriCleaner.Match(cleanLine);
string cleanTarget = targetMatcher.Groups[1].Value;
if (cleanTarget == null || cleanTarget.Length < 4 || cleanTarget.Length > 255)
{
errors.Add(String.Format("Invalid target: {0} which was cleaned as {1}", cleanLine, cleanTarget));
}
else if (!options.Targets.Contains(cleanTarget))
{
options.Targets.Add(cleanTarget);
}
}
}
}
catch (Exception)
{
errors.Add("Unable to read file: " + fileName);
}
}
return(errors);
}
/// <summary>
/// Obtain the results of a host
/// </summary>
/// <param name="options"></param>
/// <param name="hostname"></param>
/// <returns></returns>
internal static ScanResult ScanHost(ref CommandOptions options, string hostname)
{
ScanResult result = new ScanResult()
{
Hostname = hostname, StartTime = DateTime.UtcNow
};
string apiResult = null;
if (options.Verbosity > 0)
{
Console.WriteLine("[.] Scanning: {0}", hostname);
}
// server can report overload, so have a limited number of tests
int overLoadRetry = 0;
while (overLoadRetry < MaxOverloadRetries)
{
using (WebClient client = new WebClient())
{
result.Tries = 1;
Uri endpoint = BuildQueryUri(options, hostname, true);
if (options.MinVerbosity(Verbosity.Standard))
{
Console.WriteLine("[.] Requesting: " + endpoint.ToString());
}
// Iterate for up to the permitted number of tries
while (result.Tries <= options.MaxTries)
{
// Get the revised one - it can change after the first request due to the caching mode
if (result.Tries == 2)
{
endpoint = endpoint = BuildQueryUri(options, hostname, false);
if (options.MinVerbosity(Verbosity.Detailed))
{
Console.WriteLine("[.] Request string is now: " + endpoint.ToString());
}
}
// Obtain the response
try
{
apiResult = client.DownloadString(endpoint);
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.Timeout)
{
result.Status = QueryStatus.Timeout;
}
else if (ex.Response != null)
{
switch (((HttpWebResponse)ex.Response).StatusCode)
{
case (HttpStatusCode)429: // TooManyRequests - Not a const in .NET Framework target
result.Status = QueryStatus.RateLimited;
// Adapt to manage this if it's the first time seeing it
if (options.MaxParallel > 1 && overLoadRetry == 0)
{
options.MaxParallel--;
}
// Sleep for a random time as there may well be other threads also hitting this
Random sleepRandomiser = new Random();
#pragma warning disable SCS0005 // Weak random generator - this is not intended to be a strong form of random
int rateLimitedSleepDelay = sleepRandomiser.Next(1000, 10000);
#pragma warning restore SCS0005 // Weak random generator
if (options.MinVerbosity(Verbosity.Detailed))
{
Console.WriteLine("[.] Server overloaded. Pausing {0} for {1}", hostname, rateLimitedSleepDelay);
}
System.Threading.Thread.Sleep(rateLimitedSleepDelay);
continue;
case HttpStatusCode.ServiceUnavailable: // 503
result.Status = QueryStatus.Maintenance;
break;
case (HttpStatusCode)529: // 529
result.Status = QueryStatus.Overloaded;
break;
case (HttpStatusCode)441: // Unknown. Received with host of 127.0.0.1
result.Status = QueryStatus.WebError;
break;
default:
result.Status = QueryStatus.WebError;
if (options.Verbosity == Verbosity.Responses)
{
Console.WriteLine("[!] Error with the Web Server... Techie details follow:\n" + ex.ToString());
}
else if (options.MinVerbosity(Verbosity.Errors))
{
Console.WriteLine("[!] Error with the Web Server");
}
break;
}
}
if (result.Status != QueryStatus.RateLimited)
{
return(result);
}
}
if (options.Verbosity == Verbosity.Responses)
{
Console.WriteLine("[.] Raw server response: " + apiResult);
}
// Convert to a parsed JSON object
try
{
result.Content = JObject.Parse(apiResult);
}
catch (Exception)
{
result.Status = QueryStatus.ResponseError;
return(result);
}
// Check it
string responseStatus = (string)result.Content["status"];
if (responseStatus == "READY")
{
// This means it's finished, not that the test was correct. e.g. checks for a host that responds to ping but has no HTTPS endpoint (test case: amazon.ie)
bool validScan = false;
foreach (var item in result.Content["endpoints"])
{
if ((string)item["statusMessage"] == "Ready")
{
validScan = true;
}
}
if (validScan)
{
result.Status = QueryStatus.Success;
}
else
{
result.Status = QueryStatus.HostError; // e.g. "Unable to connect to the server"
}
return(result);
}
if (responseStatus == "ERROR")
{
if ((string)result.Content["statusMessage"] == "Unable to resolve domain name")
{
result.Status = QueryStatus.InvalidHostname;
}
else
{
result.Status = QueryStatus.ServiceError;
}
return(result);
}
result.Tries++;
int pauseMs = CalculatePauseTime(options, result);
if (options.MinVerbosity(Verbosity.Detailed))
{
Console.WriteLine("[.] Pausing {1} for {0} ms", pauseMs, hostname);
}
// Pause for a little mo
System.Threading.Thread.Sleep(pauseMs);
}
}
overLoadRetry++;
}
if (result.Status != QueryStatus.RateLimited)
{
result.Status = QueryStatus.Timeout;
}
return(result);
}