public async Task ExternalWebContentSource_TestAllServers()
{
var urls = await ExternalWebContentSource.LoadInternalServerListAsync();
var failedUrls = new List <Tuple <Uri, object> >();
foreach (var url in urls)
{
// Make 3 attempts to successfully connect to each server.
for (int i = 0; i < 3; i++)
{
try
{
var hc = HttpClientHelpers.Create(userAgent: UnitTestUserAgent());
var result = await hc.GetByteArrayAsync(url);
break;
}
catch (Exception ex)
{
await Task.Delay(1000);
if (i == 2)
{
failedUrls.Add(Tuple.Create(url, (object)ex));
}
}
}
}
if (failedUrls.Any())
{
throw new Exception("Urls failed to GET: " + String.Join("\n", failedUrls.Select(x => x.Item1.ToString() + " - " + x.Item2.ToString())));
}
}
public async Task <byte[]> ResetAndRun()
{
// TODO: timeout.
var hash = SHA256.Create();
var hc = HttpClientHelpers.Create(userAgent: UserAgent);
var sw = Stopwatch.StartNew();
try
{
var responseBytes = await hc.GetByteArrayAsync(Url);
sw.Stop();
Log.Trace("GET from '{0}' in {1:N2}ms, received {2:N0} bytes", Url, sw.Elapsed.TotalMilliseconds, responseBytes.Length);
var result = hash.ComputeHash(
responseBytes
.Concat(BitConverter.GetBytes(sw.ElapsedTicks))
.Concat(StaticEntropy)
.ToArray()
);
return(result);
}
catch (Exception ex)
{
Log.WarnException("Exception when trying to GET from {0}", ex, Url);
return(null);
}
}
public void TestHttpClientDefaults()
{
var http = HttpClientHelpers.Create();
Assert.AreEqual(HttpClientHelpers.DefaultTimeout, http.Timeout);
Assert.AreEqual(HttpClientHelpers.UserAgentString(), String.Join(" ", http.DefaultRequestHeaders.UserAgent.Select(x => x.ToString())));
}
/// <summary>
/// Create a user-agent string to use with HTTP requests.
/// </summary>
/// <param name="usageIdentifier">An email address, website, or other identifying mark to include.</param>
/// <exception cref="System.Exception">May throw if the usageIdentifier has invalid characters.</exception>
public static string UserAgent(string usageIdentifier)
{
var id = (usageIdentifier ?? "unconfigured").Replace("@", ".AT.");
var ua = HttpClientHelpers.UserAgentString(id);
var http = HttpClientHelpers.Create(userAgent: ua);
http.Dispose();
return(ua);
}
private async Task <byte[]> GetPublicEntropyAsync(EntropyPriority priority)
{
// https://random.org
Log.Trace("Beginning to gather entropy.");
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("https://www.random.org/cgi-bin/randbyte?nbytes=" + _BytesPerRequest + "&format=h");
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "www.random.org.html"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// The entire content is random hex bytes.
// Albeit with a bunch of whitespace.
var randomString = response.Replace("\r", "").Replace("\n", "").Replace(" ", "");
var randomBytes = randomString.ParseFromHexString()
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", randomBytes.Length);
return(randomBytes);
}
protected override async Task <byte[]> GetInternalEntropyAsync(EntropyPriority priority)
{
Log.Trace("Beginning to gather entropy.");
// This supports SSL, but the cert isn't valid (it's for the uni, rather than the correct domain).
// http://www.randomnumbers.info/content/Download.htm
// This returns HTML, which means I'm doing some hacky parsing here.
const int rangeOfNumbers = 4096 - 1; // 12 bits per number (1.5 bytes).
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("http://www.randomnumbers.info/cgibin/wqrng.cgi?amount=" + _NumberOfNumbers.ToString() + "&limit=" + rangeOfNumbers.ToString());
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "www.randomnumbers.info.html"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// Locate some pretty clear boundaries around the random numbers returned.
var startIdxString = "Download random numbers from quantum origin";
var startIdx = response.IndexOf(startIdxString, StringComparison.OrdinalIgnoreCase);
if (startIdx == -1)
{
Log.Error("Cannot locate start string in html of randomnumbers.info result: source will return nothing. Looking for '{0}'. Actual result in next message.", startIdxString);
Log.Error(response);
return(null);
}
var hrIdxString = "<hr>";
startIdx = response.IndexOf(hrIdxString, startIdx, StringComparison.OrdinalIgnoreCase);
if (startIdx == -1)
{
Log.Error("Cannot locate start string in html of randomnumbers.info result: source will return nothing. Looking for '{0}'. Actual result in next message.", hrIdxString);
Log.Error(response);
return(null);
}
var endIdxString = "</td>";
var endIdx = response.IndexOf("</td>", startIdx, StringComparison.OrdinalIgnoreCase);
if (endIdx == -1)
{
Log.Error("Cannot locate end string in html of randomnumbers.info result: source will return nothing. Looking for '{0}'. Actual result in next message.", endIdxString);
Log.Error(response);
return(null);
}
Log.Trace("Parsed beginning and end of useful entropy.");
var haystack = response.Substring(startIdx, endIdx - startIdx);
var numbersAndOtherJunk = haystack.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); // Numbers are space separated.
var numbers = numbersAndOtherJunk
.Where(x => x.All(Char.IsDigit)) // Remove non-numeric junk.
.Select(x => Int16.Parse(x)) // Parse to an int16.
.ToList();
var result = new byte[(numbers.Count() * 2) + sizeof(uint)];
for (int i = 0; i < numbers.Count; i++)
{
// Take the Int16s in the range 0..4095 (4096 possibilities) and write them into the result array.
// The top 4 bits will always be empty, but that doesn't matter too much as they will be hashed when added to a Pool.
// This means only 75% of bits are truly random, so 16 bytes is only equivalent to 12 bytes.
// TODO: some bit bashing to pack things more efficiently
var twoBytes = BitConverter.GetBytes(numbers[i]);
result[i * 2] = twoBytes[0];
result[(i * 2) + 1] = twoBytes[1];
}
var timingBytes = BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks));
result[result.Length - 4] = timingBytes[0];
result[result.Length - 3] = timingBytes[1];
result[result.Length - 2] = timingBytes[2];
result[result.Length - 1] = timingBytes[3];
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", result.Length);
return(result);
}
protected override async Task <byte[]> GetInternalEntropyAsync(EntropyPriority priority)
{
// https://beacon.nist.gov/
// Note that this will return the same result for 60 second period.
// We must mix in some local entropy to ensure differnt computers end up with different entropy.
// Yes, this reduces the effectiveness of this source, but it will still contribute over time.
Log.Trace("Beginning to gather entropy.");
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("https://beacon.nist.gov/rest/record/last");
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "beacon.nist.gov-last.xml"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
var localEntropy = (await StaticLocalEntropy.Get32()).Concat(CheapEntropy.Get16())
.Concat(BitConverter.GetBytes((uint)sw.Elapsed.Ticks))
.ToArray();
Log.Trace("Got {0:N0} bytes of local entropy to mix.", localEntropy.Length);
// Parse out the useful parts of the response.
// Keeping away from XML parsing to minimise dependencies. At least for now.
// The first two return 64 random bytes each, the signature is 256 bytes. All are hashed to 64 bytes when combined with local entropy.
var lastOutputBytes = GetWithinXmlTags(response, "previousOutputValue").ParseFromHexString();
var outputValueBytes = GetWithinXmlTags(response, "outputValue").ParseFromHexString();
var signatureBytes = GetWithinXmlTags(response, "signatureValue").ParseFromHexString();
Log.Trace("Got {0:N0} output bytes, {1:N0} last output bytes, {2:N0} signature bytes.", outputValueBytes.Length, lastOutputBytes.Length, signatureBytes.Length);
// Mix in some local entropy.
var hasher = SHA512.Create();
var result = hasher.ComputeHash(lastOutputBytes.Concat(localEntropy).ToArray())
.Concat(hasher.ComputeHash(outputValueBytes.Concat(localEntropy).ToArray()))
.Concat(hasher.ComputeHash(signatureBytes.Concat(localEntropy).ToArray()))
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy.", result.Length);
return(result);
}
protected override async Task <byte[]> GetInternalEntropyAsync(EntropyPriority priority)
{
// http://qrng.anu.edu.au/index.php
Log.Trace("Beginning to gather entropy.");
// Fetch data.
// This always returns 1024 bytes!!
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("https://qrng.anu.edu.au/RawHex.php");
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "qrng.anu.edu.au-RawHex.php.html"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// Locate some pretty clear boundaries around the random numbers returned.
var startIdxString = "1024 bytes of randomness in hexadecimal form";
var startIdx = response.IndexOf(startIdxString, StringComparison.OrdinalIgnoreCase);
if (startIdx == -1)
{
Log.Error("Cannot locate start string in html of anu result: source will return nothing. Looking for '{0}', actual result in next message.", startIdxString);
Log.Error(response);
return(null);
}
var startIdxString2 = "<td>";
startIdx = response.IndexOf(startIdxString2, startIdx, StringComparison.OrdinalIgnoreCase) + startIdxString2.Length;
if (startIdx == -1)
{
Log.Error("Cannot locate start string in html of anu result: source will return nothing. Looking for '{0}', actual result in next message.", startIdxString2);
Log.Error(response);
return(null);
}
var endIdxString = "</td>";
var endIdx = response.IndexOf(endIdxString, startIdx, StringComparison.OrdinalIgnoreCase);
if (endIdx == -1)
{
Log.Error("Cannot locate end string in html of anu result: source will return nothing. Looking for '{0}', actual result in next message.", endIdxString);
Log.Error(response);
return(null);
}
Log.Trace("Parsed beginning and end of useful entropy.");
// Trim and parse.
var randomString = response.Substring(startIdx, endIdx - startIdx).Trim();
var randomBytes = randomString.ParseFromHexString()
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", randomBytes.Length);
return(randomBytes);
}
private async Task <byte[]> GetApiEntropyAsync(EntropyPriority priority)
{
// http://www.random.org/
// https://api.random.org/json-rpc/2/introduction
// https://api.random.org/json-rpc/2/basic
// https://api.random.org/json-rpc/2/request-builder
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("https://api.random.org/json-rpc/2/invoke");
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
var requestBody = "{\"jsonrpc\":\"2.0\",\"method\":\"generateBlobs\",\"params\":{\"apiKey\":\"" + _ApiKey.ToString("D") + "\",\"n\":1,\"size\":" + (_BytesPerRequest * 8) + ",\"format\":\"base64\"},\"id\":1}";
try
{
response = await hc.PostStringAsync(apiUri, requestBody, "application/json");
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to POST to {0} with body {1}", apiUri, requestBody);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "api.random.org.html"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// To avoid using dynamic or a Json library, we do hacky string parsing!
// Check for error.
if (response.IndexOf("\"error\":") != -1)
{
Log.Error("Random.org API returned error result. Full result in next message.");
Log.Error(response);
return(null);
}
int dataIdx = response.IndexOf("\"data\":[\"");
if (dataIdx == -1)
{
Log.Error("Cannot locate random result in random.org API result: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
dataIdx = dataIdx + "\"data\":[\"".Length;
int endIdx = response.IndexOf("\"]", dataIdx);
if (endIdx == -1)
{
Log.Error("Cannot locate end of random result in random.org API result: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
Log.Trace("Parsed Json result.");
// Trim and parse.
var randomString = response.Substring(dataIdx, endIdx - dataIdx).Trim();
var randomBytes = Convert.FromBase64String(randomString)
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", randomBytes.Length);
return(randomBytes);
}
protected override async Task <byte[]> GetInternalEntropyAsync(EntropyPriority priority)
{
// http://www.fourmilab.ch/hotbits/
Log.Trace("Beginning to gather entropy.");
string pseudoSource, apiKey;
if (String.IsNullOrWhiteSpace(_ApiKey))
{
pseudoSource = "&pseudo=pseudo";
apiKey = "&apikey=";
}
else
{
pseudoSource = "";
apiKey = "&apikey=" + _ApiKey;
}
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("https://www.fourmilab.ch/cgi-bin/Hotbits.api?nbytes=" + _BytesPerRequest + "&fmt=hex&npass=1&lpass=8&pwtype=3" + apiKey + pseudoSource);
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of html in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "hotbits.html"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// Locate some pretty clear boundaries around the random numbers returned.
var startIdx = response.IndexOf("<pre>", StringComparison.OrdinalIgnoreCase) + "<pre>".Length;
if (startIdx == -1)
{
Log.Error("Cannot locate start string in html of hotbits result: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
var endIdx = response.IndexOf("</pre>", startIdx, StringComparison.OrdinalIgnoreCase);
if (endIdx == -1)
{
Log.Error("Cannot locate end string in html of hotbits result: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
var randomString = response.Substring(startIdx, endIdx - startIdx).Trim().Replace("\r", "").Replace("\n", "");
Log.Trace("Parsed beginning and end of useful entropy.");
var randomBytes = randomString.ParseFromHexString()
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", randomBytes.Length);
return(randomBytes);
}
protected override async Task <byte[]> GetInternalEntropyAsync(EntropyPriority priority)
{
// http://qrng.ethz.ch/http_api/
Log.Trace("Beginning to gather entropy.");
// Fetch data.
var response = "";
var sw = Stopwatch.StartNew();
if (!_UseDiskSourceForUnitTests)
{
var apiUri = new Uri("http://qrng.ethz.ch/api/randint?min=0&max=255&size=" + _BytesPerRequest);
var hc = HttpClientHelpers.Create(userAgent: _UserAgent);
try
{
response = await hc.GetStringAsync(apiUri);
}
catch (Exception ex)
{
Log.Warn(ex, "Unable to GET from {0}", apiUri);
return(null);
}
Log.Trace("Read {0:N0} characters of json in {1:N2}ms.", response.Length, sw.Elapsed.TotalMilliseconds);
}
else
{
using (var stream = File.OpenRead(HttpClientHelpers._BasePathToUnitTestData + "qrng.ethz.ch-randint.txt"))
{
response = await new StreamReader(stream).ReadToEndAsync();
}
}
sw.Stop();
// To avoid using dynamic or a Json library, we do hacky string parsing!
// Check for valid result.
if (response.IndexOf("\"result\":") == -1)
{
Log.Error("qrng.ethz.ch returned unknown result. Full result in next message.");
Log.Error(response);
return(null);
}
int dataIdx = response.IndexOf("[");
if (dataIdx == -1)
{
Log.Error("Cannot locate random result in qrng.ethz.ch response: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
dataIdx = dataIdx + 1;
int endIdx = response.IndexOf("]", dataIdx);
if (endIdx == -1)
{
Log.Error("Cannot locate end of random result in qrng.ethz.ch response: source will return nothing. Actual result in next message.");
Log.Error(response);
return(null);
}
Log.Trace("Parsed Json result.");
// Trim and parse.
var randomInts = response.Substring(dataIdx, endIdx - dataIdx).Trim();
var numbersAndOtherJunk = randomInts.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var result = numbersAndOtherJunk
.Select(x => (x ?? "").Trim())
.Where(x => x.All(Char.IsDigit)) // Remove non-numeric junk.
.Select(x => Byte.Parse(x)) // Parse to byte.
.Concat(BitConverter.GetBytes(unchecked ((uint)sw.Elapsed.Ticks))) // Don't forget to include network timing!
.ToArray();
Log.Trace("Read {0:N0} bytes of entropy (including 4 bytes of timing info).", result.Length);
return(result);
}