/// <summary>
/// Verify that we can create an HTTPS traffic manager rule that pre-warms items.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsCacheWarming(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that warming actually works by spinning up a [vegomatic]
// based service to accept the traffic.
//
// We'll do this by specifying warm and cold URIs that both enable
// caching. We'll specify the warm URI as a warm target but not
// the cold URI. Then we'll publish the rule and wait for a bit
// to allow it to stablize and for the [neon-proxy-cache] to
// load the warm URI.
//
// Finally, we'll verify that this worked by fetching both URIs.
// The warm URI should indicate that it came from the cache and
// the cold URI should not be cached.
var manager = hive.GetReachableManager();
var guid = Guid.NewGuid().ToString("D"); // Avoid conflicts with previous test runs
var expireSeconds = 60;
var warmUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/warm?body=text:warm&Expires={expireSeconds}");
var coldUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/cold?body=text:cold&Expires={expireSeconds}");
manager.Connect();
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1,
};
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
rule.Cache.WarmTargets.Add(
new TrafficWarmTarget()
{
UpdateSeconds = 1.0,
Uri = warmUri.ToString()
});
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = testHostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Wait a bit longer to ensure that the cache has had a chance to
// warm the URI.
await Task.Delay(TimeSpan.FromSeconds(5));
// Query for the warm and cold URIs and verify that the warm item was a
// cache hit and the cold item was not.
var warmResponse = await client.GetAsync(warmUri.PathAndQuery);
var warmBody = (await warmResponse.Content.ReadAsStringAsync()).Trim();
var coldResponse = await client.GetAsync(coldUri.PathAndQuery);
var coldBody = (await coldResponse.Content.ReadAsStringAsync()).Trim();
Assert.Equal(HttpStatusCode.OK, warmResponse.StatusCode);
Assert.Equal("warm", warmBody);
Assert.True(CacheHit(warmResponse));
Assert.Equal(HttpStatusCode.OK, coldResponse.StatusCode);
Assert.Equal("cold", coldBody);
Assert.False(CacheHit(coldResponse));
}
}
/// <summary>
/// Verify that we can create an TCP traffic manager rule for a
/// site on the public port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestTcpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an TCP traffic manager rule for a
// site on the public port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
var hostname = manager.PrivateAddress.ToString();
manager.Connect();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficTcpRule()
{
Name = "vegomatic",
CheckSeconds = 1,
};
rule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficTcpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
var uniqueResponses = new HashSet <string>();
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note that we're going to pass a new set of URLs to avoid having
// any responses cached so we'll end up seeing all of the IDs.
//
// Note also that we need to perform these requests in parallel
// to try to force Varnish to establish more than one connection
// to the [vegomatic] service. If we don't do this, Varnish will
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
Assert.Equal(2, uniqueResponses.Count);
}
}
/// <summary>
/// Verify that we can create HTTPS traffic manager rules for a
/// site on the proxy port using a specific hostname and various
/// path prefixes and then verify that that the traffic manager actually
/// works by spinning up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name prefix (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsPrefix(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var manager = hive.GetReachableManager();
var hostname = testHostname;
manager.Connect();
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = hostname;
// Create the traffic manager rules, one without a path prefix and
// some others, some with intersecting prefixes so we can verify
// that the longest prefixes are matched first.
//
// Each rule's backend will be routed to a service whose name
// will be constructed from [testName] plus the prefix with the
// slashes replaced with dashes. Each service will be configured
// to return its name.
var prefixes = new PrefixInfo[]
{
new PrefixInfo("/", $"{serviceName}"),
new PrefixInfo("/foo/", $"{serviceName}-foo"),
new PrefixInfo("/foo/bar/", $"{serviceName}-foo-bar"),
new PrefixInfo("/foobar/", $"{serviceName}-foobar"),
new PrefixInfo("/bar/", $"{serviceName}-bar")
};
// Spin the services up first in parallel (for speed). Each of
// these service will respond to requests with its service name.
var tasks = new List <Task>();
foreach (var prefix in prefixes)
{
tasks.Add(Task.Run(
() =>
{
manager.SudoCommand($"docker service create --name {prefix.ServiceName} --network {network} --replicas 1 {vegomaticImage} test-server server-id={prefix.ServiceName}").EnsureSuccess();
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
// Create the traffic manager rules.
foreach (var prefix in prefixes)
{
var rule = new TrafficHttpRule()
{
Name = prefix.ServiceName,
CheckExpect = "status 200",
CheckSeconds = 1,
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
var frontend = new TrafficHttpFrontend()
{
Host = hostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
};
if (!string.IsNullOrEmpty(prefix.Path))
{
frontend.PathPrefix = prefix.Path;
}
rule.Frontends.Add(frontend);
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = prefix.ServiceName,
Port = 80
});
trafficManager.SetRule(rule, deferUpdate: true);
}
trafficManager.Update();
// Wait for all of the services to report being ready.
await NeonHelper.WaitForAsync(
async() =>
{
foreach (var prefix in prefixes)
{
try
{
var response = await client.GetAsync(prefix.Path);
response.EnsureSuccessStatusCode();
}
catch
{
return(false);
}
}
return(true);
},
timeout : TimeSpan.FromSeconds(60),
pollTime : TimeSpan.FromSeconds(1));
// Give everything a chance to stablize.
await Task.Delay(TimeSpan.FromSeconds(5));
// Now verify that prefix rules route to the correct backend service.
foreach (var prefix in prefixes)
{
var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60");
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(prefix.ServiceName, body.Trim());
if (useCache)
{
// Verify that the request routed through Varnish.
Assert.True(ViaVarnish(response));
// This is the first request using the globally unique [testName]
// so it should not be a cache hit.
Assert.False(CacheHit(response));
}
}
// If caching is enabled, perform the requests again to ensure that
// we see cache hits.
if (useCache)
{
foreach (var prefix in prefixes)
{
// Request the item again and verify that it was a cache hit.
var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60");
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(prefix.ServiceName, body.Trim());
Assert.True(CacheHit(response));
}
}
}
}
/// <summary>
/// Verify that we can create an HTTPS traffic manager rule for a
/// site on the proxy port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces).</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsRule(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic")
{
// Verify that we can create an HTTPS traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
manager.Connect();
// We need the test hostname to point to the manager's private address
// so we can submit HTTPS requests there.
hiveFixture.LocalMachineHosts.AddHostAddress(testHostname, manager.PrivateAddress.ToString());
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
client.BaseAddress = new Uri($"https://{testHostname}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = testHostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
//
// We're going to use a different URL for each request so that we
// won't see any cache hits.
var uniqueResponses = new HashSet <string>();
var viaVarnish = false;
var cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
// Repeat the test if caching is enabled with the same URLs as last time and verify that
// we see cache hits this time.
if (useCache)
{
viaVarnish = false;
cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.True(viaVarnish);
Assert.True(cacheHit);
}
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note also that we need to perform these requests in parallel
// to try to force HAProxy to establish more than one connection
// to the [vegomatic] service. If we don't do this, HAProxy may
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 {serviceName}").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
viaVarnish = false;
cacheHit = false;
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
lock (uniqueResponses)
{
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
Assert.Equal(2, uniqueResponses.Count);
}
}
/// <summary>
/// Generates a traffic manager rule for each <see cref="Redirection"/> passed that
/// will redirect from one URI to another.
/// </summary>
/// <param name="trafficManager">The target traffic manager.</param>
/// <param name="testName">Used to name the traffic manager rules.</param>
/// <param name="singleRule">
/// Pass <c>true</c> to test a single rule with all of the redirections or
/// <c>false</c> to test with one redirection per rule.
/// </param>
/// <param name="redirections">The redirections.</param>
private async Task TestRedirect(TrafficManager trafficManager, string testName, bool singleRule, params Redirection[] redirections)
{
var manager = hive.GetReachableManager();
// We need local DNS mappings for each of the URI hosts to target a hive node.
var hosts = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var redirect in redirections)
{
if (!hosts.Contains(redirect.FromUri.Host))
{
hosts.Add(redirect.FromUri.Host);
}
if (!hosts.Contains(redirect.ToUri.Host))
{
hosts.Add(redirect.ToUri.Host);
}
}
foreach (var host in hosts)
{
hiveFixture.LocalMachineHosts.AddHostAddress(host, manager.PrivateAddress.ToString(), deferCommit: true);
}
hiveFixture.LocalMachineHosts.Commit();
// Generate and upload a self-signed certificate for each redirect host that
// uses HTTPS and upload these to the hive. Each certificate will be named
// the same as the hostname.
var hostToCertificate = new Dictionary <string, TlsCertificate>(StringComparer.InvariantCultureIgnoreCase);
foreach (var redirect in redirections.Where(r => r.FromUri.Scheme == "https"))
{
var host = redirect.FromUri.Host;
if (hostToCertificate.ContainsKey(host))
{
continue;
}
hostToCertificate[host] = TlsCertificate.CreateSelfSigned(host);
}
foreach (var item in hostToCertificate)
{
hive.Certificate.Set(item.Key, item.Value);
}
// Create the traffic manager rule(s).
if (singleRule)
{
var rule = new TrafficHttpRule()
{
Name = testName,
};
foreach (var redirect in redirections)
{
var frontend = new TrafficHttpFrontend()
{
Host = redirect.FromUri.Host,
ProxyPort = redirect.FromUri.Port,
RedirectTo = redirect.ToUri
};
if (redirect.FromUri.Scheme == "https")
{
frontend.CertName = redirect.FromUri.Host;
}
rule.Frontends.Add(frontend);
}
trafficManager.SetRule(rule);
}
else
{
var redirectIndex = 0;
foreach (var redirect in redirections)
{
var rule = new TrafficHttpRule()
{
Name = $"{testName}-{redirectIndex}",
};
var frontend = new TrafficHttpFrontend()
{
Host = redirect.FromUri.Host,
ProxyPort = redirect.FromUri.Port,
RedirectTo = redirect.ToUri
};
if (redirect.FromUri.Scheme == "https")
{
frontend.CertName = redirect.FromUri.Host;
}
rule.Frontends.Add(frontend);
trafficManager.SetRule(rule);
redirectIndex++;
}
}
// Give the new rules some time to deploy.
await Task.Delay(TimeSpan.FromSeconds(5));
// Now all we need to do is hit all of the redirect [FromUri]s
// and verify that we get redirects to the corresponding
// [ToUri]s.
// Allow self-signed certificates and disable client-side automatic redirect handling
// so we'll be able to see the redirect responses.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator,
AllowAutoRedirect = false // We need to see the redirects
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
foreach (var redirect in redirections)
{
var response = await client.GetAsync(redirect.FromUri);
Assert.Equal(HttpStatusCode.Redirect, response.StatusCode);
Assert.True(response.Headers.TryGetValues("Location", out var locations));
Assert.Equal(redirect.ToUri.ToString(), locations.Single());
}
}
}
/// <summary>
/// Verify that we can create an HTTP traffic manager rule for a
/// site on the proxy port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="hostnames">The hostnames to be used for .</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpMultipleFrontends(string testName, string[] hostnames, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic")
{
Covenant.Requires <ArgumentNullException>(hostnames != null && hostnames.Length > 0);
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
var proxyUri = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
manager.Connect();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1,
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
foreach (var hostname in hostnames)
{
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = hostname,
ProxyPort = proxyPort
});
}
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(proxyUri, hostnames.First());
// Query the service several times for each hostname to verify that we
// get a response and also that all of the responses are the same
// (because we have only a single [vegomatic] instance returning its UUID).
//
// We're going to use a different URL for each request so that we
// won't see any cache hits.
foreach (var hostname in hostnames)
{
var uniqueResponses = new HashSet <string>();
var viaVarnish = false;
var cacheHit = false;
client.BaseAddress = proxyUri;
client.DefaultRequestHeaders.Host = hostname;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/{hostname}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
// Repeat the test if caching is enabled with the same URLs as last time and verify that
// we see cache hits this time.
if (useCache)
{
viaVarnish = false;
cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/{hostname}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.True(viaVarnish);
Assert.True(cacheHit);
}
}
// Spin up a second replica and repeat the query test for each hostname
// to verify that we see two unique responses.
//
// Note that we're going to pass a new set of URLs to avoid having
// any responses cached so we'll end up seeing all of the IDs.
//
// Note also that we need to perform these requests in parallel
// to try to force Varnish to establish more than one connection
// to the [vegomatic] service. If we don't do this, Varnish will
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
await WaitUntilReadyAsync(proxyUri, hostnames.First());
foreach (var hostname in hostnames)
{
var uniqueResponses = new HashSet <string>();
var viaVarnish = false;
var cacheHit = false;
var tasks = new List <Task>();
var uris = new List <string>();
client.BaseAddress = proxyUri;
client.DefaultRequestHeaders.Host = hostname;
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
lock (uniqueResponses)
{
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
Assert.Equal(2, uniqueResponses.Count);
}
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
TrafficManager trafficManager = null;
bool isPublic = false;
string name = null;
string ruleName = null;
bool deferUpdate = false;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out name))
{
throw new ArgumentException($"[name] module argument is required.");
}
switch (name)
{
case "private":
trafficManager = HiveHelper.Hive.PrivateTraffic;
isPublic = false;
break;
case "public":
trafficManager = HiveHelper.Hive.PublicTraffic;
isPublic = true;
break;
default:
throw new ArgumentException($"[name={name}] is not a one of the valid traffic manager names: [private] or [public].");
}
if (state == "present" || state == "absent")
{
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule_name]");
if (!context.Arguments.TryGetValue <string>("rule_name", out ruleName))
{
throw new ArgumentException($"[rule_name] module argument is required.");
}
if (!HiveDefinition.IsValidName(ruleName))
{
throw new ArgumentException($"[rule_name={ruleName}] is not a valid traffic manager rule name.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [defer_update]");
if (!context.Arguments.TryGetValue <bool>("defer_update", out deferUpdate))
{
deferUpdate = false;
}
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Check if rule [{ruleName}] exists.");
if (trafficManager.GetRule(ruleName) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does exist.");
context.WriteLine(AnsibleVerbosity.Info, $"Deleting rule [{ruleName}].");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be deleted when CHECK-MODE is disabled.");
}
else
{
trafficManager.RemoveRule(ruleName, deferUpdate: deferUpdate);
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] deleted.");
context.Changed = true;
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does not exist.");
}
break;
case "present":
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule]");
if (!context.Arguments.TryGetValue <JObject>("rule", out var routeObject))
{
throw new ArgumentException($"[rule] module argument is required when [state={state}].");
}
var ruleText = routeObject.ToString();
context.WriteLine(AnsibleVerbosity.Trace, "Parsing rule");
var newRule = TrafficRule.Parse(ruleText, strict: true);
context.WriteLine(AnsibleVerbosity.Trace, "Rule parsed successfully");
// Use the name argument if the deserialized rule doesn't
// have a name. This will make it easier on operators because
// they won't need to specify the name twice.
if (string.IsNullOrWhiteSpace(newRule.Name))
{
newRule.Name = ruleName;
}
// Ensure that the name passed as an argument and the
// name within the rule definition match.
if (!string.Equals(ruleName, newRule.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new ArgumentException($"The [rule_name={ruleName}] argument and the rule's [{nameof(TrafficRule.Name)}={newRule.Name}] property are not the same.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule name matched.");
// Validate the rule.
context.WriteLine(AnsibleVerbosity.Trace, "Validating rule.");
var proxySettings = trafficManager.GetSettings();
var validationContext = new TrafficValidationContext(name, proxySettings);
// $hack(jeff.lill):
//
// This ensures that [proxySettings.Resolvers] is initialized with
// the built-in Docker DNS resolver.
proxySettings.Validate(validationContext);
// Load the TLS certificates into the validation context so we'll
// be able to verify that any referenced certificates mactually exist.
// $todo(jeff.lill):
//
// This code assumes that the operator is currently logged in with
// root Vault privileges. We'll have to do something else for
// non-root logins.
//
// One idea might be to save two versions of the certificates.
// The primary certificate with private key in Vault and then
// just the public certificate in Consul and then load just
// the public ones here.
//
// A good time to make this change might be when we convert to
// use the .NET X.509 certificate implementation.
if (!context.Login.HasVaultRootCredentials)
{
throw new ArgumentException("Access Denied: Root Vault credentials are required.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Reading hive certificates.");
using (var vault = HiveHelper.OpenVault(Program.HiveLogin.VaultCredentials.RootToken))
{
// List the certificate key/names and then fetch each one
// to capture details like the expiration date and covered
// hostnames.
foreach (var certName in vault.ListAsync("neon-secret/cert").Result)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Reading: {certName}");
var certificate = vault.ReadJsonAsync <TlsCertificate>(HiveHelper.GetVaultCertificateKey(certName)).Result;
validationContext.Certificates.Add(certName, certificate);
}
}
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Certificates.Count}] hive certificates downloaded.");
// Actually perform the rule validation.
newRule.Validate(validationContext);
if (validationContext.HasErrors)
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Errors.Count}] Route validation errors.");
foreach (var error in validationContext.Errors)
{
context.WriteLine(AnsibleVerbosity.Important, error);
context.WriteErrorLine(error);
}
context.Failed = true;
return;
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule is valid.");
// Try reading any existing rule with this name and then determine
// whether the two versions of the rule are actually different.
context.WriteLine(AnsibleVerbosity.Trace, $"Looking for existing rule [{ruleName}]");
var existingRule = trafficManager.GetRule(ruleName);
var changed = false;
if (existingRule != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule exists: checking for differences.");
// Normalize the new and existing rules so the JSON text comparision
// will work properly.
newRule.Normalize(isPublic);
existingRule.Normalize(isPublic);
changed = !NeonHelper.JsonEquals(newRule, existingRule);
if (changed)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rules are different.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Rules are the same. No need to update.");
}
}
else
{
changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [name={ruleName}] does not exist.");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Writing rule [{ruleName}].");
trafficManager.SetRule(newRule);
context.WriteLine(AnsibleVerbosity.Info, $"Rule updated.");
context.Changed = !context.CheckMode;
}
}
break;
case "update":
trafficManager.Update();
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Update signalled.");
break;
case "purge":
var purgeItems = context.ParseStringArray("purge_list");
var purgeCaseSensitive = context.ParseBool("purge_case_sensitive");
if (!purgeCaseSensitive.HasValue)
{
purgeCaseSensitive = false;
}
if (purgeItems.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"[purge_list] is missing or empty.");
break;
}
trafficManager.Purge(purgeItems.ToArray(), caseSensitive: purgeCaseSensitive.Value);
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Purge request submitted.");
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [update].");
}
}
