/// <summary>
/// Constructor.
/// </summary>
/// <param name="rule">The associated traffic manager rule.</param>
/// <param name="frontend">The frontend that generated this mapping.</param>
/// <param name="backendName">The backend name.</param>
public HostPathMapping(TrafficHttpRule rule, TrafficHttpFrontend frontend, string backendName)
{
Covenant.Requires <ArgumentNullException>(rule != null);
Covenant.Requires <ArgumentNullException>(frontend != null);
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(backendName));
this.Rule = rule;
this.Frontend = frontend;
this.BackendName = backendName;
}
/// <summary>
/// Adds the steps to configure the stateful Elasticsearch instances used to persist the log data.
/// </summary>
/// <param name="steps">The configuration step list.</param>
private void AddElasticsearchSteps(ConfigStepList steps)
{
var esNodes = new List <SshProxy <NodeDefinition> >();
foreach (var nodeDefinition in hive.Definition.Nodes.Where(n => n.Labels.LogEsData))
{
esNodes.Add(hive.GetNode(nodeDefinition.Name));
}
// Determine number of manager nodes and the quorum size.
// Note that we'll deploy an odd number of managers.
var managerCount = Math.Min(esNodes.Count, 5); // We shouldn't ever need more than 5 managers
if (!NeonHelper.IsOdd(managerCount))
{
managerCount--;
}
var quorumCount = (managerCount / 2) + 1;
// Sort the nodes by name and then separate the manager and
// worker nodes (managers will be assigned to nodes that appear
// first in the list).
var managerEsNodes = new List <SshProxy <NodeDefinition> >();
var normalEsNodes = new List <SshProxy <NodeDefinition> >();
esNodes = esNodes.OrderBy(n => n.Name).ToList();
foreach (var esNode in esNodes)
{
if (managerEsNodes.Count < managerCount)
{
managerEsNodes.Add(esNode);
}
else
{
normalEsNodes.Add(esNode);
}
}
// Figure out how much RAM to allocate to the Elasticsearch Docker containers
// as well as Java heap within. The guidance is to set the heap size to half
// the container RAM up to a maximum of 31GB.
var esContainerRam = hive.Definition.Log.EsMemoryBytes;
var esHeapBytes = Math.Min(esContainerRam / 2, 31L * NeonHelper.Giga);
// We're going to use explicit docker commands to deploy the Elasticsearch cluster
// log storage containers.
//
// We're mounting three volumes to the container:
//
// /etc/neon/host-env - Generic host specific environment variables
// /etc/neon/env-log-esdata - Elasticsearch node host specific environment variables
// neon-log-esdata-# - Persistent Elasticsearch data folder
var esBootstrapNodes = new StringBuilder();
foreach (var esMasterNode in managerEsNodes)
{
esBootstrapNodes.AppendWithSeparator($"{esMasterNode.PrivateAddress}:{HiveHostPorts.LogEsDataTcp}", ",");
}
// Create a data volume for each Elasticsearch node and then start the node container.
for (int i = 0; i < esNodes.Count; i++)
{
var esNode = esNodes[i];
var containerName = $"neon-log-esdata";
var isMaster = managerEsNodes.Contains(esNode) ? "true" : "false";
var volumeCommand = CommandStep.CreateSudo(esNode.Name, "docker-volume-create", containerName);
steps.Add(volumeCommand);
ServiceHelper.AddContainerStartSteps(hive, steps, esNode, containerName, hive.Definition.Image.Elasticsearch,
new CommandBundle(
"docker run",
"--name", containerName,
"--detach",
"--restart", "always",
"--volume", "/etc/neon/host-env:/etc/neon/host-env:ro",
"--volume", $"{containerName}:/mnt/esdata",
"--env", $"ELASTICSEARCH_CLUSTER={hive.Definition.Datacenter}.{hive.Definition.Name}.neon-log-esdata",
"--env", $"ELASTICSEARCH_NODE_MASTER={isMaster}",
"--env", $"ELASTICSEARCH_NODE_DATA=true",
"--env", $"ELASTICSEARCH_NODE_COUNT={esNodes.Count}",
"--env", $"ELASTICSEARCH_HTTP_PORT={HiveHostPorts.LogEsDataHttp}",
"--env", $"ELASTICSEARCH_TCP_PORT={HiveHostPorts.LogEsDataTcp}",
"--env", $"ELASTICSEARCH_QUORUM={quorumCount}",
"--env", $"ELASTICSEARCH_BOOTSTRAP_NODES={esBootstrapNodes}",
"--env", $"ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap",
"--memory", $"{esContainerRam / NeonHelper.Mega}M",
"--memory-reservation", $"{esContainerRam / NeonHelper.Mega}M",
"--memory-swappiness", "0",
"--network", "host",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
}
// Configure a private hive proxy route to the Elasticsearch nodes.
steps.Add(ActionStep.Create(hive.FirstManager.Name, "setup/elasticsearch-lbrule",
node =>
{
var rule = new TrafficHttpRule()
{
Name = "neon-log-esdata",
System = true,
Log = false, // This is important: we don't want to SPAM the log database with its own traffic.
Resolver = null
};
rule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHttpLogEsData
});
foreach (var esNode in esNodes)
{
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = esNode.Metadata.PrivateAddress.ToString(),
Port = HiveHostPorts.LogEsDataHttp
});
}
hive.PrivateTraffic.SetRule(rule);
}));
// Wait for the elasticsearch cluster to become ready and then save the
// [logstash-*] template. We need to do this before [neon-log-collector]
// is started so we'll be sure that no indexes will be created before
// we have a chance to persist the pattern.
//
// This works because [neon-log-collector] is the main service responsible
// for persisting events to this index.
steps.Add(ActionStep.Create(hive.FirstManager.Name, operationName: null,
node =>
{
node.Status = "wait for elasticsearch cluster";
using (var jsonClient = new JsonClient())
{
var baseLogEsDataUri = hive.Definition.LogEsDataUri;
var timeout = TimeSpan.FromMinutes(5);
var timeoutTime = DateTime.UtcNow + timeout;
var esNodeCount = hive.Definition.Nodes.Count(n => n.Labels.LogEsData);
// Wait for the Elasticsearch cluster.
jsonClient.UnsafeRetryPolicy = NoRetryPolicy.Instance;
while (true)
{
try
{
var response = jsonClient.GetUnsafeAsync($"{baseLogEsDataUri}/_cluster/health").Result;
if (response.IsSuccess)
{
var clusterStatus = response.AsDynamic();
var status = (string)(clusterStatus.status);
status = status.ToUpperInvariant();
node.Status = $"wait for [neon-log-esdata] cluster: [status={status}] [{clusterStatus.number_of_nodes}/{esNodeCount} nodes ready])";
// $todo(jeff.lill):
//
// We're accepting YELLOW status here due to this issue:
//
// https://github.com/jefflill/NeonForge/issues/257
if ((status == "GREEN" || status == "YELLOW") && clusterStatus.number_of_nodes == esNodeCount)
{
node.Status = "elasticsearch cluster is ready";
break;
}
}
}
catch
{
if (DateTime.UtcNow >= timeoutTime)
{
node.Fault($"[neon-log-esdata] cluster not ready after waiting [{timeout}].");
return;
}
}
Thread.Sleep(TimeSpan.FromSeconds(1));
}
// Save the [logstash-*] template pattern.
var templatePattern = ResourceFiles.Root.GetFolder("Elasticsearch").GetFile("logstash-template.json").Contents;
jsonClient.PutAsync($"{baseLogEsDataUri}/_template/logstash-*", templatePattern).Wait();
}
}));
}
/// <summary>
/// Configures the Kibana dashboard.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void ConfigureKibana(SshProxy <NodeDefinition> firstManager)
{
if (!hive.Definition.Log.Enabled)
{
return;
}
firstManager.InvokeIdempotentAction("setup/log-kibana",
() =>
{
using (var jsonClient = new JsonClient())
{
var baseLogEsDataUri = hive.Definition.LogEsDataUri;
var baseKibanaUri = $"http://{firstManager.PrivateAddress}:{HiveHostPorts.Kibana}";
var timeout = TimeSpan.FromMinutes(5);
var retry = new LinearRetryPolicy(TransientDetector.Http, maxAttempts: 30, retryInterval: TimeSpan.FromSeconds(2));
// The Kibana API calls below require the [kbn-xsrf] header.
jsonClient.DefaultRequestHeaders.Add("kbn-xsrf", "true");
// Ensure that Kibana is ready before we submit any API requests.
firstManager.Status = "wait for kibana";
retry.InvokeAsync(
async() =>
{
var response = await jsonClient.GetAsync <dynamic>($"{baseKibanaUri}/api/status");
if (response.status.overall.state != "green")
{
throw new TransientException($"Kibana [state={response.status.overall.state}]");
}
}).Wait();
// Add the index pattern to Kibana.
firstManager.Status = "configure kibana index pattern";
retry.InvokeAsync(
async() =>
{
dynamic indexPattern = new ExpandoObject();
dynamic attributes = new ExpandoObject();
attributes.title = "logstash-*";
attributes.timeFieldName = "@timestamp";
indexPattern.attributes = attributes;
await jsonClient.PostAsync($"{baseKibanaUri}/api/saved_objects/index-pattern/logstash-*?overwrite=true", indexPattern);
}).Wait();
// Now we need to save a Kibana config document so that [logstash-*] will be
// the default index and the timestamp will be displayed as UTC and have a
// more useful terse format.
firstManager.Status = "configure kibana defaults";
retry.InvokeAsync(
async() =>
{
dynamic setting = new ExpandoObject();
setting.value = "logstash-*";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/defaultIndex", setting);
setting.value = "HH:mm:ss.SSS MM-DD-YYYY";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat", setting);
setting.value = "UTC";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat:tz", setting);
}).Wait();
// Set the Kibana traffic manager rule.
firstManager.Status = "kibana traffic manager rule";
var rule = new TrafficHttpRule()
{
Name = "neon-log-kibana",
System = true,
Log = true,
Resolver = null
};
rule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateKibanaDashboard
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = "neon-log-kibana",
Port = NetworkPorts.Kibana
});
hive.PrivateTraffic.SetRule(rule);
firstManager.Status = string.Empty;
}
});
}
/// <summary>
/// Verify that we can create an HTTPS traffic manager rule that pre-warms items.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsCacheWarming(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that warming actually works by spinning up a [vegomatic]
// based service to accept the traffic.
//
// We'll do this by specifying warm and cold URIs that both enable
// caching. We'll specify the warm URI as a warm target but not
// the cold URI. Then we'll publish the rule and wait for a bit
// to allow it to stablize and for the [neon-proxy-cache] to
// load the warm URI.
//
// Finally, we'll verify that this worked by fetching both URIs.
// The warm URI should indicate that it came from the cache and
// the cold URI should not be cached.
var manager = hive.GetReachableManager();
var guid = Guid.NewGuid().ToString("D"); // Avoid conflicts with previous test runs
var expireSeconds = 60;
var warmUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/warm?body=text:warm&Expires={expireSeconds}");
var coldUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/cold?body=text:cold&Expires={expireSeconds}");
manager.Connect();
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1,
};
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
rule.Cache.WarmTargets.Add(
new TrafficWarmTarget()
{
UpdateSeconds = 1.0,
Uri = warmUri.ToString()
});
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = testHostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Wait a bit longer to ensure that the cache has had a chance to
// warm the URI.
await Task.Delay(TimeSpan.FromSeconds(5));
// Query for the warm and cold URIs and verify that the warm item was a
// cache hit and the cold item was not.
var warmResponse = await client.GetAsync(warmUri.PathAndQuery);
var warmBody = (await warmResponse.Content.ReadAsStringAsync()).Trim();
var coldResponse = await client.GetAsync(coldUri.PathAndQuery);
var coldBody = (await coldResponse.Content.ReadAsStringAsync()).Trim();
Assert.Equal(HttpStatusCode.OK, warmResponse.StatusCode);
Assert.Equal("warm", warmBody);
Assert.True(CacheHit(warmResponse));
Assert.Equal(HttpStatusCode.OK, coldResponse.StatusCode);
Assert.Equal("cold", coldBody);
Assert.False(CacheHit(coldResponse));
}
}
/// <summary>
/// Verify that we can create HTTPS traffic manager rules for a
/// site on the proxy port using a specific hostname and various
/// path prefixes and then verify that that the traffic manager actually
/// works by spinning up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name prefix (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsPrefix(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var manager = hive.GetReachableManager();
var hostname = testHostname;
manager.Connect();
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = hostname;
// Create the traffic manager rules, one without a path prefix and
// some others, some with intersecting prefixes so we can verify
// that the longest prefixes are matched first.
//
// Each rule's backend will be routed to a service whose name
// will be constructed from [testName] plus the prefix with the
// slashes replaced with dashes. Each service will be configured
// to return its name.
var prefixes = new PrefixInfo[]
{
new PrefixInfo("/", $"{serviceName}"),
new PrefixInfo("/foo/", $"{serviceName}-foo"),
new PrefixInfo("/foo/bar/", $"{serviceName}-foo-bar"),
new PrefixInfo("/foobar/", $"{serviceName}-foobar"),
new PrefixInfo("/bar/", $"{serviceName}-bar")
};
// Spin the services up first in parallel (for speed). Each of
// these service will respond to requests with its service name.
var tasks = new List <Task>();
foreach (var prefix in prefixes)
{
tasks.Add(Task.Run(
() =>
{
manager.SudoCommand($"docker service create --name {prefix.ServiceName} --network {network} --replicas 1 {vegomaticImage} test-server server-id={prefix.ServiceName}").EnsureSuccess();
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
// Create the traffic manager rules.
foreach (var prefix in prefixes)
{
var rule = new TrafficHttpRule()
{
Name = prefix.ServiceName,
CheckExpect = "status 200",
CheckSeconds = 1,
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
var frontend = new TrafficHttpFrontend()
{
Host = hostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
};
if (!string.IsNullOrEmpty(prefix.Path))
{
frontend.PathPrefix = prefix.Path;
}
rule.Frontends.Add(frontend);
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = prefix.ServiceName,
Port = 80
});
trafficManager.SetRule(rule, deferUpdate: true);
}
trafficManager.Update();
// Wait for all of the services to report being ready.
await NeonHelper.WaitForAsync(
async() =>
{
foreach (var prefix in prefixes)
{
try
{
var response = await client.GetAsync(prefix.Path);
response.EnsureSuccessStatusCode();
}
catch
{
return(false);
}
}
return(true);
},
timeout : TimeSpan.FromSeconds(60),
pollTime : TimeSpan.FromSeconds(1));
// Give everything a chance to stablize.
await Task.Delay(TimeSpan.FromSeconds(5));
// Now verify that prefix rules route to the correct backend service.
foreach (var prefix in prefixes)
{
var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60");
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(prefix.ServiceName, body.Trim());
if (useCache)
{
// Verify that the request routed through Varnish.
Assert.True(ViaVarnish(response));
// This is the first request using the globally unique [testName]
// so it should not be a cache hit.
Assert.False(CacheHit(response));
}
}
// If caching is enabled, perform the requests again to ensure that
// we see cache hits.
if (useCache)
{
foreach (var prefix in prefixes)
{
// Request the item again and verify that it was a cache hit.
var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60");
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(prefix.ServiceName, body.Trim());
Assert.True(CacheHit(response));
}
}
}
}
/// <summary>
/// Verify that we can create an HTTPS traffic manager rule for a
/// site on the proxy port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces).</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpsRule(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic")
{
// Verify that we can create an HTTPS traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
manager.Connect();
// We need the test hostname to point to the manager's private address
// so we can submit HTTPS requests there.
hiveFixture.LocalMachineHosts.AddHostAddress(testHostname, manager.PrivateAddress.ToString());
// Allow self-signed certificates.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
client.BaseAddress = new Uri($"https://{testHostname}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Add the test certificate.
hive.Certificate.Set("test-load-balancer", certificate);
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = testHostname,
ProxyPort = proxyPort,
CertName = "test-load-balancer"
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
//
// We're going to use a different URL for each request so that we
// won't see any cache hits.
var uniqueResponses = new HashSet <string>();
var viaVarnish = false;
var cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
// Repeat the test if caching is enabled with the same URLs as last time and verify that
// we see cache hits this time.
if (useCache)
{
viaVarnish = false;
cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.True(viaVarnish);
Assert.True(cacheHit);
}
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note also that we need to perform these requests in parallel
// to try to force HAProxy to establish more than one connection
// to the [vegomatic] service. If we don't do this, HAProxy may
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 {serviceName}").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, testHostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
viaVarnish = false;
cacheHit = false;
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
lock (uniqueResponses)
{
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
Assert.Equal(2, uniqueResponses.Count);
}
}
/// <summary>
/// Generates a traffic manager rule for each <see cref="Redirection"/> passed that
/// will redirect from one URI to another.
/// </summary>
/// <param name="trafficManager">The target traffic manager.</param>
/// <param name="testName">Used to name the traffic manager rules.</param>
/// <param name="singleRule">
/// Pass <c>true</c> to test a single rule with all of the redirections or
/// <c>false</c> to test with one redirection per rule.
/// </param>
/// <param name="redirections">The redirections.</param>
private async Task TestRedirect(TrafficManager trafficManager, string testName, bool singleRule, params Redirection[] redirections)
{
var manager = hive.GetReachableManager();
// We need local DNS mappings for each of the URI hosts to target a hive node.
var hosts = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var redirect in redirections)
{
if (!hosts.Contains(redirect.FromUri.Host))
{
hosts.Add(redirect.FromUri.Host);
}
if (!hosts.Contains(redirect.ToUri.Host))
{
hosts.Add(redirect.ToUri.Host);
}
}
foreach (var host in hosts)
{
hiveFixture.LocalMachineHosts.AddHostAddress(host, manager.PrivateAddress.ToString(), deferCommit: true);
}
hiveFixture.LocalMachineHosts.Commit();
// Generate and upload a self-signed certificate for each redirect host that
// uses HTTPS and upload these to the hive. Each certificate will be named
// the same as the hostname.
var hostToCertificate = new Dictionary <string, TlsCertificate>(StringComparer.InvariantCultureIgnoreCase);
foreach (var redirect in redirections.Where(r => r.FromUri.Scheme == "https"))
{
var host = redirect.FromUri.Host;
if (hostToCertificate.ContainsKey(host))
{
continue;
}
hostToCertificate[host] = TlsCertificate.CreateSelfSigned(host);
}
foreach (var item in hostToCertificate)
{
hive.Certificate.Set(item.Key, item.Value);
}
// Create the traffic manager rule(s).
if (singleRule)
{
var rule = new TrafficHttpRule()
{
Name = testName,
};
foreach (var redirect in redirections)
{
var frontend = new TrafficHttpFrontend()
{
Host = redirect.FromUri.Host,
ProxyPort = redirect.FromUri.Port,
RedirectTo = redirect.ToUri
};
if (redirect.FromUri.Scheme == "https")
{
frontend.CertName = redirect.FromUri.Host;
}
rule.Frontends.Add(frontend);
}
trafficManager.SetRule(rule);
}
else
{
var redirectIndex = 0;
foreach (var redirect in redirections)
{
var rule = new TrafficHttpRule()
{
Name = $"{testName}-{redirectIndex}",
};
var frontend = new TrafficHttpFrontend()
{
Host = redirect.FromUri.Host,
ProxyPort = redirect.FromUri.Port,
RedirectTo = redirect.ToUri
};
if (redirect.FromUri.Scheme == "https")
{
frontend.CertName = redirect.FromUri.Host;
}
rule.Frontends.Add(frontend);
trafficManager.SetRule(rule);
redirectIndex++;
}
}
// Give the new rules some time to deploy.
await Task.Delay(TimeSpan.FromSeconds(5));
// Now all we need to do is hit all of the redirect [FromUri]s
// and verify that we get redirects to the corresponding
// [ToUri]s.
// Allow self-signed certificates and disable client-side automatic redirect handling
// so we'll be able to see the redirect responses.
var handler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator,
AllowAutoRedirect = false // We need to see the redirects
};
using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true))
{
foreach (var redirect in redirections)
{
var response = await client.GetAsync(redirect.FromUri);
Assert.Equal(HttpStatusCode.Redirect, response.StatusCode);
Assert.True(response.Headers.TryGetValues("Location", out var locations));
Assert.Equal(redirect.ToUri.ToString(), locations.Single());
}
}
}
public async Task Cache_Purge()
{
var trafficManager = hive.PublicTraffic;
var network = HiveConst.PublicNetwork;
var proxyPort = 80;
var uuid = Guid.NewGuid().ToString("D"); // Used to avoid cache conflicts from previous test runs.
// We're going to configure a Vegomatic test instance with a
// caching load balancer rule and then fetch several different
// requests thru the cache, verify that the responses were
// cache hits. Then we'll test various purge patterns and
// verify that subsequent requests will be cache misses.
var manager = hive.GetReachableManager();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
// Configure the traffic manager rules (with caching enabled).
for (int serverId = 0; serverId < 2; serverId++)
{
var rule = new TrafficHttpRule()
{
Name = $"vegomatic-{serverId}",
CheckExpect = "status 200",
CheckSeconds = 1,
Cache = new TrafficHttpCache()
{
Enabled = true
}
};
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = $"vegomatic-{serverId}",
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = $"vegomatic-{serverId}",
Port = 80
});
trafficManager.SetRule(rule);
}
// Spin up a two [vegomatic] service instances [vegomatic-0] and [vegomatic-1]
// so that we can ensure that purging one origin server's content doesn't impact
// the other's cached content. The default response expiration time will be
// configured as 300 seconds (5 minutes) so we can test cache purging.
var expireSeconds = 300;
NeonHelper.WaitForParallel(
new Action[]
{
() =>
{
manager.SudoCommand($"docker service create --name vegomatic-0 --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess();
WaitUntilReadyAsync(client.BaseAddress, "vegomatic-0").Wait();
},
() =>
{
manager.SudoCommand($"docker service create --name vegomatic-1 --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess();
WaitUntilReadyAsync(client.BaseAddress, "vegomatic-1").Wait();
}
});
// Submit several requests to preload the cache for both origin servers.
// and verify that the items were cached.
CacheInfo status;
var testUris = new string[]
{
$"/{uuid}/test.htm",
$"/{uuid}/test0.jpg",
$"/{uuid}/test1.jpg",
$"/{uuid}/test2.jpg",
$"/{uuid}/test0.png",
$"/{uuid}/test1.png",
$"/{uuid}/test2.png",
$"/{uuid}/foo/test.htm",
$"/{uuid}/foo/test.jpg",
$"/{uuid}/foo/test.png",
$"/{uuid}/bar/test.htm",
$"/{uuid}/bar/test.jpg",
$"/{uuid}/bar/test.png",
};
for (int serverId = 0; serverId < 2; serverId++)
{
var host = $"vegomatic-{serverId}";
foreach (var uri in testUris)
{
// Load the item into the cache.
status = await GetCachingStatusAsync(client, $"http://{host}:80{uri}");
Assert.True(status.ViaVarnish);
// Ensure that it was actually cached.
status = await GetCachingStatusAsync(client, $"http://{host}:80{uri}");
Assert.True(status.ViaVarnish);
Assert.True(status.CacheHit);
}
}
//-------------------------------------------------------------
// We're going to be purging items from [vegomatic-0], leaving the [vegomatic-1]
// cache alone for now.
var purgeWaitTime = TimeSpan.FromSeconds(2); // Time to wait while cache purging completes.
// Purge a single URI and verify its no longer cached.
trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/test0.jpg" });
await Task.Delay(purgeWaitTime);
status = await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.jpg");
Assert.True(status.ViaVarnish);
Assert.False(status.CacheHit); // Shouldn't be a cache hit because we just purged it.
// Verify that a sample of the remaining [vegomatic-0] responses are still cached.
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.png")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.png")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.png")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit);
// Purge the top-level [*.png] files and verify that they are no longer cached.
trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/*.png" });
await Task.Delay(purgeWaitTime);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.png")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.png")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.png")).CacheHit);
// Verify that a sample of the remaining [vegomatic-0] responses are still cached.
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit);
// Use a glob pattern to purge all [*.jpg] responses at all nesting levels.
trafficManager.Purge(new string[] { $"http://vegomatic-0:80/**/*.jpg" });
await Task.Delay(purgeWaitTime);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.jpg")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.jpg")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.jpg")).CacheHit);
// Verify that the remaining [vegomatic-0] responses are still cached.
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.htm")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.png")).CacheHit);
// Use a glob pattern to purge all [vegomatic-0] responses.
trafficManager.Purge(new string[] { $"http://vegomatic-0:80/**" });
await Task.Delay(purgeWaitTime);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.htm")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.png")).CacheHit);
//-------------------------------------------------------------
// Verify that all the [vegomatic-0] purging didn't impact any of the cached [vegomatic-1] responses.
foreach (var uri in testUris)
{
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-1:80{uri}")).CacheHit);
}
//-------------------------------------------------------------
// Reload all responses into the cache and then do a purge ALL and verify.
for (int serverId = 0; serverId < 2; serverId++)
{
var host = $"vegomatic-{serverId}";
foreach (var uri in testUris)
{
await GetCachingStatusAsync(client, $"http://{host}:80{uri}");
}
}
trafficManager.PurgeAll();
await Task.Delay(purgeWaitTime);
for (int serverId = 0; serverId < 2; serverId++)
{
var host = $"vegomatic-{serverId}";
foreach (var uri in testUris)
{
Assert.False((await GetCachingStatusAsync(client, $"http://{host}:80{uri}")).CacheHit);
}
}
//-------------------------------------------------------------
// Test case-sensitive purge patterns.
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit);
trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/case.test" }, caseSensitive: true);
await Task.Delay(purgeWaitTime);
Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit);
Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit);
}
}
public async Task Cache_Purge_Security()
{
var trafficManager = hive.PublicTraffic;
var manager = hive.GetReachableManager();
var network = HiveConst.PublicNetwork;
var proxyPort = 80;
// Configure a traffic manager rule (with caching enabled).
var rule = new TrafficHttpRule()
{
Name = $"vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1,
Cache = new TrafficHttpCache()
{
Enabled = true
}
};
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = $"vegomatic",
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = $"vegomatic",
Port = 80
});
trafficManager.SetRule(rule);
// Verify that non-local BAN requests are rejected. This is important because
// this will block BAN based DOS attacks.
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
// Spin up a test [vegomatic] service.
var expireSeconds = 300;
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, "vegomatic");
// Load a response into the cache so we can verify that an external BAN
// didn't actually purge anything.
await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt");
Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit);
// Verify that a BAN request submitted from outside of the [neon-proxy-cache]
// container doesn't actually purge anything.
//
// Note that we're not going to see a 403 status code here because the HAProxy
// rule is going to add an [X-Neon-Frontend] header and we've configured the
// Varnish VCL to actually perform BANs only when this header is not present.
var request = new HttpRequestMessage(new HttpMethod("BAN"), $"/");
request.Headers.Host = "vegomatic";
request.Headers.Add("X-Ban-All", "yes");
await client.SendAsync(request);
Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit);
// Now we're actually going to hit Varnish directly and verify that we
// get a 403 response. The trick here is that we need to update the
// [neon-proxy-public-cache] service to publish port [61223] so we can
// hit it externally.
int externalPort = 61223;
try
{
manager.SudoCommand($"docker service update --publish-add {externalPort}:80 neon-proxy-public-cache");
// We need to reload the cached item because the cache was cleared when we updated [neon-proxy-public-cache].
await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt");
Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit);
// Send the BAN request and verify that it was rejected.
request = new HttpRequestMessage(new HttpMethod("BAN"), $"http://{manager.PrivateAddress}:{externalPort}/");
request.Headers.Add("X-Ban-All", "yes");
var response = await client.SendAsync(request);
Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode);
// Also confirm that the item wasn't purged.
Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit);
}
finally
{
// Remove the temporary test port.
manager.SudoCommand($"docker service update --publish-rm {externalPort}:80 neon-proxy-public-cache");
}
}
}
/// <summary>
/// Configures the hive services.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void Configure(SshProxy <NodeDefinition> firstManager)
{
firstManager.InvokeIdempotentAction("setup/hive-services",
() =>
{
// Ensure that Vault has been initialized.
if (!hive.HiveLogin.HasVaultRootCredentials)
{
throw new InvalidOperationException("Vault has not been initialized yet.");
}
//---------------------------------------------------------
// Persist the proxy settings.
// Obtain the AppRole credentials from Vault for the proxy manager as well as the
// public and private proxy services and persist these as Docker secrets.
firstManager.Status = "secrets: proxy services";
hive.Docker.Secret.Set("neon-proxy-manager-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-manager").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-public-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-public").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-private-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-private").Result, Formatting.Indented));
//---------------------------------------------------------
// Deploy the HiveMQ cluster.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster",
() =>
{
// We're going to list the hive nodes that will host the
// RabbitMQ cluster and sort them by node name. Then we're
// going to ensure that the first RabbitMQ node/container
// is started and ready before configuring the rest of the
// cluster so that it will bootstrap properly.
var hiveMQNodes = hive.Nodes
.Where(n => n.Metadata.Labels.HiveMQ)
.OrderBy(n => n.Name)
.ToList();
DeployHiveMQ(hiveMQNodes.First());
// Start the remaining nodes in parallel.
var actions = new List <Action>();
foreach (var node in hiveMQNodes.Skip(1))
{
actions.Add(() => DeployHiveMQ(node));
}
NeonHelper.WaitForParallel(actions);
// The RabbitMQ cluster is created with the [/] vhost and the
// [sysadmin] user by default. We need to create the [neon]
// and [app] vhosts along with the [neon] and [app] users
// and then set the appropriate permissions.
//
// We're going to run [rabbitmqctl] within the first RabbitMQ
// to accomplish this.
var hiveMQNode = hiveMQNodes.First();
// Create the vhosts.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQAppVHost}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQNeonVHost}"));
// Create the users.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQAppUser} {hive.Definition.HiveMQ.AppPassword}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQNeonUser} {hive.Definition.HiveMQ.NeonPassword}"));
// Grant the [app] account full access to the [app] vhost, the [neon] account full
// access to the [neon] vhost. Note that this doesn't need to be idempotent.
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQAppVHost} {HiveConst.HiveMQAppUser} \".*\" \".*\" \".*\"");
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQNeonVHost} {HiveConst.HiveMQNeonUser} \".*\" \".*\" \".*\"");
// Clear the UX status for the HiveMQ nodes.
foreach (var node in hiveMQNodes)
{
node.Status = string.Empty;
}
// Set the RabbitMQ cluster name to the name of the hive.
hiveMQNode.InvokeIdempotentAction("setup/hivemq-cluster-name", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_cluster_name {hive.Definition.Name}"));
});
//---------------------------------------------------------
// Initialize the public and private traffic manager managers.
hive.PublicTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PublicProxyPorts
});
hive.PrivateTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PrivateProxyPorts
});
//---------------------------------------------------------
// Deploy the HiveMQ traffic manager rules.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-traffic-manager-rules",
() =>
{
// Deploy private traffic manager for the AMQP endpoints.
var amqpRule = new TrafficTcpRule()
{
Name = "neon-hivemq-amqp",
System = true,
Resolver = null
};
// We're going to set this up to allow idle connections for up to
// five minutes. In theory, AMQP connections should never be idle
// this long because we've enabled level 7 keep-alive.
//
// https://github.com/jefflill/NeonForge/issues/new
amqpRule.Timeouts = new TrafficTimeouts()
{
ClientSeconds = 0,
ServerSeconds = 0
};
amqpRule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
});
foreach (var ampqNode in hive.Nodes.Where(n => n.Metadata.Labels.HiveMQ))
{
amqpRule.Backends.Add(
new TrafficTcpBackend()
{
Server = ampqNode.PrivateAddress.ToString(),
Port = HiveHostPorts.HiveMQAMQP
});
}
hive.PrivateTraffic.SetRule(amqpRule);
// Deploy private traffic manager for the management endpoints.
var adminRule = new TrafficHttpRule()
{
Name = "neon-hivemq-management",
System = true,
Resolver = null
};
// Initialize the frontends and backends.
adminRule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAdmin
});
adminRule.Backends.Add(
new TrafficHttpBackend()
{
Group = HiveHostGroups.HiveMQManagers,
GroupLimit = 5,
Port = HiveHostPorts.HiveMQManagement
});
hive.PrivateTraffic.SetRule(adminRule);
});
//---------------------------------------------------------
// Deploy DNS related services.
// Deploy: neon-dns-mon
ServiceHelper.StartService(hive, "neon-dns-mon", hive.Definition.Image.DnsMon,
new CommandBundle(
"docker service create",
"--name", "neon-dns-mon",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "POLL_INTERVAL=5s",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--replicas", "1",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-dns
ServiceHelper.StartService(hive, "neon-dns", hive.Definition.Image.Dns,
new CommandBundle(
"docker service create",
"--name", "neon-dns",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/etc/powerdns/hosts,dst=/etc/powerdns/hosts",
"--mount", "type=bind,src=/dev/shm/neon-dns,dst=/neon-dns",
"--env", "POLL_INTERVAL=5s",
"--env", "VERIFY_INTERVAL=5m",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--mode", "global",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
//---------------------------------------------------------
// Deploy [neon-hive-manager] as a service constrained to manager nodes.
string unsealSecretOption = null;
if (hive.Definition.Vault.AutoUnseal)
{
var vaultCredentials = NeonHelper.JsonClone <VaultCredentials>(hive.HiveLogin.VaultCredentials);
// We really don't want to include the root token in the credentials
// passed to [neon-hive-manager], which needs the unseal keys so
// we'll clear that here.
vaultCredentials.RootToken = null;
hive.Docker.Secret.Set("neon-hive-manager-vaultkeys", Encoding.UTF8.GetBytes(NeonHelper.JsonSerialize(vaultCredentials, Formatting.Indented)));
unsealSecretOption = "--secret=neon-hive-manager-vaultkeys";
}
ServiceHelper.StartService(hive, "neon-hive-manager", hive.Definition.Image.HiveManager,
new CommandBundle(
"docker service create",
"--name", "neon-hive-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-ssh-credentials",
unsealSecretOption,
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg
),
hive.SecureRunOptions | RunOptions.FaultOnError);
//---------------------------------------------------------
// Deploy proxy related services.
// Deploy the proxy manager service.
ServiceHelper.StartService(hive, "neon-proxy-manager", hive.Definition.Image.ProxyManager,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "VAULT_CREDENTIALS=neon-proxy-manager-credentials",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-proxy-manager-credentials",
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Docker mesh routing seemed unstable on versions so we're going
// to provide an option to work around this by running the PUBLIC,
// PRIVATE and VAULT proxies on all nodes and publishing the ports
// to the host (not the mesh).
//
// https://github.com/jefflill/NeonForge/issues/104
//
// Note that this mode feature is documented (somewhat poorly) here:
//
// https://docs.docker.com/engine/swarm/services/#publish-ports
var publicPublishArgs = new List <string>();
var privatePublishArgs = new List <string>();
var proxyConstraintArgs = new List <string>();
var proxyReplicasArgs = new List <string>();
var proxyModeArgs = new List <string>();
if (hive.Definition.Docker.GetAvoidIngressNetwork(hive.Definition))
{
// The parameterized [docker service create --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PublicProxyPorts.PortRange.FirstPort; port <= HiveConst.PublicProxyPorts.PortRange.LastPort; port++)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PrivateProxyPorts.PortRange.FirstPort; port <= HiveConst.PrivateProxyPorts.PortRange.LastPort; port++)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("global");
}
else
{
// The parameterized [docker run --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{port}:{port}");
}
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}:{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}");
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{port}:{port}");
}
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}:{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}");
proxyConstraintArgs.Add($"--constraint");
proxyReplicasArgs.Add("--replicas");
if (hive.Definition.Workers.Count() > 0)
{
// Constrain proxies to worker nodes if there are any.
proxyConstraintArgs.Add($"node.role!=manager");
if (hive.Definition.Workers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
else
{
// Constrain proxies to manager nodes nodes if there are no workers.
proxyConstraintArgs.Add($"node.role==manager");
if (hive.Definition.Managers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("replicated");
}
// Deploy: neon-proxy-public
ServiceHelper.StartService(hive, "neon-proxy-public", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-public-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicPublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private
ServiceHelper.StartService(hive, "neon-proxy-private", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-private-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privatePublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-public-cache
var publicCacheConstraintArgs = new List <string>();
var publicCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PublicCacheReplicas <= hive.Definition.Workers.Count())
{
publicCacheConstraintArgs.Add("--constraint");
publicCacheConstraintArgs.Add("node.role==worker");
}
publicCacheReplicaArgs.Add("--replicas");
publicCacheReplicaArgs.Add($"{hive.Definition.Proxy.PublicCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-public-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PublicCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicCacheConstraintArgs,
publicCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private-cache
var privateCacheConstraintArgs = new List <string>();
var privateCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PrivateCacheReplicas <= hive.Definition.Workers.Count())
{
privateCacheConstraintArgs.Add("--constraint");
privateCacheConstraintArgs.Add("node.role==worker");
}
privateCacheReplicaArgs.Add("--replicas");
privateCacheReplicaArgs.Add($"{hive.Definition.Proxy.PrivateCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-private-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PrivateCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privateCacheConstraintArgs,
privateCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
});
// Log the hive into any Docker registries with credentials.
firstManager.InvokeIdempotentAction("setup/registry-login",
() =>
{
foreach (var credential in hive.Definition.Docker.Registries
.Where(r => !string.IsNullOrEmpty(r.Username)))
{
hive.Registry.Login(credential.Registry, credential.Username, credential.Password);
}
});
}
/// <summary>
/// Verify that we can create an HTTP traffic manager rule for a
/// site on the proxy port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="useCache">Optionally enable caching and verify.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <param name="useIPAddress">Optionally uses an IP addrress rather than a hostname for the rule frontend.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestHttpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic", bool useIPAddress = false)
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an HTTP traffic manager rule for a
// site on the proxy port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
var hostname = useIPAddress ? manager.PrivateAddress.ToString() : testHostname;
manager.Connect();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficHttpRule()
{
Name = "vegomatic",
CheckExpect = "status 200",
CheckSeconds = 1,
};
if (useCache)
{
rule.Cache = new TrafficHttpCache()
{
Enabled = true
};
}
rule.Frontends.Add(
new TrafficHttpFrontend()
{
Host = useIPAddress ? null : hostname,
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
//
// We're going to use a different URL for each request so that we
// won't see any cache hits.
var uniqueResponses = new HashSet <string>();
var viaVarnish = false;
var cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
// Repeat the test if caching is enabled with the same URLs as last time and verify that
// we see cache hits this time.
if (useCache)
{
viaVarnish = false;
cacheHit = false;
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.True(viaVarnish);
Assert.True(cacheHit);
}
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note that we're going to pass a new set of URLs to avoid having
// any responses cached so we'll end up seeing all of the IDs.
//
// Note also that we need to perform these requests in parallel
// to try to force Varnish to establish more than one connection
// to the [vegomatic] service. If we don't do this, Varnish will
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
viaVarnish = false;
cacheHit = false;
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
if (ViaVarnish(response))
{
viaVarnish = true;
}
if (CacheHit(response))
{
cacheHit = true;
}
lock (uniqueResponses)
{
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
if (useCache)
{
// [viaVarnish] should be TRUE because we're routing through the cache.
Assert.True(viaVarnish);
// [cacheHit] should be FALSE because we used a unique URI for each request.
Assert.False(cacheHit);
}
else
{
// [viaVarnish] and [cacheHit] should both be FALSE because we're not caching.
Assert.False(viaVarnish);
Assert.False(cacheHit);
}
Assert.Equal(2, uniqueResponses.Count);
}
}
