/// <summary> /// Constructor. /// </summary> /// <param name="rule">The associated traffic manager rule.</param> /// <param name="frontend">The frontend that generated this mapping.</param> /// <param name="backendName">The backend name.</param> public HostPathMapping(TrafficHttpRule rule, TrafficHttpFrontend frontend, string backendName) { Covenant.Requires <ArgumentNullException>(rule != null); Covenant.Requires <ArgumentNullException>(frontend != null); Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(backendName)); this.Rule = rule; this.Frontend = frontend; this.BackendName = backendName; }
/// <summary> /// Adds the steps to configure the stateful Elasticsearch instances used to persist the log data. /// </summary> /// <param name="steps">The configuration step list.</param> private void AddElasticsearchSteps(ConfigStepList steps) { var esNodes = new List <SshProxy <NodeDefinition> >(); foreach (var nodeDefinition in hive.Definition.Nodes.Where(n => n.Labels.LogEsData)) { esNodes.Add(hive.GetNode(nodeDefinition.Name)); } // Determine number of manager nodes and the quorum size. // Note that we'll deploy an odd number of managers. var managerCount = Math.Min(esNodes.Count, 5); // We shouldn't ever need more than 5 managers if (!NeonHelper.IsOdd(managerCount)) { managerCount--; } var quorumCount = (managerCount / 2) + 1; // Sort the nodes by name and then separate the manager and // worker nodes (managers will be assigned to nodes that appear // first in the list). var managerEsNodes = new List <SshProxy <NodeDefinition> >(); var normalEsNodes = new List <SshProxy <NodeDefinition> >(); esNodes = esNodes.OrderBy(n => n.Name).ToList(); foreach (var esNode in esNodes) { if (managerEsNodes.Count < managerCount) { managerEsNodes.Add(esNode); } else { normalEsNodes.Add(esNode); } } // Figure out how much RAM to allocate to the Elasticsearch Docker containers // as well as Java heap within. The guidance is to set the heap size to half // the container RAM up to a maximum of 31GB. var esContainerRam = hive.Definition.Log.EsMemoryBytes; var esHeapBytes = Math.Min(esContainerRam / 2, 31L * NeonHelper.Giga); // We're going to use explicit docker commands to deploy the Elasticsearch cluster // log storage containers. // // We're mounting three volumes to the container: // // /etc/neon/host-env - Generic host specific environment variables // /etc/neon/env-log-esdata - Elasticsearch node host specific environment variables // neon-log-esdata-# - Persistent Elasticsearch data folder var esBootstrapNodes = new StringBuilder(); foreach (var esMasterNode in managerEsNodes) { esBootstrapNodes.AppendWithSeparator($"{esMasterNode.PrivateAddress}:{HiveHostPorts.LogEsDataTcp}", ","); } // Create a data volume for each Elasticsearch node and then start the node container. for (int i = 0; i < esNodes.Count; i++) { var esNode = esNodes[i]; var containerName = $"neon-log-esdata"; var isMaster = managerEsNodes.Contains(esNode) ? "true" : "false"; var volumeCommand = CommandStep.CreateSudo(esNode.Name, "docker-volume-create", containerName); steps.Add(volumeCommand); ServiceHelper.AddContainerStartSteps(hive, steps, esNode, containerName, hive.Definition.Image.Elasticsearch, new CommandBundle( "docker run", "--name", containerName, "--detach", "--restart", "always", "--volume", "/etc/neon/host-env:/etc/neon/host-env:ro", "--volume", $"{containerName}:/mnt/esdata", "--env", $"ELASTICSEARCH_CLUSTER={hive.Definition.Datacenter}.{hive.Definition.Name}.neon-log-esdata", "--env", $"ELASTICSEARCH_NODE_MASTER={isMaster}", "--env", $"ELASTICSEARCH_NODE_DATA=true", "--env", $"ELASTICSEARCH_NODE_COUNT={esNodes.Count}", "--env", $"ELASTICSEARCH_HTTP_PORT={HiveHostPorts.LogEsDataHttp}", "--env", $"ELASTICSEARCH_TCP_PORT={HiveHostPorts.LogEsDataTcp}", "--env", $"ELASTICSEARCH_QUORUM={quorumCount}", "--env", $"ELASTICSEARCH_BOOTSTRAP_NODES={esBootstrapNodes}", "--env", $"ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap", "--memory", $"{esContainerRam / NeonHelper.Mega}M", "--memory-reservation", $"{esContainerRam / NeonHelper.Mega}M", "--memory-swappiness", "0", "--network", "host", "--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events. ServiceHelper.ImagePlaceholderArg)); } // Configure a private hive proxy route to the Elasticsearch nodes. steps.Add(ActionStep.Create(hive.FirstManager.Name, "setup/elasticsearch-lbrule", node => { var rule = new TrafficHttpRule() { Name = "neon-log-esdata", System = true, Log = false, // This is important: we don't want to SPAM the log database with its own traffic. Resolver = null }; rule.Frontends.Add( new TrafficHttpFrontend() { ProxyPort = HiveHostPorts.ProxyPrivateHttpLogEsData }); foreach (var esNode in esNodes) { rule.Backends.Add( new TrafficHttpBackend() { Server = esNode.Metadata.PrivateAddress.ToString(), Port = HiveHostPorts.LogEsDataHttp }); } hive.PrivateTraffic.SetRule(rule); })); // Wait for the elasticsearch cluster to become ready and then save the // [logstash-*] template. We need to do this before [neon-log-collector] // is started so we'll be sure that no indexes will be created before // we have a chance to persist the pattern. // // This works because [neon-log-collector] is the main service responsible // for persisting events to this index. steps.Add(ActionStep.Create(hive.FirstManager.Name, operationName: null, node => { node.Status = "wait for elasticsearch cluster"; using (var jsonClient = new JsonClient()) { var baseLogEsDataUri = hive.Definition.LogEsDataUri; var timeout = TimeSpan.FromMinutes(5); var timeoutTime = DateTime.UtcNow + timeout; var esNodeCount = hive.Definition.Nodes.Count(n => n.Labels.LogEsData); // Wait for the Elasticsearch cluster. jsonClient.UnsafeRetryPolicy = NoRetryPolicy.Instance; while (true) { try { var response = jsonClient.GetUnsafeAsync($"{baseLogEsDataUri}/_cluster/health").Result; if (response.IsSuccess) { var clusterStatus = response.AsDynamic(); var status = (string)(clusterStatus.status); status = status.ToUpperInvariant(); node.Status = $"wait for [neon-log-esdata] cluster: [status={status}] [{clusterStatus.number_of_nodes}/{esNodeCount} nodes ready])"; // $todo(jeff.lill): // // We're accepting YELLOW status here due to this issue: // // https://github.com/jefflill/NeonForge/issues/257 if ((status == "GREEN" || status == "YELLOW") && clusterStatus.number_of_nodes == esNodeCount) { node.Status = "elasticsearch cluster is ready"; break; } } } catch { if (DateTime.UtcNow >= timeoutTime) { node.Fault($"[neon-log-esdata] cluster not ready after waiting [{timeout}]."); return; } } Thread.Sleep(TimeSpan.FromSeconds(1)); } // Save the [logstash-*] template pattern. var templatePattern = ResourceFiles.Root.GetFolder("Elasticsearch").GetFile("logstash-template.json").Contents; jsonClient.PutAsync($"{baseLogEsDataUri}/_template/logstash-*", templatePattern).Wait(); } })); }
/// <summary> /// Configures the Kibana dashboard. /// </summary> /// <param name="firstManager">The first hive proxy manager.</param> public void ConfigureKibana(SshProxy <NodeDefinition> firstManager) { if (!hive.Definition.Log.Enabled) { return; } firstManager.InvokeIdempotentAction("setup/log-kibana", () => { using (var jsonClient = new JsonClient()) { var baseLogEsDataUri = hive.Definition.LogEsDataUri; var baseKibanaUri = $"http://{firstManager.PrivateAddress}:{HiveHostPorts.Kibana}"; var timeout = TimeSpan.FromMinutes(5); var retry = new LinearRetryPolicy(TransientDetector.Http, maxAttempts: 30, retryInterval: TimeSpan.FromSeconds(2)); // The Kibana API calls below require the [kbn-xsrf] header. jsonClient.DefaultRequestHeaders.Add("kbn-xsrf", "true"); // Ensure that Kibana is ready before we submit any API requests. firstManager.Status = "wait for kibana"; retry.InvokeAsync( async() => { var response = await jsonClient.GetAsync <dynamic>($"{baseKibanaUri}/api/status"); if (response.status.overall.state != "green") { throw new TransientException($"Kibana [state={response.status.overall.state}]"); } }).Wait(); // Add the index pattern to Kibana. firstManager.Status = "configure kibana index pattern"; retry.InvokeAsync( async() => { dynamic indexPattern = new ExpandoObject(); dynamic attributes = new ExpandoObject(); attributes.title = "logstash-*"; attributes.timeFieldName = "@timestamp"; indexPattern.attributes = attributes; await jsonClient.PostAsync($"{baseKibanaUri}/api/saved_objects/index-pattern/logstash-*?overwrite=true", indexPattern); }).Wait(); // Now we need to save a Kibana config document so that [logstash-*] will be // the default index and the timestamp will be displayed as UTC and have a // more useful terse format. firstManager.Status = "configure kibana defaults"; retry.InvokeAsync( async() => { dynamic setting = new ExpandoObject(); setting.value = "logstash-*"; await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/defaultIndex", setting); setting.value = "HH:mm:ss.SSS MM-DD-YYYY"; await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat", setting); setting.value = "UTC"; await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat:tz", setting); }).Wait(); // Set the Kibana traffic manager rule. firstManager.Status = "kibana traffic manager rule"; var rule = new TrafficHttpRule() { Name = "neon-log-kibana", System = true, Log = true, Resolver = null }; rule.Frontends.Add( new TrafficHttpFrontend() { ProxyPort = HiveHostPorts.ProxyPrivateKibanaDashboard }); rule.Backends.Add( new TrafficHttpBackend() { Server = "neon-log-kibana", Port = NetworkPorts.Kibana }); hive.PrivateTraffic.SetRule(rule); firstManager.Status = string.Empty; } }); }
/// <summary> /// Verify that we can create an HTTPS traffic manager rule that pre-warms items. /// </summary> /// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param> /// <param name="proxyPort">The inbound proxy port.</param> /// <param name="network">The proxy network.</param> /// <param name="trafficManager">The traffic manager.</param> /// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param> /// <returns>The tracking <see cref="Task"/>.</returns> private async Task TestHttpsCacheWarming(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic") { // Append a GUID to the test name to ensure that we won't // conflict with what any previous test runs may have loaded // into the cache. testName += "-" + Guid.NewGuid().ToString("D"); // Verify that we can create an HTTP traffic manager rule for a // site on the proxy port using a specific hostname and then // verify that warming actually works by spinning up a [vegomatic] // based service to accept the traffic. // // We'll do this by specifying warm and cold URIs that both enable // caching. We'll specify the warm URI as a warm target but not // the cold URI. Then we'll publish the rule and wait for a bit // to allow it to stablize and for the [neon-proxy-cache] to // load the warm URI. // // Finally, we'll verify that this worked by fetching both URIs. // The warm URI should indicate that it came from the cache and // the cold URI should not be cached. var manager = hive.GetReachableManager(); var guid = Guid.NewGuid().ToString("D"); // Avoid conflicts with previous test runs var expireSeconds = 60; var warmUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/warm?body=text:warm&Expires={expireSeconds}"); var coldUri = new Uri($"https://{testHostname}:{proxyPort}/{guid}/cold?body=text:cold&Expires={expireSeconds}"); manager.Connect(); // Allow self-signed certificates. var handler = new HttpClientHandler() { ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator }; using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true)) { // Add the test certificate. hive.Certificate.Set("test-load-balancer", certificate); // Setup the client to query the [vegomatic] service through the // proxy without needing to configure a hive DNS entry. client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/"); client.DefaultRequestHeaders.Host = testHostname; // Configure the traffic manager rule. var rule = new TrafficHttpRule() { Name = "vegomatic", CheckExpect = "status 200", CheckSeconds = 1, }; rule.Cache = new TrafficHttpCache() { Enabled = true }; rule.Cache.WarmTargets.Add( new TrafficWarmTarget() { UpdateSeconds = 1.0, Uri = warmUri.ToString() }); rule.Frontends.Add( new TrafficHttpFrontend() { Host = testHostname, ProxyPort = proxyPort, CertName = "test-load-balancer" }); rule.Backends.Add( new TrafficHttpBackend() { Server = serviceName, Port = 80 }); trafficManager.SetRule(rule); // Spin up a [vegomatic] service instance. manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, testHostname); // Wait a bit longer to ensure that the cache has had a chance to // warm the URI. await Task.Delay(TimeSpan.FromSeconds(5)); // Query for the warm and cold URIs and verify that the warm item was a // cache hit and the cold item was not. var warmResponse = await client.GetAsync(warmUri.PathAndQuery); var warmBody = (await warmResponse.Content.ReadAsStringAsync()).Trim(); var coldResponse = await client.GetAsync(coldUri.PathAndQuery); var coldBody = (await coldResponse.Content.ReadAsStringAsync()).Trim(); Assert.Equal(HttpStatusCode.OK, warmResponse.StatusCode); Assert.Equal("warm", warmBody); Assert.True(CacheHit(warmResponse)); Assert.Equal(HttpStatusCode.OK, coldResponse.StatusCode); Assert.Equal("cold", coldBody); Assert.False(CacheHit(coldResponse)); } }
/// <summary> /// Verify that we can create HTTPS traffic manager rules for a /// site on the proxy port using a specific hostname and various /// path prefixes and then verify that that the traffic manager actually /// works by spinning up a [vegomatic] based service to accept the traffic. /// </summary> /// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param> /// <param name="proxyPort">The inbound proxy port.</param> /// <param name="network">The proxy network.</param> /// <param name="trafficManager">The traffic manager.</param> /// <param name="useCache">Optionally enable caching and verify.</param> /// <param name="serviceName">Optionally specifies the backend service name prefix (defaults to <b>vegomatic</b>).</param> /// <returns>The tracking <see cref="Task"/>.</returns> private async Task TestHttpsPrefix(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic") { // Append a GUID to the test name to ensure that we won't // conflict with what any previous test runs may have loaded // into the cache. testName += "-" + Guid.NewGuid().ToString("D"); // Verify that we can create an HTTP traffic manager rule for a // site on the proxy port using a specific hostname and then // verify that that the traffic manager actually works by spinning // up a [vegomatic] based service to accept the traffic. var manager = hive.GetReachableManager(); var hostname = testHostname; manager.Connect(); // Allow self-signed certificates. var handler = new HttpClientHandler() { ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator }; using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true)) { // Add the test certificate. hive.Certificate.Set("test-load-balancer", certificate); // Setup the client to query the [vegomatic] service through the // proxy without needing to configure a hive DNS entry. client.BaseAddress = new Uri($"https://{manager.PrivateAddress}:{proxyPort}/"); client.DefaultRequestHeaders.Host = hostname; // Create the traffic manager rules, one without a path prefix and // some others, some with intersecting prefixes so we can verify // that the longest prefixes are matched first. // // Each rule's backend will be routed to a service whose name // will be constructed from [testName] plus the prefix with the // slashes replaced with dashes. Each service will be configured // to return its name. var prefixes = new PrefixInfo[] { new PrefixInfo("/", $"{serviceName}"), new PrefixInfo("/foo/", $"{serviceName}-foo"), new PrefixInfo("/foo/bar/", $"{serviceName}-foo-bar"), new PrefixInfo("/foobar/", $"{serviceName}-foobar"), new PrefixInfo("/bar/", $"{serviceName}-bar") }; // Spin the services up first in parallel (for speed). Each of // these service will respond to requests with its service name. var tasks = new List <Task>(); foreach (var prefix in prefixes) { tasks.Add(Task.Run( () => { manager.SudoCommand($"docker service create --name {prefix.ServiceName} --network {network} --replicas 1 {vegomaticImage} test-server server-id={prefix.ServiceName}").EnsureSuccess(); })); } await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30)); // Create the traffic manager rules. foreach (var prefix in prefixes) { var rule = new TrafficHttpRule() { Name = prefix.ServiceName, CheckExpect = "status 200", CheckSeconds = 1, }; if (useCache) { rule.Cache = new TrafficHttpCache() { Enabled = true }; } var frontend = new TrafficHttpFrontend() { Host = hostname, ProxyPort = proxyPort, CertName = "test-load-balancer" }; if (!string.IsNullOrEmpty(prefix.Path)) { frontend.PathPrefix = prefix.Path; } rule.Frontends.Add(frontend); rule.Backends.Add( new TrafficHttpBackend() { Server = prefix.ServiceName, Port = 80 }); trafficManager.SetRule(rule, deferUpdate: true); } trafficManager.Update(); // Wait for all of the services to report being ready. await NeonHelper.WaitForAsync( async() => { foreach (var prefix in prefixes) { try { var response = await client.GetAsync(prefix.Path); response.EnsureSuccessStatusCode(); } catch { return(false); } } return(true); }, timeout : TimeSpan.FromSeconds(60), pollTime : TimeSpan.FromSeconds(1)); // Give everything a chance to stablize. await Task.Delay(TimeSpan.FromSeconds(5)); // Now verify that prefix rules route to the correct backend service. foreach (var prefix in prefixes) { var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60"); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(); Assert.Equal(prefix.ServiceName, body.Trim()); if (useCache) { // Verify that the request routed through Varnish. Assert.True(ViaVarnish(response)); // This is the first request using the globally unique [testName] // so it should not be a cache hit. Assert.False(CacheHit(response)); } } // If caching is enabled, perform the requests again to ensure that // we see cache hits. if (useCache) { foreach (var prefix in prefixes) { // Request the item again and verify that it was a cache hit. var response = await client.GetAsync($"{prefix.Path}{testName}?expires=60"); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(); Assert.Equal(prefix.ServiceName, body.Trim()); Assert.True(CacheHit(response)); } } } }
/// <summary> /// Verify that we can create an HTTPS traffic manager rule for a /// site on the proxy port using a specific hostname and then /// verify that that the traffic manager actually works by spinning /// up a [vegomatic] based service to accept the traffic. /// </summary> /// <param name="testName">Simple name (without spaces).</param> /// <param name="proxyPort">The inbound proxy port.</param> /// <param name="network">The proxy network.</param> /// <param name="trafficManager">The traffic manager.</param> /// <param name="useCache">Optionally enable caching and verify.</param> /// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param> /// <returns>The tracking <see cref="Task"/>.</returns> private async Task TestHttpsRule(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic") { // Verify that we can create an HTTPS traffic manager rule for a // site on the proxy port using a specific hostname and then // verify that that the traffic manager actually works by spinning // up a [vegomatic] based service to accept the traffic. var queryCount = 100; var manager = hive.GetReachableManager(); manager.Connect(); // We need the test hostname to point to the manager's private address // so we can submit HTTPS requests there. hiveFixture.LocalMachineHosts.AddHostAddress(testHostname, manager.PrivateAddress.ToString()); // Allow self-signed certificates. var handler = new HttpClientHandler() { ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator }; using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true)) { client.BaseAddress = new Uri($"https://{testHostname}:{proxyPort}/"); client.DefaultRequestHeaders.Host = testHostname; // Add the test certificate. hive.Certificate.Set("test-load-balancer", certificate); // Configure the traffic manager rule. var rule = new TrafficHttpRule() { Name = "vegomatic", CheckExpect = "status 200", CheckSeconds = 1 }; if (useCache) { rule.Cache = new TrafficHttpCache() { Enabled = true }; } rule.Frontends.Add( new TrafficHttpFrontend() { Host = testHostname, ProxyPort = proxyPort, CertName = "test-load-balancer" }); rule.Backends.Add( new TrafficHttpBackend() { Server = serviceName, Port = 80 }); trafficManager.SetRule(rule); // Spin up a single [vegomatic] service instance. manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, testHostname); // Query the service several times to verify that we get a response and // also that all of the responses are the same (because we have only // a single [vegomatic] instance returning its UUID). // // We're going to use a different URL for each request so that we // won't see any cache hits. var uniqueResponses = new HashSet <string>(); var viaVarnish = false; var cacheHit = false; for (int i = 0; i < queryCount; i++) { var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60"); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } var body = await response.Content.ReadAsStringAsync(); if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } Assert.Single(uniqueResponses); if (useCache) { // [viaVarnish] should be TRUE because we're routing through the cache. Assert.True(viaVarnish); // [cacheHit] should be FALSE because we used a unique URI for each request. Assert.False(cacheHit); } else { // [viaVarnish] and [cacheHit] should both be FALSE because we're not caching. Assert.False(viaVarnish); Assert.False(cacheHit); } // Repeat the test if caching is enabled with the same URLs as last time and verify that // we see cache hits this time. if (useCache) { viaVarnish = false; cacheHit = false; for (int i = 0; i < queryCount; i++) { var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60"); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } var body = await response.Content.ReadAsStringAsync(); if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } Assert.True(viaVarnish); Assert.True(cacheHit); } // Spin up a second replica and repeat the query test to verify // that we see two unique responses. // // Note also that we need to perform these requests in parallel // to try to force HAProxy to establish more than one connection // to the [vegomatic] service. If we don't do this, HAProxy may // establish a single connection to one of the service instances // and keep sending traffic there resulting in us seeing only // one response UUID. manager.SudoCommand($"docker service update --replicas 2 {serviceName}").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, testHostname); // Reset the response info and do the requests. uniqueResponses.Clear(); viaVarnish = false; cacheHit = false; var tasks = new List <Task>(); var uris = new List <string>(); for (int i = 0; i < queryCount; i++) { uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250"); } foreach (var uri in uris) { tasks.Add(Task.Run( async() => { var response = await client.GetAsync(uri); var body = await response.Content.ReadAsStringAsync(); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } lock (uniqueResponses) { if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } })); } await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30)); if (useCache) { // [viaVarnish] should be TRUE because we're routing through the cache. Assert.True(viaVarnish); // [cacheHit] should be FALSE because we used a unique URI for each request. Assert.False(cacheHit); } else { // [viaVarnish] and [cacheHit] should both be FALSE because we're not caching. Assert.False(viaVarnish); Assert.False(cacheHit); } Assert.Equal(2, uniqueResponses.Count); } }
/// <summary> /// Generates a traffic manager rule for each <see cref="Redirection"/> passed that /// will redirect from one URI to another. /// </summary> /// <param name="trafficManager">The target traffic manager.</param> /// <param name="testName">Used to name the traffic manager rules.</param> /// <param name="singleRule"> /// Pass <c>true</c> to test a single rule with all of the redirections or /// <c>false</c> to test with one redirection per rule. /// </param> /// <param name="redirections">The redirections.</param> private async Task TestRedirect(TrafficManager trafficManager, string testName, bool singleRule, params Redirection[] redirections) { var manager = hive.GetReachableManager(); // We need local DNS mappings for each of the URI hosts to target a hive node. var hosts = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); foreach (var redirect in redirections) { if (!hosts.Contains(redirect.FromUri.Host)) { hosts.Add(redirect.FromUri.Host); } if (!hosts.Contains(redirect.ToUri.Host)) { hosts.Add(redirect.ToUri.Host); } } foreach (var host in hosts) { hiveFixture.LocalMachineHosts.AddHostAddress(host, manager.PrivateAddress.ToString(), deferCommit: true); } hiveFixture.LocalMachineHosts.Commit(); // Generate and upload a self-signed certificate for each redirect host that // uses HTTPS and upload these to the hive. Each certificate will be named // the same as the hostname. var hostToCertificate = new Dictionary <string, TlsCertificate>(StringComparer.InvariantCultureIgnoreCase); foreach (var redirect in redirections.Where(r => r.FromUri.Scheme == "https")) { var host = redirect.FromUri.Host; if (hostToCertificate.ContainsKey(host)) { continue; } hostToCertificate[host] = TlsCertificate.CreateSelfSigned(host); } foreach (var item in hostToCertificate) { hive.Certificate.Set(item.Key, item.Value); } // Create the traffic manager rule(s). if (singleRule) { var rule = new TrafficHttpRule() { Name = testName, }; foreach (var redirect in redirections) { var frontend = new TrafficHttpFrontend() { Host = redirect.FromUri.Host, ProxyPort = redirect.FromUri.Port, RedirectTo = redirect.ToUri }; if (redirect.FromUri.Scheme == "https") { frontend.CertName = redirect.FromUri.Host; } rule.Frontends.Add(frontend); } trafficManager.SetRule(rule); } else { var redirectIndex = 0; foreach (var redirect in redirections) { var rule = new TrafficHttpRule() { Name = $"{testName}-{redirectIndex}", }; var frontend = new TrafficHttpFrontend() { Host = redirect.FromUri.Host, ProxyPort = redirect.FromUri.Port, RedirectTo = redirect.ToUri }; if (redirect.FromUri.Scheme == "https") { frontend.CertName = redirect.FromUri.Host; } rule.Frontends.Add(frontend); trafficManager.SetRule(rule); redirectIndex++; } } // Give the new rules some time to deploy. await Task.Delay(TimeSpan.FromSeconds(5)); // Now all we need to do is hit all of the redirect [FromUri]s // and verify that we get redirects to the corresponding // [ToUri]s. // Allow self-signed certificates and disable client-side automatic redirect handling // so we'll be able to see the redirect responses. var handler = new HttpClientHandler() { ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator, AllowAutoRedirect = false // We need to see the redirects }; using (var client = new TestHttpClient(disableConnectionReuse: true, handler: handler, disposeHandler: true)) { foreach (var redirect in redirections) { var response = await client.GetAsync(redirect.FromUri); Assert.Equal(HttpStatusCode.Redirect, response.StatusCode); Assert.True(response.Headers.TryGetValues("Location", out var locations)); Assert.Equal(redirect.ToUri.ToString(), locations.Single()); } } }
public async Task Cache_Purge() { var trafficManager = hive.PublicTraffic; var network = HiveConst.PublicNetwork; var proxyPort = 80; var uuid = Guid.NewGuid().ToString("D"); // Used to avoid cache conflicts from previous test runs. // We're going to configure a Vegomatic test instance with a // caching load balancer rule and then fetch several different // requests thru the cache, verify that the responses were // cache hits. Then we'll test various purge patterns and // verify that subsequent requests will be cache misses. var manager = hive.GetReachableManager(); using (var client = new TestHttpClient(disableConnectionReuse: true)) { // Setup the client to query the [vegomatic] service through the // proxy without needing to configure a hive DNS entry. client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/"); // Configure the traffic manager rules (with caching enabled). for (int serverId = 0; serverId < 2; serverId++) { var rule = new TrafficHttpRule() { Name = $"vegomatic-{serverId}", CheckExpect = "status 200", CheckSeconds = 1, Cache = new TrafficHttpCache() { Enabled = true } }; rule.Frontends.Add( new TrafficHttpFrontend() { Host = $"vegomatic-{serverId}", ProxyPort = proxyPort }); rule.Backends.Add( new TrafficHttpBackend() { Server = $"vegomatic-{serverId}", Port = 80 }); trafficManager.SetRule(rule); } // Spin up a two [vegomatic] service instances [vegomatic-0] and [vegomatic-1] // so that we can ensure that purging one origin server's content doesn't impact // the other's cached content. The default response expiration time will be // configured as 300 seconds (5 minutes) so we can test cache purging. var expireSeconds = 300; NeonHelper.WaitForParallel( new Action[] { () => { manager.SudoCommand($"docker service create --name vegomatic-0 --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess(); WaitUntilReadyAsync(client.BaseAddress, "vegomatic-0").Wait(); }, () => { manager.SudoCommand($"docker service create --name vegomatic-1 --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess(); WaitUntilReadyAsync(client.BaseAddress, "vegomatic-1").Wait(); } }); // Submit several requests to preload the cache for both origin servers. // and verify that the items were cached. CacheInfo status; var testUris = new string[] { $"/{uuid}/test.htm", $"/{uuid}/test0.jpg", $"/{uuid}/test1.jpg", $"/{uuid}/test2.jpg", $"/{uuid}/test0.png", $"/{uuid}/test1.png", $"/{uuid}/test2.png", $"/{uuid}/foo/test.htm", $"/{uuid}/foo/test.jpg", $"/{uuid}/foo/test.png", $"/{uuid}/bar/test.htm", $"/{uuid}/bar/test.jpg", $"/{uuid}/bar/test.png", }; for (int serverId = 0; serverId < 2; serverId++) { var host = $"vegomatic-{serverId}"; foreach (var uri in testUris) { // Load the item into the cache. status = await GetCachingStatusAsync(client, $"http://{host}:80{uri}"); Assert.True(status.ViaVarnish); // Ensure that it was actually cached. status = await GetCachingStatusAsync(client, $"http://{host}:80{uri}"); Assert.True(status.ViaVarnish); Assert.True(status.CacheHit); } } //------------------------------------------------------------- // We're going to be purging items from [vegomatic-0], leaving the [vegomatic-1] // cache alone for now. var purgeWaitTime = TimeSpan.FromSeconds(2); // Time to wait while cache purging completes. // Purge a single URI and verify its no longer cached. trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/test0.jpg" }); await Task.Delay(purgeWaitTime); status = await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.jpg"); Assert.True(status.ViaVarnish); Assert.False(status.CacheHit); // Shouldn't be a cache hit because we just purged it. // Verify that a sample of the remaining [vegomatic-0] responses are still cached. Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.png")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.png")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.png")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit); // Purge the top-level [*.png] files and verify that they are no longer cached. trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/*.png" }); await Task.Delay(purgeWaitTime); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.png")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.png")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.png")).CacheHit); // Verify that a sample of the remaining [vegomatic-0] responses are still cached. Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit); // Use a glob pattern to purge all [*.jpg] responses at all nesting levels. trafficManager.Purge(new string[] { $"http://vegomatic-0:80/**/*.jpg" }); await Task.Delay(purgeWaitTime); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test0.jpg")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test1.jpg")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test2.jpg")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.jpg")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.jpg")).CacheHit); // Verify that the remaining [vegomatic-0] responses are still cached. Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.htm")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.png")).CacheHit); // Use a glob pattern to purge all [vegomatic-0] responses. trafficManager.Purge(new string[] { $"http://vegomatic-0:80/**" }); await Task.Delay(purgeWaitTime); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/test.htm")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.htm")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/foo/test.png")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.htm")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/bar/test.png")).CacheHit); //------------------------------------------------------------- // Verify that all the [vegomatic-0] purging didn't impact any of the cached [vegomatic-1] responses. foreach (var uri in testUris) { Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-1:80{uri}")).CacheHit); } //------------------------------------------------------------- // Reload all responses into the cache and then do a purge ALL and verify. for (int serverId = 0; serverId < 2; serverId++) { var host = $"vegomatic-{serverId}"; foreach (var uri in testUris) { await GetCachingStatusAsync(client, $"http://{host}:80{uri}"); } } trafficManager.PurgeAll(); await Task.Delay(purgeWaitTime); for (int serverId = 0; serverId < 2; serverId++) { var host = $"vegomatic-{serverId}"; foreach (var uri in testUris) { Assert.False((await GetCachingStatusAsync(client, $"http://{host}:80{uri}")).CacheHit); } } //------------------------------------------------------------- // Test case-sensitive purge patterns. Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit); trafficManager.Purge(new string[] { $"http://vegomatic-0:80/{uuid}/case.test" }, caseSensitive: true); await Task.Delay(purgeWaitTime); Assert.False((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/case.test")).CacheHit); Assert.True((await GetCachingStatusAsync(client, $"http://vegomatic-0:80/{uuid}/CASE.TEST")).CacheHit); } }
public async Task Cache_Purge_Security() { var trafficManager = hive.PublicTraffic; var manager = hive.GetReachableManager(); var network = HiveConst.PublicNetwork; var proxyPort = 80; // Configure a traffic manager rule (with caching enabled). var rule = new TrafficHttpRule() { Name = $"vegomatic", CheckExpect = "status 200", CheckSeconds = 1, Cache = new TrafficHttpCache() { Enabled = true } }; rule.Frontends.Add( new TrafficHttpFrontend() { Host = $"vegomatic", ProxyPort = proxyPort }); rule.Backends.Add( new TrafficHttpBackend() { Server = $"vegomatic", Port = 80 }); trafficManager.SetRule(rule); // Verify that non-local BAN requests are rejected. This is important because // this will block BAN based DOS attacks. using (var client = new TestHttpClient(disableConnectionReuse: true)) { client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/"); // Spin up a test [vegomatic] service. var expireSeconds = 300; manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server expires={expireSeconds}").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, "vegomatic"); // Load a response into the cache so we can verify that an external BAN // didn't actually purge anything. await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt"); Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit); // Verify that a BAN request submitted from outside of the [neon-proxy-cache] // container doesn't actually purge anything. // // Note that we're not going to see a 403 status code here because the HAProxy // rule is going to add an [X-Neon-Frontend] header and we've configured the // Varnish VCL to actually perform BANs only when this header is not present. var request = new HttpRequestMessage(new HttpMethod("BAN"), $"/"); request.Headers.Host = "vegomatic"; request.Headers.Add("X-Ban-All", "yes"); await client.SendAsync(request); Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit); // Now we're actually going to hit Varnish directly and verify that we // get a 403 response. The trick here is that we need to update the // [neon-proxy-public-cache] service to publish port [61223] so we can // hit it externally. int externalPort = 61223; try { manager.SudoCommand($"docker service update --publish-add {externalPort}:80 neon-proxy-public-cache"); // We need to reload the cached item because the cache was cleared when we updated [neon-proxy-public-cache]. await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt"); Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit); // Send the BAN request and verify that it was rejected. request = new HttpRequestMessage(new HttpMethod("BAN"), $"http://{manager.PrivateAddress}:{externalPort}/"); request.Headers.Add("X-Ban-All", "yes"); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode); // Also confirm that the item wasn't purged. Assert.True((await GetCachingStatusAsync(client, "http://vegomatic:80/foo.txt")).CacheHit); } finally { // Remove the temporary test port. manager.SudoCommand($"docker service update --publish-rm {externalPort}:80 neon-proxy-public-cache"); } } }
/// <summary> /// Configures the hive services. /// </summary> /// <param name="firstManager">The first hive proxy manager.</param> public void Configure(SshProxy <NodeDefinition> firstManager) { firstManager.InvokeIdempotentAction("setup/hive-services", () => { // Ensure that Vault has been initialized. if (!hive.HiveLogin.HasVaultRootCredentials) { throw new InvalidOperationException("Vault has not been initialized yet."); } //--------------------------------------------------------- // Persist the proxy settings. // Obtain the AppRole credentials from Vault for the proxy manager as well as the // public and private proxy services and persist these as Docker secrets. firstManager.Status = "secrets: proxy services"; hive.Docker.Secret.Set("neon-proxy-manager-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-manager").Result, Formatting.Indented)); hive.Docker.Secret.Set("neon-proxy-public-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-public").Result, Formatting.Indented)); hive.Docker.Secret.Set("neon-proxy-private-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-private").Result, Formatting.Indented)); //--------------------------------------------------------- // Deploy the HiveMQ cluster. hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster", () => { // We're going to list the hive nodes that will host the // RabbitMQ cluster and sort them by node name. Then we're // going to ensure that the first RabbitMQ node/container // is started and ready before configuring the rest of the // cluster so that it will bootstrap properly. var hiveMQNodes = hive.Nodes .Where(n => n.Metadata.Labels.HiveMQ) .OrderBy(n => n.Name) .ToList(); DeployHiveMQ(hiveMQNodes.First()); // Start the remaining nodes in parallel. var actions = new List <Action>(); foreach (var node in hiveMQNodes.Skip(1)) { actions.Add(() => DeployHiveMQ(node)); } NeonHelper.WaitForParallel(actions); // The RabbitMQ cluster is created with the [/] vhost and the // [sysadmin] user by default. We need to create the [neon] // and [app] vhosts along with the [neon] and [app] users // and then set the appropriate permissions. // // We're going to run [rabbitmqctl] within the first RabbitMQ // to accomplish this. var hiveMQNode = hiveMQNodes.First(); // Create the vhosts. hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQAppVHost}")); hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQNeonVHost}")); // Create the users. hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQAppUser} {hive.Definition.HiveMQ.AppPassword}")); hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQNeonUser} {hive.Definition.HiveMQ.NeonPassword}")); // Grant the [app] account full access to the [app] vhost, the [neon] account full // access to the [neon] vhost. Note that this doesn't need to be idempotent. hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQAppVHost} {HiveConst.HiveMQAppUser} \".*\" \".*\" \".*\""); hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQNeonVHost} {HiveConst.HiveMQNeonUser} \".*\" \".*\" \".*\""); // Clear the UX status for the HiveMQ nodes. foreach (var node in hiveMQNodes) { node.Status = string.Empty; } // Set the RabbitMQ cluster name to the name of the hive. hiveMQNode.InvokeIdempotentAction("setup/hivemq-cluster-name", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_cluster_name {hive.Definition.Name}")); }); //--------------------------------------------------------- // Initialize the public and private traffic manager managers. hive.PublicTraffic.UpdateSettings( new TrafficSettings() { ProxyPorts = HiveConst.PublicProxyPorts }); hive.PrivateTraffic.UpdateSettings( new TrafficSettings() { ProxyPorts = HiveConst.PrivateProxyPorts }); //--------------------------------------------------------- // Deploy the HiveMQ traffic manager rules. hive.FirstManager.InvokeIdempotentAction("setup/hivemq-traffic-manager-rules", () => { // Deploy private traffic manager for the AMQP endpoints. var amqpRule = new TrafficTcpRule() { Name = "neon-hivemq-amqp", System = true, Resolver = null }; // We're going to set this up to allow idle connections for up to // five minutes. In theory, AMQP connections should never be idle // this long because we've enabled level 7 keep-alive. // // https://github.com/jefflill/NeonForge/issues/new amqpRule.Timeouts = new TrafficTimeouts() { ClientSeconds = 0, ServerSeconds = 0 }; amqpRule.Frontends.Add( new TrafficTcpFrontend() { ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP }); foreach (var ampqNode in hive.Nodes.Where(n => n.Metadata.Labels.HiveMQ)) { amqpRule.Backends.Add( new TrafficTcpBackend() { Server = ampqNode.PrivateAddress.ToString(), Port = HiveHostPorts.HiveMQAMQP }); } hive.PrivateTraffic.SetRule(amqpRule); // Deploy private traffic manager for the management endpoints. var adminRule = new TrafficHttpRule() { Name = "neon-hivemq-management", System = true, Resolver = null }; // Initialize the frontends and backends. adminRule.Frontends.Add( new TrafficHttpFrontend() { ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAdmin }); adminRule.Backends.Add( new TrafficHttpBackend() { Group = HiveHostGroups.HiveMQManagers, GroupLimit = 5, Port = HiveHostPorts.HiveMQManagement }); hive.PrivateTraffic.SetRule(adminRule); }); //--------------------------------------------------------- // Deploy DNS related services. // Deploy: neon-dns-mon ServiceHelper.StartService(hive, "neon-dns-mon", hive.Definition.Image.DnsMon, new CommandBundle( "docker service create", "--name", "neon-dns-mon", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--env", "POLL_INTERVAL=5s", "--env", "LOG_LEVEL=INFO", "--constraint", "node.role==manager", "--replicas", "1", "--restart-delay", hive.Definition.Docker.RestartDelay, ServiceHelper.ImagePlaceholderArg)); // Deploy: neon-dns ServiceHelper.StartService(hive, "neon-dns", hive.Definition.Image.Dns, new CommandBundle( "docker service create", "--name", "neon-dns", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--mount", "type=bind,src=/etc/powerdns/hosts,dst=/etc/powerdns/hosts", "--mount", "type=bind,src=/dev/shm/neon-dns,dst=/neon-dns", "--env", "POLL_INTERVAL=5s", "--env", "VERIFY_INTERVAL=5m", "--env", "LOG_LEVEL=INFO", "--constraint", "node.role==manager", "--mode", "global", "--restart-delay", hive.Definition.Docker.RestartDelay, ServiceHelper.ImagePlaceholderArg)); //--------------------------------------------------------- // Deploy [neon-hive-manager] as a service constrained to manager nodes. string unsealSecretOption = null; if (hive.Definition.Vault.AutoUnseal) { var vaultCredentials = NeonHelper.JsonClone <VaultCredentials>(hive.HiveLogin.VaultCredentials); // We really don't want to include the root token in the credentials // passed to [neon-hive-manager], which needs the unseal keys so // we'll clear that here. vaultCredentials.RootToken = null; hive.Docker.Secret.Set("neon-hive-manager-vaultkeys", Encoding.UTF8.GetBytes(NeonHelper.JsonSerialize(vaultCredentials, Formatting.Indented))); unsealSecretOption = "--secret=neon-hive-manager-vaultkeys"; } ServiceHelper.StartService(hive, "neon-hive-manager", hive.Definition.Image.HiveManager, new CommandBundle( "docker service create", "--name", "neon-hive-manager", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock", "--env", "LOG_LEVEL=INFO", "--secret", "neon-ssh-credentials", unsealSecretOption, "--constraint", "node.role==manager", "--replicas", 1, "--restart-delay", hive.Definition.Docker.RestartDelay, ServiceHelper.ImagePlaceholderArg ), hive.SecureRunOptions | RunOptions.FaultOnError); //--------------------------------------------------------- // Deploy proxy related services. // Deploy the proxy manager service. ServiceHelper.StartService(hive, "neon-proxy-manager", hive.Definition.Image.ProxyManager, new CommandBundle( "docker service create", "--name", "neon-proxy-manager", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock", "--env", "VAULT_CREDENTIALS=neon-proxy-manager-credentials", "--env", "LOG_LEVEL=INFO", "--secret", "neon-proxy-manager-credentials", "--constraint", "node.role==manager", "--replicas", 1, "--restart-delay", hive.Definition.Docker.RestartDelay, ServiceHelper.ImagePlaceholderArg)); // Docker mesh routing seemed unstable on versions so we're going // to provide an option to work around this by running the PUBLIC, // PRIVATE and VAULT proxies on all nodes and publishing the ports // to the host (not the mesh). // // https://github.com/jefflill/NeonForge/issues/104 // // Note that this mode feature is documented (somewhat poorly) here: // // https://docs.docker.com/engine/swarm/services/#publish-ports var publicPublishArgs = new List <string>(); var privatePublishArgs = new List <string>(); var proxyConstraintArgs = new List <string>(); var proxyReplicasArgs = new List <string>(); var proxyModeArgs = new List <string>(); if (hive.Definition.Docker.GetAvoidIngressNetwork(hive.Definition)) { // The parameterized [docker service create --publish] option doesn't handle port ranges so we need to // specify multiple publish options. foreach (var port in HiveConst.PublicProxyPorts.Ports) { publicPublishArgs.Add($"--publish"); publicPublishArgs.Add($"mode=host,published={port},target={port}"); } for (int port = HiveConst.PublicProxyPorts.PortRange.FirstPort; port <= HiveConst.PublicProxyPorts.PortRange.LastPort; port++) { publicPublishArgs.Add($"--publish"); publicPublishArgs.Add($"mode=host,published={port},target={port}"); } foreach (var port in HiveConst.PrivateProxyPorts.Ports) { privatePublishArgs.Add($"--publish"); privatePublishArgs.Add($"mode=host,published={port},target={port}"); } for (int port = HiveConst.PrivateProxyPorts.PortRange.FirstPort; port <= HiveConst.PrivateProxyPorts.PortRange.LastPort; port++) { privatePublishArgs.Add($"--publish"); privatePublishArgs.Add($"mode=host,published={port},target={port}"); } proxyModeArgs.Add("--mode"); proxyModeArgs.Add("global"); } else { // The parameterized [docker run --publish] option doesn't handle port ranges so we need to // specify multiple publish options. foreach (var port in HiveConst.PublicProxyPorts.Ports) { publicPublishArgs.Add($"--publish"); publicPublishArgs.Add($"{port}:{port}"); } publicPublishArgs.Add($"--publish"); publicPublishArgs.Add($"{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}:{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}"); foreach (var port in HiveConst.PrivateProxyPorts.Ports) { privatePublishArgs.Add($"--publish"); privatePublishArgs.Add($"{port}:{port}"); } privatePublishArgs.Add($"--publish"); privatePublishArgs.Add($"{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}:{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}"); proxyConstraintArgs.Add($"--constraint"); proxyReplicasArgs.Add("--replicas"); if (hive.Definition.Workers.Count() > 0) { // Constrain proxies to worker nodes if there are any. proxyConstraintArgs.Add($"node.role!=manager"); if (hive.Definition.Workers.Count() == 1) { proxyReplicasArgs.Add("1"); } else { proxyReplicasArgs.Add("2"); } } else { // Constrain proxies to manager nodes nodes if there are no workers. proxyConstraintArgs.Add($"node.role==manager"); if (hive.Definition.Managers.Count() == 1) { proxyReplicasArgs.Add("1"); } else { proxyReplicasArgs.Add("2"); } } proxyModeArgs.Add("--mode"); proxyModeArgs.Add("replicated"); } // Deploy: neon-proxy-public ServiceHelper.StartService(hive, "neon-proxy-public", hive.Definition.Image.Proxy, new CommandBundle( "docker service create", "--name", "neon-proxy-public", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf", "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash", "--env", "VAULT_CREDENTIALS=neon-proxy-public-credentials", "--env", "WARN_SECONDS=300", "--env", "START_SECONDS=10", "--env", "LOG_LEVEL=INFO", "--env", "DEBUG=false", "--secret", "neon-proxy-public-credentials", publicPublishArgs, proxyConstraintArgs, proxyReplicasArgs, proxyModeArgs, "--restart-delay", hive.Definition.Docker.RestartDelay, "--network", HiveConst.PublicNetwork, ServiceHelper.ImagePlaceholderArg)); // Deploy: neon-proxy-private ServiceHelper.StartService(hive, "neon-proxy-private", hive.Definition.Image.Proxy, new CommandBundle( "docker service create", "--name", "neon-proxy-private", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf", "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash", "--env", "VAULT_CREDENTIALS=neon-proxy-private-credentials", "--env", "WARN_SECONDS=300", "--env", "START_SECONDS=10", "--env", "LOG_LEVEL=INFO", "--env", "DEBUG=false", "--secret", "neon-proxy-private-credentials", privatePublishArgs, proxyConstraintArgs, proxyReplicasArgs, proxyModeArgs, "--restart-delay", hive.Definition.Docker.RestartDelay, "--network", HiveConst.PrivateNetwork, ServiceHelper.ImagePlaceholderArg)); // Deploy: neon-proxy-public-cache var publicCacheConstraintArgs = new List <string>(); var publicCacheReplicaArgs = new List <string>(); if (hive.Definition.Proxy.PublicCacheReplicas <= hive.Definition.Workers.Count()) { publicCacheConstraintArgs.Add("--constraint"); publicCacheConstraintArgs.Add("node.role==worker"); } publicCacheReplicaArgs.Add("--replicas"); publicCacheReplicaArgs.Add($"{hive.Definition.Proxy.PublicCacheReplicas}"); ServiceHelper.StartService(hive, "neon-proxy-public-cache", hive.Definition.Image.ProxyCache, new CommandBundle( "docker service create", "--name", "neon-proxy-public-cache", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755", "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf", "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash", "--env", "WARN_SECONDS=300", "--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PublicCacheSize}", "--env", "LOG_LEVEL=INFO", "--env", "DEBUG=false", "--secret", "neon-proxy-public-credentials", publicCacheConstraintArgs, publicCacheReplicaArgs, "--restart-delay", hive.Definition.Docker.RestartDelay, "--network", HiveConst.PublicNetwork, ServiceHelper.ImagePlaceholderArg)); // Deploy: neon-proxy-private-cache var privateCacheConstraintArgs = new List <string>(); var privateCacheReplicaArgs = new List <string>(); if (hive.Definition.Proxy.PrivateCacheReplicas <= hive.Definition.Workers.Count()) { privateCacheConstraintArgs.Add("--constraint"); privateCacheConstraintArgs.Add("node.role==worker"); } privateCacheReplicaArgs.Add("--replicas"); privateCacheReplicaArgs.Add($"{hive.Definition.Proxy.PrivateCacheReplicas}"); ServiceHelper.StartService(hive, "neon-proxy-private-cache", hive.Definition.Image.ProxyCache, new CommandBundle( "docker service create", "--name", "neon-proxy-private-cache", "--detach=false", "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true", "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true", "--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755", "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf", "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash", "--env", "WARN_SECONDS=300", "--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PrivateCacheSize}", "--env", "LOG_LEVEL=INFO", "--env", "DEBUG=false", "--secret", "neon-proxy-private-credentials", privateCacheConstraintArgs, privateCacheReplicaArgs, "--restart-delay", hive.Definition.Docker.RestartDelay, "--network", HiveConst.PrivateNetwork, ServiceHelper.ImagePlaceholderArg)); }); // Log the hive into any Docker registries with credentials. firstManager.InvokeIdempotentAction("setup/registry-login", () => { foreach (var credential in hive.Definition.Docker.Registries .Where(r => !string.IsNullOrEmpty(r.Username))) { hive.Registry.Login(credential.Registry, credential.Username, credential.Password); } }); }
/// <summary> /// Verify that we can create an HTTP traffic manager rule for a /// site on the proxy port using a specific hostname and then /// verify that that the traffic manager actually works by spinning /// up a [vegomatic] based service to accept the traffic. /// </summary> /// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param> /// <param name="proxyPort">The inbound proxy port.</param> /// <param name="network">The proxy network.</param> /// <param name="trafficManager">The traffic manager.</param> /// <param name="useCache">Optionally enable caching and verify.</param> /// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param> /// <param name="useIPAddress">Optionally uses an IP addrress rather than a hostname for the rule frontend.</param> /// <returns>The tracking <see cref="Task"/>.</returns> private async Task TestHttpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, bool useCache = false, string serviceName = "vegomatic", bool useIPAddress = false) { // Append a GUID to the test name to ensure that we won't // conflict with what any previous test runs may have loaded // into the cache. testName += "-" + Guid.NewGuid().ToString("D"); // Verify that we can create an HTTP traffic manager rule for a // site on the proxy port using a specific hostname and then // verify that that the traffic manager actually works by spinning // up a [vegomatic] based service to accept the traffic. var queryCount = 100; var manager = hive.GetReachableManager(); var hostname = useIPAddress ? manager.PrivateAddress.ToString() : testHostname; manager.Connect(); using (var client = new TestHttpClient(disableConnectionReuse: true)) { // Setup the client to query the [vegomatic] service through the // proxy without needing to configure a hive DNS entry. client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/"); client.DefaultRequestHeaders.Host = testHostname; // Configure the traffic manager rule. var rule = new TrafficHttpRule() { Name = "vegomatic", CheckExpect = "status 200", CheckSeconds = 1, }; if (useCache) { rule.Cache = new TrafficHttpCache() { Enabled = true }; } rule.Frontends.Add( new TrafficHttpFrontend() { Host = useIPAddress ? null : hostname, ProxyPort = proxyPort }); rule.Backends.Add( new TrafficHttpBackend() { Server = serviceName, Port = 80 }); trafficManager.SetRule(rule); // Spin up a single [vegomatic] service instance. manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, hostname); // Query the service several times to verify that we get a response and // also that all of the responses are the same (because we have only // a single [vegomatic] instance returning its UUID). // // We're going to use a different URL for each request so that we // won't see any cache hits. var uniqueResponses = new HashSet <string>(); var viaVarnish = false; var cacheHit = false; for (int i = 0; i < queryCount; i++) { var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60"); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } var body = await response.Content.ReadAsStringAsync(); if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } Assert.Single(uniqueResponses); if (useCache) { // [viaVarnish] should be TRUE because we're routing through the cache. Assert.True(viaVarnish); // [cacheHit] should be FALSE because we used a unique URI for each request. Assert.False(cacheHit); } else { // [viaVarnish] and [cacheHit] should both be FALSE because we're not caching. Assert.False(viaVarnish); Assert.False(cacheHit); } // Repeat the test if caching is enabled with the same URLs as last time and verify that // we see cache hits this time. if (useCache) { viaVarnish = false; cacheHit = false; for (int i = 0; i < queryCount; i++) { var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60"); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } var body = await response.Content.ReadAsStringAsync(); if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } Assert.True(viaVarnish); Assert.True(cacheHit); } // Spin up a second replica and repeat the query test to verify // that we see two unique responses. // // Note that we're going to pass a new set of URLs to avoid having // any responses cached so we'll end up seeing all of the IDs. // // Note also that we need to perform these requests in parallel // to try to force Varnish to establish more than one connection // to the [vegomatic] service. If we don't do this, Varnish will // establish a single connection to one of the service instances // and keep sending traffic there resulting in us seeing only // one response UUID. manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess(); await WaitUntilReadyAsync(client.BaseAddress, hostname); // Reset the response info and do the requests. uniqueResponses.Clear(); viaVarnish = false; cacheHit = false; var tasks = new List <Task>(); var uris = new List <string>(); for (int i = 0; i < queryCount; i++) { uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250"); } foreach (var uri in uris) { tasks.Add(Task.Run( async() => { var response = await client.GetAsync(uri); var body = await response.Content.ReadAsStringAsync(); Assert.Equal(HttpStatusCode.OK, response.StatusCode); if (ViaVarnish(response)) { viaVarnish = true; } if (CacheHit(response)) { cacheHit = true; } lock (uniqueResponses) { if (!uniqueResponses.Contains(body)) { uniqueResponses.Add(body); } } })); } await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30)); if (useCache) { // [viaVarnish] should be TRUE because we're routing through the cache. Assert.True(viaVarnish); // [cacheHit] should be FALSE because we used a unique URI for each request. Assert.False(cacheHit); } else { // [viaVarnish] and [cacheHit] should both be FALSE because we're not caching. Assert.False(viaVarnish); Assert.False(cacheHit); } Assert.Equal(2, uniqueResponses.Count); } }