/// <summary>
/// Adds the steps required to configure the hive log collector which aggregates log events received
/// from all hive nodes via their [neon-log-host] containers.
/// </summary>
/// <param name="steps">The configuration step list.</param>
private void AddCollectorSteps(ConfigStepList steps)
{
// Add the steps to create the service.
ServiceHelper.AddServiceStartSteps(hive, steps, "neon-log-collector", hive.Definition.Image.LogCollector,
new CommandBundle(
"docker service create",
"--name", "neon-log-collector",
"--detach=false",
"--mode", "global",
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--endpoint-mode", "vip",
"--network", $"{HiveConst.PrivateNetwork}",
"--constraint", $"node.role==manager",
"--mount", "type=bind,source=/etc/neon/host-env,destination=/etc/neon/host-env,readonly=true",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
// Deploy the [neon-log-collector] traffic manager rule.
steps.Add(ActionStep.Create(hive.FirstManager.Name, "setup/neon-log-collection-lbrule",
node =>
{
node.Status = "set neon-log-collector traffic manager rule";
// Configure a private hive proxy TCP route so the [neon-log-host] containers
// will be able to reach the collectors.
var rule = new TrafficTcpRule()
{
Name = "neon-log-collector",
System = true,
Log = false // This is important: we don't want to SPAM the log database with its own traffic.
};
rule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateTcpLogCollector
});
rule.Backends.Add(
new TrafficTcpBackend()
{
Server = "neon-log-collector",
Port = NetworkPorts.TDAgentForward
});
hive.PrivateTraffic.SetRule(rule);
}));
}
/// <summary>
/// Verify that we can create an TCP traffic manager rule for a
/// site on the public port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestTcpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an TCP traffic manager rule for a
// site on the public port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
var hostname = manager.PrivateAddress.ToString();
manager.Connect();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficTcpRule()
{
Name = "vegomatic",
CheckSeconds = 1,
};
rule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficTcpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
var uniqueResponses = new HashSet <string>();
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note that we're going to pass a new set of URLs to avoid having
// any responses cached so we'll end up seeing all of the IDs.
//
// Note also that we need to perform these requests in parallel
// to try to force Varnish to establish more than one connection
// to the [vegomatic] service. If we don't do this, Varnish will
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
Assert.Equal(2, uniqueResponses.Count);
}
}
public Test_HiveFixture(HiveFixture hive)
{
// We're passing [login=null] below to connect to the hive specified
// by the NEON_TEST_HIVE environment variable. This needs to be
// initialized with the login for a deployed hive.
if (hive.LoginAndInitialize())
{
hive.Reset();
}
this.hiveFixture = hive;
this.hive = hive.Hive;
// Initialize the hive state.
NeonHelper.WaitForParallel(
new Action[]
{
() => hive.CreateSecret("secret_text", "hello"),
() => hive.CreateSecret("secret_data", new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }),
() => hive.CreateConfig("config_text", "hello"),
() => hive.CreateConfig("config_data", new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }),
() => hive.CreateNetwork("test-network"),
() => hive.CreateService("test-service", "nhive/test"),
async() =>
{
// Create the HiveMQ [test] user, [test] vhost along with the [test-queue].
using (var mqManager = hive.ConnectHiveMQManager())
{
var mqUser = await mqManager.CreateUserAsync(new UserInfo("test-user", "password"));
var mqVHost = await mqManager.CreateVirtualHostAsync("test-vhost");
var mqQueue = await mqManager.CreateQueueAsync(new QueueInfo("test-queue", autoDelete: false, durable: true, arguments: new InputArguments()), mqVHost);
await mqManager.CreatePermissionAsync(new PermissionInfo(mqUser, mqVHost));
}
},
() =>
{
var composeText =
@"version: '3'
services:
sleeper:
image: nhive/test
deploy:
replicas: 2
";
hive.DeployStack("test-stack", composeText);
},
() =>
{
var publicRule = new TrafficTcpRule();
publicRule.Name = "test-rule";
publicRule.Frontends.Add(new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPublicFirstUser
});
publicRule.Backends.Add(new TrafficTcpBackend()
{
Server = "127.0.0.1", Port = 10000
});
hive.PutTrafficManagerRule("public", publicRule);
},
() =>
{
var privateRule = new TrafficTcpRule();
privateRule.Name = "test-rule";
privateRule.Frontends.Add(new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateFirstUser
});
privateRule.Backends.Add(new TrafficTcpBackend()
{
Server = "127.0.0.1", Port = 10000
});
hive.PutTrafficManagerRule("private", privateRule);
hive.PutCertificate("test-certificate", TestCertificate.CombinedPem);
hive.SetSelfSignedCertificate("test-certificate2", "*.foo.com");
},
async() => await hive.Consul.KV.PutString("test/value1", "one"),
async() => await hive.Consul.KV.PutString("test/value2", "two"),
async() => await hive.Consul.KV.PutString("test/folder/value3", "three"),
async() => await hive.Consul.KV.PutString("test/folder/value4", "four")
});
}
/// <summary>
/// Configures the hive services.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void Configure(SshProxy <NodeDefinition> firstManager)
{
firstManager.InvokeIdempotentAction("setup/hive-services",
() =>
{
// Ensure that Vault has been initialized.
if (!hive.HiveLogin.HasVaultRootCredentials)
{
throw new InvalidOperationException("Vault has not been initialized yet.");
}
//---------------------------------------------------------
// Persist the proxy settings.
// Obtain the AppRole credentials from Vault for the proxy manager as well as the
// public and private proxy services and persist these as Docker secrets.
firstManager.Status = "secrets: proxy services";
hive.Docker.Secret.Set("neon-proxy-manager-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-manager").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-public-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-public").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-private-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-private").Result, Formatting.Indented));
//---------------------------------------------------------
// Deploy the HiveMQ cluster.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster",
() =>
{
// We're going to list the hive nodes that will host the
// RabbitMQ cluster and sort them by node name. Then we're
// going to ensure that the first RabbitMQ node/container
// is started and ready before configuring the rest of the
// cluster so that it will bootstrap properly.
var hiveMQNodes = hive.Nodes
.Where(n => n.Metadata.Labels.HiveMQ)
.OrderBy(n => n.Name)
.ToList();
DeployHiveMQ(hiveMQNodes.First());
// Start the remaining nodes in parallel.
var actions = new List <Action>();
foreach (var node in hiveMQNodes.Skip(1))
{
actions.Add(() => DeployHiveMQ(node));
}
NeonHelper.WaitForParallel(actions);
// The RabbitMQ cluster is created with the [/] vhost and the
// [sysadmin] user by default. We need to create the [neon]
// and [app] vhosts along with the [neon] and [app] users
// and then set the appropriate permissions.
//
// We're going to run [rabbitmqctl] within the first RabbitMQ
// to accomplish this.
var hiveMQNode = hiveMQNodes.First();
// Create the vhosts.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQAppVHost}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQNeonVHost}"));
// Create the users.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQAppUser} {hive.Definition.HiveMQ.AppPassword}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQNeonUser} {hive.Definition.HiveMQ.NeonPassword}"));
// Grant the [app] account full access to the [app] vhost, the [neon] account full
// access to the [neon] vhost. Note that this doesn't need to be idempotent.
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQAppVHost} {HiveConst.HiveMQAppUser} \".*\" \".*\" \".*\"");
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQNeonVHost} {HiveConst.HiveMQNeonUser} \".*\" \".*\" \".*\"");
// Clear the UX status for the HiveMQ nodes.
foreach (var node in hiveMQNodes)
{
node.Status = string.Empty;
}
// Set the RabbitMQ cluster name to the name of the hive.
hiveMQNode.InvokeIdempotentAction("setup/hivemq-cluster-name", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_cluster_name {hive.Definition.Name}"));
});
//---------------------------------------------------------
// Initialize the public and private traffic manager managers.
hive.PublicTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PublicProxyPorts
});
hive.PrivateTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PrivateProxyPorts
});
//---------------------------------------------------------
// Deploy the HiveMQ traffic manager rules.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-traffic-manager-rules",
() =>
{
// Deploy private traffic manager for the AMQP endpoints.
var amqpRule = new TrafficTcpRule()
{
Name = "neon-hivemq-amqp",
System = true,
Resolver = null
};
// We're going to set this up to allow idle connections for up to
// five minutes. In theory, AMQP connections should never be idle
// this long because we've enabled level 7 keep-alive.
//
// https://github.com/jefflill/NeonForge/issues/new
amqpRule.Timeouts = new TrafficTimeouts()
{
ClientSeconds = 0,
ServerSeconds = 0
};
amqpRule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
});
foreach (var ampqNode in hive.Nodes.Where(n => n.Metadata.Labels.HiveMQ))
{
amqpRule.Backends.Add(
new TrafficTcpBackend()
{
Server = ampqNode.PrivateAddress.ToString(),
Port = HiveHostPorts.HiveMQAMQP
});
}
hive.PrivateTraffic.SetRule(amqpRule);
// Deploy private traffic manager for the management endpoints.
var adminRule = new TrafficHttpRule()
{
Name = "neon-hivemq-management",
System = true,
Resolver = null
};
// Initialize the frontends and backends.
adminRule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAdmin
});
adminRule.Backends.Add(
new TrafficHttpBackend()
{
Group = HiveHostGroups.HiveMQManagers,
GroupLimit = 5,
Port = HiveHostPorts.HiveMQManagement
});
hive.PrivateTraffic.SetRule(adminRule);
});
//---------------------------------------------------------
// Deploy DNS related services.
// Deploy: neon-dns-mon
ServiceHelper.StartService(hive, "neon-dns-mon", hive.Definition.Image.DnsMon,
new CommandBundle(
"docker service create",
"--name", "neon-dns-mon",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "POLL_INTERVAL=5s",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--replicas", "1",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-dns
ServiceHelper.StartService(hive, "neon-dns", hive.Definition.Image.Dns,
new CommandBundle(
"docker service create",
"--name", "neon-dns",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/etc/powerdns/hosts,dst=/etc/powerdns/hosts",
"--mount", "type=bind,src=/dev/shm/neon-dns,dst=/neon-dns",
"--env", "POLL_INTERVAL=5s",
"--env", "VERIFY_INTERVAL=5m",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--mode", "global",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
//---------------------------------------------------------
// Deploy [neon-hive-manager] as a service constrained to manager nodes.
string unsealSecretOption = null;
if (hive.Definition.Vault.AutoUnseal)
{
var vaultCredentials = NeonHelper.JsonClone <VaultCredentials>(hive.HiveLogin.VaultCredentials);
// We really don't want to include the root token in the credentials
// passed to [neon-hive-manager], which needs the unseal keys so
// we'll clear that here.
vaultCredentials.RootToken = null;
hive.Docker.Secret.Set("neon-hive-manager-vaultkeys", Encoding.UTF8.GetBytes(NeonHelper.JsonSerialize(vaultCredentials, Formatting.Indented)));
unsealSecretOption = "--secret=neon-hive-manager-vaultkeys";
}
ServiceHelper.StartService(hive, "neon-hive-manager", hive.Definition.Image.HiveManager,
new CommandBundle(
"docker service create",
"--name", "neon-hive-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-ssh-credentials",
unsealSecretOption,
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg
),
hive.SecureRunOptions | RunOptions.FaultOnError);
//---------------------------------------------------------
// Deploy proxy related services.
// Deploy the proxy manager service.
ServiceHelper.StartService(hive, "neon-proxy-manager", hive.Definition.Image.ProxyManager,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "VAULT_CREDENTIALS=neon-proxy-manager-credentials",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-proxy-manager-credentials",
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Docker mesh routing seemed unstable on versions so we're going
// to provide an option to work around this by running the PUBLIC,
// PRIVATE and VAULT proxies on all nodes and publishing the ports
// to the host (not the mesh).
//
// https://github.com/jefflill/NeonForge/issues/104
//
// Note that this mode feature is documented (somewhat poorly) here:
//
// https://docs.docker.com/engine/swarm/services/#publish-ports
var publicPublishArgs = new List <string>();
var privatePublishArgs = new List <string>();
var proxyConstraintArgs = new List <string>();
var proxyReplicasArgs = new List <string>();
var proxyModeArgs = new List <string>();
if (hive.Definition.Docker.GetAvoidIngressNetwork(hive.Definition))
{
// The parameterized [docker service create --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PublicProxyPorts.PortRange.FirstPort; port <= HiveConst.PublicProxyPorts.PortRange.LastPort; port++)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PrivateProxyPorts.PortRange.FirstPort; port <= HiveConst.PrivateProxyPorts.PortRange.LastPort; port++)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("global");
}
else
{
// The parameterized [docker run --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{port}:{port}");
}
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}:{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}");
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{port}:{port}");
}
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}:{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}");
proxyConstraintArgs.Add($"--constraint");
proxyReplicasArgs.Add("--replicas");
if (hive.Definition.Workers.Count() > 0)
{
// Constrain proxies to worker nodes if there are any.
proxyConstraintArgs.Add($"node.role!=manager");
if (hive.Definition.Workers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
else
{
// Constrain proxies to manager nodes nodes if there are no workers.
proxyConstraintArgs.Add($"node.role==manager");
if (hive.Definition.Managers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("replicated");
}
// Deploy: neon-proxy-public
ServiceHelper.StartService(hive, "neon-proxy-public", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-public-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicPublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private
ServiceHelper.StartService(hive, "neon-proxy-private", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-private-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privatePublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-public-cache
var publicCacheConstraintArgs = new List <string>();
var publicCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PublicCacheReplicas <= hive.Definition.Workers.Count())
{
publicCacheConstraintArgs.Add("--constraint");
publicCacheConstraintArgs.Add("node.role==worker");
}
publicCacheReplicaArgs.Add("--replicas");
publicCacheReplicaArgs.Add($"{hive.Definition.Proxy.PublicCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-public-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PublicCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicCacheConstraintArgs,
publicCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private-cache
var privateCacheConstraintArgs = new List <string>();
var privateCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PrivateCacheReplicas <= hive.Definition.Workers.Count())
{
privateCacheConstraintArgs.Add("--constraint");
privateCacheConstraintArgs.Add("node.role==worker");
}
privateCacheReplicaArgs.Add("--replicas");
privateCacheReplicaArgs.Add($"{hive.Definition.Proxy.PrivateCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-private-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PrivateCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privateCacheConstraintArgs,
privateCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
});
// Log the hive into any Docker registries with credentials.
firstManager.InvokeIdempotentAction("setup/registry-login",
() =>
{
foreach (var credential in hive.Definition.Docker.Registries
.Where(r => !string.IsNullOrEmpty(r.Username)))
{
hive.Registry.Login(credential.Registry, credential.Username, credential.Password);
}
});
}
/// <summary>
/// <para>
/// This release relocated the HiveMQ account settings from Docker secrets
/// to hive global Consul keys so that these can be available to containers
/// too (e.g. running on pets) and also to make deploying services that use
/// queuing more transparent.
/// </para>
/// <para>
/// This update copies these settings from the Docker secrets to Consul and
/// then removes these secret references from the built-in hive service
/// scripts and also updates the services by pulling the latest images and
/// removing the secrets there too.
/// </para>
/// <note>
/// We're going to leave the old Docker secrets in place on the off chance
/// that a user service is still referencing them.
/// </note>
/// </summary>
private void UpdateHiveMQSettings()
{
var firstManager = Hive.FirstManager;
// Fetch the current HiveMQ settings from the Docker secrets.
// This is slow, so we'll capture these in parallel.
firstManager.Status = "reading hivemq secrets";
var appSettings = (HiveMQSettings)null;
var neonSettings = (HiveMQSettings)null;
var sysadminSettings = (HiveMQSettings)null;
NeonHelper.WaitForParallel(
new Action[]
{
new Action(() => appSettings = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-app")),
new Action(() => neonSettings = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-neon")),
new Action(() => sysadminSettings = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-sysadmin")),
},
TimeSpan.FromSeconds(120));
firstManager.Status = string.Empty;
// Edit the service start scripts by removing any lines that
// attach a HiveMQ account secret.
var services = new string[]
{
"neon-hive-manager",
"neon-proxy-manager",
"neon-proxy-public",
"neon-proxy-private"
};
foreach (var manager in Hive.Managers)
{
foreach (var service in services)
{
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, $"{service}.sh");
manager.Status = $"edit: {scriptPath}";
// Edit the service start scripts by removing any lines that
// attach a HiveMQ account secret.
var curScript = manager.DownloadText(scriptPath);
var newScript = new StringBuilder();
using (var reader = new StringReader(curScript))
{
foreach (var line in reader.Lines())
{
if (!line.Trim().StartsWith("--secret neon-hivemq-settings-"))
{
newScript.AppendLine(line);
}
}
}
manager.UploadText(scriptPath, newScript.ToString(), permissions: "660");
manager.Status = string.Empty;
}
}
// Update the impacted services by removing the secret and pulling the latest image.
firstManager.Status = "neon-hive-manager";
firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.HiveManager)} neon-hive-manager");
firstManager.Status = "neon-proxy-manager";
firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.ProxyManager)} neon-proxy-manager");
firstManager.Status = "neon-proxy-public";
firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.Proxy)} neon-proxy-public");
firstManager.Status = "neon-proxy-private";
firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.Proxy)} neon-proxy-private");
firstManager.Status = string.Empty;
// Redeploy the [neon-hivemq-ampq] private traffic manager to be a TCP rather than an HTTP proxy.
var ampqRule = new TrafficTcpRule()
{
Name = "neon-hivemq-ampq",
System = true,
Resolver = null
};
ampqRule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
});
ampqRule.Backends.Add(
new TrafficTcpBackend()
{
Group = HiveHostGroups.HiveMQ,
GroupLimit = 5,
Port = HiveHostPorts.HiveMQAMQP
});
Hive.PrivateTraffic.SetRule(ampqRule);
}