/// <summary>
/// Configures the hive logging related services.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void Configure(SshProxy <NodeDefinition> firstManager)
{
if (!hive.Definition.Log.Enabled)
{
return;
}
firstManager.InvokeIdempotentAction("setup/log-services",
() =>
{
var steps = new ConfigStepList();
AddElasticsearchSteps(steps);
if (hive.Definition.Dashboard.Kibana)
{
AddKibanaSteps(steps);
}
AddCollectorSteps(steps);
hive.Configure(steps);
firstManager.Status = string.Empty;
});
}
/// <summary>
/// <para>
/// Edits the [/etc/hosts] file on all hive nodes so that the line:
/// </para>
/// <code>
/// 127.0.1.1 {hostname}
/// </code>
/// <para>
/// is changed to:
/// </para>
/// <code>
/// {node.PrivateAddress} {hostname}
/// </code>
/// <para>
/// Hashicorp Vault cannot restart with the old setting, complaining about a
/// <b>""missing API address</b>.
/// </para>
/// </summary>
/// <param name="node">The target node.</param>
private void EditEtcHosts(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("edit-etc-hosts"),
() =>
{
var etcHosts = node.DownloadText("/etc/hosts");
var sbEtcHosts = new StringBuilder();
using (var reader = new StringReader(etcHosts))
{
foreach (var line in reader.Lines())
{
if (line.StartsWith("127.0.1.1"))
{
var nodeAddress = node.PrivateAddress.ToString();
var separator = new string(' ', Math.Max(16 - nodeAddress.Length, 1));
sbEtcHosts.AppendLine($"{nodeAddress}{separator}{node.Name}");
}
else
{
sbEtcHosts.AppendLine(line);
}
}
}
node.UploadText("/etc/hosts", sbEtcHosts.ToString(), permissions: "644");
node.SudoCommand("systemctl restart vault");
});
}
/// <summary>
/// Removes the Docker python module from all nodes because it conflicts with
/// Docker related Ansible playbooks.
/// </summary>
/// <param name="node">The target node.</param>
private void RemoveDockerPython(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("remove-docker-py"),
() =>
{
node.SudoCommand("su sysadmin -c 'pip uninstall -y docker'", RunOptions.LogOnErrorOnly);
});
}
/// <summary>
/// Update the Elasticsearch container launch scripts to enable automatic
/// memory settings based on any cgroup limits.
/// </summary>
/// <param name="node">The target node.</param>
private void UpdateElasticsearch(SshProxy <NodeDefinition> node)
{
// This method is called for all cluster nodes, even those
// that aren't currently hosting Elasticsearch, so we can
// update any scripts that may have been orphaned (for
// consistency).
//
// The update consists of replacing the script line that
// sets the [ES_JAVA_OPTS] environment variable with:
//
// --env ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
//
// To ensure that this feature is enabled in favor of the
// old hacked memory level settings.
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, "neon-log-esdata.sh");
node.InvokeIdempotentAction(GetIdempotentTag("neon-log-esdata"),
() =>
{
if (node.FileExists(scriptPath))
{
node.Status = $"edit: {scriptPath}";
var orgScript = node.DownloadText(scriptPath);
var newScript = new StringBuilder();
foreach (var line in new StringReader(orgScript).Lines())
{
if (line.Contains("ES_JAVA_OPTS="))
{
newScript.AppendLine(" --env \"ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap\" \\");
}
else
{
newScript.AppendLine(line);
}
}
node.UploadText(scriptPath, newScript.ToString(), permissions: "");
node.Status = string.Empty;
}
});
}
/// <summary>
/// Updates the <b>/etc/systemd/system/ceph-fuse-hivefs.service</b> to adjust restart
/// behavior: https://github.com/jefflill/NeonForge/issues/364
/// </summary>
/// <param name="node">The target node.</param>
private void UpdateCephFuse(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("ceph-fuse"),
() =>
{
node.UploadText("/etc/systemd/system/ceph-fuse-hivefs.service",
@"[Unit]
Description=Ceph FUSE client (for /mnt/hivefs)
After=network-online.target local-fs.target time-sync.target
Wants=network-online.target local-fs.target time-sync.target
Conflicts=umount.target
PartOf=ceph-fuse.target
[Service]
EnvironmentFile=-/etc/default/ceph
Environment=CLUSTER=ceph
ExecStart=/usr/bin/ceph-fuse -f -o nonempty --cluster ${CLUSTER} /mnt/hivefs
TasksMax=infinity
# These settings configure the service to restart always after
# waiting 5 seconds between attempts for up to a 365 days (effectively
# forever). [StartLimitIntervalSec] is set to the number of seconds
# in a year and [StartLimitBurst] is set to the number of 5 second
# intervals in [StartLimitIntervalSec].
Restart=always
RestartSec=5
StartLimitIntervalSec=31536000
StartLimitBurst=6307200
[Install]
WantedBy=ceph-fuse.target
WantedBy=docker.service
",
permissions: "644");
// Tell systemd to regenerate its configuration.
node.SudoCommand("systemctl daemon-reload");
});
}
/// <summary>
/// Configures the Kibana dashboard.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void ConfigureKibana(SshProxy <NodeDefinition> firstManager)
{
if (!hive.Definition.Log.Enabled)
{
return;
}
firstManager.InvokeIdempotentAction("setup/log-kibana",
() =>
{
using (var jsonClient = new JsonClient())
{
var baseLogEsDataUri = hive.Definition.LogEsDataUri;
var baseKibanaUri = $"http://{firstManager.PrivateAddress}:{HiveHostPorts.Kibana}";
var timeout = TimeSpan.FromMinutes(5);
var retry = new LinearRetryPolicy(TransientDetector.Http, maxAttempts: 30, retryInterval: TimeSpan.FromSeconds(2));
// The Kibana API calls below require the [kbn-xsrf] header.
jsonClient.DefaultRequestHeaders.Add("kbn-xsrf", "true");
// Ensure that Kibana is ready before we submit any API requests.
firstManager.Status = "wait for kibana";
retry.InvokeAsync(
async() =>
{
var response = await jsonClient.GetAsync <dynamic>($"{baseKibanaUri}/api/status");
if (response.status.overall.state != "green")
{
throw new TransientException($"Kibana [state={response.status.overall.state}]");
}
}).Wait();
// Add the index pattern to Kibana.
firstManager.Status = "configure kibana index pattern";
retry.InvokeAsync(
async() =>
{
dynamic indexPattern = new ExpandoObject();
dynamic attributes = new ExpandoObject();
attributes.title = "logstash-*";
attributes.timeFieldName = "@timestamp";
indexPattern.attributes = attributes;
await jsonClient.PostAsync($"{baseKibanaUri}/api/saved_objects/index-pattern/logstash-*?overwrite=true", indexPattern);
}).Wait();
// Now we need to save a Kibana config document so that [logstash-*] will be
// the default index and the timestamp will be displayed as UTC and have a
// more useful terse format.
firstManager.Status = "configure kibana defaults";
retry.InvokeAsync(
async() =>
{
dynamic setting = new ExpandoObject();
setting.value = "logstash-*";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/defaultIndex", setting);
setting.value = "HH:mm:ss.SSS MM-DD-YYYY";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat", setting);
setting.value = "UTC";
await jsonClient.PostAsync($"{baseKibanaUri}/api/kibana/settings/dateFormat:tz", setting);
}).Wait();
// Set the Kibana traffic manager rule.
firstManager.Status = "kibana traffic manager rule";
var rule = new TrafficHttpRule()
{
Name = "neon-log-kibana",
System = true,
Log = true,
Resolver = null
};
rule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateKibanaDashboard
});
rule.Backends.Add(
new TrafficHttpBackend()
{
Server = "neon-log-kibana",
Port = NetworkPorts.Kibana
});
hive.PrivateTraffic.SetRule(rule);
firstManager.Status = string.Empty;
}
});
}
/// <summary>
/// Initializes a near virgin server with the basic capabilities required
/// for a cluster host node.
/// </summary>
/// <param name="node">The target cluster node.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">Kubernetes setup details.</param>
/// <param name="shutdown">Optionally shuts down the node.</param>
public static void PrepareNode(SshProxy <NodeDefinition> node, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, bool shutdown = false)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentNullException>(clusterDefinition != null);
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null);
if (node.FileExists($"{KubeHostFolders.State}/setup/prepared"))
{
return; // Already prepared
}
//-----------------------------------------------------------------
// Ensure that the cluster host folders exist.
node.CreateHostFolders();
//-----------------------------------------------------------------
// Package manager configuration.
if (!clusterDefinition.NodeOptions.AllowPackageManagerIPv6)
{
// Restrict the [apt] package manager to using IPv4 to communicate
// with the package mirrors, since IPv6 often doesn't work.
node.UploadText("/etc/apt/apt.conf.d/99-force-ipv4-transport", "Acquire::ForceIPv4 \"true\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-force-ipv4-transport");
}
// Configure [apt] to retry.
node.UploadText("/etc/apt/apt.conf.d/99-retries", $"APT::Acquire::Retries \"{clusterDefinition.NodeOptions.PackageManagerRetries}\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-retries");
//-----------------------------------------------------------------
// Other configuration.
ConfigureOpenSSH(node, TimeSpan.Zero);
node.UploadConfigFiles(clusterDefinition, kubeSetupInfo);
node.UploadResources(clusterDefinition, kubeSetupInfo);
if (clusterDefinition != null)
{
ConfigureEnvironmentVariables(node, clusterDefinition);
}
node.SudoCommand("safe-apt-get update");
node.InvokeIdempotentAction("setup/prep-node",
() =>
{
node.Status = "preparing";
node.SudoCommand("setup-prep.sh");
node.Reboot(wait: true);
});
// We need to upload the cluster configuration and initialize drives attached
// to the node. We're going to assume that these are not already initialized.
// $todo(jeff.lill):
//
// We may need an option that allows an operator to pre-build a hardware
// based drive array or something. I'm going to defer this to later and
// concentrate on commodity hardware and cloud deployments for now.
CommonSteps.ConfigureEnvironmentVariables(node, clusterDefinition);
node.Status = "setup: disk";
node.SudoCommand("setup-disk.sh");
// Clear any DHCP leases to be super sure that cloned node
// VMs will obtain fresh IP addresses.
node.Status = "clear: DHCP leases";
node.SudoCommand("rm -f /var/lib/dhcp/*");
// Indicate that the node has been fully prepared.
node.SudoCommand($"touch {KubeHostFolders.State}/setup/prepared");
// Shutdown the node if requested.
if (shutdown)
{
node.Status = "shutdown";
node.SudoCommand("shutdown 0", RunOptions.Defaults | RunOptions.Shutdown);
}
}
/// <summary>
/// Performs the Docker registry cache related configuration of the node.
/// </summary>
public void Configure(SshProxy <NodeDefinition> node)
{
// NOTE:
//
// We're going to configure the certificates even if the registry cache
// isn't enabled so it'll be easier to upgrade the hive later.
// For managers, upload the individual cache certificate and
// private key files for managers [cache.crt] and [cache.key] at
// [/etc/neon-registry-cache/]. This directory will be
// mapped into the cache container.
//
// Then create the cache's data volume and start the manager's
// Registry cache container.
if (node.Metadata.IsManager)
{
node.InvokeIdempotentAction("setup/registrycache",
() =>
{
// Copy the registry cache certificate and private key to
//
// /etc/neon-registry-cache
node.Status = "run: registry-cache-server-certs.sh";
var copyCommand = new CommandBundle("./registry-cache-server-certs.sh");
var sbCopyScript = new StringBuilder();
sbCopyScript.AppendLine("mkdir -p /etc/neon-registry-cache");
sbCopyScript.AppendLine("chmod 750 /etc/neon-registry-cache");
copyCommand.AddFile($"cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
copyCommand.AddFile($"cache.key", hive.HiveLogin.HiveCertificate.KeyPem);
sbCopyScript.AppendLine($"cp cache.crt /etc/neon-registry-cache/cache.crt");
sbCopyScript.AppendLine($"cp cache.key /etc/neon-registry-cache/cache.key");
sbCopyScript.AppendLine($"chmod 640 /etc/neon-registry-cache/*");
copyCommand.AddFile("registry-cache-server-certs.sh", sbCopyScript.ToString(), isExecutable: true);
node.SudoCommand(copyCommand);
// Upload the cache certificates to every hive node at:
//
// /etc/docker/certs.d/<hostname>:{HiveHostPorts.RegistryCache}/ca.crt
//
// and then have Linux reload the trusted certificates.
node.InvokeIdempotentAction("setup/registrycache-cert",
() =>
{
node.Status = "upload: registry cache certs";
var uploadCommand = new CommandBundle("./registry-cache-client-certs.sh");
var sbUploadScript = new StringBuilder();
uploadCommand.AddFile($"hive-neon-registry-cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
foreach (var manager in hive.Definition.SortedManagers)
{
var cacheHostName = hive.Definition.GetRegistryCacheHost(manager);
sbUploadScript.AppendLine($"mkdir -p /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}");
sbUploadScript.AppendLine($"cp hive-neon-registry-cache.crt /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}/ca.crt");
}
uploadCommand.AddFile("registry-cache-client-certs.sh", sbUploadScript.ToString(), isExecutable: true);
node.SudoCommand(uploadCommand);
});
// Start the registry cache containers if enabled for the hive.
if (hive.Definition.Docker.RegistryCache)
{
// Create the registry data volume.
node.Status = "create: registry cache volume";
node.SudoCommand(new CommandBundle("docker-volume-create \"neon-registry-cache\""));
// Start the registry cache using the required Docker public registry
// credentials, if any.
var publicRegistryCredentials = hive.Definition.Docker.Registries.SingleOrDefault(r => HiveHelper.IsDockerPublicRegistry(r.Registry));
publicRegistryCredentials = publicRegistryCredentials ?? new RegistryCredentials()
{
Registry = HiveConst.DockerPublicRegistry
};
publicRegistryCredentials.Username = publicRegistryCredentials.Username ?? string.Empty;
publicRegistryCredentials.Password = publicRegistryCredentials.Password ?? string.Empty;
node.Status = "start: neon-registry-cache";
var registry = publicRegistryCredentials.Registry;
if (string.IsNullOrEmpty(registry) || registry.Equals("docker.io", StringComparison.InvariantCultureIgnoreCase))
{
registry = "registry-1.docker.io";
}
ServiceHelper.StartContainer(node, "neon-registry-cache", hive.Definition.Image.RegistryCache, RunOptions.FaultOnError | hive.SecureRunOptions,
new CommandBundle(
"docker run",
"--name", "neon-registry-cache",
"--detach",
"--restart", "always",
"--publish", $"{HiveHostPorts.DockerRegistryCache}:5000",
"--volume", "/etc/neon-registry-cache:/etc/neon-registry-cache:ro", // Registry cache certificates folder
"--volume", "neon-registry-cache:/var/lib/neon-registry-cache",
"--env", $"HOSTNAME={node.Name}.{hive.Definition.Hostnames.RegistryCache}",
"--env", $"REGISTRY=https://{registry}",
"--env", $"USERNAME={publicRegistryCredentials.Username}",
"--env", $"PASSWORD={publicRegistryCredentials.Password}",
"--env", "LOG_LEVEL=info",
ServiceHelper.ImagePlaceholderArg));
}
});
node.Status = string.Empty;
}
}
/// <summary>
/// Configures the hive services.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void Configure(SshProxy <NodeDefinition> firstManager)
{
firstManager.InvokeIdempotentAction("setup/hive-services",
() =>
{
// Ensure that Vault has been initialized.
if (!hive.HiveLogin.HasVaultRootCredentials)
{
throw new InvalidOperationException("Vault has not been initialized yet.");
}
//---------------------------------------------------------
// Persist the proxy settings.
// Obtain the AppRole credentials from Vault for the proxy manager as well as the
// public and private proxy services and persist these as Docker secrets.
firstManager.Status = "secrets: proxy services";
hive.Docker.Secret.Set("neon-proxy-manager-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-manager").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-public-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-public").Result, Formatting.Indented));
hive.Docker.Secret.Set("neon-proxy-private-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-private").Result, Formatting.Indented));
//---------------------------------------------------------
// Deploy the HiveMQ cluster.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster",
() =>
{
// We're going to list the hive nodes that will host the
// RabbitMQ cluster and sort them by node name. Then we're
// going to ensure that the first RabbitMQ node/container
// is started and ready before configuring the rest of the
// cluster so that it will bootstrap properly.
var hiveMQNodes = hive.Nodes
.Where(n => n.Metadata.Labels.HiveMQ)
.OrderBy(n => n.Name)
.ToList();
DeployHiveMQ(hiveMQNodes.First());
// Start the remaining nodes in parallel.
var actions = new List <Action>();
foreach (var node in hiveMQNodes.Skip(1))
{
actions.Add(() => DeployHiveMQ(node));
}
NeonHelper.WaitForParallel(actions);
// The RabbitMQ cluster is created with the [/] vhost and the
// [sysadmin] user by default. We need to create the [neon]
// and [app] vhosts along with the [neon] and [app] users
// and then set the appropriate permissions.
//
// We're going to run [rabbitmqctl] within the first RabbitMQ
// to accomplish this.
var hiveMQNode = hiveMQNodes.First();
// Create the vhosts.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQAppVHost}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQNeonVHost}"));
// Create the users.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQAppUser} {hive.Definition.HiveMQ.AppPassword}"));
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQNeonUser} {hive.Definition.HiveMQ.NeonPassword}"));
// Grant the [app] account full access to the [app] vhost, the [neon] account full
// access to the [neon] vhost. Note that this doesn't need to be idempotent.
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQAppVHost} {HiveConst.HiveMQAppUser} \".*\" \".*\" \".*\"");
hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQNeonVHost} {HiveConst.HiveMQNeonUser} \".*\" \".*\" \".*\"");
// Clear the UX status for the HiveMQ nodes.
foreach (var node in hiveMQNodes)
{
node.Status = string.Empty;
}
// Set the RabbitMQ cluster name to the name of the hive.
hiveMQNode.InvokeIdempotentAction("setup/hivemq-cluster-name", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_cluster_name {hive.Definition.Name}"));
});
//---------------------------------------------------------
// Initialize the public and private traffic manager managers.
hive.PublicTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PublicProxyPorts
});
hive.PrivateTraffic.UpdateSettings(
new TrafficSettings()
{
ProxyPorts = HiveConst.PrivateProxyPorts
});
//---------------------------------------------------------
// Deploy the HiveMQ traffic manager rules.
hive.FirstManager.InvokeIdempotentAction("setup/hivemq-traffic-manager-rules",
() =>
{
// Deploy private traffic manager for the AMQP endpoints.
var amqpRule = new TrafficTcpRule()
{
Name = "neon-hivemq-amqp",
System = true,
Resolver = null
};
// We're going to set this up to allow idle connections for up to
// five minutes. In theory, AMQP connections should never be idle
// this long because we've enabled level 7 keep-alive.
//
// https://github.com/jefflill/NeonForge/issues/new
amqpRule.Timeouts = new TrafficTimeouts()
{
ClientSeconds = 0,
ServerSeconds = 0
};
amqpRule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
});
foreach (var ampqNode in hive.Nodes.Where(n => n.Metadata.Labels.HiveMQ))
{
amqpRule.Backends.Add(
new TrafficTcpBackend()
{
Server = ampqNode.PrivateAddress.ToString(),
Port = HiveHostPorts.HiveMQAMQP
});
}
hive.PrivateTraffic.SetRule(amqpRule);
// Deploy private traffic manager for the management endpoints.
var adminRule = new TrafficHttpRule()
{
Name = "neon-hivemq-management",
System = true,
Resolver = null
};
// Initialize the frontends and backends.
adminRule.Frontends.Add(
new TrafficHttpFrontend()
{
ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAdmin
});
adminRule.Backends.Add(
new TrafficHttpBackend()
{
Group = HiveHostGroups.HiveMQManagers,
GroupLimit = 5,
Port = HiveHostPorts.HiveMQManagement
});
hive.PrivateTraffic.SetRule(adminRule);
});
//---------------------------------------------------------
// Deploy DNS related services.
// Deploy: neon-dns-mon
ServiceHelper.StartService(hive, "neon-dns-mon", hive.Definition.Image.DnsMon,
new CommandBundle(
"docker service create",
"--name", "neon-dns-mon",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "POLL_INTERVAL=5s",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--replicas", "1",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-dns
ServiceHelper.StartService(hive, "neon-dns", hive.Definition.Image.Dns,
new CommandBundle(
"docker service create",
"--name", "neon-dns",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/etc/powerdns/hosts,dst=/etc/powerdns/hosts",
"--mount", "type=bind,src=/dev/shm/neon-dns,dst=/neon-dns",
"--env", "POLL_INTERVAL=5s",
"--env", "VERIFY_INTERVAL=5m",
"--env", "LOG_LEVEL=INFO",
"--constraint", "node.role==manager",
"--mode", "global",
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
//---------------------------------------------------------
// Deploy [neon-hive-manager] as a service constrained to manager nodes.
string unsealSecretOption = null;
if (hive.Definition.Vault.AutoUnseal)
{
var vaultCredentials = NeonHelper.JsonClone <VaultCredentials>(hive.HiveLogin.VaultCredentials);
// We really don't want to include the root token in the credentials
// passed to [neon-hive-manager], which needs the unseal keys so
// we'll clear that here.
vaultCredentials.RootToken = null;
hive.Docker.Secret.Set("neon-hive-manager-vaultkeys", Encoding.UTF8.GetBytes(NeonHelper.JsonSerialize(vaultCredentials, Formatting.Indented)));
unsealSecretOption = "--secret=neon-hive-manager-vaultkeys";
}
ServiceHelper.StartService(hive, "neon-hive-manager", hive.Definition.Image.HiveManager,
new CommandBundle(
"docker service create",
"--name", "neon-hive-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-ssh-credentials",
unsealSecretOption,
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg
),
hive.SecureRunOptions | RunOptions.FaultOnError);
//---------------------------------------------------------
// Deploy proxy related services.
// Deploy the proxy manager service.
ServiceHelper.StartService(hive, "neon-proxy-manager", hive.Definition.Image.ProxyManager,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-manager",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--env", "VAULT_CREDENTIALS=neon-proxy-manager-credentials",
"--env", "LOG_LEVEL=INFO",
"--secret", "neon-proxy-manager-credentials",
"--constraint", "node.role==manager",
"--replicas", 1,
"--restart-delay", hive.Definition.Docker.RestartDelay,
ServiceHelper.ImagePlaceholderArg));
// Docker mesh routing seemed unstable on versions so we're going
// to provide an option to work around this by running the PUBLIC,
// PRIVATE and VAULT proxies on all nodes and publishing the ports
// to the host (not the mesh).
//
// https://github.com/jefflill/NeonForge/issues/104
//
// Note that this mode feature is documented (somewhat poorly) here:
//
// https://docs.docker.com/engine/swarm/services/#publish-ports
var publicPublishArgs = new List <string>();
var privatePublishArgs = new List <string>();
var proxyConstraintArgs = new List <string>();
var proxyReplicasArgs = new List <string>();
var proxyModeArgs = new List <string>();
if (hive.Definition.Docker.GetAvoidIngressNetwork(hive.Definition))
{
// The parameterized [docker service create --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PublicProxyPorts.PortRange.FirstPort; port <= HiveConst.PublicProxyPorts.PortRange.LastPort; port++)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"mode=host,published={port},target={port}");
}
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
for (int port = HiveConst.PrivateProxyPorts.PortRange.FirstPort; port <= HiveConst.PrivateProxyPorts.PortRange.LastPort; port++)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"mode=host,published={port},target={port}");
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("global");
}
else
{
// The parameterized [docker run --publish] option doesn't handle port ranges so we need to
// specify multiple publish options.
foreach (var port in HiveConst.PublicProxyPorts.Ports)
{
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{port}:{port}");
}
publicPublishArgs.Add($"--publish");
publicPublishArgs.Add($"{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}:{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}");
foreach (var port in HiveConst.PrivateProxyPorts.Ports)
{
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{port}:{port}");
}
privatePublishArgs.Add($"--publish");
privatePublishArgs.Add($"{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}:{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}");
proxyConstraintArgs.Add($"--constraint");
proxyReplicasArgs.Add("--replicas");
if (hive.Definition.Workers.Count() > 0)
{
// Constrain proxies to worker nodes if there are any.
proxyConstraintArgs.Add($"node.role!=manager");
if (hive.Definition.Workers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
else
{
// Constrain proxies to manager nodes nodes if there are no workers.
proxyConstraintArgs.Add($"node.role==manager");
if (hive.Definition.Managers.Count() == 1)
{
proxyReplicasArgs.Add("1");
}
else
{
proxyReplicasArgs.Add("2");
}
}
proxyModeArgs.Add("--mode");
proxyModeArgs.Add("replicated");
}
// Deploy: neon-proxy-public
ServiceHelper.StartService(hive, "neon-proxy-public", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-public-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicPublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private
ServiceHelper.StartService(hive, "neon-proxy-private", hive.Definition.Image.Proxy,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "VAULT_CREDENTIALS=neon-proxy-private-credentials",
"--env", "WARN_SECONDS=300",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privatePublishArgs,
proxyConstraintArgs,
proxyReplicasArgs,
proxyModeArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-public-cache
var publicCacheConstraintArgs = new List <string>();
var publicCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PublicCacheReplicas <= hive.Definition.Workers.Count())
{
publicCacheConstraintArgs.Add("--constraint");
publicCacheConstraintArgs.Add("node.role==worker");
}
publicCacheReplicaArgs.Add("--replicas");
publicCacheReplicaArgs.Add($"{hive.Definition.Proxy.PublicCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-public-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-public-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PublicCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-public-credentials",
publicCacheConstraintArgs,
publicCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PublicNetwork,
ServiceHelper.ImagePlaceholderArg));
// Deploy: neon-proxy-private-cache
var privateCacheConstraintArgs = new List <string>();
var privateCacheReplicaArgs = new List <string>();
if (hive.Definition.Proxy.PrivateCacheReplicas <= hive.Definition.Workers.Count())
{
privateCacheConstraintArgs.Add("--constraint");
privateCacheConstraintArgs.Add("node.role==worker");
}
privateCacheReplicaArgs.Add("--replicas");
privateCacheReplicaArgs.Add($"{hive.Definition.Proxy.PrivateCacheReplicas}");
ServiceHelper.StartService(hive, "neon-proxy-private-cache", hive.Definition.Image.ProxyCache,
new CommandBundle(
"docker service create",
"--name", "neon-proxy-private-cache",
"--detach=false",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PrivateCacheSize}",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--secret", "neon-proxy-private-credentials",
privateCacheConstraintArgs,
privateCacheReplicaArgs,
"--restart-delay", hive.Definition.Docker.RestartDelay,
"--network", HiveConst.PrivateNetwork,
ServiceHelper.ImagePlaceholderArg));
});
// Log the hive into any Docker registries with credentials.
firstManager.InvokeIdempotentAction("setup/registry-login",
() =>
{
foreach (var credential in hive.Definition.Docker.Registries
.Where(r => !string.IsNullOrEmpty(r.Username)))
{
hive.Registry.Login(credential.Registry, credential.Username, credential.Password);
}
});
}