Ejemplo n.º 1
0
        /// <summary>
        /// Adds the steps required to configure the hive log collector which aggregates log events received
        /// from all hive nodes via their [neon-log-host] containers.
        /// </summary>
        /// <param name="steps">The configuration step list.</param>
        private void AddCollectorSteps(ConfigStepList steps)
        {
            // Add the steps to create the service.

            ServiceHelper.AddServiceStartSteps(hive, steps, "neon-log-collector", hive.Definition.Image.LogCollector,
                                               new CommandBundle(
                                                   "docker service create",
                                                   "--name", "neon-log-collector",
                                                   "--detach=false",
                                                   "--mode", "global",
                                                   "--restart-delay", hive.Definition.Docker.RestartDelay,
                                                   "--endpoint-mode", "vip",
                                                   "--network", $"{HiveConst.PrivateNetwork}",
                                                   "--constraint", $"node.role==manager",
                                                   "--mount", "type=bind,source=/etc/neon/host-env,destination=/etc/neon/host-env,readonly=true",
                                                   "--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
                                                   ServiceHelper.ImagePlaceholderArg));

            // Deploy the [neon-log-collector] traffic manager rule.

            steps.Add(ActionStep.Create(hive.FirstManager.Name, "setup/neon-log-collection-lbrule",
                                        node =>
            {
                node.Status = "set neon-log-collector traffic manager rule";

                // Configure a private hive proxy TCP route so the [neon-log-host] containers
                // will be able to reach the collectors.

                var rule = new TrafficTcpRule()
                {
                    Name   = "neon-log-collector",
                    System = true,
                    Log    = false        // This is important: we don't want to SPAM the log database with its own traffic.
                };

                rule.Frontends.Add(
                    new TrafficTcpFrontend()
                {
                    ProxyPort = HiveHostPorts.ProxyPrivateTcpLogCollector
                });

                rule.Backends.Add(
                    new TrafficTcpBackend()
                {
                    Server = "neon-log-collector",
                    Port   = NetworkPorts.TDAgentForward
                });

                hive.PrivateTraffic.SetRule(rule);
            }));
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Verify that we can create an TCP traffic manager rule for a
        /// site on the public port using a specific hostname and then
        /// verify that that the traffic manager actually works by spinning
        /// up a [vegomatic] based service to accept the traffic.
        /// </summary>
        /// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
        /// <param name="proxyPort">The inbound proxy port.</param>
        /// <param name="network">The proxy network.</param>
        /// <param name="trafficManager">The traffic manager.</param>
        /// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
        /// <returns>The tracking <see cref="Task"/>.</returns>
        private async Task TestTcpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
        {
            // Append a GUID to the test name to ensure that we won't
            // conflict with what any previous test runs may have loaded
            // into the cache.

            testName += "-" + Guid.NewGuid().ToString("D");

            // Verify that we can create an TCP traffic manager rule for a
            // site on the public port using a specific hostname and then
            // verify that that the traffic manager actually works by spinning
            // up a [vegomatic] based service to accept the traffic.

            var queryCount = 100;
            var manager    = hive.GetReachableManager();
            var hostname   = manager.PrivateAddress.ToString();

            manager.Connect();

            using (var client = new TestHttpClient(disableConnectionReuse: true))
            {
                // Setup the client to query the [vegomatic] service through the
                // proxy without needing to configure a hive DNS entry.

                client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
                client.DefaultRequestHeaders.Host = testHostname;

                // Configure the traffic manager rule.

                var rule = new TrafficTcpRule()
                {
                    Name         = "vegomatic",
                    CheckSeconds = 1,
                };

                rule.Frontends.Add(
                    new TrafficTcpFrontend()
                {
                    ProxyPort = proxyPort
                });

                rule.Backends.Add(
                    new TrafficTcpBackend()
                {
                    Server = serviceName,
                    Port   = 80
                });

                trafficManager.SetRule(rule);

                // Spin up a single [vegomatic] service instance.

                manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
                await WaitUntilReadyAsync(client.BaseAddress, hostname);

                // Query the service several times to verify that we get a response and
                // also that all of the responses are the same (because we have only
                // a single [vegomatic] instance returning its UUID).

                var uniqueResponses = new HashSet <string>();

                for (int i = 0; i < queryCount; i++)
                {
                    var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");

                    Assert.Equal(HttpStatusCode.OK, response.StatusCode);

                    var body = await response.Content.ReadAsStringAsync();

                    if (!uniqueResponses.Contains(body))
                    {
                        uniqueResponses.Add(body);
                    }
                }

                Assert.Single(uniqueResponses);

                // Spin up a second replica and repeat the query test to verify
                // that we see two unique responses.
                //
                // Note that we're going to pass a new set of URLs to avoid having
                // any responses cached so we'll end up seeing all of the IDs.
                //
                // Note also that we need to perform these requests in parallel
                // to try to force Varnish to establish more than one connection
                // to the [vegomatic] service.  If we don't do this, Varnish will
                // establish a single connection to one of the service instances
                // and keep sending traffic there resulting in us seeing only
                // one response UUID.

                manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
                await WaitUntilReadyAsync(client.BaseAddress, hostname);

                // Reset the response info and do the requests.

                uniqueResponses.Clear();

                var tasks = new List <Task>();
                var uris  = new List <string>();

                for (int i = 0; i < queryCount; i++)
                {
                    uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
                }

                foreach (var uri in uris)
                {
                    tasks.Add(Task.Run(
                                  async() =>
                    {
                        var response = await client.GetAsync(uri);
                        var body     = await response.Content.ReadAsStringAsync();

                        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
                    }));
                }

                await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));

                Assert.Equal(2, uniqueResponses.Count);
            }
        }
Ejemplo n.º 3
0
        public Test_HiveFixture(HiveFixture hive)
        {
            // We're passing [login=null] below to connect to the hive specified
            // by the NEON_TEST_HIVE environment variable.  This needs to be
            // initialized with the login for a deployed hive.

            if (hive.LoginAndInitialize())
            {
                hive.Reset();
            }

            this.hiveFixture = hive;
            this.hive        = hive.Hive;

            // Initialize the hive state.

            NeonHelper.WaitForParallel(
                new Action[]
            {
                () => hive.CreateSecret("secret_text", "hello"),
                () => hive.CreateSecret("secret_data", new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }),
                () => hive.CreateConfig("config_text", "hello"),
                () => hive.CreateConfig("config_data", new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }),
                () => hive.CreateNetwork("test-network"),
                () => hive.CreateService("test-service", "nhive/test"),
                async() =>
                {
                    // Create the HiveMQ [test] user, [test] vhost along with the [test-queue].

                    using (var mqManager = hive.ConnectHiveMQManager())
                    {
                        var mqUser  = await mqManager.CreateUserAsync(new UserInfo("test-user", "password"));
                        var mqVHost = await mqManager.CreateVirtualHostAsync("test-vhost");
                        var mqQueue = await mqManager.CreateQueueAsync(new QueueInfo("test-queue", autoDelete: false, durable: true, arguments: new InputArguments()), mqVHost);

                        await mqManager.CreatePermissionAsync(new PermissionInfo(mqUser, mqVHost));
                    }
                },
                () =>
                {
                    var composeText =
                        @"version: '3'

services:
  sleeper:
    image: nhive/test
    deploy:
      replicas: 2
";
                    hive.DeployStack("test-stack", composeText);
                },
                () =>
                {
                    var publicRule = new TrafficTcpRule();

                    publicRule.Name = "test-rule";
                    publicRule.Frontends.Add(new TrafficTcpFrontend()
                    {
                        ProxyPort = HiveHostPorts.ProxyPublicFirstUser
                    });
                    publicRule.Backends.Add(new TrafficTcpBackend()
                    {
                        Server = "127.0.0.1", Port = 10000
                    });

                    hive.PutTrafficManagerRule("public", publicRule);
                },
                () =>
                {
                    var privateRule = new TrafficTcpRule();

                    privateRule.Name = "test-rule";
                    privateRule.Frontends.Add(new TrafficTcpFrontend()
                    {
                        ProxyPort = HiveHostPorts.ProxyPrivateFirstUser
                    });
                    privateRule.Backends.Add(new TrafficTcpBackend()
                    {
                        Server = "127.0.0.1", Port = 10000
                    });

                    hive.PutTrafficManagerRule("private", privateRule);
                    hive.PutCertificate("test-certificate", TestCertificate.CombinedPem);
                    hive.SetSelfSignedCertificate("test-certificate2", "*.foo.com");
                },
                async() => await hive.Consul.KV.PutString("test/value1", "one"),
                async() => await hive.Consul.KV.PutString("test/value2", "two"),
                async() => await hive.Consul.KV.PutString("test/folder/value3", "three"),
                async() => await hive.Consul.KV.PutString("test/folder/value4", "four")
            });
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Configures the hive services.
        /// </summary>
        /// <param name="firstManager">The first hive proxy manager.</param>
        public void Configure(SshProxy <NodeDefinition> firstManager)
        {
            firstManager.InvokeIdempotentAction("setup/hive-services",
                                                () =>
            {
                // Ensure that Vault has been initialized.

                if (!hive.HiveLogin.HasVaultRootCredentials)
                {
                    throw new InvalidOperationException("Vault has not been initialized yet.");
                }

                //---------------------------------------------------------
                // Persist the proxy settings.

                // Obtain the AppRole credentials from Vault for the proxy manager as well as the
                // public and private proxy services and persist these as Docker secrets.

                firstManager.Status = "secrets: proxy services";

                hive.Docker.Secret.Set("neon-proxy-manager-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-manager").Result, Formatting.Indented));
                hive.Docker.Secret.Set("neon-proxy-public-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-public").Result, Formatting.Indented));
                hive.Docker.Secret.Set("neon-proxy-private-credentials", NeonHelper.JsonSerialize(hive.Vault.Client.GetAppRoleCredentialsAsync("neon-proxy-private").Result, Formatting.Indented));

                //---------------------------------------------------------
                // Deploy the HiveMQ cluster.

                hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster",
                                                         () =>
                {
                    // We're going to list the hive nodes that will host the
                    // RabbitMQ cluster and sort them by node name.  Then we're
                    // going to ensure that the first RabbitMQ node/container
                    // is started and ready before configuring the rest of the
                    // cluster so that it will bootstrap properly.

                    var hiveMQNodes = hive.Nodes
                                      .Where(n => n.Metadata.Labels.HiveMQ)
                                      .OrderBy(n => n.Name)
                                      .ToList();

                    DeployHiveMQ(hiveMQNodes.First());

                    // Start the remaining nodes in parallel.

                    var actions = new List <Action>();

                    foreach (var node in hiveMQNodes.Skip(1))
                    {
                        actions.Add(() => DeployHiveMQ(node));
                    }

                    NeonHelper.WaitForParallel(actions);

                    // The RabbitMQ cluster is created with the [/] vhost and the
                    // [sysadmin] user by default.  We need to create the [neon]
                    // and [app] vhosts along with the [neon] and [app] users
                    // and then set the appropriate permissions.
                    //
                    // We're going to run [rabbitmqctl] within the first RabbitMQ
                    // to accomplish this.

                    var hiveMQNode = hiveMQNodes.First();

                    // Create the vhosts.

                    hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQAppVHost}"));
                    hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-vhost-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_vhost {HiveConst.HiveMQNeonVHost}"));

                    // Create the users.

                    hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-app", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQAppUser} {hive.Definition.HiveMQ.AppPassword}"));
                    hive.FirstManager.InvokeIdempotentAction("setup/hivemq-cluster-user-neon", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl add_user {HiveConst.HiveMQNeonUser} {hive.Definition.HiveMQ.NeonPassword}"));

                    // Grant the [app] account full access to the [app] vhost, the [neon] account full
                    // access to the [neon] vhost.  Note that this doesn't need to be idempotent.

                    hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQAppVHost} {HiveConst.HiveMQAppUser} \".*\" \".*\" \".*\"");
                    hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_permissions -p {HiveConst.HiveMQNeonVHost} {HiveConst.HiveMQNeonUser} \".*\" \".*\" \".*\"");

                    // Clear the UX status for the HiveMQ nodes.

                    foreach (var node in hiveMQNodes)
                    {
                        node.Status = string.Empty;
                    }

                    // Set the RabbitMQ cluster name to the name of the hive.

                    hiveMQNode.InvokeIdempotentAction("setup/hivemq-cluster-name", () => hiveMQNode.SudoCommand($"docker exec neon-hivemq rabbitmqctl set_cluster_name {hive.Definition.Name}"));
                });

                //---------------------------------------------------------
                // Initialize the public and private traffic manager managers.

                hive.PublicTraffic.UpdateSettings(
                    new TrafficSettings()
                {
                    ProxyPorts = HiveConst.PublicProxyPorts
                });

                hive.PrivateTraffic.UpdateSettings(
                    new TrafficSettings()
                {
                    ProxyPorts = HiveConst.PrivateProxyPorts
                });

                //---------------------------------------------------------
                // Deploy the HiveMQ traffic manager rules.

                hive.FirstManager.InvokeIdempotentAction("setup/hivemq-traffic-manager-rules",
                                                         () =>
                {
                    // Deploy private traffic manager for the AMQP endpoints.

                    var amqpRule = new TrafficTcpRule()
                    {
                        Name     = "neon-hivemq-amqp",
                        System   = true,
                        Resolver = null
                    };

                    // We're going to set this up to allow idle connections for up to
                    // five minutes.  In theory, AMQP connections should never be idle
                    // this long because we've enabled level 7 keep-alive.
                    //
                    //      https://github.com/jefflill/NeonForge/issues/new

                    amqpRule.Timeouts = new TrafficTimeouts()
                    {
                        ClientSeconds = 0,
                        ServerSeconds = 0
                    };

                    amqpRule.Frontends.Add(
                        new TrafficTcpFrontend()
                    {
                        ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
                    });

                    foreach (var ampqNode in hive.Nodes.Where(n => n.Metadata.Labels.HiveMQ))
                    {
                        amqpRule.Backends.Add(
                            new TrafficTcpBackend()
                        {
                            Server = ampqNode.PrivateAddress.ToString(),
                            Port   = HiveHostPorts.HiveMQAMQP
                        });
                    }

                    hive.PrivateTraffic.SetRule(amqpRule);

                    // Deploy private traffic manager for the management endpoints.

                    var adminRule = new TrafficHttpRule()
                    {
                        Name     = "neon-hivemq-management",
                        System   = true,
                        Resolver = null
                    };

                    // Initialize the frontends and backends.

                    adminRule.Frontends.Add(
                        new TrafficHttpFrontend()
                    {
                        ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAdmin
                    });

                    adminRule.Backends.Add(
                        new TrafficHttpBackend()
                    {
                        Group      = HiveHostGroups.HiveMQManagers,
                        GroupLimit = 5,
                        Port       = HiveHostPorts.HiveMQManagement
                    });

                    hive.PrivateTraffic.SetRule(adminRule);
                });

                //---------------------------------------------------------
                // Deploy DNS related services.

                // Deploy: neon-dns-mon

                ServiceHelper.StartService(hive, "neon-dns-mon", hive.Definition.Image.DnsMon,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-dns-mon",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--env", "POLL_INTERVAL=5s",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--constraint", "node.role==manager",
                                               "--replicas", "1",
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               ServiceHelper.ImagePlaceholderArg));

                // Deploy: neon-dns

                ServiceHelper.StartService(hive, "neon-dns", hive.Definition.Image.Dns,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-dns",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--mount", "type=bind,src=/etc/powerdns/hosts,dst=/etc/powerdns/hosts",
                                               "--mount", "type=bind,src=/dev/shm/neon-dns,dst=/neon-dns",
                                               "--env", "POLL_INTERVAL=5s",
                                               "--env", "VERIFY_INTERVAL=5m",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--constraint", "node.role==manager",
                                               "--mode", "global",
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               ServiceHelper.ImagePlaceholderArg));

                //---------------------------------------------------------
                // Deploy [neon-hive-manager] as a service constrained to manager nodes.

                string unsealSecretOption = null;

                if (hive.Definition.Vault.AutoUnseal)
                {
                    var vaultCredentials = NeonHelper.JsonClone <VaultCredentials>(hive.HiveLogin.VaultCredentials);

                    // We really don't want to include the root token in the credentials
                    // passed to [neon-hive-manager], which needs the unseal keys so
                    // we'll clear that here.

                    vaultCredentials.RootToken = null;

                    hive.Docker.Secret.Set("neon-hive-manager-vaultkeys", Encoding.UTF8.GetBytes(NeonHelper.JsonSerialize(vaultCredentials, Formatting.Indented)));

                    unsealSecretOption = "--secret=neon-hive-manager-vaultkeys";
                }

                ServiceHelper.StartService(hive, "neon-hive-manager", hive.Definition.Image.HiveManager,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-hive-manager",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--secret", "neon-ssh-credentials",
                                               unsealSecretOption,
                                               "--constraint", "node.role==manager",
                                               "--replicas", 1,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               ServiceHelper.ImagePlaceholderArg
                                               ),
                                           hive.SecureRunOptions | RunOptions.FaultOnError);

                //---------------------------------------------------------
                // Deploy proxy related services.

                // Deploy the proxy manager service.

                ServiceHelper.StartService(hive, "neon-proxy-manager", hive.Definition.Image.ProxyManager,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-proxy-manager",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
                                               "--env", "VAULT_CREDENTIALS=neon-proxy-manager-credentials",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--secret", "neon-proxy-manager-credentials",
                                               "--constraint", "node.role==manager",
                                               "--replicas", 1,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               ServiceHelper.ImagePlaceholderArg));

                // Docker mesh routing seemed unstable on versions so we're going
                // to provide an option to work around this by running the PUBLIC,
                // PRIVATE and VAULT proxies on all nodes and  publishing the ports
                // to the host (not the mesh).
                //
                //      https://github.com/jefflill/NeonForge/issues/104
                //
                // Note that this mode feature is documented (somewhat poorly) here:
                //
                //      https://docs.docker.com/engine/swarm/services/#publish-ports

                var publicPublishArgs   = new List <string>();
                var privatePublishArgs  = new List <string>();
                var proxyConstraintArgs = new List <string>();
                var proxyReplicasArgs   = new List <string>();
                var proxyModeArgs       = new List <string>();

                if (hive.Definition.Docker.GetAvoidIngressNetwork(hive.Definition))
                {
                    // The parameterized [docker service create --publish] option doesn't handle port ranges so we need to
                    // specify multiple publish options.

                    foreach (var port in HiveConst.PublicProxyPorts.Ports)
                    {
                        publicPublishArgs.Add($"--publish");
                        publicPublishArgs.Add($"mode=host,published={port},target={port}");
                    }

                    for (int port = HiveConst.PublicProxyPorts.PortRange.FirstPort; port <= HiveConst.PublicProxyPorts.PortRange.LastPort; port++)
                    {
                        publicPublishArgs.Add($"--publish");
                        publicPublishArgs.Add($"mode=host,published={port},target={port}");
                    }

                    foreach (var port in HiveConst.PrivateProxyPorts.Ports)
                    {
                        privatePublishArgs.Add($"--publish");
                        privatePublishArgs.Add($"mode=host,published={port},target={port}");
                    }

                    for (int port = HiveConst.PrivateProxyPorts.PortRange.FirstPort; port <= HiveConst.PrivateProxyPorts.PortRange.LastPort; port++)
                    {
                        privatePublishArgs.Add($"--publish");
                        privatePublishArgs.Add($"mode=host,published={port},target={port}");
                    }

                    proxyModeArgs.Add("--mode");
                    proxyModeArgs.Add("global");
                }
                else
                {
                    // The parameterized [docker run --publish] option doesn't handle port ranges so we need to
                    // specify multiple publish options.

                    foreach (var port in HiveConst.PublicProxyPorts.Ports)
                    {
                        publicPublishArgs.Add($"--publish");
                        publicPublishArgs.Add($"{port}:{port}");
                    }

                    publicPublishArgs.Add($"--publish");
                    publicPublishArgs.Add($"{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}:{HiveConst.PublicProxyPorts.PortRange.FirstPort}-{HiveConst.PublicProxyPorts.PortRange.LastPort}");

                    foreach (var port in HiveConst.PrivateProxyPorts.Ports)
                    {
                        privatePublishArgs.Add($"--publish");
                        privatePublishArgs.Add($"{port}:{port}");
                    }

                    privatePublishArgs.Add($"--publish");
                    privatePublishArgs.Add($"{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}:{HiveConst.PrivateProxyPorts.PortRange.FirstPort}-{HiveConst.PrivateProxyPorts.PortRange.LastPort}");

                    proxyConstraintArgs.Add($"--constraint");
                    proxyReplicasArgs.Add("--replicas");

                    if (hive.Definition.Workers.Count() > 0)
                    {
                        // Constrain proxies to worker nodes if there are any.

                        proxyConstraintArgs.Add($"node.role!=manager");

                        if (hive.Definition.Workers.Count() == 1)
                        {
                            proxyReplicasArgs.Add("1");
                        }
                        else
                        {
                            proxyReplicasArgs.Add("2");
                        }
                    }
                    else
                    {
                        // Constrain proxies to manager nodes nodes if there are no workers.

                        proxyConstraintArgs.Add($"node.role==manager");

                        if (hive.Definition.Managers.Count() == 1)
                        {
                            proxyReplicasArgs.Add("1");
                        }
                        else
                        {
                            proxyReplicasArgs.Add("2");
                        }
                    }

                    proxyModeArgs.Add("--mode");
                    proxyModeArgs.Add("replicated");
                }

                // Deploy: neon-proxy-public

                ServiceHelper.StartService(hive, "neon-proxy-public", hive.Definition.Image.Proxy,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-proxy-public",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
                                               "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
                                               "--env", "VAULT_CREDENTIALS=neon-proxy-public-credentials",
                                               "--env", "WARN_SECONDS=300",
                                               "--env", "START_SECONDS=10",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--env", "DEBUG=false",
                                               "--secret", "neon-proxy-public-credentials",
                                               publicPublishArgs,
                                               proxyConstraintArgs,
                                               proxyReplicasArgs,
                                               proxyModeArgs,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               "--network", HiveConst.PublicNetwork,
                                               ServiceHelper.ImagePlaceholderArg));

                // Deploy: neon-proxy-private

                ServiceHelper.StartService(hive, "neon-proxy-private", hive.Definition.Image.Proxy,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-proxy-private",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
                                               "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
                                               "--env", "VAULT_CREDENTIALS=neon-proxy-private-credentials",
                                               "--env", "WARN_SECONDS=300",
                                               "--env", "START_SECONDS=10",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--env", "DEBUG=false",
                                               "--secret", "neon-proxy-private-credentials",
                                               privatePublishArgs,
                                               proxyConstraintArgs,
                                               proxyReplicasArgs,
                                               proxyModeArgs,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               "--network", HiveConst.PrivateNetwork,
                                               ServiceHelper.ImagePlaceholderArg));

                // Deploy: neon-proxy-public-cache

                var publicCacheConstraintArgs = new List <string>();
                var publicCacheReplicaArgs    = new List <string>();

                if (hive.Definition.Proxy.PublicCacheReplicas <= hive.Definition.Workers.Count())
                {
                    publicCacheConstraintArgs.Add("--constraint");
                    publicCacheConstraintArgs.Add("node.role==worker");
                }

                publicCacheReplicaArgs.Add("--replicas");
                publicCacheReplicaArgs.Add($"{hive.Definition.Proxy.PublicCacheReplicas}");

                ServiceHelper.StartService(hive, "neon-proxy-public-cache", hive.Definition.Image.ProxyCache,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-proxy-public-cache",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
                                               "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-conf",
                                               "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public/proxy-hash",
                                               "--env", "WARN_SECONDS=300",
                                               "--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PublicCacheSize}",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--env", "DEBUG=false",
                                               "--secret", "neon-proxy-public-credentials",
                                               publicCacheConstraintArgs,
                                               publicCacheReplicaArgs,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               "--network", HiveConst.PublicNetwork,
                                               ServiceHelper.ImagePlaceholderArg));

                // Deploy: neon-proxy-private-cache

                var privateCacheConstraintArgs = new List <string>();
                var privateCacheReplicaArgs    = new List <string>();

                if (hive.Definition.Proxy.PrivateCacheReplicas <= hive.Definition.Workers.Count())
                {
                    privateCacheConstraintArgs.Add("--constraint");
                    privateCacheConstraintArgs.Add("node.role==worker");
                }

                privateCacheReplicaArgs.Add("--replicas");
                privateCacheReplicaArgs.Add($"{hive.Definition.Proxy.PrivateCacheReplicas}");

                ServiceHelper.StartService(hive, "neon-proxy-private-cache", hive.Definition.Image.ProxyCache,
                                           new CommandBundle(
                                               "docker service create",
                                               "--name", "neon-proxy-private-cache",
                                               "--detach=false",
                                               "--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
                                               "--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
                                               "--mount", "type=tmpfs,dst=/var/lib/varnish/_.vsm_mgt,tmpfs-size=90M,tmpfs-mode=755",
                                               "--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-conf",
                                               "--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private/proxy-hash",
                                               "--env", "WARN_SECONDS=300",
                                               "--env", $"MEMORY-LIMIT={hive.Definition.Proxy.PrivateCacheSize}",
                                               "--env", "LOG_LEVEL=INFO",
                                               "--env", "DEBUG=false",
                                               "--secret", "neon-proxy-private-credentials",
                                               privateCacheConstraintArgs,
                                               privateCacheReplicaArgs,
                                               "--restart-delay", hive.Definition.Docker.RestartDelay,
                                               "--network", HiveConst.PrivateNetwork,
                                               ServiceHelper.ImagePlaceholderArg));
            });

            // Log the hive into any Docker registries with credentials.

            firstManager.InvokeIdempotentAction("setup/registry-login",
                                                () =>
            {
                foreach (var credential in hive.Definition.Docker.Registries
                         .Where(r => !string.IsNullOrEmpty(r.Username)))
                {
                    hive.Registry.Login(credential.Registry, credential.Username, credential.Password);
                }
            });
        }
Ejemplo n.º 5
0
        /// <summary>
        /// <para>
        /// This release relocated the HiveMQ account settings from Docker secrets
        /// to hive global Consul keys so that these can be available to containers
        /// too (e.g. running on pets) and also to make deploying services that use
        /// queuing more transparent.
        /// </para>
        /// <para>
        /// This update copies these settings from the Docker secrets to Consul and
        /// then removes these secret references from the built-in hive service
        /// scripts and also updates the services by pulling the latest images and
        /// removing the secrets there too.
        /// </para>
        /// <note>
        /// We're going to leave the old Docker secrets in place on the off chance
        /// that a user service is still referencing them.
        /// </note>
        /// </summary>
        private void UpdateHiveMQSettings()
        {
            var firstManager = Hive.FirstManager;

            // Fetch the current HiveMQ settings from the Docker secrets.
            // This is slow, so we'll capture these in parallel.

            firstManager.Status = "reading hivemq secrets";

            var appSettings      = (HiveMQSettings)null;
            var neonSettings     = (HiveMQSettings)null;
            var sysadminSettings = (HiveMQSettings)null;

            NeonHelper.WaitForParallel(
                new Action[]
            {
                new Action(() => appSettings      = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-app")),
                new Action(() => neonSettings     = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-neon")),
                new Action(() => sysadminSettings = Hive.Docker.Secret.Get <HiveMQSettings>("neon-hivemq-settings-sysadmin")),
            },
                TimeSpan.FromSeconds(120));

            firstManager.Status = string.Empty;

            // Edit the service start scripts by removing any lines that
            // attach a HiveMQ account secret.

            var services = new string[]
            {
                "neon-hive-manager",
                "neon-proxy-manager",
                "neon-proxy-public",
                "neon-proxy-private"
            };

            foreach (var manager in Hive.Managers)
            {
                foreach (var service in services)
                {
                    var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, $"{service}.sh");

                    manager.Status = $"edit: {scriptPath}";

                    // Edit the service start scripts by removing any lines that
                    // attach a HiveMQ account secret.

                    var curScript = manager.DownloadText(scriptPath);
                    var newScript = new StringBuilder();

                    using (var reader = new StringReader(curScript))
                    {
                        foreach (var line in reader.Lines())
                        {
                            if (!line.Trim().StartsWith("--secret neon-hivemq-settings-"))
                            {
                                newScript.AppendLine(line);
                            }
                        }
                    }

                    manager.UploadText(scriptPath, newScript.ToString(), permissions: "660");

                    manager.Status = string.Empty;
                }
            }

            // Update the impacted services by removing the secret and pulling the latest image.

            firstManager.Status = "neon-hive-manager";
            firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.HiveManager)} neon-hive-manager");
            firstManager.Status = "neon-proxy-manager";
            firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.ProxyManager)} neon-proxy-manager");
            firstManager.Status = "neon-proxy-public";
            firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.Proxy)} neon-proxy-public");
            firstManager.Status = "neon-proxy-private";
            firstManager.DockerCommand(RunOptions.FaultOnError, $"docker service update --secret-rm=neon-hivemq-settings-neon --image {Program.ResolveDockerImage(Hive.Definition.Image.Proxy)} neon-proxy-private");
            firstManager.Status = string.Empty;

            // Redeploy the [neon-hivemq-ampq] private traffic manager to be a TCP rather than an HTTP proxy.

            var ampqRule = new TrafficTcpRule()
            {
                Name     = "neon-hivemq-ampq",
                System   = true,
                Resolver = null
            };

            ampqRule.Frontends.Add(
                new TrafficTcpFrontend()
            {
                ProxyPort = HiveHostPorts.ProxyPrivateHiveMQAMQP
            });

            ampqRule.Backends.Add(
                new TrafficTcpBackend()
            {
                Group      = HiveHostGroups.HiveMQ,
                GroupLimit = 5,
                Port       = HiveHostPorts.HiveMQAMQP
            });

            Hive.PrivateTraffic.SetRule(ampqRule);
        }