예제 #1
0
        public async Task RunNodeAgent(Guid nodeId, CancellationToken ct = default)
        {
            using var activity  = Extensions.SuperComposeActivitySource.StartActivity("RunNodeAgent");
            await using var ctx = ctxFactory.CreateDbContext();

            var node = await ctx.Nodes.FirstOrDefaultAsync(x => x.Id == nodeId, ct);

            activity?.AddTag(Extensions.ActivityNodeIdName, nodeId.ToString());
            activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString());

            activity?.AddTag(Extensions.ActivityTenantIdName, node.TenantId.ToString());
            activity?.AddBaggage(Extensions.ActivityTenantIdName, node.TenantId.ToString());

            using var _ = logger.BeginScope(new { nodeId });

            try
            {
                var credentials = await cryptoService.GetNodeCredentials(node);

                await ReloadContainersForNode(credentials, nodeId, ct);

                var events = ListenForEvents(credentials, ct);
                await foreach (var ee in events.WithCancellation(ct))
                {
                    logger.LogDebug("Received node event {type} {compose}", ee.Message.Type, ee.Compose);

                    await HandleDockerEvent(credentials, ee, nodeId, ct);
                }
            }
            catch (TaskCanceledException)
            {
            }
            catch (ProxyClientException ex)
            {
                logger.LogInformation("Node agent connection failed {why}", ex.Message);
                connectionLog.Error($"Node connection failed", ex);
                activity.RecordException(ex);
            }
            catch (ContainerInfoException ex)
            {
                logger.LogInformation("Failed to get container info {why}", ex.Message);
                connectionLog.Error($"Failed to get container info", ex);
            }
            catch (Exception ex)
            {
                logger.LogWarning(ex, "Unknown error in node agent {nodeId}", nodeId);
                connectionLog.Error($"Unknown error", ex);

                throw;
            }
        }
예제 #2
0
        private void LogError(ProxyClientException ex)
        {
            var currentActivity = Activity.Current;

            connectionLog.Error(
                currentActivity != null && ex.ErrorResponse != null ? $"{currentActivity.DisplayName} failed" : ex.Message,
                ex,
                ex.ErrorResponse != null ? new Dictionary <string, dynamic>(
                    new[]
            {
                new KeyValuePair <string, dynamic>("detail", ex.ErrorResponse.Detail)
            }) : null
                );

            currentActivity?.AddEvent(new ActivityEvent("Request failed", tags: new ActivityTagsCollection
            {
                ["detail"] = ex.ErrorResponse?.Detail,
                ["title"]  = ex.ErrorResponse?.Title,
                ["type"]   = ex.ErrorResponse?.Type
            }));
        }
예제 #3
0
        public async Task ProcessNodeUpdates(Guid nodeId, CancellationToken ct)
        {
            using var activity = Extensions.SuperComposeActivitySource.StartActivity("ProcessNodeUpdates");

            activity?.AddTag(Extensions.ActivityNodeIdName, nodeId.ToString());
            activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString());

            var tenantId = await ctx.Nodes
                           .Where(x => x.Id == nodeId)
                           .Select(x => x.TenantId)
                           .FirstOrDefaultAsync(ct);

            activity?.AddTag(Extensions.ActivityTenantIdName, tenantId.ToString());
            activity?.AddBaggage(Extensions.ActivityTenantIdName, tenantId.ToString());

            using var _ = logger.BeginScope(new { nodeId });

            try
            {
                while (!ct.IsCancellationRequested)
                {
                    Node?node;
                    using (Extensions.SuperComposeActivitySource.StartActivity("Node with details query"))
                    {
                        node = await ctx.Nodes
                               .Where(x => x.Id == nodeId)
                               .Include(x => x.Deployments)
                               .ThenInclude(x => x.Compose)
                               .ThenInclude(x => x.Current)
                               .Include(x => x.Deployments)
                               .ThenInclude(x => x.LastDeployedComposeVersion)
                               .FirstOrDefaultAsync(ct);
                    }

                    if (node == null)
                    {
                        throw new NodeNotFoundException();
                    }

                    var deployments = node.Deployments.Where(ShouldUpdateDeployment).ToList();
                    if (!deployments.Any())
                    {
                        return;
                    }

                    var connectionParams = await cryptoService.GetNodeCredentials(node);


                    if (deployments.Any(x => x.LastDeployedNodeVersion != node.Version))
                    {
                        if (!await VerifyNode(connectionParams, ct))
                        {
                            return;
                        }
                    }


                    foreach (var deployment in deployments)
                    {
                        await ApplyDeployment(connectionParams, deployment, ct);
                    }
                }
            }
            catch (TaskCanceledException)
            {
            }
            catch (NodeReconciliationFailedException ex)
            {
                logger.LogInformation("Node reconciliation failed {why}", ex.Message);
                connectionLog.Error("Node reconciliation failed", ex);
                await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
                    ReconciliationFailed = true
                }, ct);

                activity.RecordException(ex);
            }
            catch (NodeConnectionFailedException ex)
            {
                logger.LogInformation("Node reconciliation failed {why}", ex.Message);
                connectionLog.Error($"Node connection failed", ex);
                await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
                    ReconciliationFailed = true
                }, ct);

                activity.RecordException(ex);
            }
            catch (SshException ex)
            {
                logger.LogInformation("Node reconciliation failed {why}", ex.Message);
                connectionLog.Error($"SSH error", ex);
                await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
                    ReconciliationFailed = true
                }, ct);

                activity.RecordException(ex);
            }
            catch (ProxyClientException ex)
            {
                logger.LogInformation("Node agent connection failed {why}", ex.Message);
                connectionLog.Error($"Node connection failed", ex);
                activity.RecordException(ex);
            }
            catch (Exception ex)
            {
                logger.LogWarning(ex, "Unknown error when reconciling node {nodeId}", nodeId);
                connectionLog.Error($"Unknown error", ex);

                await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
                    ReconciliationFailed = true
                }, ct);

                activity.RecordException(ex);

                throw;
            }
        }