public async Task RunNodeAgent(Guid nodeId, CancellationToken ct = default)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("RunNodeAgent");
await using var ctx = ctxFactory.CreateDbContext();
var node = await ctx.Nodes.FirstOrDefaultAsync(x => x.Id == nodeId, ct);
activity?.AddTag(Extensions.ActivityNodeIdName, nodeId.ToString());
activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString());
activity?.AddTag(Extensions.ActivityTenantIdName, node.TenantId.ToString());
activity?.AddBaggage(Extensions.ActivityTenantIdName, node.TenantId.ToString());
using var _ = logger.BeginScope(new { nodeId });
try
{
var credentials = await cryptoService.GetNodeCredentials(node);
await ReloadContainersForNode(credentials, nodeId, ct);
var events = ListenForEvents(credentials, ct);
await foreach (var ee in events.WithCancellation(ct))
{
logger.LogDebug("Received node event {type} {compose}", ee.Message.Type, ee.Compose);
await HandleDockerEvent(credentials, ee, nodeId, ct);
}
}
catch (TaskCanceledException)
{
}
catch (ProxyClientException ex)
{
logger.LogInformation("Node agent connection failed {why}", ex.Message);
connectionLog.Error($"Node connection failed", ex);
activity.RecordException(ex);
}
catch (ContainerInfoException ex)
{
logger.LogInformation("Failed to get container info {why}", ex.Message);
connectionLog.Error($"Failed to get container info", ex);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Unknown error in node agent {nodeId}", nodeId);
connectionLog.Error($"Unknown error", ex);
throw;
}
}
private void LogError(ProxyClientException ex)
{
var currentActivity = Activity.Current;
connectionLog.Error(
currentActivity != null && ex.ErrorResponse != null ? $"{currentActivity.DisplayName} failed" : ex.Message,
ex,
ex.ErrorResponse != null ? new Dictionary <string, dynamic>(
new[]
{
new KeyValuePair <string, dynamic>("detail", ex.ErrorResponse.Detail)
}) : null
);
currentActivity?.AddEvent(new ActivityEvent("Request failed", tags: new ActivityTagsCollection
{
["detail"] = ex.ErrorResponse?.Detail,
["title"] = ex.ErrorResponse?.Title,
["type"] = ex.ErrorResponse?.Type
}));
}
public async Task ProcessNodeUpdates(Guid nodeId, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("ProcessNodeUpdates");
activity?.AddTag(Extensions.ActivityNodeIdName, nodeId.ToString());
activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString());
var tenantId = await ctx.Nodes
.Where(x => x.Id == nodeId)
.Select(x => x.TenantId)
.FirstOrDefaultAsync(ct);
activity?.AddTag(Extensions.ActivityTenantIdName, tenantId.ToString());
activity?.AddBaggage(Extensions.ActivityTenantIdName, tenantId.ToString());
using var _ = logger.BeginScope(new { nodeId });
try
{
while (!ct.IsCancellationRequested)
{
Node?node;
using (Extensions.SuperComposeActivitySource.StartActivity("Node with details query"))
{
node = await ctx.Nodes
.Where(x => x.Id == nodeId)
.Include(x => x.Deployments)
.ThenInclude(x => x.Compose)
.ThenInclude(x => x.Current)
.Include(x => x.Deployments)
.ThenInclude(x => x.LastDeployedComposeVersion)
.FirstOrDefaultAsync(ct);
}
if (node == null)
{
throw new NodeNotFoundException();
}
var deployments = node.Deployments.Where(ShouldUpdateDeployment).ToList();
if (!deployments.Any())
{
return;
}
var connectionParams = await cryptoService.GetNodeCredentials(node);
if (deployments.Any(x => x.LastDeployedNodeVersion != node.Version))
{
if (!await VerifyNode(connectionParams, ct))
{
return;
}
}
foreach (var deployment in deployments)
{
await ApplyDeployment(connectionParams, deployment, ct);
}
}
}
catch (TaskCanceledException)
{
}
catch (NodeReconciliationFailedException ex)
{
logger.LogInformation("Node reconciliation failed {why}", ex.Message);
connectionLog.Error("Node reconciliation failed", ex);
await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
ReconciliationFailed = true
}, ct);
activity.RecordException(ex);
}
catch (NodeConnectionFailedException ex)
{
logger.LogInformation("Node reconciliation failed {why}", ex.Message);
connectionLog.Error($"Node connection failed", ex);
await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
ReconciliationFailed = true
}, ct);
activity.RecordException(ex);
}
catch (SshException ex)
{
logger.LogInformation("Node reconciliation failed {why}", ex.Message);
connectionLog.Error($"SSH error", ex);
await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
ReconciliationFailed = true
}, ct);
activity.RecordException(ex);
}
catch (ProxyClientException ex)
{
logger.LogInformation("Node agent connection failed {why}", ex.Message);
connectionLog.Error($"Node connection failed", ex);
activity.RecordException(ex);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Unknown error when reconciling node {nodeId}", nodeId);
connectionLog.Error($"Unknown error", ex);
await ctx.Nodes.Where(x => x.Id == nodeId).UpdateAsync(x => new Node {
ReconciliationFailed = true
}, ct);
activity.RecordException(ex);
throw;
}
}