private async Task <bool> VerifyNode(NodeCredentials credentials, CancellationToken ct) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("Verify node"); connectionLog.Info("Verifying node"); var systemctlVersion = await proxyClient.RunCommand(credentials, "systemctl --version", ct); if (systemctlVersion.Code != 0) { logger.LogDebug("systemd unavailable"); throw new NodeReconciliationFailedException("systemd unavailable, stopping node configuration"); } var dockerVersion = await proxyClient.RunCommand(credentials, "docker --version", ct); if (dockerVersion.Code != 0) { logger.LogDebug("docker unavailable"); connectionLog.Error("docker unavailable, stopping node configuration"); throw new NodeReconciliationFailedException("docker unavailable, stopping node configuration"); } var dockerComposeVersion = await proxyClient.RunCommand(credentials, "docker-compose --version", ct); if (dockerComposeVersion.Code != 0) { logger.LogDebug("docker-compose unavailable"); connectionLog.Error("docker-compose unavailable, stopping node configuration"); throw new NodeReconciliationFailedException("docker-compose unavailable, stopping node configuration"); } return(true); }
public async Task <Guid> Create( Guid tenantId, string name, NodeCredentials conn ) { await connectionService.TestConnection(conn); var node = new Node { TenantId = tenantId, Host = conn.host, Enabled = true, Id = Guid.NewGuid(), Name = name, Port = conn.port, Username = conn.username, Password = string.IsNullOrWhiteSpace(conn.password) ? null : await crypto.EncryptSecret(conn.password), PrivateKey = string.IsNullOrWhiteSpace(conn.privateKey) ? null : await crypto.EncryptSecret(conn.privateKey) }; await ctx.Nodes.AddAsync(node); await ctx.SaveChangesAsync(); await nodeUpdater.NotifyAboutNodeChange(node.Id); return(node.Id); }
private async Task <string> GetDockerComposePath(NodeCredentials credentials, CancellationToken ct = default) { var key = $"{credentials.username}@{credentials.host}:{credentials.port}"; var path = await cache.GetStringAsync(key, ct); if (path != null) { return(path); } var result = await proxyClient.RunCommand(credentials, "which docker-compose", ct); if (string.IsNullOrEmpty(result.Error) && result.Code == 0 && result.Stdout != null) { path = Encoding.UTF8.GetString(result.Stdout).Trim(); await cache.SetStringAsync(key, path, new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1) }, ct); } else { throw new DeploymentReconciliationFailedException("Unable to find docker-compose"); } return(path); }
private async Task <TResp> GetDocker <TResp>(string path, NodeCredentials credentials, CancellationToken ct = default) { var resp = await Request(path, null, HttpMethod.Get, credentials, ct); var str = await resp.ReadAsStringAsync(cancellationToken : ct) ?? throw new InvalidOperationException("Could not parse response from proxy call"); return(dockerSerializer.DeserializeObject <TResp>(str)); }
private HttpClient ClientFor(NodeCredentials credentials) { var client = clientFactory.CreateClient("proxy"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("bearer", MintJwtFromCredentials(credentials)); return(client); }
private async Task StartDockerCompose(NodeCredentials credentials, DeploymentInfo composeVersion, CancellationToken ct) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("StartDockerCompose"); var cmd = await ComposeCmdPrefix(credentials, composeVersion.ComposeDirectory, composeVersion.ServiceName, ct) + " up -d --remove-orphans"; await proxyClient.RunCommand(credentials, cmd, ct); }
private async Task <bool> GetComposeIsRunning(NodeCredentials credentials, ComposeVersion version, CancellationToken ct = default) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("GetComposeNeedsToStop"); activity?.AddTag("supercompose.path", version.Directory); var cmd = await ComposeCmdPrefix(credentials, version.Directory, version.ServiceName, ct) + " ps --quiet"; var res = await proxyClient.RunCommand(credentials, cmd, ct); return(res.Code == 0 && res.Stdout != null && !string.IsNullOrEmpty(Encoding.UTF8.GetString(res.Stdout).Trim())); }
private async Task RemoveCompose(NodeCredentials credentials, DeploymentInfo deployment, CancellationToken ct = default) { if (!string.IsNullOrEmpty(deployment.ComposePath)) { var removed = await proxyClient.DeleteFile(credentials, deployment.ComposePath, ct); if (removed) { connectionLog.Info($"Removed old compose file"); } } }
private async Task <bool> UpdateComposeFile(NodeCredentials credentials, DeploymentInfo composeVersion, CancellationToken ct) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("UpdateComposeFile"); var resp = await proxyClient.UpsertFile(credentials, composeVersion.ComposePath, composeVersion.ComposeContent, true, ct); if (resp.Updated) { connectionLog.Info($"docker-compose.yaml has been updated"); } return(resp.Updated); }
private async IAsyncEnumerable <TResp> GetDockerSSE <TResp>(string path, NodeCredentials credentials, [EnumeratorCancellation] CancellationToken ct = default) where TResp : class { using var client = ClientFor(credentials); using var reader = new StreamReader(await client.GetStreamAsync(path, ct)); while (!ct.IsCancellationRequested) { var line = await reader.ReadLineAsync(); if (!string.IsNullOrEmpty(line) && line.StartsWith("data: ")) { line = line[6..];
private async Task ReloadContainersForNode(NodeCredentials credentials, Guid nodeId, CancellationToken ct = default) { await using var ctx = ctxFactory.CreateDbContext(); using var activity = Extensions.SuperComposeActivitySource.StartActivity("ReloadContainersForNode"); activity?.SetTag(Extensions.ActivityNodeIdName, nodeId); activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString()); var deployments = await ctx.Deployments .Where(x => x.NodeId == nodeId) .Include(x => x.Containers) .Select(x => new { Deployment = x, ServiceName = x.LastDeployedComposeVersion !.ServiceName ?? x.Compose !.Current.ServiceName })
private async Task <string> GenerateSystemdServiceFile(NodeCredentials credentials, ComposeVersion composeVersion, CancellationToken ct) { return($@" [Unit] Description={composeVersion.Compose.Name} service with docker compose managed by supercompose Requires=docker.service After=docker.service [Service] Type=oneshot RemainAfterExit=true WorkingDirectory={composeVersion.Directory} ExecStart={await ComposeCmdPrefix(credentials, composeVersion.Directory, composeVersion.ServiceName, ct)} up -d --remove-orphans ExecStop={await ComposeCmdPrefix(credentials, composeVersion.Directory, composeVersion.ServiceName, ct)} down [Install] WantedBy=multi-user.target".Trim().Replace("\r\n", "\n")); }
private string MintJwtFromCredentials(NodeCredentials credentials) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("ProxyClient.MintJwtFromCredentials"); try { var tokenHandler = new JwtSecurityTokenHandler(); var claims = new List <Claim> { new("host", $"{credentials.host}:{credentials.port}"), new("username", credentials.username), }; if (!string.IsNullOrEmpty(credentials.password)) { claims.Add(new Claim("password", credentials.password)); } if (!string.IsNullOrEmpty(credentials.privateKey)) { claims.Add(new Claim("pkey", credentials.privateKey)); } var tokenDescriptor = new SecurityTokenDescriptor { Subject = new ClaimsIdentity(claims), Expires = DateTime.UtcNow + TimeSpan.FromHours(2), SigningCredentials = new SigningCredentials( new SymmetricSecurityKey(Encoding.UTF8.GetBytes(config["Proxy:JWT"])), SecurityAlgorithms.HmacSha256Signature ), Audience = "proxy" }; return(tokenHandler.CreateEncodedJwt(tokenDescriptor)); } catch (Exception ex) { logger.LogCritical(ex, "Failed to mint JWT for proxy"); activity.RecordException(ex); throw; } }
private async Task <bool> UpdateSystemdFile(NodeCredentials credentials, ComposeVersion compose, CancellationToken ct) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("UpdateSystemdFile"); var serviceFile = await GenerateSystemdServiceFile(credentials, compose, ct); var resp = await proxyClient.UpsertFile( credentials, compose.ServicePath, Encoding.UTF8.GetBytes(serviceFile), false, ct ); if (resp.Updated) { connectionLog.Info($"systemd service file has been updated"); } return(resp.Updated); }
/// <exception cref="NodeConnectionFailedException"></exception> public async Task <SftpClient> CreateSftpConnection(NodeCredentials conn, TimeSpan timeout, CancellationToken ct) { logger.BeginScope(new { conn.username, conn.host, conn.port, passwordAvailable = !string.IsNullOrEmpty(conn.password), privateKeyAvailable = !string.IsNullOrEmpty(conn.privateKey), kind = "sftp_client" }); var connectionInfo = await PrepareConnectionInfo(conn, timeout); var client = new SftpClient(connectionInfo); await CreateConnectionInner(client, conn, ct); return(client); }
private async Task <ConnectionInfo> PrepareConnectionInfo(NodeCredentials conn, TimeSpan timeout) { var authMethods = new List <AuthenticationMethod>(); if (!string.IsNullOrEmpty(conn.password)) { authMethods.Add(new PasswordAuthenticationMethod(conn.username, conn.password)); } if (!string.IsNullOrEmpty(conn.privateKey)) { await using var pkMs = new MemoryStream(Encoding.UTF8.GetBytes(conn.privateKey)); try { var pkFile = new PrivateKeyFile(pkMs); authMethods.Add(new PrivateKeyAuthenticationMethod(conn.username, pkFile)); } catch (Exception ex) { logger.LogDebug("Failed to parse private key {error}", ex.Message); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.PrivateKey }; } } var connectionInfo = new ConnectionInfo( conn.host, conn.port, conn.username, authMethods.ToArray() ) { Timeout = timeout }; return(connectionInfo); }
public async Task TestConnection(NodeCredentials conn, Guid?nodeId = null, CancellationToken ct = default) { if (nodeId != null) { var node = await ctx.Nodes.FirstOrDefaultAsync(x => x.Id == nodeId, ct); if (node == null) { throw new NodeNotFoundException(); } if (string.IsNullOrWhiteSpace(conn.password) && string.IsNullOrWhiteSpace(conn.privateKey)) { conn = conn with { password = node.Password != null ? await crypto.DecryptSecret(node.Password) : null, privateKey = node.PrivateKey != null ? await crypto.DecryptSecret(node.PrivateKey) : null } } ; } using var client = await CreateSshConnection(conn, TimeSpan.FromSeconds(5), ct); }
private async Task <HttpContent> Request(string path, object?body, HttpMethod method, NodeCredentials credentials, CancellationToken ct = default) { using var client = ClientFor(credentials); var request = new HttpRequestMessage(); if (body != null) { request.Content = new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8); } request.Method = method; request.RequestUri = new Uri(path, UriKind.Relative); var resp = await client.SendAsync(request, ct); if (!resp.IsSuccessStatusCode) { var errorResponse = await resp.Content.ReadFromJsonAsync <ProxyClientErrorResponse>(cancellationToken : ct) ?? throw new InvalidOperationException("Could not read error response"); var exp = new ProxyClientException(errorResponse.Title) { ErrorResponse = errorResponse }; LogError(exp); throw exp; } return(resp.Content); }
/// <exception cref="NodeConnectionFailedException"></exception> private async Task CreateConnectionInner(BaseClient client, NodeCredentials conn, CancellationToken ct) { try { logger.LogDebug("Resolving host"); await Dns.GetHostEntryAsync(conn.host); logger.LogDebug("Resolved"); } catch (Exception ex) { logger.LogDebug("Host resolution failed"); client.Dispose(); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.DNS }; } try { logger.LogDebug("Connecting"); client.Connect(); logger.LogDebug("Connected"); } catch (SshAuthenticationException ex) { logger.LogDebug("Connection failed because to authenticate {error}", ex.Message); client.Dispose(); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.Authentication }; } catch (SshConnectionException ex) { logger.LogDebug("Connection failed due to SSH connection error {error}", ex.Message); client.Dispose(); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.Connection }; } catch (SocketException ex) { logger.LogDebug("Connection failed due to socket exception {error}", ex.Message); client.Dispose(); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.Connection }; } catch (TaskCanceledException) { logger.LogDebug("Connection cancelled"); client.Dispose(); throw; } catch (Exception ex) { logger.LogDebug("Connection failed for unknown reason {error}", ex.Message); client.Dispose(); throw new NodeConnectionFailedException(ex.Message, ex) { Kind = NodeConnectionFailedException.ConnectionErrorKind.Unknown }; } }
private async Task <string> ComposeCmdPrefix(NodeCredentials credentials, string dir, string service, CancellationToken ct) { var composePath = await GetDockerComposePath(credentials, ct); return($"{composePath} --project-directory '{dir}' --project-name '{service}' --file '{dir}/docker-compose.yml'"); }
private async Task ApplyDeployment(NodeCredentials credentials, Deployment deployment, CancellationToken ct) { using var activity = Extensions.SuperComposeActivitySource.StartActivity("ApplyDeployment"); activity?.AddTag(Extensions.ActivityDeploymentIdName, deployment.Id.ToString()); activity?.AddBaggage(Extensions.ActivityDeploymentIdName, deployment.Id.ToString()); activity?.AddTag(Extensions.ActivityComposeIdName, deployment.Compose !.Id.ToString()); activity?.AddBaggage(Extensions.ActivityComposeIdName, deployment.Compose !.Id.ToString()); try { var(target, last) = CalculateDeploymentDiff(deployment); activity?.AddTag("deployment.target.enabled", target.DeploymentEnabled); activity?.AddTag("deployment.target.service", target.UseService); activity?.AddTag("deployment.target.service_path", target.ServicePath); activity?.AddTag("deployment.target.compose_path", target.ComposePath); activity?.AddTag("deployment.last.enabled", last?.DeploymentEnabled); activity?.AddTag("deployment.last.service", last?.UseService); activity?.AddTag("deployment.last.service_path", last?.ServicePath); activity?.AddTag("deployment.last.compose_path", last?.ComposePath); activity?.AddEvent(new ActivityEvent("Getting current system status and updating files")); var targetComposeUpdatedTask = UpdateComposeFile(credentials, target, ct); var targetSystemdUpdatedTask = target.UseService ? UpdateSystemdFile(credentials, deployment.Compose !.Current, ct) // TODO if service path equal, but compose paths changed, we first need to stop the old service : Task.FromResult(false); var lastComposeStatusTask = deployment.LastDeployedComposeVersion != null ? GetComposeIsRunning(credentials, deployment.LastDeployedComposeVersion, ct) : Task.FromResult(false); var targetServiceStatusTask = target.UseService ? proxyClient.SystemdGetService(credentials, target.ServiceId, ct) : Task.FromResult <ProxyClient.SystemdGetServiceResponse?>(null) !; var lastServiceStatusTask = last != null && last.UseService ? (target.UseService && target.ServiceId == last.ServiceId) ? targetServiceStatusTask : proxyClient.SystemdGetService(credentials, last.ServiceId, ct) : Task.FromResult <ProxyClient.SystemdGetServiceResponse?>(null) !; await Task.WhenAll(targetComposeUpdatedTask, targetSystemdUpdatedTask, lastComposeStatusTask, targetServiceStatusTask, lastServiceStatusTask); var composeChanged = await targetComposeUpdatedTask; var serviceChanged = await targetSystemdUpdatedTask; var bothAreServices = last != null && target.UseService && last.UseService; var bothAreComposes = last != null && !target.UseService && !last.UseService; var composePathChanged = last == null || target.ComposePath != last.ComposePath; var servicePathChanged = last == null || target.ComposePath != last.ComposePath; var targetEnabled = target.DeploymentEnabled; var lastServiceStatus = await targetServiceStatusTask; var lastServiceRunning = lastServiceStatus != null && lastServiceStatus.IsRunning; var lastServiceEnabled = lastServiceStatus != null && lastServiceStatus.IsEnabled; var lastDockerRunning = await lastComposeStatusTask; var redeploymentRequested = RedeployRequested(deployment); var restartRequired = composeChanged || serviceChanged || redeploymentRequested; activity?.AddTag("startStop.composeChanged", composeChanged); activity?.AddTag("startStop.serviceChanged", serviceChanged); activity?.AddTag("startStop.bothAreServices", bothAreServices); activity?.AddTag("startStop.bothAreComposes", bothAreComposes); activity?.AddTag("startStop.composePathChanged", composePathChanged); activity?.AddTag("startStop.servicePathChanged", servicePathChanged); activity?.AddTag("startStop.enabled", targetEnabled); activity?.AddTag("startStop.lastServiceRunning", lastServiceRunning); activity?.AddTag("startStop.lastServiceEnabled", lastServiceEnabled); activity?.AddTag("startStop.lastDockerRunning", lastDockerRunning); activity?.AddTag("startStop.redeploymentRequested", redeploymentRequested); activity?.AddTag("startStop.restartRequired", restartRequired); activity?.AddEvent(new ActivityEvent("System status acquired, updating resource enablement state")); connectionLog.Info($"System status acquired, updating resource enablement state"); if (serviceChanged) { await proxyClient.SystemdReload(credentials, ct); } using (var disableEnableActivity = Extensions.SuperComposeActivitySource.StartActivity("Disable old services and enable new")) { var pendingTasks = new List <Task>(); var shouldDisableOldService = last != null && lastServiceEnabled && ( (last.UseService && !target.UseService) || (target.UseService && servicePathChanged) || (!targetEnabled) ); if (shouldDisableOldService) { disableEnableActivity?.AddEvent(new ActivityEvent("shouldDisableOldService")); var serviceStatus = await lastServiceStatusTask; if (serviceStatus != null && serviceStatus.IsEnabled) { pendingTasks.Add(proxyClient.SystemdDisableService(credentials, last !.ServiceId, ct)); } } var shouldEnableNewService = targetEnabled && ( last != null ? (target.UseService && !last.UseService) || (target.UseService && last.UseService && target.ServicePath != last.ServicePath) : target.UseService ); if (shouldEnableNewService) { disableEnableActivity?.AddEvent(new ActivityEvent("shouldEnableNewService")); var serviceStatus = await targetServiceStatusTask; if (serviceStatus != null && serviceStatus.IsEnabled) { pendingTasks.Add(proxyClient.SystemdEnableService(credentials, target.ServiceId, ct)); } } await Task.WhenAll(pendingTasks.ToArray()); } activity?.AddEvent(new ActivityEvent("Successfully changed resource enabled status, restarting resources")); connectionLog.Info($"Successfully changed resource enabled status, restarting resources"); if (last != null && (lastServiceRunning || lastDockerRunning)) { using var stopActivity = Extensions.SuperComposeActivitySource.StartActivity("Stop old"); activity?.AddEvent(new ActivityEvent("Stopping old resources")); connectionLog.Info($"Stopping old resources"); if (lastServiceRunning && !target.UseService) { stopActivity?.AddEvent(new ActivityEvent("old service is running, but new is compose")); await proxyClient.SystemdStopService(credentials, last.ServiceId, ct); } else if (lastServiceRunning && servicePathChanged) { stopActivity?.AddEvent(new ActivityEvent("old service is running, but service path has changed")); await proxyClient.SystemdStopService(credentials, last.ServiceId, ct); } else if (!targetEnabled && lastServiceRunning) { stopActivity?.AddEvent(new ActivityEvent("old service should not be running, but is")); await proxyClient.SystemdStopService(credentials, last.ServiceId, ct); } else if (lastDockerRunning && composePathChanged) { stopActivity?.AddEvent(new ActivityEvent("old docker is running, but compose path has changed")); await StopDockerCompose(credentials, last, ct); } else if (!targetEnabled && lastDockerRunning) { stopActivity?.AddEvent(new ActivityEvent("old docker should not be running, but is")); await StopDockerCompose(credentials, last, ct); } } if (targetEnabled && (!lastDockerRunning || composePathChanged || servicePathChanged || restartRequired || last.UseService != target.UseService)) { using var startActivity = Extensions.SuperComposeActivitySource.StartActivity("Start new"); activity?.AddEvent(new ActivityEvent("Starting new resources")); connectionLog.Info($"Starting new resources"); if (bothAreServices && !composePathChanged && !servicePathChanged && restartRequired) { startActivity?.AddEvent(new ActivityEvent("restart unchanged service")); await proxyClient.SystemdRestartService(credentials, target.ServiceId, ct); } else if (lastDockerRunning && bothAreServices && !composePathChanged && !servicePathChanged && !restartRequired) { startActivity?.AddEvent(new ActivityEvent("nothing changed, no restart required")); } else if (lastDockerRunning && bothAreComposes && !composePathChanged && !restartRequired) { startActivity?.AddEvent(new ActivityEvent("nothing changed, no restart required")); } else if (bothAreComposes && !composePathChanged && restartRequired) { startActivity?.AddEvent(new ActivityEvent("restart docker")); await RestartDockerCompose(credentials, target, ct); } else if (target.UseService) { startActivity?.AddEvent(new ActivityEvent("starting service")); await proxyClient.SystemdRestartService(credentials, target.ServiceId, ct); } else if (!target.UseService) { startActivity?.AddEvent(new ActivityEvent("starting compose")); await StartDockerCompose(credentials, target, ct); } } if (last != null && (composePathChanged || servicePathChanged || !targetEnabled)) { connectionLog.Info($"Restart successful, removing outdated resources"); using var removeActivity = Extensions.SuperComposeActivitySource.StartActivity("Remove old configurations"); var pendingTasks = new List <Task>(); if (servicePathChanged || !targetEnabled) { removeActivity?.AddEvent(new ActivityEvent("shouldRemoveOldService")); pendingTasks.Add(RemoveService(credentials, last, ct)); } if (composePathChanged || !targetEnabled) { removeActivity?.AddEvent(new ActivityEvent("shouldRemoveOldCompose")); pendingTasks.Add(RemoveCompose(credentials, last, ct)); } await Task.WhenAll(pendingTasks.ToArray()); } deployment.LastDeployedComposeVersionId = deployment.Compose.CurrentId; deployment.LastDeployedNodeVersion = deployment.Node.Version; deployment.LastCheck = DateTime.UtcNow; deployment.LastDeployedAsEnabled = target.DeploymentEnabled; await ctx.SaveChangesAsync(ct); connectionLog.Info($"Deployment applied successfully"); } catch (DeploymentReconciliationFailedException ex) { connectionLog.Error($"Deployment reconciliation failed", ex); deployment.ReconciliationFailed = true; await ctx.SaveChangesAsync(ct); } }
private async Task <TResp> Get <TResp>(string path, NodeCredentials credentials, CancellationToken ct = default) { var resp = await Request(path, null, HttpMethod.Get, credentials, ct); return(await resp.ReadFromJsonAsync <TResp>(cancellationToken : ct) ?? throw new InvalidOperationException("Could not parse response from proxy call")); }