private async Task <bool> VerifyNode(NodeCredentials credentials, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("Verify node");
connectionLog.Info("Verifying node");
var systemctlVersion = await proxyClient.RunCommand(credentials, "systemctl --version", ct);
if (systemctlVersion.Code != 0)
{
logger.LogDebug("systemd unavailable");
throw new NodeReconciliationFailedException("systemd unavailable, stopping node configuration");
}
var dockerVersion = await proxyClient.RunCommand(credentials, "docker --version", ct);
if (dockerVersion.Code != 0)
{
logger.LogDebug("docker unavailable");
connectionLog.Error("docker unavailable, stopping node configuration");
throw new NodeReconciliationFailedException("docker unavailable, stopping node configuration");
}
var dockerComposeVersion = await proxyClient.RunCommand(credentials, "docker-compose --version", ct);
if (dockerComposeVersion.Code != 0)
{
logger.LogDebug("docker-compose unavailable");
connectionLog.Error("docker-compose unavailable, stopping node configuration");
throw new NodeReconciliationFailedException("docker-compose unavailable, stopping node configuration");
}
return(true);
}
public async Task <Guid> Create(
Guid tenantId,
string name,
NodeCredentials conn
)
{
await connectionService.TestConnection(conn);
var node = new Node
{
TenantId = tenantId,
Host = conn.host,
Enabled = true,
Id = Guid.NewGuid(),
Name = name,
Port = conn.port,
Username = conn.username,
Password = string.IsNullOrWhiteSpace(conn.password) ? null : await crypto.EncryptSecret(conn.password),
PrivateKey = string.IsNullOrWhiteSpace(conn.privateKey) ? null : await crypto.EncryptSecret(conn.privateKey)
};
await ctx.Nodes.AddAsync(node);
await ctx.SaveChangesAsync();
await nodeUpdater.NotifyAboutNodeChange(node.Id);
return(node.Id);
}
private async Task <string> GetDockerComposePath(NodeCredentials credentials, CancellationToken ct = default)
{
var key = $"{credentials.username}@{credentials.host}:{credentials.port}";
var path = await cache.GetStringAsync(key, ct);
if (path != null)
{
return(path);
}
var result = await proxyClient.RunCommand(credentials, "which docker-compose", ct);
if (string.IsNullOrEmpty(result.Error) && result.Code == 0 && result.Stdout != null)
{
path = Encoding.UTF8.GetString(result.Stdout).Trim();
await cache.SetStringAsync(key, path, new DistributedCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1)
}, ct);
}
else
{
throw new DeploymentReconciliationFailedException("Unable to find docker-compose");
}
return(path);
}
private async Task <TResp> GetDocker <TResp>(string path, NodeCredentials credentials, CancellationToken ct = default)
{
var resp = await Request(path, null, HttpMethod.Get, credentials, ct);
var str = await resp.ReadAsStringAsync(cancellationToken : ct) ?? throw new InvalidOperationException("Could not parse response from proxy call");
return(dockerSerializer.DeserializeObject <TResp>(str));
}
private HttpClient ClientFor(NodeCredentials credentials)
{
var client = clientFactory.CreateClient("proxy");
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("bearer", MintJwtFromCredentials(credentials));
return(client);
}
private async Task StartDockerCompose(NodeCredentials credentials, DeploymentInfo composeVersion, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("StartDockerCompose");
var cmd = await ComposeCmdPrefix(credentials, composeVersion.ComposeDirectory, composeVersion.ServiceName, ct) + " up -d --remove-orphans";
await proxyClient.RunCommand(credentials, cmd, ct);
}
private async Task <bool> GetComposeIsRunning(NodeCredentials credentials, ComposeVersion version, CancellationToken ct = default)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("GetComposeNeedsToStop");
activity?.AddTag("supercompose.path", version.Directory);
var cmd = await ComposeCmdPrefix(credentials, version.Directory, version.ServiceName, ct) + " ps --quiet";
var res = await proxyClient.RunCommand(credentials, cmd, ct);
return(res.Code == 0 && res.Stdout != null && !string.IsNullOrEmpty(Encoding.UTF8.GetString(res.Stdout).Trim()));
}
private async Task RemoveCompose(NodeCredentials credentials, DeploymentInfo deployment, CancellationToken ct = default)
{
if (!string.IsNullOrEmpty(deployment.ComposePath))
{
var removed = await proxyClient.DeleteFile(credentials, deployment.ComposePath, ct);
if (removed)
{
connectionLog.Info($"Removed old compose file");
}
}
}
private async Task <bool> UpdateComposeFile(NodeCredentials credentials, DeploymentInfo composeVersion, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("UpdateComposeFile");
var resp = await proxyClient.UpsertFile(credentials, composeVersion.ComposePath, composeVersion.ComposeContent, true, ct);
if (resp.Updated)
{
connectionLog.Info($"docker-compose.yaml has been updated");
}
return(resp.Updated);
}
private async IAsyncEnumerable <TResp> GetDockerSSE <TResp>(string path, NodeCredentials credentials, [EnumeratorCancellation] CancellationToken ct =
default) where TResp : class
{
using var client = ClientFor(credentials);
using var reader = new StreamReader(await client.GetStreamAsync(path, ct));
while (!ct.IsCancellationRequested)
{
var line = await reader.ReadLineAsync();
if (!string.IsNullOrEmpty(line) && line.StartsWith("data: "))
{
line = line[6..];
private async Task ReloadContainersForNode(NodeCredentials credentials, Guid nodeId, CancellationToken ct = default)
{
await using var ctx = ctxFactory.CreateDbContext();
using var activity = Extensions.SuperComposeActivitySource.StartActivity("ReloadContainersForNode");
activity?.SetTag(Extensions.ActivityNodeIdName, nodeId);
activity?.AddBaggage(Extensions.ActivityNodeIdName, nodeId.ToString());
var deployments = await ctx.Deployments
.Where(x => x.NodeId == nodeId)
.Include(x => x.Containers)
.Select(x => new
{
Deployment = x,
ServiceName = x.LastDeployedComposeVersion !.ServiceName ?? x.Compose !.Current.ServiceName
})
private async Task <string> GenerateSystemdServiceFile(NodeCredentials credentials, ComposeVersion composeVersion, CancellationToken ct)
{
return($@"
[Unit]
Description={composeVersion.Compose.Name} service with docker compose managed by supercompose
Requires=docker.service
After=docker.service
[Service]
Type=oneshot
RemainAfterExit=true
WorkingDirectory={composeVersion.Directory}
ExecStart={await ComposeCmdPrefix(credentials, composeVersion.Directory, composeVersion.ServiceName, ct)} up -d --remove-orphans
ExecStop={await ComposeCmdPrefix(credentials, composeVersion.Directory, composeVersion.ServiceName, ct)} down
[Install]
WantedBy=multi-user.target".Trim().Replace("\r\n", "\n"));
}
private string MintJwtFromCredentials(NodeCredentials credentials)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("ProxyClient.MintJwtFromCredentials");
try
{
var tokenHandler = new JwtSecurityTokenHandler();
var claims = new List <Claim>
{
new("host", $"{credentials.host}:{credentials.port}"),
new("username", credentials.username),
};
if (!string.IsNullOrEmpty(credentials.password))
{
claims.Add(new Claim("password", credentials.password));
}
if (!string.IsNullOrEmpty(credentials.privateKey))
{
claims.Add(new Claim("pkey", credentials.privateKey));
}
var tokenDescriptor = new SecurityTokenDescriptor
{
Subject = new ClaimsIdentity(claims),
Expires = DateTime.UtcNow + TimeSpan.FromHours(2),
SigningCredentials = new SigningCredentials(
new SymmetricSecurityKey(Encoding.UTF8.GetBytes(config["Proxy:JWT"])),
SecurityAlgorithms.HmacSha256Signature
),
Audience = "proxy"
};
return(tokenHandler.CreateEncodedJwt(tokenDescriptor));
}
catch (Exception ex)
{
logger.LogCritical(ex, "Failed to mint JWT for proxy");
activity.RecordException(ex);
throw;
}
}
private async Task <bool> UpdateSystemdFile(NodeCredentials credentials, ComposeVersion compose, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("UpdateSystemdFile");
var serviceFile = await GenerateSystemdServiceFile(credentials, compose, ct);
var resp = await proxyClient.UpsertFile(
credentials,
compose.ServicePath,
Encoding.UTF8.GetBytes(serviceFile),
false,
ct
);
if (resp.Updated)
{
connectionLog.Info($"systemd service file has been updated");
}
return(resp.Updated);
}
/// <exception cref="NodeConnectionFailedException"></exception>
public async Task <SftpClient> CreateSftpConnection(NodeCredentials conn, TimeSpan timeout,
CancellationToken ct)
{
logger.BeginScope(new
{
conn.username,
conn.host,
conn.port,
passwordAvailable = !string.IsNullOrEmpty(conn.password),
privateKeyAvailable = !string.IsNullOrEmpty(conn.privateKey),
kind = "sftp_client"
});
var connectionInfo = await PrepareConnectionInfo(conn, timeout);
var client = new SftpClient(connectionInfo);
await CreateConnectionInner(client, conn, ct);
return(client);
}
private async Task <ConnectionInfo> PrepareConnectionInfo(NodeCredentials conn, TimeSpan timeout)
{
var authMethods = new List <AuthenticationMethod>();
if (!string.IsNullOrEmpty(conn.password))
{
authMethods.Add(new PasswordAuthenticationMethod(conn.username, conn.password));
}
if (!string.IsNullOrEmpty(conn.privateKey))
{
await using var pkMs = new MemoryStream(Encoding.UTF8.GetBytes(conn.privateKey));
try
{
var pkFile = new PrivateKeyFile(pkMs);
authMethods.Add(new PrivateKeyAuthenticationMethod(conn.username, pkFile));
}
catch (Exception ex)
{
logger.LogDebug("Failed to parse private key {error}", ex.Message);
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.PrivateKey
};
}
}
var connectionInfo = new ConnectionInfo(
conn.host,
conn.port,
conn.username,
authMethods.ToArray()
)
{
Timeout = timeout
};
return(connectionInfo);
}
public async Task TestConnection(NodeCredentials conn, Guid?nodeId = null, CancellationToken ct = default)
{
if (nodeId != null)
{
var node = await ctx.Nodes.FirstOrDefaultAsync(x => x.Id == nodeId, ct);
if (node == null)
{
throw new NodeNotFoundException();
}
if (string.IsNullOrWhiteSpace(conn.password) && string.IsNullOrWhiteSpace(conn.privateKey))
{
conn = conn with
{
password = node.Password != null ? await crypto.DecryptSecret(node.Password) : null,
privateKey = node.PrivateKey != null ? await crypto.DecryptSecret(node.PrivateKey) : null
}
}
;
}
using var client = await CreateSshConnection(conn, TimeSpan.FromSeconds(5), ct);
}
private async Task <HttpContent> Request(string path, object?body, HttpMethod method, NodeCredentials credentials, CancellationToken ct = default)
{
using var client = ClientFor(credentials);
var request = new HttpRequestMessage();
if (body != null)
{
request.Content = new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8);
}
request.Method = method;
request.RequestUri = new Uri(path, UriKind.Relative);
var resp = await client.SendAsync(request, ct);
if (!resp.IsSuccessStatusCode)
{
var errorResponse = await resp.Content.ReadFromJsonAsync <ProxyClientErrorResponse>(cancellationToken : ct)
?? throw new InvalidOperationException("Could not read error response");
var exp = new ProxyClientException(errorResponse.Title)
{
ErrorResponse = errorResponse
};
LogError(exp);
throw exp;
}
return(resp.Content);
}
/// <exception cref="NodeConnectionFailedException"></exception>
private async Task CreateConnectionInner(BaseClient client, NodeCredentials conn, CancellationToken ct)
{
try
{
logger.LogDebug("Resolving host");
await Dns.GetHostEntryAsync(conn.host);
logger.LogDebug("Resolved");
}
catch (Exception ex)
{
logger.LogDebug("Host resolution failed");
client.Dispose();
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.DNS
};
}
try
{
logger.LogDebug("Connecting");
client.Connect();
logger.LogDebug("Connected");
}
catch (SshAuthenticationException ex)
{
logger.LogDebug("Connection failed because to authenticate {error}", ex.Message);
client.Dispose();
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.Authentication
};
}
catch (SshConnectionException ex)
{
logger.LogDebug("Connection failed due to SSH connection error {error}", ex.Message);
client.Dispose();
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.Connection
};
}
catch (SocketException ex)
{
logger.LogDebug("Connection failed due to socket exception {error}", ex.Message);
client.Dispose();
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.Connection
};
}
catch (TaskCanceledException)
{
logger.LogDebug("Connection cancelled");
client.Dispose();
throw;
}
catch (Exception ex)
{
logger.LogDebug("Connection failed for unknown reason {error}", ex.Message);
client.Dispose();
throw new NodeConnectionFailedException(ex.Message, ex)
{
Kind = NodeConnectionFailedException.ConnectionErrorKind.Unknown
};
}
}
private async Task <string> ComposeCmdPrefix(NodeCredentials credentials, string dir, string service, CancellationToken ct)
{
var composePath = await GetDockerComposePath(credentials, ct);
return($"{composePath} --project-directory '{dir}' --project-name '{service}' --file '{dir}/docker-compose.yml'");
}
private async Task ApplyDeployment(NodeCredentials credentials, Deployment deployment, CancellationToken ct)
{
using var activity = Extensions.SuperComposeActivitySource.StartActivity("ApplyDeployment");
activity?.AddTag(Extensions.ActivityDeploymentIdName, deployment.Id.ToString());
activity?.AddBaggage(Extensions.ActivityDeploymentIdName, deployment.Id.ToString());
activity?.AddTag(Extensions.ActivityComposeIdName, deployment.Compose !.Id.ToString());
activity?.AddBaggage(Extensions.ActivityComposeIdName, deployment.Compose !.Id.ToString());
try
{
var(target, last) = CalculateDeploymentDiff(deployment);
activity?.AddTag("deployment.target.enabled", target.DeploymentEnabled);
activity?.AddTag("deployment.target.service", target.UseService);
activity?.AddTag("deployment.target.service_path", target.ServicePath);
activity?.AddTag("deployment.target.compose_path", target.ComposePath);
activity?.AddTag("deployment.last.enabled", last?.DeploymentEnabled);
activity?.AddTag("deployment.last.service", last?.UseService);
activity?.AddTag("deployment.last.service_path", last?.ServicePath);
activity?.AddTag("deployment.last.compose_path", last?.ComposePath);
activity?.AddEvent(new ActivityEvent("Getting current system status and updating files"));
var targetComposeUpdatedTask = UpdateComposeFile(credentials, target, ct);
var targetSystemdUpdatedTask = target.UseService
? UpdateSystemdFile(credentials, deployment.Compose !.Current, ct) // TODO if service path equal, but compose paths changed, we first need to stop the old service
: Task.FromResult(false);
var lastComposeStatusTask = deployment.LastDeployedComposeVersion != null
? GetComposeIsRunning(credentials, deployment.LastDeployedComposeVersion, ct)
: Task.FromResult(false);
var targetServiceStatusTask = target.UseService
? proxyClient.SystemdGetService(credentials, target.ServiceId, ct)
: Task.FromResult <ProxyClient.SystemdGetServiceResponse?>(null) !;
var lastServiceStatusTask = last != null && last.UseService
? (target.UseService && target.ServiceId == last.ServiceId)
? targetServiceStatusTask
: proxyClient.SystemdGetService(credentials, last.ServiceId, ct)
: Task.FromResult <ProxyClient.SystemdGetServiceResponse?>(null) !;
await Task.WhenAll(targetComposeUpdatedTask, targetSystemdUpdatedTask, lastComposeStatusTask, targetServiceStatusTask, lastServiceStatusTask);
var composeChanged = await targetComposeUpdatedTask;
var serviceChanged = await targetSystemdUpdatedTask;
var bothAreServices = last != null && target.UseService && last.UseService;
var bothAreComposes = last != null && !target.UseService && !last.UseService;
var composePathChanged = last == null || target.ComposePath != last.ComposePath;
var servicePathChanged = last == null || target.ComposePath != last.ComposePath;
var targetEnabled = target.DeploymentEnabled;
var lastServiceStatus = await targetServiceStatusTask;
var lastServiceRunning = lastServiceStatus != null && lastServiceStatus.IsRunning;
var lastServiceEnabled = lastServiceStatus != null && lastServiceStatus.IsEnabled;
var lastDockerRunning = await lastComposeStatusTask;
var redeploymentRequested = RedeployRequested(deployment);
var restartRequired = composeChanged || serviceChanged || redeploymentRequested;
activity?.AddTag("startStop.composeChanged", composeChanged);
activity?.AddTag("startStop.serviceChanged", serviceChanged);
activity?.AddTag("startStop.bothAreServices", bothAreServices);
activity?.AddTag("startStop.bothAreComposes", bothAreComposes);
activity?.AddTag("startStop.composePathChanged", composePathChanged);
activity?.AddTag("startStop.servicePathChanged", servicePathChanged);
activity?.AddTag("startStop.enabled", targetEnabled);
activity?.AddTag("startStop.lastServiceRunning", lastServiceRunning);
activity?.AddTag("startStop.lastServiceEnabled", lastServiceEnabled);
activity?.AddTag("startStop.lastDockerRunning", lastDockerRunning);
activity?.AddTag("startStop.redeploymentRequested", redeploymentRequested);
activity?.AddTag("startStop.restartRequired", restartRequired);
activity?.AddEvent(new ActivityEvent("System status acquired, updating resource enablement state"));
connectionLog.Info($"System status acquired, updating resource enablement state");
if (serviceChanged)
{
await proxyClient.SystemdReload(credentials, ct);
}
using (var disableEnableActivity = Extensions.SuperComposeActivitySource.StartActivity("Disable old services and enable new"))
{
var pendingTasks = new List <Task>();
var shouldDisableOldService = last != null && lastServiceEnabled && (
(last.UseService && !target.UseService) ||
(target.UseService && servicePathChanged) ||
(!targetEnabled)
);
if (shouldDisableOldService)
{
disableEnableActivity?.AddEvent(new ActivityEvent("shouldDisableOldService"));
var serviceStatus = await lastServiceStatusTask;
if (serviceStatus != null && serviceStatus.IsEnabled)
{
pendingTasks.Add(proxyClient.SystemdDisableService(credentials, last !.ServiceId, ct));
}
}
var shouldEnableNewService = targetEnabled && (
last != null
? (target.UseService && !last.UseService) || (target.UseService && last.UseService && target.ServicePath != last.ServicePath)
: target.UseService
);
if (shouldEnableNewService)
{
disableEnableActivity?.AddEvent(new ActivityEvent("shouldEnableNewService"));
var serviceStatus = await targetServiceStatusTask;
if (serviceStatus != null && serviceStatus.IsEnabled)
{
pendingTasks.Add(proxyClient.SystemdEnableService(credentials, target.ServiceId, ct));
}
}
await Task.WhenAll(pendingTasks.ToArray());
}
activity?.AddEvent(new ActivityEvent("Successfully changed resource enabled status, restarting resources"));
connectionLog.Info($"Successfully changed resource enabled status, restarting resources");
if (last != null && (lastServiceRunning || lastDockerRunning))
{
using var stopActivity = Extensions.SuperComposeActivitySource.StartActivity("Stop old");
activity?.AddEvent(new ActivityEvent("Stopping old resources"));
connectionLog.Info($"Stopping old resources");
if (lastServiceRunning && !target.UseService)
{
stopActivity?.AddEvent(new ActivityEvent("old service is running, but new is compose"));
await proxyClient.SystemdStopService(credentials, last.ServiceId, ct);
}
else if (lastServiceRunning && servicePathChanged)
{
stopActivity?.AddEvent(new ActivityEvent("old service is running, but service path has changed"));
await proxyClient.SystemdStopService(credentials, last.ServiceId, ct);
}
else if (!targetEnabled && lastServiceRunning)
{
stopActivity?.AddEvent(new ActivityEvent("old service should not be running, but is"));
await proxyClient.SystemdStopService(credentials, last.ServiceId, ct);
}
else if (lastDockerRunning && composePathChanged)
{
stopActivity?.AddEvent(new ActivityEvent("old docker is running, but compose path has changed"));
await StopDockerCompose(credentials, last, ct);
}
else if (!targetEnabled && lastDockerRunning)
{
stopActivity?.AddEvent(new ActivityEvent("old docker should not be running, but is"));
await StopDockerCompose(credentials, last, ct);
}
}
if (targetEnabled && (!lastDockerRunning || composePathChanged || servicePathChanged || restartRequired || last.UseService != target.UseService))
{
using var startActivity = Extensions.SuperComposeActivitySource.StartActivity("Start new");
activity?.AddEvent(new ActivityEvent("Starting new resources"));
connectionLog.Info($"Starting new resources");
if (bothAreServices && !composePathChanged && !servicePathChanged && restartRequired)
{
startActivity?.AddEvent(new ActivityEvent("restart unchanged service"));
await proxyClient.SystemdRestartService(credentials, target.ServiceId, ct);
}
else if (lastDockerRunning && bothAreServices && !composePathChanged && !servicePathChanged && !restartRequired)
{
startActivity?.AddEvent(new ActivityEvent("nothing changed, no restart required"));
}
else if (lastDockerRunning && bothAreComposes && !composePathChanged && !restartRequired)
{
startActivity?.AddEvent(new ActivityEvent("nothing changed, no restart required"));
}
else if (bothAreComposes && !composePathChanged && restartRequired)
{
startActivity?.AddEvent(new ActivityEvent("restart docker"));
await RestartDockerCompose(credentials, target, ct);
}
else if (target.UseService)
{
startActivity?.AddEvent(new ActivityEvent("starting service"));
await proxyClient.SystemdRestartService(credentials, target.ServiceId, ct);
}
else if (!target.UseService)
{
startActivity?.AddEvent(new ActivityEvent("starting compose"));
await StartDockerCompose(credentials, target, ct);
}
}
if (last != null && (composePathChanged || servicePathChanged || !targetEnabled))
{
connectionLog.Info($"Restart successful, removing outdated resources");
using var removeActivity = Extensions.SuperComposeActivitySource.StartActivity("Remove old configurations");
var pendingTasks = new List <Task>();
if (servicePathChanged || !targetEnabled)
{
removeActivity?.AddEvent(new ActivityEvent("shouldRemoveOldService"));
pendingTasks.Add(RemoveService(credentials, last, ct));
}
if (composePathChanged || !targetEnabled)
{
removeActivity?.AddEvent(new ActivityEvent("shouldRemoveOldCompose"));
pendingTasks.Add(RemoveCompose(credentials, last, ct));
}
await Task.WhenAll(pendingTasks.ToArray());
}
deployment.LastDeployedComposeVersionId = deployment.Compose.CurrentId;
deployment.LastDeployedNodeVersion = deployment.Node.Version;
deployment.LastCheck = DateTime.UtcNow;
deployment.LastDeployedAsEnabled = target.DeploymentEnabled;
await ctx.SaveChangesAsync(ct);
connectionLog.Info($"Deployment applied successfully");
}
catch (DeploymentReconciliationFailedException ex)
{
connectionLog.Error($"Deployment reconciliation failed", ex);
deployment.ReconciliationFailed = true;
await ctx.SaveChangesAsync(ct);
}
}
private async Task <TResp> Get <TResp>(string path, NodeCredentials credentials, CancellationToken ct = default)
{
var resp = await Request(path, null, HttpMethod.Get, credentials, ct);
return(await resp.ReadFromJsonAsync <TResp>(cancellationToken : ct) ?? throw new InvalidOperationException("Could not parse response from proxy call"));
}
