public async Task PerformHealthChecksAsync(CancellationToken cancellationToken)
{
if (_healthChecks.Length == 0)
{
_logger.Debug("No health checks are registered");
return;
}
_logger.Debug("{HealthCheckCount} health checks are registered", _healthChecks.Length);
foreach (IHealthCheck healthCheck in _healthChecks)
{
try
{
using CancellationTokenSource cts =
_timeoutHelper.CreateCancellationTokenSource(
TimeSpan.FromSeconds(healthCheck.TimeoutInSeconds));
using var combined =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cts.Token);
_logger.Debug("Making health check with {Check}", healthCheck.Description);
await healthCheck.CheckHealthAsync(combined.Token);
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Debug(ex, "Health check error for check {Check}", healthCheck.Description);
}
}
_logger.Debug("Health checks done");
}
public async Task <WebHookResult> HandleRequest(
HttpRequest request,
string content,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(content))
{
_logger.Debug("Cannot process empty web hook request body");
return(new WebHookResult(false));
}
bool handled = false;
foreach (IPackageWebHook packageWebHook in _packageWebHooks)
{
CancellationTokenSource?cancellationTokenSource = default;
if (cancellationToken == CancellationToken.None)
{
cancellationTokenSource = _timeoutHelper.CreateCancellationTokenSource(TimeSpan.FromSeconds(10));
cancellationToken = cancellationTokenSource.Token;
}
try
{
PackageUpdatedEvent?webHook =
await packageWebHook.TryGetWebHookNotification(request, content, cancellationToken);
if (webHook is null)
{
continue;
}
handled = true;
_logger.Information("Web hook successfully handled by {Handler}",
packageWebHook.GetType().FullName);
await Task.Run(() => _mediator.Publish(webHook, cancellationToken), cancellationToken);
break;
}
catch (Exception ex)
{
_logger.Error(
ex,
"Could not get web hook notification from hook {Hook}",
packageWebHook.GetType().FullName);
throw;
}
finally
{
cancellationTokenSource?.Dispose();
}
}
return(new WebHookResult(handled));
}
public async Task <ExitCode> RunAsync(string deploymentTaskId,
DeploymentTargetId deploymentTargetId,
CancellationToken cancellationToken = default)
{
_logger.Information("Received deployment task {DeploymentTaskId}", deploymentTaskId);
IHttpClient client = _logHttpClientFactory.CreateClient(deploymentTaskId, deploymentTargetId, AgentId, _logger);
Logger logger = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.Logger(_logger)
.WriteTo.DurableHttpUsingTimeRolledBuffers(AgentConstants.DeploymentTaskLogRoute,
period: TimeSpan.FromMilliseconds(100), httpClient: client)
.CreateLogger(); //TODO create job logger in agent
ExitCode exitCode;
try
{
using CancellationTokenSource cancellationTokenSource =
_timeoutHelper.CreateCancellationTokenSource(TimeSpan.FromMinutes(30));
var deploymentTaskPackage =
await _deploymentTaskPackageService.GetDeploymentTaskPackageAsync(deploymentTaskId,
cancellationTokenSource.Token);
if (deploymentTaskPackage is null)
{
_logger.Error("Could not get deployment task package for deployment task id {DeploymentTaskId}",
deploymentTaskId);
return(ExitCode.Failure);
}
if (string.IsNullOrWhiteSpace(deploymentTaskPackage.DeploymentTaskId))
{
_logger.Error(
"Deployment task package for deployment task id {DeploymentTaskId} is missing deployment task id",
deploymentTaskId);
return(ExitCode.Failure);
}
exitCode =
await _deploymentPackageHandler.RunAsync(deploymentTaskPackage, logger,
cancellationTokenSource.Token);
logger.Dispose();
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Error(ex, "Failed to deploy {DeploymentTaskId}", deploymentTaskId);
return(ExitCode.Failure);
}
return(exitCode);
}
private void TryStartWorker(IDeploymentTargetWorker deploymentTargetWorker, CancellationToken stoppingToken)
{
_logger.Debug("Trying to start worker for target id {TargetId}", deploymentTargetWorker.TargetId);
if (_tasks.ContainsKey(deploymentTargetWorker.TargetId))
{
if (deploymentTargetWorker.IsRunning)
{
_logger.Debug("Worker for target id {TargetId} is already running",
deploymentTargetWorker.TargetId);
return;
}
Task task = _tasks[deploymentTargetWorker.TargetId];
if (!task.IsCompleted && _cancellations.ContainsKey(deploymentTargetWorker.TargetId))
{
CancellationTokenSource tokenSource = _cancellations[deploymentTargetWorker.TargetId];
try
{
if (!tokenSource.IsCancellationRequested)
{
tokenSource.Cancel();
}
tokenSource.Dispose();
}
catch (ObjectDisposedException)
{
// ignore
}
_cancellations.Remove(deploymentTargetWorker.TargetId);
}
_tasks.Remove(deploymentTargetWorker.TargetId);
}
else
{
_logger.Debug("Start worker task was not found for target id {TargetId}",
deploymentTargetWorker.TargetId);
}
var cancellationTokenSource = _timeoutHelper.CreateCancellationTokenSource();
var linked = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, cancellationTokenSource.Token);
_cancellations.TryAdd(deploymentTargetWorker.TargetId, cancellationTokenSource);
_tasks.Add(deploymentTargetWorker.TargetId,
Task.Run(() => deploymentTargetWorker.ExecuteAsync(linked.Token), linked.Token));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await Task.Yield();
IReadOnlyCollection <DeploymentTargetId> targetIds;
try
{
if (!int.TryParse(
_configuration[DeployerAppConstants.StartupTargetsTimeoutInSeconds],
out int startupTimeoutInSeconds) ||
startupTimeoutInSeconds <= 0)
{
startupTimeoutInSeconds = 30;
}
using CancellationTokenSource startupToken =
_timeoutHelper.CreateCancellationTokenSource(TimeSpan.FromSeconds(startupTimeoutInSeconds));
using var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(
stoppingToken,
startupToken.Token);
targetIds =
(await _deploymentTargetReadService.GetDeploymentTargetsAsync(stoppingToken: linkedToken.Token))
.Select(deploymentTarget => deploymentTarget.Id)
.ToArray();
_logger.Debug("Found deployment target IDs {IDs}", targetIds);
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Warning(ex, "Could not get target ids");
IsCompleted = true;
return;
}
foreach (var targetId in targetIds)
{
var deploymentTargetWorker = new DeploymentTargetWorker(targetId, _logger, _mediator,
_workerConfiguration, _timeoutHelper, _clock, _serviceProvider);
_holder.Add(new NamedInstance <DeploymentTargetWorker>(
deploymentTargetWorker,
targetId.TargetId));
await _mediator.Send(new StartWorker(deploymentTargetWorker), stoppingToken);
}
IsCompleted = true;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await Task.Yield();
string?nugetExePath = "";
_logger.Debug("Ensuring nuget.exe exists");
if (!int.TryParse(_configuration[DeployerAppConstants.NuGetDownloadTimeoutInSeconds],
out int initialNuGetDownloadTimeoutInSeconds) || initialNuGetDownloadTimeoutInSeconds <= 0)
{
initialNuGetDownloadTimeoutInSeconds = 100;
}
try
{
var fromSeconds = TimeSpan.FromSeconds(initialNuGetDownloadTimeoutInSeconds);
using CancellationTokenSource cts = _timeoutHelper.CreateCancellationTokenSource(fromSeconds);
string?downloadDirectory = _configuration[DeployerAppConstants.NuGetExeDirectory].WithDefault();
string?exeVersion = _configuration[DeployerAppConstants.NuGetExeVersion].WithDefault();
HttpClient httpClient = _httpClientFactory.CreateClient();
var nuGetDownloadClient = new NuGetDownloadClient();
NuGetDownloadResult nuGetDownloadResult = await nuGetDownloadClient.DownloadNuGetAsync(
new NuGetDownloadSettings(downloadDirectory : downloadDirectory, nugetExeVersion : exeVersion),
_logger,
httpClient,
cts.Token);
if (nuGetDownloadResult.Succeeded)
{
nugetExePath = nuGetDownloadResult.NuGetExePath;
}
}
catch (Exception ex)
{
_logger.Warning(ex, "Could not download nuget.exe");
}
if (_configuration is { } && _nugetConfiguration is {})
public async Task <IActionResult> Index()
{
IReadOnlyCollection <DeploymentTarget> targets;
try
{
using CancellationTokenSource cts =
_timeoutHelper.CreateCancellationTokenSource(TimeSpan.FromSeconds(30));
targets =
(await _getTargets.GetOrganizationsAsync(cts.Token)).SelectMany(
organization => organization.Projects.SelectMany(project => project.DeploymentTargets))
.SafeToReadOnlyCollection();
}
catch (Exception ex)
{
_logger.Error(ex, "Could not get organizations");
targets = ImmutableArray <DeploymentTarget> .Empty;
}
IReadOnlyCollection <PackageVersion> items = ImmutableArray <PackageVersion> .Empty;
return(View(new DeploymentViewOutputModel(items, targets)));
}
public async Task <AppVersion?> GetAppMetadataAsync(
[NotNull] DeploymentTarget target,
CancellationToken cancellationToken)
{
if (target is null)
{
throw new ArgumentNullException(nameof(target));
}
string cacheKey = GetCacheKey(target.Id);
if (_customMemoryCache.TryGetValue(cacheKey, out AppVersion? appMetadata))
{
_logger.Verbose("App metadata fetched from cache {CacheKey}", cacheKey);
return(appMetadata);
}
ApplicationSettings applicationSettings =
await _applicationSettingsStore.GetApplicationSettings(cancellationToken);
var targetMetadataTimeout = target.MetadataTimeout ?? applicationSettings.ApplicationSettingsCacheTimeout;
using (CancellationTokenSource cancellationTokenSource = _timeoutHelper.CreateCancellationTokenSource(
targetMetadataTimeout))
{
if (_logger.IsEnabled(LogEventLevel.Verbose))
{
cancellationTokenSource.Token.Register(() =>
{
_logger.Verbose(
"{Method} for {Target}, cancellation token invoked out after {Seconds} seconds",
nameof(GetAppMetadataAsync),
target,
targetMetadataTimeout.TotalSeconds.ToString("F1"));
});
}
using var linkedTokenSource =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cancellationTokenSource.Token);
Task <(HttpResponseMessage?, string?)> metadataTask =
GetApplicationMetadataTask(target, linkedTokenSource.Token);
Task <IReadOnlyCollection <PackageVersion> > allowedPackagesTask =
GetAllowedPackagesAsync(target, linkedTokenSource.Token);
await Task.WhenAll(metadataTask, allowedPackagesTask);
(var response, string?message) = await metadataTask;
IReadOnlyCollection <PackageVersion> packages = await allowedPackagesTask;
if (response is null)
{
return(new AppVersion(
target,
message ?? $"Could not get application metadata from target {target.Url}, no response",
packages));
}
if (!response.IsSuccessStatusCode)
{
return(new AppVersion(
target,
message ??
$"Could not get application metadata from target {target.Url}, status code not successful {response.StatusCode}",
packages));
}
appMetadata =
await GetAppVersionAsync(response, target, packages, linkedTokenSource.Token);
response.Dispose();
}
if (appMetadata.SemanticVersion is { })
public async Task Handle(DeploymentMetadataLog notification, CancellationToken cancellationToken)
{
if (!_emailConfiguration.IsValid)
{
_logger.Warning("Email configuration is invalid {Configuration}", _emailConfiguration);
return;
}
if (!_emailConfiguration.EmailEnabled)
{
_logger.Debug(
"Email is disabled, skipping sending deployment finished email for notification {Notification}",
notification);
return;
}
using CancellationTokenSource cancellationTokenSource =
_timeoutHelper.CreateCancellationTokenSource(
TimeSpan.FromSeconds(_emailConfiguration.NotificationTimeOutInSeconds));
DeploymentTarget?target =
await _targetSource.GetDeploymentTargetAsync(notification.DeploymentTask.DeploymentTargetId,
cancellationTokenSource.Token);
if (target is null)
{
return;
}
if (!target.EmailNotificationAddresses.Any())
{
return;
}
var mimeMessage = new MimeMessage();
foreach (string targetEmailNotificationAddress in target.EmailNotificationAddresses)
{
try
{
mimeMessage.To.Add(new MailboxAddress("", targetEmailNotificationAddress));
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Error(ex,
"Could not add email address {EmailAddress} when sending deployment finished notification email",
targetEmailNotificationAddress);
}
}
mimeMessage.Body = new TextPart
{
Text =
$@"Deployment finished for {notification.DeploymentTask}
{notification.Result.Metadata}"
};
mimeMessage.Subject = $"Deployment finished for {notification.DeploymentTask}";
try
{
await _smtpService.SendAsync(mimeMessage, cancellationTokenSource.Token);
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Error(ex,
"Could not send email to '{To}'",
string.Join(", ", target.EmailNotificationAddresses));
}
}
