private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
try
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup.",
msg,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
}
private void ScheduleNextBackup(PeriodicBackup periodicBackup)
{
try
{
_serverStore.ConcurrentBackupsCounter.FinishBackup();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var message = $"Failed to schedule next backup for task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
try
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
if (periodicBackup.UpdateBackupTaskSemaphore.Wait(0) == false)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
try
{
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
finally
{
periodicBackup.UpdateBackupTaskSemaphore.Release();
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task exceeds the maximum number of concurrent backup tasks configured. " +
$"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}")
{
DelayPeriod = TimeSpan.FromMinutes(1)
};
}
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// releasing the semaphore because we failed to start the backup task
_serverStore.ConcurrentBackupsSemaphore.Release();
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}