private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
try
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup.",
msg,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
}
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
ScheduleNextBackup(periodicBackup);
}
}
public void FinishBackup(string backupName, PeriodicBackupStatus backupStatus, TimeSpan?elapsed, Logger logger)
{
lock (_locker)
{
_concurrentBackups++;
}
if (logger.IsOperationsEnabled)
{
string backupTypeString = "backup";
string extendedBackupTimings = string.Empty;
if (backupStatus != null)
{
backupTypeString = BackupTask.GetBackupDescription(backupStatus.BackupType, backupStatus.IsFull);
var first = true;
AddBackupTimings(backupStatus.LocalBackup, "local");
AddBackupTimings(backupStatus.UploadToS3, "Amazon S3");
AddBackupTimings(backupStatus.UploadToGlacier, "Amazon Glacier");
AddBackupTimings(backupStatus.UploadToAzure, "Azure");
AddBackupTimings(backupStatus.UploadToGoogleCloud, "Google Cloud");
AddBackupTimings(backupStatus.UploadToFtp, "FTP");
void AddBackupTimings(BackupStatus perDestinationBackupStatus, string backupTypeName)
{
if (perDestinationBackupStatus == null ||
perDestinationBackupStatus is CloudUploadStatus cus && cus.Skipped)
{
return;
}
if (first == false)
{
extendedBackupTimings += ", ";
}
first = false;
extendedBackupTimings +=
$"backup to {backupTypeName} took: " +
$"{(backupStatus.IsFull ? perDestinationBackupStatus.FullBackupDurationInMs : perDestinationBackupStatus.IncrementalBackupDurationInMs)}ms";
}
}
var message = $"Finished {backupTypeString} task '{backupName}'";
if (elapsed != null)
{
message += $", took: {elapsed}";
}
message += $" {extendedBackupTimings}";
logger.Operations(message);
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
try
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private Task <IOperationResult> StartBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress)
{
var tcs = new TaskCompletionSource <IOperationResult>(TaskCreationOptions.RunContinuationsAsynchronously);
PoolOfThreads.GlobalRavenThreadPool.LongRunning(x => RunBackupThread(periodicBackup, backupTask, onProgress, tcs), null, $"Backup task {periodicBackup.Configuration.Name} for database '{_database.Name}'");
return(tcs.Task);
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task exceeds the maximum number of concurrent backup tasks configured. " +
$"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}")
{
DelayPeriod = TimeSpan.FromMinutes(1)
};
}
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// releasing the semaphore because we failed to start the backup task
_serverStore.ConcurrentBackupsSemaphore.Release();
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
if (periodicBackup.UpdateBackupTaskSemaphore.Wait(0) == false)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
try
{
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
finally
{
periodicBackup.UpdateBackupTaskSemaphore.Release();
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.DirtyMemoryState.IsHighDirty)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in high dirty memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
_serverStore.ConcurrentBackupsCounter.StartBackup(periodicBackup.Configuration.Name);
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// we failed to START the backup, need to update the status anyway
// in order to reschedule the next full/incremental backup
periodicBackup.BackupStatus.Version++;
periodicBackup.BackupStatus.Error = new Error
{
Exception = e.ToString(),
At = DateTime.UtcNow
};
if (isFullBackup)
{
periodicBackup.BackupStatus.LastFullBackupInternal = startTimeInUtc;
}
else
{
periodicBackup.BackupStatus.LastIncrementalBackupInternal = startTimeInUtc;
}
BackupTask.SaveBackupStatus(periodicBackup.BackupStatus, _database, _logger);
ScheduleNextBackup(periodicBackup);
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
message,
"The next backup will be rescheduled",
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}