private void ScheduleNextBackup(PeriodicBackup periodicBackup)
{
try
{
_serverStore.ConcurrentBackupsCounter.FinishBackup();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var message = $"Failed to schedule next backup for task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
try
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup.",
msg,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
}
public BackupTask(
ServerStore serverStore,
DocumentDatabase database,
PeriodicBackup periodicBackup,
bool isFullBackup,
bool backupToLocalFolder,
long operationId,
PathSetting tempBackupPath,
Logger logger,
CancellationToken databaseShutdownCancellationToken)
{
_serverStore = serverStore;
_database = database;
_startTime = periodicBackup.StartTime;
_periodicBackup = periodicBackup;
_configuration = periodicBackup.Configuration;
_previousBackupStatus = periodicBackup.BackupStatus;
_isFullBackup = isFullBackup;
_backupToLocalFolder = backupToLocalFolder;
_operationId = operationId;
_tempBackupPath = tempBackupPath;
_logger = logger;
_databaseShutdownCancellationToken = databaseShutdownCancellationToken;
TaskCancelToken = new OperationCancelToken(_databaseShutdownCancellationToken);
_backupResult = GenerateBackupResult();
}
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
ScheduleNextBackup(periodicBackup);
}
}
private bool ShouldRunBackupAfterTimerCallback(NextBackup backupInfo, out PeriodicBackup periodicBackup)
{
if (_periodicBackups.TryGetValue(backupInfo.TaskId, out periodicBackup) == false)
{
// periodic backup doesn't exist anymore
return(false);
}
DatabaseTopology topology;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
using (var rawRecord = _serverStore.Cluster.ReadRawDatabaseRecord(context, _database.Name))
{
if (rawRecord == null)
{
return(false);
}
topology = rawRecord.GetTopology();
}
var taskStatus = GetTaskStatus(topology, periodicBackup.Configuration);
return(taskStatus == TaskStatus.ActiveByCurrentNode);
}
private Task <IOperationResult> StartBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress)
{
var tcs = new TaskCompletionSource <IOperationResult>(TaskCreationOptions.RunContinuationsAsynchronously);
PoolOfThreads.GlobalRavenThreadPool.LongRunning(x => RunBackupThread(periodicBackup, backupTask, onProgress, tcs), null, $"Backup task {periodicBackup.Configuration.Name} for database '{_database.Name}'");
return(tcs.Task);
}
private void StartBackupTaskAndRescheduleIfNeeded(PeriodicBackup periodicBackup, NextBackup currentBackup)
{
try
{
CreateBackupTask(periodicBackup, currentBackup.IsFull, currentBackup.DateTime);
}
catch (BackupDelayException e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Backup task will be retried in {(int)e.DelayPeriod.TotalSeconds} seconds.", e);
}
// we'll retry in one minute
var backupTaskDetails = new NextBackup
{
IsFull = currentBackup.IsFull,
TaskId = periodicBackup.Configuration.TaskId,
DateTime = DateTime.UtcNow.Add(e.DelayPeriod),
TimeSpan = e.DelayPeriod
};
var timer = new Timer(TimerCallback, backupTaskDetails, backupTaskDetails.TimeSpan, Timeout.InfiniteTimeSpan);
periodicBackup.UpdateTimer(timer);
}
}
private void UpdatePeriodicBackup(long taskId,
PeriodicBackupConfiguration newConfiguration,
TaskStatus taskState)
{
Debug.Assert(taskId == newConfiguration.TaskId);
var backupStatus = GetBackupStatus(taskId, inMemoryBackupStatus: null);
if (_periodicBackups.TryGetValue(taskId, out var existingBackupState) == false)
{
var newPeriodicBackup = new PeriodicBackup(_inactiveRunningPeriodicBackupsTasks)
{
Configuration = newConfiguration
};
var periodicBackup = _periodicBackups.GetOrAdd(taskId, newPeriodicBackup);
if (periodicBackup != newPeriodicBackup)
{
newPeriodicBackup.Dispose();
}
if (taskState == TaskStatus.ActiveByCurrentNode)
{
periodicBackup.UpdateTimer(GetTimer(newConfiguration, backupStatus));
}
return;
}
var previousConfiguration = existingBackupState.Configuration;
existingBackupState.Configuration = newConfiguration;
if (taskState != TaskStatus.ActiveByCurrentNode)
{
// this node isn't responsible for the backup task
existingBackupState.DisableFutureBackups();
return;
}
if (existingBackupState.RunningTask != null)
{
// a backup is already running
// the next one will be re-scheduled by the backup task
return;
}
if (previousConfiguration.HasBackupFrequencyChanged(newConfiguration) == false &&
existingBackupState.HasScheduledBackup())
{
// backup frequency hasn't changed
// and we have a scheduled backup
return;
}
existingBackupState.UpdateTimer(GetTimer(newConfiguration, backupStatus));
}
private void UpdatePeriodicBackups(long taskId,
PeriodicBackupConfiguration newConfiguration,
TaskStatus taskState)
{
Debug.Assert(taskId == newConfiguration.TaskId);
var backupStatus = GetBackupStatus(taskId, inMemoryBackupStatus: null);
if (_periodicBackups.TryGetValue(taskId, out PeriodicBackup existingBackupState) == false)
{
var newPeriodicBackup = new PeriodicBackup
{
Configuration = newConfiguration
};
if (taskState == TaskStatus.ActiveByCurrentNode)
{
newPeriodicBackup.BackupTimer = GetTimer(newConfiguration, backupStatus);
}
_periodicBackups.TryAdd(taskId, newPeriodicBackup);
return;
}
if (existingBackupState.Configuration.Equals(newConfiguration))
{
// the username/password for the cloud backups might have changed,
// and it will be reloaded on the next backup re-scheduling
existingBackupState.Configuration = newConfiguration;
if (taskState == TaskStatus.ActiveByCurrentNode)
{
existingBackupState.BackupTimer = GetTimer(newConfiguration, backupStatus);
}
return;
}
// the backup configuration changed
existingBackupState.DisableFutureBackups();
TryAddInactiveRunningPeriodicBackups(existingBackupState.RunningTask);
_periodicBackups.TryRemove(taskId, out _);
var periodicBackup = new PeriodicBackup
{
Configuration = newConfiguration
};
if (taskState == TaskStatus.ActiveByCurrentNode)
{
periodicBackup.BackupTimer = GetTimer(newConfiguration, backupStatus);
}
_periodicBackups.TryAdd(taskId, periodicBackup);
}
private bool ShouldRunBackupAfterTimerCallback(BackupTaskDetails backupInfo, out PeriodicBackup periodicBackup)
{
if (_periodicBackups.TryGetValue(backupInfo.TaskId, out periodicBackup) == false)
{
// periodic backup doesn't exist anymore
return(false);
}
if (periodicBackup.Disposed)
{
// this periodic backup was canceled
return(false);
}
var databaseRecord = GetDatabaseRecord();
var taskStatus = GetTaskStatus(databaseRecord, periodicBackup.Configuration);
return(taskStatus == TaskStatus.ActiveByCurrentNode);
}
private void CreateBackupTask(PeriodicBackup periodicBackup, BackupTaskDetails backupDetails)
{
periodicBackup.RunningTask = Task.Run(async() =>
{
periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
try
{
await RunPeriodicBackup(periodicBackup.Configuration,
periodicBackup.BackupStatus, backupDetails.IsFullBackup);
}
finally
{
if (_cancellationToken.IsCancellationRequested == false &&
periodicBackup.Disposed == false)
{
periodicBackup.BackupTimer.Dispose();
periodicBackup.BackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
}
}
}, _database.DatabaseShutdown);
}
public BackupTask(
ServerStore serverStore,
DocumentDatabase database,
PeriodicBackup periodicBackup,
bool isFullBackup,
bool backupToLocalFolder,
long operationId,
PathSetting tempBackupPath,
Logger logger,
CancellationToken databaseShutdownCancellationToken,
PeriodicBackupRunner.TestingStuff forTestingPurposes = null)
{
_serverStore = serverStore;
_database = database;
_periodicBackup = periodicBackup;
_configuration = periodicBackup.Configuration;
_isServerWide = _configuration.Name?.StartsWith(ServerWideBackupConfiguration.NamePrefix, StringComparison.OrdinalIgnoreCase) ?? false;
_isBackupEncrypted = IsBackupEncrypted(_database, _configuration);
_previousBackupStatus = periodicBackup.BackupStatus;
_isFullBackup = isFullBackup;
_backupToLocalFolder = backupToLocalFolder;
_operationId = operationId;
_tempBackupPath = tempBackupPath;
_logger = logger;
_databaseShutdownCancellationToken = databaseShutdownCancellationToken;
_forTestingPurposes = forTestingPurposes;
TaskCancelToken = new OperationCancelToken(_databaseShutdownCancellationToken);
_backupResult = GenerateBackupResult();
_retentionPolicyParameters = new RetentionPolicyBaseParameters
{
RetentionPolicy = _configuration.RetentionPolicy,
DatabaseName = _database.Name,
IsFullBackup = _isFullBackup,
OnProgress = AddInfo,
CancellationToken = TaskCancelToken.Token
};
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.DirtyMemoryState.IsHighDirty)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in high dirty memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
_serverStore.ConcurrentBackupsCounter.StartBackup(periodicBackup.Configuration.Name);
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// we failed to START the backup, need to update the status anyway
// in order to reschedule the next full/incremental backup
periodicBackup.BackupStatus.Version++;
periodicBackup.BackupStatus.Error = new Error
{
Exception = e.ToString(),
At = DateTime.UtcNow
};
if (isFullBackup)
{
periodicBackup.BackupStatus.LastFullBackupInternal = startTimeInUtc;
}
else
{
periodicBackup.BackupStatus.LastIncrementalBackupInternal = startTimeInUtc;
}
BackupTask.SaveBackupStatus(periodicBackup.BackupStatus, _database, _logger);
ScheduleNextBackup(periodicBackup);
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
message,
"The next backup will be rescheduled",
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
try
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task exceeds the maximum number of concurrent backup tasks configured. " +
$"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}")
{
DelayPeriod = TimeSpan.FromMinutes(1)
};
}
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// releasing the semaphore because we failed to start the backup task
_serverStore.ConcurrentBackupsSemaphore.Release();
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private void UpdatePeriodicBackup(long taskId,
PeriodicBackupConfiguration newConfiguration,
TaskStatus taskState)
{
Debug.Assert(taskId == newConfiguration.TaskId);
var backupStatus = GetBackupStatus(taskId, inMemoryBackupStatus: null);
if (_periodicBackups.TryGetValue(taskId, out var existingBackupState) == false)
{
var newPeriodicBackup = new PeriodicBackup(_inactiveRunningPeriodicBackupsTasks)
{
Configuration = newConfiguration
};
var periodicBackup = _periodicBackups.GetOrAdd(taskId, newPeriodicBackup);
if (periodicBackup != newPeriodicBackup)
{
newPeriodicBackup.Dispose();
}
if (taskState == TaskStatus.ActiveByCurrentNode)
{
periodicBackup.UpdateTimer(GetTimer(newConfiguration, backupStatus));
}
return;
}
var previousConfiguration = existingBackupState.Configuration;
existingBackupState.Configuration = newConfiguration;
switch (taskState)
{
case TaskStatus.Disabled:
case TaskStatus.ActiveByOtherNode:
// the task is disabled or this node isn't responsible for the backup task
existingBackupState.DisableFutureBackups();
return;
case TaskStatus.ClusterDown:
// this node cannot connect to cluster, the task will continue on this node
return;
case TaskStatus.ActiveByCurrentNode:
// a backup is already running, the next one will be re-scheduled by the backup task if needed
if (existingBackupState.RunningTask != null)
{
return;
}
// backup frequency hasn't changed, and we have a scheduled backup
if (previousConfiguration.HasBackupFrequencyChanged(newConfiguration) == false && existingBackupState.HasScheduledBackup())
{
return;
}
existingBackupState.UpdateTimer(GetTimer(newConfiguration, backupStatus));
return;
default:
throw new ArgumentOutOfRangeException(nameof(taskState), taskState, null);
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
if (periodicBackup.UpdateBackupTaskSemaphore.Wait(0) == false)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
try
{
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
finally
{
periodicBackup.UpdateBackupTaskSemaphore.Release();
}
}
