private static void CreateUploadTaskIfNeeded <S, T>(
S settings,
List <Task> tasks,
string backupPath,
bool isFullBackup,
Func <S, FileStream, UploadProgress, Task> uploadToServer,
ref T uploadStatus,
string backupDestination)
where S : BackupSettings
where T : CloudUploadStatus
{
if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
{
return;
}
if (uploadStatus == null)
{
uploadStatus = (T)Activator.CreateInstance(typeof(T));
}
var localUploadStatus = uploadStatus;
tasks.Add(Task.Run(async() =>
{
using (localUploadStatus.UpdateStats(isFullBackup))
using (var fileStream = File.OpenRead(backupPath))
{
var uploadProgress = localUploadStatus.UploadProgress;
uploadProgress.ChangeState(UploadState.PendingUpload);
uploadProgress.SetTotal(fileStream.Length);
try
{
await uploadToServer(settings, fileStream, uploadProgress);
}
catch (OperationCanceledException e)
{
// shutting down
localUploadStatus.Exception = e;
}
catch (Exception e)
{
localUploadStatus.Exception = e;
throw new InvalidOperationException($"Failed to backup to {backupDestination}", e);
}
finally
{
uploadProgress.ChangeState(UploadState.Done);
}
}
}));
}
private void CreateUploadTaskIfNeeded <S, T>(S settings, Action <S, FileStream, Progress> uploadToServer, T uploadStatus, string targetName)
where S : BackupSettings
where T : CloudUploadStatus
{
if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
{
return;
}
Debug.Assert(uploadStatus != null);
var localUploadStatus = uploadStatus;
var thread = PoolOfThreads.GlobalRavenThreadPool.LongRunning(_ =>
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (localUploadStatus.UpdateStats(_isFullBackup))
using (var fileStream = File.OpenRead(_settings.BackupPath))
{
var uploadProgress = localUploadStatus.UploadProgress;
try
{
localUploadStatus.Skipped = false;
uploadProgress.ChangeState(UploadState.PendingUpload);
uploadProgress.SetTotal(fileStream.Length);
AddInfo($"Starting the upload of backup file to {targetName}.");
var bytesPutsPerSec = new MeterMetric();
long lastUploadedInBytes = 0;
var totalToUpload = new Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
var sw = Stopwatch.StartNew();
var progress = new Progress(uploadProgress)
{
OnUploadProgress = () =>
{
if (sw.ElapsedMilliseconds <= 1000)
{
return;
}
var totalUploadedInBytes = uploadProgress.UploadedInBytes;
bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
lastUploadedInBytes = totalUploadedInBytes;
var uploaded = new Size(totalUploadedInBytes, SizeUnit.Bytes);
uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
AddInfo($"Uploaded: {uploaded} / {totalToUpload}");
sw.Restart();
}
};
uploadToServer(settings, fileStream, progress);
AddInfo($"Total uploaded: {totalToUpload}, took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}");
}
finally
{
uploadProgress.ChangeState(UploadState.Done);
}
}
}
catch (Exception e)
{
var extracted = e.ExtractSingleInnerException();
var error = $"Failed to upload the backup file to {targetName}.";
Exception exception = null;
if (extracted is OperationCanceledException)
{
// shutting down or HttpClient timeout
exception = TaskCancelToken.Token.IsCancellationRequested ? extracted : new TimeoutException(error, e);
}
localUploadStatus.Exception = (exception ?? e).ToString();
_exceptions.Add(exception ?? new InvalidOperationException(error, e));
}
}, null, $"Upload backup file of database '{_settings.DatabaseName}' to {targetName} (task: '{_settings.TaskName}')");
_threads.Add(thread);
}
private void CreateUploadTaskIfNeeded <S, T>(
S settings,
List <Task> tasks,
string backupPath,
bool isFullBackup,
Func <S, FileStream, Progress, Task> uploadToServer,
T uploadStatus,
Action <IOperationProgress> onProgress)
where S : BackupSettings
where T : CloudUploadStatus
{
if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
{
return;
}
Debug.Assert(uploadStatus != null);
var localUploadStatus = uploadStatus;
tasks.Add(Task.Run(async() =>
{
using (localUploadStatus.UpdateStats(isFullBackup))
using (var fileStream = File.OpenRead(backupPath))
{
var uploadProgress = localUploadStatus.UploadProgress;
localUploadStatus.Skipped = false;
uploadProgress.ChangeState(UploadState.PendingUpload);
uploadProgress.SetTotal(fileStream.Length);
AddInfo($"Starting {uploadStatus.GetType().AssemblyQualifiedName}", onProgress);
try
{
var bytesPutsPerSec = new MeterMetric();
long lastUploadedInBytes = 0;
var totalToUpload = new Sparrow.Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
var sw = Stopwatch.StartNew();
var progress = new Progress(uploadProgress)
{
OnUploadProgress = () =>
{
if (sw.ElapsedMilliseconds <= 1000)
{
return;
}
var totalUploadedInBytes = uploadProgress.UploadedInBytes;
bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
lastUploadedInBytes = totalUploadedInBytes;
var uploaded = new Sparrow.Size(totalUploadedInBytes, SizeUnit.Bytes);
uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
AddInfo($"Uploaded: {uploaded} / {totalToUpload}", onProgress);
sw.Restart();
}
};
await uploadToServer(settings, fileStream, progress);
AddInfo($"Total uploaded: {totalToUpload}, " +
$"took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}", onProgress);
}
catch (OperationCanceledException e)
{
// shutting down
localUploadStatus.Exception = e.ToString();
throw;
}
catch (Exception e)
{
localUploadStatus.Exception = e.ToString();
throw new InvalidOperationException($"Failed to backup to {uploadStatus.GetType().FullName}", e);
}
finally
{
uploadProgress.ChangeState(UploadState.Done);
}
}
}));
}
private async Task RunPeriodicBackup(
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus status,
bool isFullBackup)
{
var backupStarted = SystemTime.UtcNow;
var totalSw = Stopwatch.StartNew();
status.BackupType = configuration.BackupType;
try
{
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(configuration.LocalSettings);
var now = SystemTime.UtcNow.ToString(DateTimeFormat, CultureInfo.InvariantCulture);
if (status.LocalBackup == null)
{
status.LocalBackup = new LocalBackup();
}
PathSetting backupDirectory;
string folderName;
// check if we need to do a new full backup
if (isFullBackup ||
status.LastFullBackup == null || // no full backup was previously performed
status.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
status.BackupType != configuration.BackupType || // backup type has changed
status.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && DirectoryContainsFullBackupOrSnapshot(status.LocalBackup.BackupDirectory, configuration.BackupType) == false)
// the local folder has a missing full backup
{
isFullBackup = true;
folderName = $"{now}.ravendb-{_database.Name}-{_serverStore.NodeTag}-{configuration.BackupType.ToString().ToLower()}";
backupDirectory = backupToLocalFolder ?
new PathSetting(configuration.LocalSettings.FolderPath).Combine(folderName) :
_tempBackupPath;
if (Directory.Exists(backupDirectory.FullPath) == false)
{
Directory.CreateDirectory(backupDirectory.FullPath);
}
status.LocalBackup.TempFolderUsed = backupToLocalFolder == false;
status.LocalBackup.BackupDirectory = backupToLocalFolder ? backupDirectory.FullPath : null;
}
else
{
backupDirectory = backupToLocalFolder ? new PathSetting(status.LocalBackup.BackupDirectory) : _tempBackupPath;
folderName = status.FolderName;
}
if (_logger.IsInfoEnabled)
{
var fullBackupText = "a " + (configuration.BackupType == BackupType.Backup ? "full backup" : "snapshot");
_logger.Info($"Creating {(isFullBackup ? fullBackupText : "an incremental backup")}");
}
if (isFullBackup == false)
{
// no-op if nothing has changed
var currentLastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
if (currentLastEtag == status.LastEtag)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Skipping incremental backup because " +
$"last etag ({currentLastEtag}) hasn't changed since last backup");
}
status.DurationInMs = totalSw.ElapsedMilliseconds;
status.LastIncrementalBackup = backupStarted;
return;
}
}
var startDocumentEtag = isFullBackup == false ? status.LastEtag : null;
var fileName = GetFileName(isFullBackup, backupDirectory.FullPath, now, configuration.BackupType, out string backupFilePath);
var lastEtag = CreateLocalBackupOrSnapshot(configuration,
isFullBackup, status, backupFilePath, startDocumentEtag, context, tx);
try
{
await UploadToServer(configuration, status, backupFilePath, folderName, fileName, isFullBackup);
}
finally
{
// if user did not specify local folder we delete temporary file
if (backupToLocalFolder == false)
{
IOExtensions.DeleteFile(backupFilePath);
}
}
status.LastEtag = lastEtag;
status.FolderName = folderName;
}
totalSw.Stop();
if (_logger.IsInfoEnabled)
{
var fullBackupText = "a " + (configuration.BackupType == BackupType.Backup ? " full backup" : " snapshot");
_logger.Info($"Successfully created {(isFullBackup ? fullBackupText : "an incremental backup")} " +
$"in {totalSw.ElapsedMilliseconds:#,#;;0} ms");
}
}
catch (OperationCanceledException)
{
// shutting down, probably
}
catch (ObjectDisposedException)
{
// shutting down, probably
}
catch (Exception e)
{
const string message = "Error when performing periodic backup";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create("Periodic Backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
}
finally
{
// whether we succeded or not,
// we need to update the last backup time to avoid
// starting a new backup right after this one
if (isFullBackup)
{
status.LastFullBackup = backupStarted;
}
else
{
status.LastIncrementalBackup = backupStarted;
}
status.NodeTag = _serverStore.NodeTag;
status.DurationInMs = totalSw.ElapsedMilliseconds;
status.Version++;
// save the backup status
await WriteStatus(status);
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
using (periodicBackup.UpdateBackupTask())
{
try
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
using (_database.PreventFromUnloading())
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private async Task CanDeleteBackupsByDate(
int backupAgeInSeconds,
int numberOfBackupsToCreate,
Action <PeriodicBackupConfiguration, string> modifyConfiguration,
Func <string, Task <int> > getDirectoriesCount,
int timeout, bool checkIncremental = false)
{
var minimumBackupAgeToKeep = TimeSpan.FromSeconds(backupAgeInSeconds);
using (var store = GetDocumentStore())
{
var config = Backup.CreateBackupConfiguration(incrementalBackupFrequency: "30 3 L * ?", retentionPolicy: new RetentionPolicy
{
MinimumBackupAgeToKeep = minimumBackupAgeToKeep
});
modifyConfiguration(config, store.Database);
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
var userId = "";
for (var i = 0; i < numberOfBackupsToCreate; i++)
{
using (var session = store.OpenAsyncSession())
{
var user = new User {
Name = "Grisha"
};
await session.StoreAsync(user);
userId = user.Id;
await session.SaveChangesAsync();
}
// create full backup
var etagForFullBackup = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : true, expectedEtag : etagForFullBackup, timeout : timeout);
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>(userId);
user.Age = 33;
await session.SaveChangesAsync();
}
// create incremental backup
var etagForIncBackup = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
Assert.NotEqual(etagForFullBackup, etagForIncBackup);
await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : etagForIncBackup, timeout : timeout);
}
await Task.Delay(minimumBackupAgeToKeep + TimeSpan.FromSeconds(3));
if (checkIncremental)
{
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>(userId);
user.Name = "Egor";
user.Age = 322;
await session.SaveChangesAsync();
}
// create incremental backup with retention policy
var etagForIncBackup = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : etagForIncBackup, timeout : timeout);
}
var sp1 = Stopwatch.StartNew();
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Grisha" });
await session.SaveChangesAsync();
}
sp1.Stop();
var sp2 = Stopwatch.StartNew();
var etag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
sp2.Stop();
var sp3 = Stopwatch.StartNew();
var status = await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : true, expectedEtag : etag, timeout : timeout);
sp3.Stop();
var directoriesCount = await getDirectoriesCount(store.Database);
var expectedNumberOfDirectories = checkIncremental ? 2 : 1;
Assert.True(expectedNumberOfDirectories == directoriesCount,
$"ExpectedNumberOfDirectories: {expectedNumberOfDirectories}, ActualNumberOfDirectories: {directoriesCount}, SaveChanges() duration: {sp1.Elapsed}, GetStatisticsOperation duration: {sp2.Elapsed}, RunBackupAndReturnStatusAsync duration: {sp3.Elapsed}," +
$" Backup duration: {status.DurationInMs}, LocalRetentionDurationInMs: {status.LocalRetentionDurationInMs}");
if (PeriodicBackupConfiguration.CanBackupUsing(config.LocalSettings))
{
Assert.NotNull(status.LocalRetentionDurationInMs);
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task exceeds the maximum number of concurrent backup tasks configured. " +
$"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}")
{
DelayPeriod = TimeSpan.FromMinutes(1)
};
}
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// releasing the semaphore because we failed to start the backup task
_serverStore.ConcurrentBackupsSemaphore.Release();
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{periodicBackup.Configuration.Name}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup)
{
if (periodicBackup.UpdateBackupTaskSemaphore.Wait(0) == false)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
try
{
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTime = SystemTime.UtcNow;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => Task.Run(async() =>
{
try
{
return(await backupTask.RunPeriodicBackup(onProgress));
}
finally
{
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() &&
_cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
}, backupTask.TaskCancelToken.Token),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
finally
{
periodicBackup.UpdateBackupTaskSemaphore.Release();
}
}
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup, DateTime startTimeInUtc)
{
using (periodicBackup.UpdateBackupTask())
{
if (periodicBackup.Disposed)
{
throw new InvalidOperationException("Backup task was already disposed");
}
if (periodicBackup.RunningTask != null)
{
return(periodicBackup.RunningBackupTaskId ?? -1);
}
if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised())
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.")
{
DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.LowMemoryState)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in low memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
if (LowMemoryNotification.Instance.DirtyMemoryState.IsHighDirty)
{
throw new BackupDelayException(
$"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " +
$"The task cannot run because the server is in high dirty memory state.")
{
DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan
};
}
_serverStore.ConcurrentBackupsCounter.StartBackup(periodicBackup.Configuration.Name);
try
{
var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus);
var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings);
// check if we need to do a new full backup
if (backupStatus.LastFullBackup == null || // no full backup was previously performed
backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node
backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed
backupStatus.LastEtag == null || // last document etag wasn't updated
backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false)
// the local folder already includes a full backup or snapshot
{
isFullBackup = true;
}
var operationId = _database.Operations.GetNextOperationId();
var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType);
periodicBackup.StartTimeInUtc = startTimeInUtc;
var backupTask = new BackupTask(
_serverStore,
_database,
periodicBackup,
isFullBackup,
backupToLocalFolder,
operationId,
_tempBackupPath,
_logger,
_cancellationToken.Token);
periodicBackup.RunningBackupTaskId = operationId;
periodicBackup.CancelToken = backupTask.TaskCancelToken;
var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'. Database: '{_database.Name}'";
var task = _database.Operations.AddOperation(
null,
backupTaskName,
Operations.Operations.OperationType.DatabaseBackup,
taskFactory: onProgress => StartBackupThread(periodicBackup, backupTask, onProgress),
id: operationId,
token: backupTask.TaskCancelToken);
periodicBackup.RunningTask = task;
task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose());
return(operationId);
}
catch (Exception e)
{
// we failed to START the backup, need to update the status anyway
// in order to reschedule the next full/incremental backup
periodicBackup.BackupStatus.Version++;
periodicBackup.BackupStatus.Error = new Error
{
Exception = e.ToString(),
At = DateTime.UtcNow
};
if (isFullBackup)
{
periodicBackup.BackupStatus.LastFullBackupInternal = startTimeInUtc;
}
else
{
periodicBackup.BackupStatus.LastIncrementalBackupInternal = startTimeInUtc;
}
BackupTask.SaveBackupStatus(periodicBackup.BackupStatus, _database, _logger);
ScheduleNextBackup(periodicBackup);
var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
message,
"The next backup will be rescheduled",
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
}
}