private static void ValidateBackupConfiguration(ServerWideBackupConfiguration serverWideConfiguration, PeriodicBackupConfiguration backupConfiguration, string databaseName)
{
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(serverWideConfiguration.Name), backupConfiguration.Name);
Assert.Equal(serverWideConfiguration.Disabled, backupConfiguration.Disabled);
Assert.Equal(serverWideConfiguration.FullBackupFrequency, backupConfiguration.FullBackupFrequency);
Assert.Equal(serverWideConfiguration.IncrementalBackupFrequency, backupConfiguration.IncrementalBackupFrequency);
Assert.Equal($"{serverWideConfiguration.LocalSettings.FolderPath}{Path.DirectorySeparatorChar}{databaseName}", backupConfiguration.LocalSettings.FolderPath);
Assert.Equal(serverWideConfiguration.S3Settings.BucketName, backupConfiguration.S3Settings.BucketName);
Assert.Equal($"{serverWideConfiguration.S3Settings.RemoteFolderName}/{databaseName}", backupConfiguration.S3Settings.RemoteFolderName);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountKey, backupConfiguration.AzureSettings.AccountKey);
Assert.Equal(serverWideConfiguration.AzureSettings.AccountName, backupConfiguration.AzureSettings.AccountName);
Assert.Equal($"{serverWideConfiguration.AzureSettings.RemoteFolderName}/{databaseName}", backupConfiguration.AzureSettings.RemoteFolderName);
Assert.Equal($"{serverWideConfiguration.FtpSettings.Url}/{databaseName}", backupConfiguration.FtpSettings.Url);
}
public async Task CreatePeriodicBackupFailsWhenUsingReservedName()
{
using (var store = GetDocumentStore())
{
var putConfiguration = new ServerWideBackupConfiguration
{
Disabled = true,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var currentBackupConfiguration = databaseRecord.PeriodicBackups.First();
var serverWideBackupTaskId = currentBackupConfiguration.TaskId;
var backupConfiguration = new PeriodicBackupConfiguration
{
Disabled = true,
TaskId = currentBackupConfiguration.TaskId,
FullBackupFrequency = "0 2 * * 0",
IncrementalBackupFrequency = "0 2 * * 1"
};
var taskName = PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(putConfiguration.GetDefaultTaskName());
var e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration)));
var expectedError = $"Can't delete task id: {currentBackupConfiguration.TaskId}, name: '{taskName}', because it is a server-wide backup task";
Assert.Contains(expectedError, e.Message);
backupConfiguration.TaskId = 0;
backupConfiguration.Name = currentBackupConfiguration.Name;
e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration)));
expectedError = $"Can't create task: '{taskName}'. A regular (non server-wide) backup task name can't start with prefix '{ServerWideBackupConfiguration.NamePrefix}'";
Assert.Contains(expectedError, e.Message);
e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new DeleteOngoingTaskOperation(serverWideBackupTaskId, OngoingTaskType.Backup)));
expectedError = $"Can't delete task id: {serverWideBackupTaskId}, name: '{taskName}', because it is a server-wide backup task";
Assert.Contains(expectedError, e.Message);
}
}
private void UploadToServer(string backupPath, string folderName, string fileName)
{
var s3Settings = GetBackupConfigurationFromScript(_configuration.S3Settings, x => JsonDeserializationServer.S3Settings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForS3(settings, _database.Name));
var glacierSettings = GetBackupConfigurationFromScript(_configuration.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGlacier(settings, _database.Name));
var azureSettings = GetBackupConfigurationFromScript(_configuration.AzureSettings, x => JsonDeserializationServer.AzureSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForAzure(settings, _database.Name));
var googleCloudSettings = GetBackupConfigurationFromScript(_configuration.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGoogleCloud(settings, _database.Name));
var ftpSettings = GetBackupConfigurationFromScript(_configuration.FtpSettings, x => JsonDeserializationServer.FtpSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForFtp(settings, _database.Name));
TaskCancelToken.Token.ThrowIfCancellationRequested();
var uploaderSettings = new BackupUploaderSettings(_database.Configuration.Backup)
{
S3Settings = s3Settings,
GlacierSettings = glacierSettings,
AzureSettings = azureSettings,
GoogleCloudSettings = googleCloudSettings,
FtpSettings = ftpSettings,
BackupPath = backupPath,
FolderName = folderName,
FileName = fileName,
DatabaseName = _database.Name,
TaskName = _taskName,
BackupType = _configuration.BackupType
};
var backupUploader = new BackupUploader(uploaderSettings, _retentionPolicyParameters, _logger, _backupResult, _onProgress, TaskCancelToken);
backupUploader.Execute();
}
public BackupResult RunPeriodicBackup(Action <IOperationProgress> onProgress, ref PeriodicBackupStatus runningBackupStatus)
{
_onProgress = onProgress;
AddInfo($"Started task: '{_taskName}'");
var totalSw = Stopwatch.StartNew();
var operationCanceled = false;
try
{
if (_forTestingPurposes != null && _forTestingPurposes.SimulateFailedBackup)
{
throw new Exception(nameof(_forTestingPurposes.SimulateFailedBackup));
}
if (_forTestingPurposes != null && _forTestingPurposes.OnBackupTaskRunHoldBackupExecution != null)
{
_forTestingPurposes.OnBackupTaskRunHoldBackupExecution.Task.Wait();
}
if (runningBackupStatus.LocalBackup == null)
{
runningBackupStatus.LocalBackup = new LocalBackup();
}
if (runningBackupStatus.LastRaftIndex == null)
{
runningBackupStatus.LastRaftIndex = new LastRaftIndex();
}
runningBackupStatus.IsFull = _isFullBackup;
if (_logger.IsInfoEnabled)
{
var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? "full backup" : "snapshot");
_logger.Info($"Creating {(_isFullBackup ? fullBackupText : "an incremental backup")}");
}
if (_isFullBackup == false)
{
// if we come from old version the _previousBackupStatus won't have LastRaftIndex
_previousBackupStatus.LastRaftIndex ??= new LastRaftIndex();
// no-op if nothing has changed
var(currentLastEtag, currentChangeVector) = _database.ReadLastEtagAndChangeVector();
var currentLastRaftIndex = GetDatabaseEtagForBackup();
// if we come from old version the _previousBackupStatus won't have LastRaftIndex
_previousBackupStatus.LastRaftIndex ??= new LastRaftIndex();
if (currentLastEtag == _previousBackupStatus.LastEtag &&
currentChangeVector == _previousBackupStatus.LastDatabaseChangeVector &&
currentLastRaftIndex == _previousBackupStatus.LastRaftIndex.LastEtag)
{
var message = $"Skipping incremental backup because no changes were made from last full backup on {_previousBackupStatus.LastFullBackup}.";
if (_logger.IsInfoEnabled)
{
_logger.Info(message);
}
runningBackupStatus.LastIncrementalBackup = _startTimeUtc;
runningBackupStatus.LocalBackup.LastIncrementalBackup = _startTimeUtc;
runningBackupStatus.LocalBackup.IncrementalBackupDurationInMs = 0;
DatabaseSmuggler.EnsureProcessed(_backupResult);
AddInfo(message);
return(_backupResult);
}
}
// update the local configuration before starting the local backup
var localSettings = GetBackupConfigurationFromScript(_configuration.LocalSettings, x => JsonDeserializationServer.LocalSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForLocal(settings, _database.Name));
GenerateFolderNameAndBackupDirectory(localSettings, _startTimeUtc, out var nowAsString, out var folderName, out var backupDirectory);
var startDocumentEtag = _isFullBackup == false ? _previousBackupStatus.LastEtag : null;
var startRaftIndex = _isFullBackup == false ? _previousBackupStatus.LastRaftIndex.LastEtag : null;
var fileName = GetFileName(_isFullBackup, backupDirectory.FullPath, nowAsString, _configuration.BackupType, out string backupFilePath);
var internalBackupResult = CreateLocalBackupOrSnapshot(runningBackupStatus, backupFilePath, startDocumentEtag, startRaftIndex);
runningBackupStatus.LocalBackup.BackupDirectory = _backupToLocalFolder ? backupDirectory.FullPath : null;
runningBackupStatus.LocalBackup.TempFolderUsed = _backupToLocalFolder == false;
runningBackupStatus.IsEncrypted = _isBackupEncrypted;
try
{
UploadToServer(backupFilePath, folderName, fileName);
}
finally
{
runningBackupStatus.UploadToS3 = _backupResult.S3Backup;
runningBackupStatus.UploadToAzure = _backupResult.AzureBackup;
runningBackupStatus.UploadToGoogleCloud = _backupResult.GoogleCloudBackup;
runningBackupStatus.UploadToGlacier = _backupResult.GlacierBackup;
runningBackupStatus.UploadToFtp = _backupResult.FtpBackup;
_backupResult.LocalBackup = new LocalBackup
{
BackupDirectory = folderName,
FileName = fileName
};
// if user did not specify local folder we delete the temporary file
if (_backupToLocalFolder == false)
{
DeleteFile(backupFilePath);
}
}
runningBackupStatus.LastEtag = internalBackupResult.LastDocumentEtag;
runningBackupStatus.LastDatabaseChangeVector = internalBackupResult.LastDatabaseChangeVector;
runningBackupStatus.LastRaftIndex.LastEtag = internalBackupResult.LastRaftIndex;
runningBackupStatus.FolderName = folderName;
if (_isFullBackup)
{
runningBackupStatus.LastFullBackup = _startTimeUtc;
}
else
{
runningBackupStatus.LastIncrementalBackup = _startTimeUtc;
}
totalSw.Stop();
if (_logger.IsInfoEnabled)
{
var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? " full backup" : " snapshot");
_logger.Info($"Successfully created {(_isFullBackup ? fullBackupText : "an incremental backup")} " +
$"in {totalSw.ElapsedMilliseconds:#,#;;0} ms");
}
return(_backupResult);
}
catch (OperationCanceledException)
{
operationCanceled = TaskCancelToken.Token.IsCancellationRequested;
throw;
}
catch (ObjectDisposedException)
{
// shutting down, probably
operationCanceled = true;
throw;
}
catch (Exception e)
{
const string message = "Error when performing periodic backup";
runningBackupStatus.Error = new Error
{
Exception = e.ToString(),
At = DateTime.UtcNow
};
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Periodic Backup task: '{_taskName}'",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
finally
{
if (operationCanceled == false)
{
// whether we succeeded or not,
// in periodic backup we need to update the last backup time to avoid
// starting a new backup right after this one
if (_isFullBackup)
{
runningBackupStatus.LastFullBackupInternal = _startTimeUtc;
}
else
{
runningBackupStatus.LastIncrementalBackupInternal = _startTimeUtc;
}
runningBackupStatus.NodeTag = _database.ServerStore.NodeTag;
runningBackupStatus.DurationInMs = totalSw.ElapsedMilliseconds;
UpdateOperationId(runningBackupStatus);
if (_isOneTimeBackup == false)
{
runningBackupStatus.Version = ++_previousBackupStatus.Version;
// save the backup status
AddInfo("Saving backup status");
SaveBackupStatus(runningBackupStatus, _database, _logger, _backupResult);
}
}
}
}
public async Task CanStoreAndEditServerWideBackupForIdleDatabase()
{
using var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
});
using (var store = GetDocumentStore(new Options {
Server = server, RunInMemory = false
}))
{
var fullFreq = "0 2 1 1 *";
var incFreq = "0 2 * * 0";
var putConfiguration = new ServerWideBackupConfiguration
{
FullBackupFrequency = fullFreq,
IncrementalBackupFrequency = incFreq,
LocalSettings = new LocalSettings
{
FolderPath = "test/folder"
}
};
Assert.Equal(1, WaitForValue(() => server.ServerStore.IdleDatabases.Count, 1, timeout: 60000, interval: 1000));
var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
Assert.NotNull(serverWideConfiguration);
Assert.Equal(fullFreq, serverWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
// update the backup configuration
putConfiguration.Name = serverWideConfiguration.Name;
putConfiguration.TaskId = serverWideConfiguration.TaskId;
putConfiguration.FullBackupFrequency = "0 2 * * 0";
var oldName = result.Name;
result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration));
Exception ex = null;
try
{
await server.ServerStore.Cluster.WaitForIndexNotification(result.RaftCommandIndex, TimeSpan.FromMinutes(1));
}
catch (Exception e)
{
ex = e;
}
finally
{
Assert.Null(ex);
}
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
Assert.Equal(1, record.PeriodicBackups.Count);
PeriodicBackupConfiguration periodicBackupConfiguration = record.PeriodicBackups.First();
var newServerWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name));
// compare with periodic backup task
Assert.NotEqual(newServerWideConfiguration.TaskId, periodicBackupConfiguration.TaskId); // backup task id in db record doesn't change
Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(oldName), periodicBackupConfiguration.Name);
Assert.Equal(incFreq, periodicBackupConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, periodicBackupConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, periodicBackupConfiguration.FullBackupFrequency);
// compare with previous server wide backup
Assert.NotEqual(serverWideConfiguration.TaskId, newServerWideConfiguration.TaskId); // task id in server storage get increased with each change
Assert.Equal(oldName, result.Name);
Assert.Equal(incFreq, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(incFreq, newServerWideConfiguration.IncrementalBackupFrequency);
Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, newServerWideConfiguration.FullBackupFrequency);
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
}
}
