private static void ValidateBackupConfiguration(ServerWideBackupConfiguration serverWideConfiguration, PeriodicBackupConfiguration backupConfiguration, string databaseName) { Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(serverWideConfiguration.Name), backupConfiguration.Name); Assert.Equal(serverWideConfiguration.Disabled, backupConfiguration.Disabled); Assert.Equal(serverWideConfiguration.FullBackupFrequency, backupConfiguration.FullBackupFrequency); Assert.Equal(serverWideConfiguration.IncrementalBackupFrequency, backupConfiguration.IncrementalBackupFrequency); Assert.Equal($"{serverWideConfiguration.LocalSettings.FolderPath}{Path.DirectorySeparatorChar}{databaseName}", backupConfiguration.LocalSettings.FolderPath); Assert.Equal(serverWideConfiguration.S3Settings.BucketName, backupConfiguration.S3Settings.BucketName); Assert.Equal($"{serverWideConfiguration.S3Settings.RemoteFolderName}/{databaseName}", backupConfiguration.S3Settings.RemoteFolderName); Assert.Equal(serverWideConfiguration.AzureSettings.AccountKey, backupConfiguration.AzureSettings.AccountKey); Assert.Equal(serverWideConfiguration.AzureSettings.AccountName, backupConfiguration.AzureSettings.AccountName); Assert.Equal($"{serverWideConfiguration.AzureSettings.RemoteFolderName}/{databaseName}", backupConfiguration.AzureSettings.RemoteFolderName); Assert.Equal($"{serverWideConfiguration.FtpSettings.Url}/{databaseName}", backupConfiguration.FtpSettings.Url); }
public async Task CreatePeriodicBackupFailsWhenUsingReservedName() { using (var store = GetDocumentStore()) { var putConfiguration = new ServerWideBackupConfiguration { Disabled = true, FullBackupFrequency = "0 2 * * 0", IncrementalBackupFrequency = "0 2 * * 1" }; await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration)); var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); var currentBackupConfiguration = databaseRecord.PeriodicBackups.First(); var serverWideBackupTaskId = currentBackupConfiguration.TaskId; var backupConfiguration = new PeriodicBackupConfiguration { Disabled = true, TaskId = currentBackupConfiguration.TaskId, FullBackupFrequency = "0 2 * * 0", IncrementalBackupFrequency = "0 2 * * 1" }; var taskName = PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(putConfiguration.GetDefaultTaskName()); var e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration))); var expectedError = $"Can't delete task id: {currentBackupConfiguration.TaskId}, name: '{taskName}', because it is a server-wide backup task"; Assert.Contains(expectedError, e.Message); backupConfiguration.TaskId = 0; backupConfiguration.Name = currentBackupConfiguration.Name; e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfiguration))); expectedError = $"Can't create task: '{taskName}'. A regular (non server-wide) backup task name can't start with prefix '{ServerWideBackupConfiguration.NamePrefix}'"; Assert.Contains(expectedError, e.Message); e = await Assert.ThrowsAsync <RavenException>(() => store.Maintenance.SendAsync(new DeleteOngoingTaskOperation(serverWideBackupTaskId, OngoingTaskType.Backup))); expectedError = $"Can't delete task id: {serverWideBackupTaskId}, name: '{taskName}', because it is a server-wide backup task"; Assert.Contains(expectedError, e.Message); } }
private void UploadToServer(string backupPath, string folderName, string fileName) { var s3Settings = GetBackupConfigurationFromScript(_configuration.S3Settings, x => JsonDeserializationServer.S3Settings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForS3(settings, _database.Name)); var glacierSettings = GetBackupConfigurationFromScript(_configuration.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGlacier(settings, _database.Name)); var azureSettings = GetBackupConfigurationFromScript(_configuration.AzureSettings, x => JsonDeserializationServer.AzureSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForAzure(settings, _database.Name)); var googleCloudSettings = GetBackupConfigurationFromScript(_configuration.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGoogleCloud(settings, _database.Name)); var ftpSettings = GetBackupConfigurationFromScript(_configuration.FtpSettings, x => JsonDeserializationServer.FtpSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForFtp(settings, _database.Name)); TaskCancelToken.Token.ThrowIfCancellationRequested(); var uploaderSettings = new BackupUploaderSettings(_database.Configuration.Backup) { S3Settings = s3Settings, GlacierSettings = glacierSettings, AzureSettings = azureSettings, GoogleCloudSettings = googleCloudSettings, FtpSettings = ftpSettings, BackupPath = backupPath, FolderName = folderName, FileName = fileName, DatabaseName = _database.Name, TaskName = _taskName, BackupType = _configuration.BackupType }; var backupUploader = new BackupUploader(uploaderSettings, _retentionPolicyParameters, _logger, _backupResult, _onProgress, TaskCancelToken); backupUploader.Execute(); }
public BackupResult RunPeriodicBackup(Action <IOperationProgress> onProgress, ref PeriodicBackupStatus runningBackupStatus) { _onProgress = onProgress; AddInfo($"Started task: '{_taskName}'"); var totalSw = Stopwatch.StartNew(); var operationCanceled = false; try { if (_forTestingPurposes != null && _forTestingPurposes.SimulateFailedBackup) { throw new Exception(nameof(_forTestingPurposes.SimulateFailedBackup)); } if (_forTestingPurposes != null && _forTestingPurposes.OnBackupTaskRunHoldBackupExecution != null) { _forTestingPurposes.OnBackupTaskRunHoldBackupExecution.Task.Wait(); } if (runningBackupStatus.LocalBackup == null) { runningBackupStatus.LocalBackup = new LocalBackup(); } if (runningBackupStatus.LastRaftIndex == null) { runningBackupStatus.LastRaftIndex = new LastRaftIndex(); } runningBackupStatus.IsFull = _isFullBackup; if (_logger.IsInfoEnabled) { var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? "full backup" : "snapshot"); _logger.Info($"Creating {(_isFullBackup ? fullBackupText : "an incremental backup")}"); } if (_isFullBackup == false) { // if we come from old version the _previousBackupStatus won't have LastRaftIndex _previousBackupStatus.LastRaftIndex ??= new LastRaftIndex(); // no-op if nothing has changed var(currentLastEtag, currentChangeVector) = _database.ReadLastEtagAndChangeVector(); var currentLastRaftIndex = GetDatabaseEtagForBackup(); // if we come from old version the _previousBackupStatus won't have LastRaftIndex _previousBackupStatus.LastRaftIndex ??= new LastRaftIndex(); if (currentLastEtag == _previousBackupStatus.LastEtag && currentChangeVector == _previousBackupStatus.LastDatabaseChangeVector && currentLastRaftIndex == _previousBackupStatus.LastRaftIndex.LastEtag) { var message = $"Skipping incremental backup because no changes were made from last full backup on {_previousBackupStatus.LastFullBackup}."; if (_logger.IsInfoEnabled) { _logger.Info(message); } runningBackupStatus.LastIncrementalBackup = _startTimeUtc; runningBackupStatus.LocalBackup.LastIncrementalBackup = _startTimeUtc; runningBackupStatus.LocalBackup.IncrementalBackupDurationInMs = 0; DatabaseSmuggler.EnsureProcessed(_backupResult); AddInfo(message); return(_backupResult); } } // update the local configuration before starting the local backup var localSettings = GetBackupConfigurationFromScript(_configuration.LocalSettings, x => JsonDeserializationServer.LocalSettings(x), settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForLocal(settings, _database.Name)); GenerateFolderNameAndBackupDirectory(localSettings, _startTimeUtc, out var nowAsString, out var folderName, out var backupDirectory); var startDocumentEtag = _isFullBackup == false ? _previousBackupStatus.LastEtag : null; var startRaftIndex = _isFullBackup == false ? _previousBackupStatus.LastRaftIndex.LastEtag : null; var fileName = GetFileName(_isFullBackup, backupDirectory.FullPath, nowAsString, _configuration.BackupType, out string backupFilePath); var internalBackupResult = CreateLocalBackupOrSnapshot(runningBackupStatus, backupFilePath, startDocumentEtag, startRaftIndex); runningBackupStatus.LocalBackup.BackupDirectory = _backupToLocalFolder ? backupDirectory.FullPath : null; runningBackupStatus.LocalBackup.TempFolderUsed = _backupToLocalFolder == false; runningBackupStatus.IsEncrypted = _isBackupEncrypted; try { UploadToServer(backupFilePath, folderName, fileName); } finally { runningBackupStatus.UploadToS3 = _backupResult.S3Backup; runningBackupStatus.UploadToAzure = _backupResult.AzureBackup; runningBackupStatus.UploadToGoogleCloud = _backupResult.GoogleCloudBackup; runningBackupStatus.UploadToGlacier = _backupResult.GlacierBackup; runningBackupStatus.UploadToFtp = _backupResult.FtpBackup; _backupResult.LocalBackup = new LocalBackup { BackupDirectory = folderName, FileName = fileName }; // if user did not specify local folder we delete the temporary file if (_backupToLocalFolder == false) { DeleteFile(backupFilePath); } } runningBackupStatus.LastEtag = internalBackupResult.LastDocumentEtag; runningBackupStatus.LastDatabaseChangeVector = internalBackupResult.LastDatabaseChangeVector; runningBackupStatus.LastRaftIndex.LastEtag = internalBackupResult.LastRaftIndex; runningBackupStatus.FolderName = folderName; if (_isFullBackup) { runningBackupStatus.LastFullBackup = _startTimeUtc; } else { runningBackupStatus.LastIncrementalBackup = _startTimeUtc; } totalSw.Stop(); if (_logger.IsInfoEnabled) { var fullBackupText = "a " + (_configuration.BackupType == BackupType.Backup ? " full backup" : " snapshot"); _logger.Info($"Successfully created {(_isFullBackup ? fullBackupText : "an incremental backup")} " + $"in {totalSw.ElapsedMilliseconds:#,#;;0} ms"); } return(_backupResult); } catch (OperationCanceledException) { operationCanceled = TaskCancelToken.Token.IsCancellationRequested; throw; } catch (ObjectDisposedException) { // shutting down, probably operationCanceled = true; throw; } catch (Exception e) { const string message = "Error when performing periodic backup"; runningBackupStatus.Error = new Error { Exception = e.ToString(), At = DateTime.UtcNow }; if (_logger.IsOperationsEnabled) { _logger.Operations(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, $"Periodic Backup task: '{_taskName}'", message, AlertType.PeriodicBackup, NotificationSeverity.Error, details: new ExceptionDetails(e))); throw; } finally { if (operationCanceled == false) { // whether we succeeded or not, // in periodic backup we need to update the last backup time to avoid // starting a new backup right after this one if (_isFullBackup) { runningBackupStatus.LastFullBackupInternal = _startTimeUtc; } else { runningBackupStatus.LastIncrementalBackupInternal = _startTimeUtc; } runningBackupStatus.NodeTag = _database.ServerStore.NodeTag; runningBackupStatus.DurationInMs = totalSw.ElapsedMilliseconds; UpdateOperationId(runningBackupStatus); if (_isOneTimeBackup == false) { runningBackupStatus.Version = ++_previousBackupStatus.Version; // save the backup status AddInfo("Saving backup status"); SaveBackupStatus(runningBackupStatus, _database, _logger, _backupResult); } } } }
public async Task CanStoreAndEditServerWideBackupForIdleDatabase() { using var server = GetNewServer(new ServerCreationOptions { CustomSettings = new Dictionary <string, string> { [RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10", [RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3", [RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false" } }); using (var store = GetDocumentStore(new Options { Server = server, RunInMemory = false })) { var fullFreq = "0 2 1 1 *"; var incFreq = "0 2 * * 0"; var putConfiguration = new ServerWideBackupConfiguration { FullBackupFrequency = fullFreq, IncrementalBackupFrequency = incFreq, LocalSettings = new LocalSettings { FolderPath = "test/folder" } }; Assert.Equal(1, WaitForValue(() => server.ServerStore.IdleDatabases.Count, 1, timeout: 60000, interval: 1000)); var result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration)); var serverWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name)); Assert.NotNull(serverWideConfiguration); Assert.Equal(fullFreq, serverWideConfiguration.FullBackupFrequency); Assert.Equal(incFreq, serverWideConfiguration.IncrementalBackupFrequency); Assert.Equal(1, server.ServerStore.IdleDatabases.Count); // update the backup configuration putConfiguration.Name = serverWideConfiguration.Name; putConfiguration.TaskId = serverWideConfiguration.TaskId; putConfiguration.FullBackupFrequency = "0 2 * * 0"; var oldName = result.Name; result = await store.Maintenance.Server.SendAsync(new PutServerWideBackupConfigurationOperation(putConfiguration)); Exception ex = null; try { await server.ServerStore.Cluster.WaitForIndexNotification(result.RaftCommandIndex, TimeSpan.FromMinutes(1)); } catch (Exception e) { ex = e; } finally { Assert.Null(ex); } var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); Assert.Equal(1, record.PeriodicBackups.Count); PeriodicBackupConfiguration periodicBackupConfiguration = record.PeriodicBackups.First(); var newServerWideConfiguration = await store.Maintenance.Server.SendAsync(new GetServerWideBackupConfigurationOperation(result.Name)); // compare with periodic backup task Assert.NotEqual(newServerWideConfiguration.TaskId, periodicBackupConfiguration.TaskId); // backup task id in db record doesn't change Assert.Equal(PutServerWideBackupConfigurationCommand.GetTaskNameForDatabase(oldName), periodicBackupConfiguration.Name); Assert.Equal(incFreq, periodicBackupConfiguration.FullBackupFrequency); Assert.Equal(incFreq, periodicBackupConfiguration.IncrementalBackupFrequency); Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, periodicBackupConfiguration.FullBackupFrequency); // compare with previous server wide backup Assert.NotEqual(serverWideConfiguration.TaskId, newServerWideConfiguration.TaskId); // task id in server storage get increased with each change Assert.Equal(oldName, result.Name); Assert.Equal(incFreq, newServerWideConfiguration.FullBackupFrequency); Assert.Equal(incFreq, newServerWideConfiguration.IncrementalBackupFrequency); Assert.NotEqual(serverWideConfiguration.FullBackupFrequency, newServerWideConfiguration.FullBackupFrequency); Assert.Equal(1, server.ServerStore.IdleDatabases.Count); } }