public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental) { await Locker.WaitAsync(); try { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true; await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate, (configuration, databaseName) => { configuration.S3Settings = GetS3Settings(databaseName); }, async databaseName => { using (var client = new RavenAwsS3Client(GetS3Settings(databaseName))) { var folders = await client.ListObjectsAsync($"{client.RemoteFolderName}/", "/", listFolders: true); return(folders.FileInfoDetails.Count); } }, timeout : 120000, checkIncremental); } finally { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false; Locker.Release(); } }
public override void Dispose() { base.Dispose(); var s3Settings = GetS3Settings(); if (s3Settings == null) { return; } try { using (var s3Client = new RavenAwsS3Client(s3Settings)) { var cloudObjects = s3Client.ListObjectsAsync(s3Settings.RemoteFolderName, string.Empty, false).GetAwaiter().GetResult(); var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList(); s3Client.DeleteMultipleObjects(pathsToDelete); } } catch (Exception) { // ignored } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var folderPathOptions = new FolderPathOptions(); ; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) || string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) || string.IsNullOrWhiteSpace(s3Settings.BucketName) || string.IsNullOrWhiteSpace(s3Settings.AwsRegionName)) { break; } using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64); if (folders != null) { foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } } break; case PeriodicBackupConnectionType.Azure: var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null) { throw new BadRequestException("No JSON was posted."); } var azureSettings = JsonDeserializationServer.AzureSettings(azureJson); if (azureSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(azureSettings.AccountName) || string.IsNullOrWhiteSpace(azureSettings.AccountKey) || string.IsNullOrWhiteSpace(azureSettings.StorageContainer)) { break; } using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true)); foreach (var folder in folders.List) { var fullPath = folder.Name; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.GoogleCloud: var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null) { throw new BadRequestException("No JSON was posted."); } var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson); if (googleCloudSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) || string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson)) { break; } using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName)); var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length; foreach (var folder in folders) { const char separator = '/'; var splitted = folder.Name.Split(separator); var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator; if (string.IsNullOrWhiteSpace(result)) { continue; } folderPathOptions.List.Add(result); } } break; case PeriodicBackupConnectionType.FTP: case PeriodicBackupConnectionType.Glacier: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
protected async Task incremental_and_full_check_last_file_for_backup_internal() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 */6 * * *", }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus backupStatus = null; var value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: 1, timeout: 30_000); Assert.True(1 == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); string lastFileToRestore; using (var client = new RavenAwsS3Client(defaultS3Settings)) { var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(backupStatus.FolderName); using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = subfolderS3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load <User>("users/1"); Assert.NotNull(users); users = session.Load <User>("users/2"); Assert.NotNull(users); users = session.Load <User>("users/3"); Assert.Null(users); } } } }
public async Task incremental_and_full_check_last_file_for_backup() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 */6 * * *", }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); var value = WaitForValue(() => { var getPeriodicBackupResult = store.Maintenance.Send(operation); return getPeriodicBackupResult.Status?.LastEtag; }, 1); Assert.Equal(1, value); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); string lastFileToRestore; var backupStatus = store.Maintenance.Send(operation); using (var client = new RavenAwsS3Client(AmazonS3FactAttribute.S3Settings)) { var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.Status.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); backupStatus = store.Maintenance.Send(operation); var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName); using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = subfolderS3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load<User>("users/1"); Assert.NotNull(users); users = session.Load<User>("users/2"); Assert.NotNull(users); users = session.Load<User>("users/3"); Assert.Null(users); } } } }
protected async Task incremental_and_full_check_last_file_for_backup_internal() { var s3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = Backup.CreateBackupConfiguration(s3Settings: s3Settings, fullBackupFrequency: null, incrementalBackupFrequency: "0 */6 * * *"); var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; var backupStatus = await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag); string lastFileToRestore; using (var client = new RavenAwsS3Client(s3Settings, DefaultConfiguration)) { var fullBackupPath = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag); var databaseName = $"restored_database-{Guid.NewGuid()}"; s3Settings.RemoteFolderName = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}"; using (Backup.RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = s3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load <User>("users/1"); Assert.NotNull(users); users = session.Load <User>("users/2"); Assert.NotNull(users); users = session.Load <User>("users/3"); Assert.Null(users); } } } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { FolderPathOptions folderPathOptions; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } using (var client = new RavenAwsS3Client(s3Settings)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, 64); folderPathOptions = new FolderPathOptions(); foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.Glacier: case PeriodicBackupConnectionType.Azure: case PeriodicBackupConnectionType.GoogleCloud: case PeriodicBackupConnectionType.FTP: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }