protected override GetFoldersResult GetSortedFolders() { var prefix = $"{_client.RemoteFolderName}{Delimiter}"; var result = _client.ListObjects(prefix, Delimiter, true, continuationToken: _folderContinuationToken); _folderContinuationToken = result.ContinuationToken; return(new GetFoldersResult { List = result.FileInfoDetails.Select(x => x.FullPath).ToList(), HasMore = result.ContinuationToken != null }); }
public override void Dispose() { base.Dispose(); var s3Settings = _isCustom ? CustomS3FactAttribute.S3Settings : AmazonS3FactAttribute.S3Settings; if (s3Settings == null) { return; } try { using (var s3Client = new RavenAwsS3Client(s3Settings, DefaultConfiguration)) { var cloudObjects = s3Client.ListObjects($"{s3Settings.RemoteFolderName}/{_remoteFolderName}/", string.Empty, false, includeFolders: true); var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList(); s3Client.DeleteMultipleObjects(pathsToDelete); } } catch (Exception) { // ignored } }
public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental) { await Locker.WaitAsync(); try { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true; await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate, (configuration, databaseName) => { configuration.S3Settings = GetS3Settings(databaseName); }, async databaseName => { using (var client = new RavenAwsS3Client(GetS3Settings(databaseName))) { var folders = await client.ListObjects($"{client.RemoteFolderName}/", "/", listFolders: true); return(folders.FileInfoDetails.Count); } }, timeout : 120000, checkIncremental); } finally { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false; Locker.Release(); } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { FolderPathOptions folderPathOptions; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } using (var client = new RavenAwsS3Client(s3Settings)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjects(s3Settings.RemoteFolderName, "/", true, 64); folderPathOptions = new FolderPathOptions(); foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.Glacier: case PeriodicBackupConnectionType.Azure: case PeriodicBackupConnectionType.GoogleCloud: case PeriodicBackupConnectionType.FTP: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
public async Task incremental_and_full_check_last_file_for_backup() { var localS3Settings = CopyLocalS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = localS3Settings, IncrementalBackupFrequency = "0 */6 * * *", }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); var value = WaitForValue(() => { var getPeriodicBackupResult = store.Maintenance.Send(operation); return(getPeriodicBackupResult.Status?.LastEtag); }, 1); Assert.Equal(1, value); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); string lastFileToRestore; var backupStatus = store.Maintenance.Send(operation); using (var client = new RavenAwsS3Client(S3Fact.S3Settings)) { lastFileToRestore = (await client.ListObjects(backupStatus.Status.FolderName, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); backupStatus = store.Maintenance.Send(operation); var databaseName = $"restored_database-{Guid.NewGuid()}"; localS3Settings.RemoteFolderName = $"{backupStatus.Status.FolderName}"; using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = localS3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load <User>("users/1"); Assert.NotNull(users); users = session.Load <User>("users/2"); Assert.NotNull(users); users = session.Load <User>("users/3"); Assert.Null(users); } } } }