protected override GetFoldersResult GetSortedFolders()
{
var prefix = $"{_client.RemoteFolderName}{Delimiter}";
var result = _client.ListObjects(prefix, Delimiter, true, continuationToken: _folderContinuationToken);
_folderContinuationToken = result.ContinuationToken;
return(new GetFoldersResult
{
List = result.FileInfoDetails.Select(x => x.FullPath).ToList(),
HasMore = result.ContinuationToken != null
});
}
public override void Dispose()
{
base.Dispose();
var s3Settings = _isCustom ? CustomS3FactAttribute.S3Settings : AmazonS3FactAttribute.S3Settings;
if (s3Settings == null)
{
return;
}
try
{
using (var s3Client = new RavenAwsS3Client(s3Settings, DefaultConfiguration))
{
var cloudObjects = s3Client.ListObjects($"{s3Settings.RemoteFolderName}/{_remoteFolderName}/", string.Empty, false, includeFolders: true);
var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList();
s3Client.DeleteMultipleObjects(pathsToDelete);
}
}
catch (Exception)
{
// ignored
}
}
public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental)
{
await Locker.WaitAsync();
try
{
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true;
await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate,
(configuration, databaseName) =>
{
configuration.S3Settings = GetS3Settings(databaseName);
},
async databaseName =>
{
using (var client = new RavenAwsS3Client(GetS3Settings(databaseName)))
{
var folders = await client.ListObjects($"{client.RemoteFolderName}/", "/", listFolders: true);
return(folders.FileInfoDetails.Count);
}
}, timeout : 120000, checkIncremental);
}
finally
{
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false;
Locker.Release();
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
FolderPathOptions folderPathOptions;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
using (var client = new RavenAwsS3Client(s3Settings))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjects(s3Settings.RemoteFolderName, "/", true, 64);
folderPathOptions = new FolderPathOptions();
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.Glacier:
case PeriodicBackupConnectionType.Azure:
case PeriodicBackupConnectionType.GoogleCloud:
case PeriodicBackupConnectionType.FTP:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
public async Task incremental_and_full_check_last_file_for_backup()
{
var localS3Settings = CopyLocalS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-1" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = localS3Settings,
IncrementalBackupFrequency = "0 */6 * * *",
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return(getPeriodicBackupResult.Status?.LastEtag);
}, 1);
Assert.Equal(1, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-2" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
string lastFileToRestore;
var backupStatus = store.Maintenance.Send(operation);
using (var client = new RavenAwsS3Client(S3Fact.S3Settings))
{
lastFileToRestore = (await client.ListObjects(backupStatus.Status.FolderName, string.Empty, false)).FileInfoDetails.Last().FullPath;
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-3" }, "users/3");
await session.SaveChangesAsync();
}
lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
backupStatus = store.Maintenance.Send(operation);
var databaseName = $"restored_database-{Guid.NewGuid()}";
localS3Settings.RemoteFolderName = $"{backupStatus.Status.FolderName}";
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
Settings = localS3Settings,
DatabaseName = databaseName,
LastFileNameToRestore = lastFileToRestore
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load <User>("users/1");
Assert.NotNull(users);
users = session.Load <User>("users/2");
Assert.NotNull(users);
users = session.Load <User>("users/3");
Assert.Null(users);
}
}
}
}