public override void Dispose()
{
base.Dispose();
var s3Settings = GetS3Settings();
if (s3Settings == null)
{
return;
}
try
{
using (var s3Client = new RavenAwsS3Client(s3Settings))
{
var cloudObjects = s3Client.ListObjectsAsync(s3Settings.RemoteFolderName, string.Empty, false).GetAwaiter().GetResult();
var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList();
s3Client.DeleteMultipleObjects(pathsToDelete);
}
}
catch (Exception)
{
// ignored
}
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly2()
{
using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", "examplebucket"))
{
var date = new DateTime(2013, 5, 24);
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", payloadHash);
var url = client.GetUrl() + "/test.txt";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "Date", date.ToString("R") },
{ "Host", "s3.amazonaws.com" },
{ "Range", "bytes=0-9" }
};
var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Get, url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=819484c483cfb97d16522b1ac156f87e61677cc8f1f2545c799650ef178f4aa8", auth.Parameter);
}
}
public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental)
{
await Locker.WaitAsync();
try
{
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true;
await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate,
(configuration, databaseName) =>
{
configuration.S3Settings = GetS3Settings(databaseName);
},
async databaseName =>
{
using (var client = new RavenAwsS3Client(GetS3Settings(databaseName)))
{
var folders = await client.ListObjectsAsync($"{client.RemoteFolderName}/", "/", listFolders: true);
return(folders.FileInfoDetails.Count);
}
}, timeout : 120000, checkIncremental);
}
finally
{
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false;
Locker.Release();
}
}
private async Task UploadToS3(
S3Settings settings,
Stream stream,
string folderName,
string fileName,
Progress progress,
string archiveDescription)
{
using (var client = new RavenAwsS3Client(settings.AwsAccessKey, settings.AwsSecretKey,
settings.AwsRegionName, settings.BucketName, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName, folderName, fileName);
await client.PutObject(key, stream, new Dictionary <string, string>
{
{ "Description", archiveDescription }
});
if (_logger.IsInfoEnabled)
{
_logger.Info(string.Format($"Successfully uploaded backup file '{fileName}' " +
$"to S3 bucket named: {settings.BucketName}, " +
$"with key: {key}"));
}
}
}
public override void Dispose()
{
base.Dispose();
var s3Settings = _isCustom ? CustomS3FactAttribute.S3Settings : AmazonS3FactAttribute.S3Settings;
if (s3Settings == null)
{
return;
}
try
{
using (var s3Client = new RavenAwsS3Client(s3Settings, DefaultConfiguration))
{
var cloudObjects = s3Client.ListObjects($"{s3Settings.RemoteFolderName}/{_remoteFolderName}/", string.Empty, false, includeFolders: true);
var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList();
s3Client.DeleteMultipleObjects(pathsToDelete);
}
}
catch (Exception)
{
// ignored
}
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly2()
{
using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1"))
{
var date = new DateTime(2013, 5, 24);
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", payloadHash);
var url = client.GetUrl("examplebucket") + "/" + "test.txt";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "Date", date.ToString("R") },
{ "Host", "examplebucket.s3.amazonaws.com" },
{ "Range", "bytes=0-9" }
};
var auth = client.CalculateAuthorizationHeaderValue("GET", url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", auth.Parameter);
}
}
public void can_get_correct_error_s3()
{
var settings = GetS3Settings();
string region1 = settings.AwsRegionName;
string region2 = settings.AwsRegionName = WestRegion2;
var bucketName = settings.BucketName;
using (var clientRegion2 = new RavenAwsS3Client(settings, DefaultConfiguration))
{
var sb = new StringBuilder();
for (var i = 0; i < 1 * 1024 * 1024; i++)
{
sb.Append("a");
}
var blobs = GenerateBlobNames(settings, 1, out _);
Assert.Equal(1, blobs.Count);
var key = blobs[0];
var error2 = Assert.Throws <InvalidOperationException>(() =>
{
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
clientRegion2.PutObject(key,
memoryStream,
new Dictionary <string, string>());
}
});
Assert.Equal($"AWS location is set to '{region2}', but the bucket named: '{bucketName}' is located in: {region1}", error2.Message);
}
}
public void PutObject()
{
var bucketName = "ravendb";
var key = "testKey";
using (var client = new RavenAwsS3Client("<aws_access_key>", "<aws_secret_key>", "<aws_region_for_bucket>"))
{
client.PutObject(bucketName, key, new MemoryStream(Encoding.UTF8.GetBytes("321")), new Dictionary <string, string>
{
{ "property1", "value1" },
{ "property2", "value2" }
}, 60 * 60);
var @object = client.GetObject(bucketName, key);
Assert.NotNull(@object);
using (var reader = new StreamReader(@object.Data))
Assert.Equal("321", reader.ReadToEnd());
var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal("value1", @object.Metadata[property1]);
Assert.Equal("value2", @object.Metadata[property2]);
}
}
public async Task put_object()
{
var settings = GetS3Settings();
using (var client = new RavenAwsS3Client(settings, DefaultConfiguration))
{
var blobs = GenerateBlobNames(settings, 1, out _);
Assert.Equal(1, blobs.Count);
var key = blobs[0];
var value1 = Guid.NewGuid().ToString();
var value2 = Guid.NewGuid().ToString();
client.PutObject(key, new MemoryStream(Encoding.UTF8.GetBytes("231")), new Dictionary <string, string>
{
{ "property1", value1 },
{ "property2", value2 }
});
var @object = await client.GetObjectAsync(key);
Assert.NotNull(@object);
using (var reader = new StreamReader(@object.Data))
Assert.Equal("231", reader.ReadToEnd());
var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal(value1, @object.Metadata[property1]);
Assert.Equal(value2, @object.Metadata[property2]);
}
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly1()
{
var s3Settings = new S3Settings
{
AwsAccessKey = "AKIAIOSFODNN7EXAMPLE",
AwsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
AwsRegionName = "us-east-1",
BucketName = "examplebucket"
};
using (var client = new RavenAwsS3Client(s3Settings))
{
var date = new DateTime(2013, 5, 24);
var stream = new MemoryStream(Encoding.UTF8.GetBytes("Welcome to Amazon S3."));
var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream);
Assert.Equal("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash);
var url = client.GetUrl() + "/test%24file.text";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-storage-class", "REDUCED_REDUNDANCY" },
{ "Date", date.ToString("R") },
{ "Host", "s3.amazonaws.com" }
};
var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Put, url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=d2dd2e48b10d2cb89c271a6464d0748686c158b5fde44e8d83936fd9b30b5c4c", auth.Parameter);
}
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly1()
{
using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1"))
{
var date = new DateTime(2013, 5, 24);
var stream = new MemoryStream(Encoding.UTF8.GetBytes("Welcome to Amazon S3."));
var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream);
Assert.Equal("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash);
var url = client.GetUrl("examplebucket") + "/" + "test%24file.text";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-storage-class", "REDUCED_REDUNDANCY" },
{ "Date", date.ToString("R") },
{ "Host", "examplebucket.s3.amazonaws.com" }
};
var auth = client.CalculateAuthorizationHeaderValue("PUT", url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd", auth.Parameter);
}
}
private void DeleteFromS3(S3Settings settings)
{
using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.DeleteObject(key);
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportDeletion(S3Name)} bucket named: {settings.BucketName}, with key: {key}");
}
}
}
public async Task can_get_correct_error_s3(string region1, string region2)
{
var bucketName = $"testing-{Guid.NewGuid()}";
var key = Guid.NewGuid().ToString();
using (var clientRegion1 = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region1, bucketName))
using (var clientRegion2 = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region2, bucketName))
{
// make sure that the bucket doesn't exist
await clientRegion1.DeleteBucket();
try
{
var sb = new StringBuilder();
for (var i = 0; i < 1 * 1024 * 1024; i++)
{
sb.Append("a");
}
var error1 = await Assert.ThrowsAsync <BucketNotFoundException>(async() =>
{
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
await clientRegion1.PutObject(key,
memoryStream,
new Dictionary <string, string>());
}
});
Assert.Equal($"Bucket name '{bucketName}' doesn't exist!", error1.Message);
await clientRegion1.PutBucket();
var error2 = await Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
await clientRegion2.PutObject(key,
memoryStream,
new Dictionary <string, string>());
}
});
Assert.Equal($"AWS location as set to {region2}, but the bucket named: '{bucketName}' is located in: {region1}", error2.Message);
}
finally
{
await clientRegion1.DeleteBucket();
}
}
}
private void UploadToS3(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
{
using (var client = new RavenAwsS3Client(awsAccessKey, awsSecretKey, localExportConfigs.AwsRegionEndpoint ?? RavenAwsClient.DefaultRegion))
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
client.PutObject(localExportConfigs.S3BucketName, key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullBackup) }
}, 60 * 60);
logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}",
Path.GetFileName(backupPath), localExportConfigs.S3BucketName, key));
}
}
public async Task put_object(string region)
{
var bucketName = $"testing-{Guid.NewGuid()}";
var key = $"test-key-{Guid.NewGuid()}";
using (var client = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region, bucketName))
{
// make sure that the bucket doesn't exist
await client.DeleteBucket();
try
{
await client.PutBucket();
var value1 = Guid.NewGuid().ToString();
var value2 = Guid.NewGuid().ToString();
await client.PutObject(key, new MemoryStream(Encoding.UTF8.GetBytes("231")), new Dictionary <string, string>
{
{ "property1", value1 },
{ "property2", value2 }
});
// can't delete a bucket with existing objects
var e = await Assert.ThrowsAsync <StorageException>(async() => await client.DeleteBucket());
Assert.True(e.Message.Contains("The bucket you tried to delete is not empty"));
var @object = await client.GetObject(key);
Assert.NotNull(@object);
using (var reader = new StreamReader(@object.Data))
Assert.Equal("231", reader.ReadToEnd());
var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal(value1, @object.Metadata[property1]);
Assert.Equal(value2, @object.Metadata[property2]);
}
finally
{
await client.DeleteObject(key);
await client.DeleteBucket();
}
}
}
private void UploadToS3(S3Settings settings, Stream stream, Progress progress)
{
using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.PutObject(key, stream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription() }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportSuccess(S3Name)} bucket named: {settings.BucketName}, with key: {key}");
}
var runner = new S3RetentionPolicyRunner(_retentionPolicyParameters, client);
runner.Execute();
}
}
private void UploadToS3(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
{
if (awsAccessKey == Constants.DataCouldNotBeDecrypted ||
awsSecretKey == Constants.DataCouldNotBeDecrypted)
{
throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true.");
}
using (var client = new RavenAwsS3Client(awsAccessKey, awsSecretKey, localExportConfigs.AwsRegionEndpoint ?? RavenAwsClient.DefaultRegion))
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
client.PutObject(localExportConfigs.S3BucketName, key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullBackup) }
}, 60 * 60);
logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}",
Path.GetFileName(backupPath), localExportConfigs.S3BucketName, key));
}
}
private async Task UploadToS3(string exportPath, string fileName, bool isFullExport)
{
if (_awsAccessKey == Constants.DataCouldNotBeDecrypted ||
_awsSecretKey == Constants.DataCouldNotBeDecrypted)
{
throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true.");
}
using (var client = new RavenAwsS3Client(_awsAccessKey, _awsSecretKey, _configuration.AwsRegionName ?? RavenAwsClient.DefaultRegion))
using (var fileStream = File.OpenRead(exportPath))
{
var key = CombinePathAndKey(_configuration.S3RemoteFolderName, fileName);
await client.PutObject(_configuration.S3BucketName, key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullExport) }
}, 60 *60);
if (_logger.IsInfoEnabled)
{
_logger.Info(string.Format("Successfully uploaded export {0} to S3 bucket {1}, with key {2}", fileName, _configuration.S3BucketName, key));
}
}
}
public S3RestorePoints(Config.Categories.BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, S3Settings s3Settings) : base(sortedList, context)
{
_configuration = configuration;
_client = new RavenAwsS3Client(s3Settings, configuration);
}
protected async Task incremental_and_full_check_last_file_for_backup_internal()
{
var s3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-1" }, "users/1");
await session.SaveChangesAsync();
}
var config = Backup.CreateBackupConfiguration(s3Settings: s3Settings, fullBackupFrequency: null, incrementalBackupFrequency: "0 */6 * * *");
var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-2" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
var backupStatus = await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag);
string lastFileToRestore;
using (var client = new RavenAwsS3Client(s3Settings, DefaultConfiguration))
{
var fullBackupPath = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}";
lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath;
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-3" }, "users/3");
await session.SaveChangesAsync();
}
lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag);
var databaseName = $"restored_database-{Guid.NewGuid()}";
s3Settings.RemoteFolderName = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}";
using (Backup.RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
Settings = s3Settings,
DatabaseName = databaseName,
LastFileNameToRestore = lastFileToRestore
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load <User>("users/1");
Assert.NotNull(users);
users = session.Load <User>("users/2");
Assert.NotNull(users);
users = session.Load <User>("users/3");
Assert.Null(users);
}
}
}
}
public async Task incremental_and_full_check_last_file_for_backup()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-1" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 */6 * * *",
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var getPeriodicBackupResult = store.Maintenance.Send(operation);
return getPeriodicBackupResult.Status?.LastEtag;
}, 1);
Assert.Equal(1, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-2" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
string lastFileToRestore;
var backupStatus = store.Maintenance.Send(operation);
using (var client = new RavenAwsS3Client(AmazonS3FactAttribute.S3Settings))
{
var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.Status.FolderName}";
lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath;
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-3" }, "users/3");
await session.SaveChangesAsync();
}
lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
backupStatus = store.Maintenance.Send(operation);
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName);
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
Settings = subfolderS3Settings,
DatabaseName = databaseName,
LastFileNameToRestore = lastFileToRestore
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load<User>("users/1");
Assert.NotNull(users);
users = session.Load<User>("users/2");
Assert.NotNull(users);
users = session.Load<User>("users/3");
Assert.Null(users);
}
}
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var folderPathOptions = new FolderPathOptions();
;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
break;
}
using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);
if (folders != null)
{
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
}
break;
case PeriodicBackupConnectionType.Azure:
var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
if (azureSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
{
break;
}
using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));
foreach (var folder in folders.List)
{
var fullPath = folder.Name;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
if (googleCloudSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
{
break;
}
using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;
foreach (var folder in folders)
{
const char separator = '/';
var splitted = folder.Name.Split(separator);
var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator;
if (string.IsNullOrWhiteSpace(result))
{
continue;
}
folderPathOptions.List.Add(result);
}
}
break;
case PeriodicBackupConnectionType.FTP:
case PeriodicBackupConnectionType.Glacier:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
public async Task TestPeriodicBackupCredentials()
{
var type = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");
if (Enum.TryParse(type, out PeriodicBackupConnectionType connectionType) == false)
{
throw new ArgumentException($"Unknown backup connection: {type}");
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
DynamicJsonValue result;
try
{
var connectionInfo = await context.ReadForMemoryAsync(RequestBodyStream(), "test-connection");
switch (connectionType)
{
case PeriodicBackupConnectionType.S3:
var s3Settings = JsonDeserializationClient.S3Settings(connectionInfo);
using (var awsClient = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
awsClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Glacier:
var glacierSettings = JsonDeserializationClient.GlacierSettings(connectionInfo);
using (var glacierClient = new RavenAwsGlacierClient(glacierSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
glacierClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Azure:
var azureSettings = JsonDeserializationClient.AzureSettings(connectionInfo);
using (var azureClient = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
azureClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudSettings = JsonDeserializationClient.GoogleCloudSettings(connectionInfo);
using (var googleCloudClient = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
await googleCloudClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.FTP:
var ftpSettings = JsonDeserializationClient.FtpSettings(connectionInfo);
using (var ftpClient = new RavenFtpClient(ftpSettings))
{
ftpClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Local:
case PeriodicBackupConnectionType.None:
default:
throw new ArgumentOutOfRangeException();
}
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = true,
};
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = e.ToString()
};
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
FolderPathOptions folderPathOptions;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
using (var client = new RavenAwsS3Client(s3Settings))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjects(s3Settings.RemoteFolderName, "/", true, 64);
folderPathOptions = new FolderPathOptions();
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.Glacier:
case PeriodicBackupConnectionType.Azure:
case PeriodicBackupConnectionType.GoogleCloud:
case PeriodicBackupConnectionType.FTP:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
public S3RetentionPolicyRunner(RetentionPolicyBaseParameters parameters, RavenAwsS3Client client)
: base(parameters)
{
_client = client;
}
public RestoreFromS3(ServerStore serverStore, RestoreFromS3Configuration restoreFromConfiguration, string nodeTag, OperationCancelToken operationCancelToken) : base(serverStore, restoreFromConfiguration, nodeTag, operationCancelToken)
{
_client = new RavenAwsS3Client(restoreFromConfiguration.Settings);
_remoteFolderName = restoreFromConfiguration.Settings.RemoteFolderName;
}
public S3RestorePoints(SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, S3Settings s3Settings) : base(sortedList, context)
{
_client = new RavenAwsS3Client(s3Settings);
}
// ReSharper disable once InconsistentNaming
private static async Task PutObject(string region,
int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType)
{
var bucketName = $"testing-{Guid.NewGuid()}";
var key = testBlobKeyAsFolder == false?
Guid.NewGuid().ToString() :
$"{Guid.NewGuid()}/folder/testKey";
var uploadProgress = new UploadProgress();
var maxUploadPutObjectInBytesSetter = ExpressionHelper.CreateFieldSetter <RavenAwsS3Client, int>("MaxUploadPutObjectSizeInBytes");
var minOnePartUploadSizeLimitInBytesSetter = ExpressionHelper.CreateFieldSetter <RavenAwsS3Client, int>("MinOnePartUploadSizeLimitInBytes");
using (var client = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region, bucketName, uploadProgress))
{
maxUploadPutObjectInBytesSetter(client, 10 * 1024 * 1024); // 10MB
minOnePartUploadSizeLimitInBytesSetter(client, 7 * 1024 * 1024); // 7MB
// make sure that the bucket doesn't exist
await client.DeleteBucket();
try
{
await client.PutBucket();
var value1 = Guid.NewGuid().ToString();
var value2 = Guid.NewGuid().ToString();
var sb = new StringBuilder();
for (var i = 0; i < sizeInMB * 1024 * 1024; i++)
{
sb.Append("a");
}
long streamLength;
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
streamLength = memoryStream.Length;
await client.PutObject(key,
memoryStream,
new Dictionary <string, string>
{
{ "property1", value1 },
{ "property2", value2 }
});
}
var @object = await client.GetObject(key);
Assert.NotNull(@object);
using (var reader = new StreamReader(@object.Data))
Assert.Equal(sb.ToString(), reader.ReadToEnd());
var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal(value1, @object.Metadata[property1]);
Assert.Equal(value2, @object.Metadata[property2]);
Assert.Equal(UploadState.Done, uploadProgress.UploadState);
Assert.Equal(uploadType, uploadProgress.UploadType);
Assert.Equal(streamLength, uploadProgress.TotalInBytes);
Assert.Equal(streamLength, uploadProgress.UploadedInBytes);
}
finally
{
await client.DeleteObject(key);
await client.DeleteBucket();
}
}
}
protected async Task incremental_and_full_check_last_file_for_backup_internal()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-1" }, "users/1");
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 */6 * * *",
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus backupStatus = null;
var value = WaitForValue(() =>
{
backupStatus = store.Maintenance.Send(operation).Status;
return(backupStatus?.LastEtag);
}, expectedVal: 1, timeout: 30_000);
Assert.True(1 == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," +
$" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}");
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-2" }, "users/2");
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() =>
{
backupStatus = store.Maintenance.Send(operation).Status;
return(backupStatus?.LastEtag);
}, expectedVal: lastEtag, timeout: 30_000);
Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," +
$" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}");
string lastFileToRestore;
using (var client = new RavenAwsS3Client(defaultS3Settings))
{
var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.FolderName}";
lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath;
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "user-3" }, "users/3");
await session.SaveChangesAsync();
}
lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() =>
{
backupStatus = store.Maintenance.Send(operation).Status;
return(backupStatus?.LastEtag);
}, expectedVal: lastEtag, timeout: 30_000);
Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," +
$" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}");
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.FolderName);
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
Settings = subfolderS3Settings,
DatabaseName = databaseName,
LastFileNameToRestore = lastFileToRestore
}))
{
using (var session = store.OpenSession(databaseName))
{
var users = session.Load <User>("users/1");
Assert.NotNull(users);
users = session.Load <User>("users/2");
Assert.NotNull(users);
users = session.Load <User>("users/3");
Assert.Null(users);
}
}
}
}
// ReSharper disable once InconsistentNaming
private async Task PutObject(int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType)
{
var settings = GetS3Settings();
var blobs = GenerateBlobNames(settings, 1, out _);
Assert.Equal(1, blobs.Count);
var key = $"{blobs[0]}";
if (testBlobKeyAsFolder)
{
key += "/";
}
var progress = new Progress();
using (var client = new RavenAwsS3Client(settings, DefaultConfiguration, progress))
{
client.MaxUploadPutObject = new Sparrow.Size(10, SizeUnit.Megabytes);
client.MinOnePartUploadSizeLimit = new Sparrow.Size(7, SizeUnit.Megabytes);
var value1 = Guid.NewGuid().ToString();
var value2 = Guid.NewGuid().ToString();
var sb = new StringBuilder();
for (var i = 0; i < sizeInMB * 1024 * 1024; i++)
{
sb.Append("a");
}
long streamLength;
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
streamLength = memoryStream.Length;
client.PutObject(key,
memoryStream,
new Dictionary <string, string>
{
{ "property1", value1 },
{ "property2", value2 }
});
}
var @object = await client.GetObjectAsync(key);
Assert.NotNull(@object);
using (var reader = new StreamReader(@object.Data))
Assert.Equal(sb.ToString(), reader.ReadToEnd());
var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal(value1, @object.Metadata[property1]);
Assert.Equal(value2, @object.Metadata[property2]);
Assert.Equal(UploadState.Done, progress.UploadProgress.UploadState);
Assert.Equal(uploadType, progress.UploadProgress.UploadType);
Assert.Equal(streamLength, progress.UploadProgress.TotalInBytes);
Assert.Equal(streamLength, progress.UploadProgress.UploadedInBytes);
}
}
