public override void Dispose() { base.Dispose(); var s3Settings = GetS3Settings(); if (s3Settings == null) { return; } try { using (var s3Client = new RavenAwsS3Client(s3Settings)) { var cloudObjects = s3Client.ListObjectsAsync(s3Settings.RemoteFolderName, string.Empty, false).GetAwaiter().GetResult(); var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList(); s3Client.DeleteMultipleObjects(pathsToDelete); } } catch (Exception) { // ignored } }
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly2() { using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", "examplebucket")) { var date = new DateTime(2013, 5, 24); var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", payloadHash); var url = client.GetUrl() + "/test.txt"; var headers = new Dictionary <string, string> { { "x-amz-date", RavenAwsHelper.ConvertToString(date) }, { "x-amz-content-sha256", payloadHash }, { "Date", date.ToString("R") }, { "Host", "s3.amazonaws.com" }, { "Range", "bytes=0-9" } }; var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Get, url, date, headers); Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme); Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=819484c483cfb97d16522b1ac156f87e61677cc8f1f2545c799650ef178f4aa8", auth.Parameter); } }
public async Task can_delete_backups_by_date_s3(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental) { await Locker.WaitAsync(); try { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true; await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate, (configuration, databaseName) => { configuration.S3Settings = GetS3Settings(databaseName); }, async databaseName => { using (var client = new RavenAwsS3Client(GetS3Settings(databaseName))) { var folders = await client.ListObjectsAsync($"{client.RemoteFolderName}/", "/", listFolders: true); return(folders.FileInfoDetails.Count); } }, timeout : 120000, checkIncremental); } finally { BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false; Locker.Release(); } }
private async Task UploadToS3( S3Settings settings, Stream stream, string folderName, string fileName, Progress progress, string archiveDescription) { using (var client = new RavenAwsS3Client(settings.AwsAccessKey, settings.AwsSecretKey, settings.AwsRegionName, settings.BucketName, progress, TaskCancelToken.Token)) { var key = CombinePathAndKey(settings.RemoteFolderName, folderName, fileName); await client.PutObject(key, stream, new Dictionary <string, string> { { "Description", archiveDescription } }); if (_logger.IsInfoEnabled) { _logger.Info(string.Format($"Successfully uploaded backup file '{fileName}' " + $"to S3 bucket named: {settings.BucketName}, " + $"with key: {key}")); } } }
public override void Dispose() { base.Dispose(); var s3Settings = _isCustom ? CustomS3FactAttribute.S3Settings : AmazonS3FactAttribute.S3Settings; if (s3Settings == null) { return; } try { using (var s3Client = new RavenAwsS3Client(s3Settings, DefaultConfiguration)) { var cloudObjects = s3Client.ListObjects($"{s3Settings.RemoteFolderName}/{_remoteFolderName}/", string.Empty, false, includeFolders: true); var pathsToDelete = cloudObjects.FileInfoDetails.Select(x => x.FullPath).ToList(); s3Client.DeleteMultipleObjects(pathsToDelete); } } catch (Exception) { // ignored } }
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly2() { using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1")) { var date = new DateTime(2013, 5, 24); var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", payloadHash); var url = client.GetUrl("examplebucket") + "/" + "test.txt"; var headers = new Dictionary <string, string> { { "x-amz-date", RavenAwsHelper.ConvertToString(date) }, { "x-amz-content-sha256", payloadHash }, { "Date", date.ToString("R") }, { "Host", "examplebucket.s3.amazonaws.com" }, { "Range", "bytes=0-9" } }; var auth = client.CalculateAuthorizationHeaderValue("GET", url, date, headers); Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme); Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", auth.Parameter); } }
public void can_get_correct_error_s3() { var settings = GetS3Settings(); string region1 = settings.AwsRegionName; string region2 = settings.AwsRegionName = WestRegion2; var bucketName = settings.BucketName; using (var clientRegion2 = new RavenAwsS3Client(settings, DefaultConfiguration)) { var sb = new StringBuilder(); for (var i = 0; i < 1 * 1024 * 1024; i++) { sb.Append("a"); } var blobs = GenerateBlobNames(settings, 1, out _); Assert.Equal(1, blobs.Count); var key = blobs[0]; var error2 = Assert.Throws <InvalidOperationException>(() => { using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { clientRegion2.PutObject(key, memoryStream, new Dictionary <string, string>()); } }); Assert.Equal($"AWS location is set to '{region2}', but the bucket named: '{bucketName}' is located in: {region1}", error2.Message); } }
public void PutObject() { var bucketName = "ravendb"; var key = "testKey"; using (var client = new RavenAwsS3Client("<aws_access_key>", "<aws_secret_key>", "<aws_region_for_bucket>")) { client.PutObject(bucketName, key, new MemoryStream(Encoding.UTF8.GetBytes("321")), new Dictionary <string, string> { { "property1", "value1" }, { "property2", "value2" } }, 60 * 60); var @object = client.GetObject(bucketName, key); Assert.NotNull(@object); using (var reader = new StreamReader(@object.Data)) Assert.Equal("321", reader.ReadToEnd()); var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1")); var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2")); Assert.Equal("value1", @object.Metadata[property1]); Assert.Equal("value2", @object.Metadata[property2]); } }
public async Task put_object() { var settings = GetS3Settings(); using (var client = new RavenAwsS3Client(settings, DefaultConfiguration)) { var blobs = GenerateBlobNames(settings, 1, out _); Assert.Equal(1, blobs.Count); var key = blobs[0]; var value1 = Guid.NewGuid().ToString(); var value2 = Guid.NewGuid().ToString(); client.PutObject(key, new MemoryStream(Encoding.UTF8.GetBytes("231")), new Dictionary <string, string> { { "property1", value1 }, { "property2", value2 } }); var @object = await client.GetObjectAsync(key); Assert.NotNull(@object); using (var reader = new StreamReader(@object.Data)) Assert.Equal("231", reader.ReadToEnd()); var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1")); var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2")); Assert.Equal(value1, @object.Metadata[property1]); Assert.Equal(value2, @object.Metadata[property2]); } }
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly1() { var s3Settings = new S3Settings { AwsAccessKey = "AKIAIOSFODNN7EXAMPLE", AwsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", AwsRegionName = "us-east-1", BucketName = "examplebucket" }; using (var client = new RavenAwsS3Client(s3Settings)) { var date = new DateTime(2013, 5, 24); var stream = new MemoryStream(Encoding.UTF8.GetBytes("Welcome to Amazon S3.")); var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream); Assert.Equal("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash); var url = client.GetUrl() + "/test%24file.text"; var headers = new Dictionary <string, string> { { "x-amz-date", RavenAwsHelper.ConvertToString(date) }, { "x-amz-content-sha256", payloadHash }, { "x-amz-storage-class", "REDUCED_REDUNDANCY" }, { "Date", date.ToString("R") }, { "Host", "s3.amazonaws.com" } }; var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Put, url, date, headers); Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme); Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=d2dd2e48b10d2cb89c271a6464d0748686c158b5fde44e8d83936fd9b30b5c4c", auth.Parameter); } }
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly1() { using (var client = new RavenAwsS3Client("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1")) { var date = new DateTime(2013, 5, 24); var stream = new MemoryStream(Encoding.UTF8.GetBytes("Welcome to Amazon S3.")); var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream); Assert.Equal("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash); var url = client.GetUrl("examplebucket") + "/" + "test%24file.text"; var headers = new Dictionary <string, string> { { "x-amz-date", RavenAwsHelper.ConvertToString(date) }, { "x-amz-content-sha256", payloadHash }, { "x-amz-storage-class", "REDUCED_REDUNDANCY" }, { "Date", date.ToString("R") }, { "Host", "examplebucket.s3.amazonaws.com" } }; var auth = client.CalculateAuthorizationHeaderValue("PUT", url, date, headers); Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme); Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd", auth.Parameter); } }
private void DeleteFromS3(S3Settings settings) { using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress: null, TaskCancelToken.Token)) { var key = CombinePathAndKey(settings.RemoteFolderName); client.DeleteObject(key); if (_logger.IsInfoEnabled) { _logger.Info($"{ReportDeletion(S3Name)} bucket named: {settings.BucketName}, with key: {key}"); } } }
public async Task can_get_correct_error_s3(string region1, string region2) { var bucketName = $"testing-{Guid.NewGuid()}"; var key = Guid.NewGuid().ToString(); using (var clientRegion1 = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region1, bucketName)) using (var clientRegion2 = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region2, bucketName)) { // make sure that the bucket doesn't exist await clientRegion1.DeleteBucket(); try { var sb = new StringBuilder(); for (var i = 0; i < 1 * 1024 * 1024; i++) { sb.Append("a"); } var error1 = await Assert.ThrowsAsync <BucketNotFoundException>(async() => { using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { await clientRegion1.PutObject(key, memoryStream, new Dictionary <string, string>()); } }); Assert.Equal($"Bucket name '{bucketName}' doesn't exist!", error1.Message); await clientRegion1.PutBucket(); var error2 = await Assert.ThrowsAsync <InvalidOperationException>(async() => { using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { await clientRegion2.PutObject(key, memoryStream, new Dictionary <string, string>()); } }); Assert.Equal($"AWS location as set to {region2}, but the bucket named: '{bucketName}' is located in: {region1}", error2.Message); } finally { await clientRegion1.DeleteBucket(); } } }
private void UploadToS3(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup) { using (var client = new RavenAwsS3Client(awsAccessKey, awsSecretKey, localExportConfigs.AwsRegionEndpoint ?? RavenAwsClient.DefaultRegion)) using (var fileStream = File.OpenRead(backupPath)) { var key = Path.GetFileName(backupPath); client.PutObject(localExportConfigs.S3BucketName, key, fileStream, new Dictionary <string, string> { { "Description", GetArchiveDescription(isFullBackup) } }, 60 * 60); logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}", Path.GetFileName(backupPath), localExportConfigs.S3BucketName, key)); } }
public async Task put_object(string region) { var bucketName = $"testing-{Guid.NewGuid()}"; var key = $"test-key-{Guid.NewGuid()}"; using (var client = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region, bucketName)) { // make sure that the bucket doesn't exist await client.DeleteBucket(); try { await client.PutBucket(); var value1 = Guid.NewGuid().ToString(); var value2 = Guid.NewGuid().ToString(); await client.PutObject(key, new MemoryStream(Encoding.UTF8.GetBytes("231")), new Dictionary <string, string> { { "property1", value1 }, { "property2", value2 } }); // can't delete a bucket with existing objects var e = await Assert.ThrowsAsync <StorageException>(async() => await client.DeleteBucket()); Assert.True(e.Message.Contains("The bucket you tried to delete is not empty")); var @object = await client.GetObject(key); Assert.NotNull(@object); using (var reader = new StreamReader(@object.Data)) Assert.Equal("231", reader.ReadToEnd()); var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1")); var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2")); Assert.Equal(value1, @object.Metadata[property1]); Assert.Equal(value2, @object.Metadata[property2]); } finally { await client.DeleteObject(key); await client.DeleteBucket(); } } }
private void UploadToS3(S3Settings settings, Stream stream, Progress progress) { using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress, TaskCancelToken.Token)) { var key = CombinePathAndKey(settings.RemoteFolderName); client.PutObject(key, stream, new Dictionary <string, string> { { "Description", GetArchiveDescription() } }); if (_logger.IsInfoEnabled) { _logger.Info($"{ReportSuccess(S3Name)} bucket named: {settings.BucketName}, with key: {key}"); } var runner = new S3RetentionPolicyRunner(_retentionPolicyParameters, client); runner.Execute(); } }
private void UploadToS3(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup) { if (awsAccessKey == Constants.DataCouldNotBeDecrypted || awsSecretKey == Constants.DataCouldNotBeDecrypted) { throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true."); } using (var client = new RavenAwsS3Client(awsAccessKey, awsSecretKey, localExportConfigs.AwsRegionEndpoint ?? RavenAwsClient.DefaultRegion)) using (var fileStream = File.OpenRead(backupPath)) { var key = Path.GetFileName(backupPath); client.PutObject(localExportConfigs.S3BucketName, key, fileStream, new Dictionary <string, string> { { "Description", GetArchiveDescription(isFullBackup) } }, 60 * 60); logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}", Path.GetFileName(backupPath), localExportConfigs.S3BucketName, key)); } }
private async Task UploadToS3(string exportPath, string fileName, bool isFullExport) { if (_awsAccessKey == Constants.DataCouldNotBeDecrypted || _awsSecretKey == Constants.DataCouldNotBeDecrypted) { throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true."); } using (var client = new RavenAwsS3Client(_awsAccessKey, _awsSecretKey, _configuration.AwsRegionName ?? RavenAwsClient.DefaultRegion)) using (var fileStream = File.OpenRead(exportPath)) { var key = CombinePathAndKey(_configuration.S3RemoteFolderName, fileName); await client.PutObject(_configuration.S3BucketName, key, fileStream, new Dictionary <string, string> { { "Description", GetArchiveDescription(isFullExport) } }, 60 *60); if (_logger.IsInfoEnabled) { _logger.Info(string.Format("Successfully uploaded export {0} to S3 bucket {1}, with key {2}", fileName, _configuration.S3BucketName, key)); } } }
public S3RestorePoints(Config.Categories.BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, S3Settings s3Settings) : base(sortedList, context) { _configuration = configuration; _client = new RavenAwsS3Client(s3Settings, configuration); }
protected async Task incremental_and_full_check_last_file_for_backup_internal() { var s3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = Backup.CreateBackupConfiguration(s3Settings: s3Settings, fullBackupFrequency: null, incrementalBackupFrequency: "0 */6 * * *"); var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; var backupStatus = await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag); string lastFileToRestore; using (var client = new RavenAwsS3Client(s3Settings, DefaultConfiguration)) { var fullBackupPath = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await Backup.RunBackupAndReturnStatusAsync(Server, backupTaskId, store, isFullBackup : false, expectedEtag : lastEtag); var databaseName = $"restored_database-{Guid.NewGuid()}"; s3Settings.RemoteFolderName = $"{s3Settings.RemoteFolderName}/{backupStatus.FolderName}"; using (Backup.RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = s3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load <User>("users/1"); Assert.NotNull(users); users = session.Load <User>("users/2"); Assert.NotNull(users); users = session.Load <User>("users/3"); Assert.Null(users); } } } }
public async Task incremental_and_full_check_last_file_for_backup() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 */6 * * *", }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); var value = WaitForValue(() => { var getPeriodicBackupResult = store.Maintenance.Send(operation); return getPeriodicBackupResult.Status?.LastEtag; }, 1); Assert.Equal(1, value); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); string lastFileToRestore; var backupStatus = store.Maintenance.Send(operation); using (var client = new RavenAwsS3Client(AmazonS3FactAttribute.S3Settings)) { var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.Status.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); backupStatus = store.Maintenance.Send(operation); var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(backupStatus.Status.FolderName); using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = subfolderS3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load<User>("users/1"); Assert.NotNull(users); users = session.Load<User>("users/2"); Assert.NotNull(users); users = session.Load<User>("users/3"); Assert.Null(users); } } } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var folderPathOptions = new FolderPathOptions(); ; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) || string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) || string.IsNullOrWhiteSpace(s3Settings.BucketName) || string.IsNullOrWhiteSpace(s3Settings.AwsRegionName)) { break; } using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64); if (folders != null) { foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } } break; case PeriodicBackupConnectionType.Azure: var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null) { throw new BadRequestException("No JSON was posted."); } var azureSettings = JsonDeserializationServer.AzureSettings(azureJson); if (azureSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(azureSettings.AccountName) || string.IsNullOrWhiteSpace(azureSettings.AccountKey) || string.IsNullOrWhiteSpace(azureSettings.StorageContainer)) { break; } using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true)); foreach (var folder in folders.List) { var fullPath = folder.Name; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.GoogleCloud: var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null) { throw new BadRequestException("No JSON was posted."); } var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson); if (googleCloudSettings == null) { throw new BadRequestException("No AzureSettings were found."); } if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) || string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson)) { break; } using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup)) { var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName)); var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length; foreach (var folder in folders) { const char separator = '/'; var splitted = folder.Name.Split(separator); var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator; if (string.IsNullOrWhiteSpace(result)) { continue; } folderPathOptions.List.Add(result); } } break; case PeriodicBackupConnectionType.FTP: case PeriodicBackupConnectionType.Glacier: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
public async Task TestPeriodicBackupCredentials() { var type = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type"); if (Enum.TryParse(type, out PeriodicBackupConnectionType connectionType) == false) { throw new ArgumentException($"Unknown backup connection: {type}"); } using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { DynamicJsonValue result; try { var connectionInfo = await context.ReadForMemoryAsync(RequestBodyStream(), "test-connection"); switch (connectionType) { case PeriodicBackupConnectionType.S3: var s3Settings = JsonDeserializationClient.S3Settings(connectionInfo); using (var awsClient = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown)) { awsClient.TestConnection(); } break; case PeriodicBackupConnectionType.Glacier: var glacierSettings = JsonDeserializationClient.GlacierSettings(connectionInfo); using (var glacierClient = new RavenAwsGlacierClient(glacierSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown)) { glacierClient.TestConnection(); } break; case PeriodicBackupConnectionType.Azure: var azureSettings = JsonDeserializationClient.AzureSettings(connectionInfo); using (var azureClient = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown)) { azureClient.TestConnection(); } break; case PeriodicBackupConnectionType.GoogleCloud: var googleCloudSettings = JsonDeserializationClient.GoogleCloudSettings(connectionInfo); using (var googleCloudClient = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown)) { await googleCloudClient.TestConnection(); } break; case PeriodicBackupConnectionType.FTP: var ftpSettings = JsonDeserializationClient.FtpSettings(connectionInfo); using (var ftpClient = new RavenFtpClient(ftpSettings)) { ftpClient.TestConnection(); } break; case PeriodicBackupConnectionType.Local: case PeriodicBackupConnectionType.None: default: throw new ArgumentOutOfRangeException(); } result = new DynamicJsonValue { [nameof(NodeConnectionTestResult.Success)] = true, }; } catch (Exception e) { result = new DynamicJsonValue { [nameof(NodeConnectionTestResult.Success)] = false, [nameof(NodeConnectionTestResult.Error)] = e.ToString() }; } await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, result); } } }
public async Task GetFolderPathOptions() { PeriodicBackupConnectionType connectionType; var type = GetStringValuesQueryString("type", false).FirstOrDefault(); if (type == null) { //Backward compatibility connectionType = PeriodicBackupConnectionType.Local; } else if (Enum.TryParse(type, out connectionType) == false) { throw new ArgumentException($"Query string '{type}' was not recognized as valid type"); } using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { FolderPathOptions folderPathOptions; switch (connectionType) { case PeriodicBackupConnectionType.Local: var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false; var path = GetStringQueryString("path", required: false); folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration); break; case PeriodicBackupConnectionType.S3: var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format"); if (connectionType != PeriodicBackupConnectionType.Local && json == null) { throw new BadRequestException("No JSON was posted."); } var s3Settings = JsonDeserializationServer.S3Settings(json); if (s3Settings == null) { throw new BadRequestException("No S3Settings were found."); } using (var client = new RavenAwsS3Client(s3Settings)) { // fetching only the first 64 results for the auto complete var folders = await client.ListObjects(s3Settings.RemoteFolderName, "/", true, 64); folderPathOptions = new FolderPathOptions(); foreach (var folder in folders.FileInfoDetails) { var fullPath = folder.FullPath; if (string.IsNullOrWhiteSpace(fullPath)) { continue; } folderPathOptions.List.Add(fullPath); } } break; case PeriodicBackupConnectionType.Glacier: case PeriodicBackupConnectionType.Azure: case PeriodicBackupConnectionType.GoogleCloud: case PeriodicBackupConnectionType.FTP: throw new NotSupportedException(); default: throw new ArgumentOutOfRangeException(); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List) }); } } }
public S3RetentionPolicyRunner(RetentionPolicyBaseParameters parameters, RavenAwsS3Client client) : base(parameters) { _client = client; }
public RestoreFromS3(ServerStore serverStore, RestoreFromS3Configuration restoreFromConfiguration, string nodeTag, OperationCancelToken operationCancelToken) : base(serverStore, restoreFromConfiguration, nodeTag, operationCancelToken) { _client = new RavenAwsS3Client(restoreFromConfiguration.Settings); _remoteFolderName = restoreFromConfiguration.Settings.RemoteFolderName; }
public S3RestorePoints(SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, S3Settings s3Settings) : base(sortedList, context) { _client = new RavenAwsS3Client(s3Settings); }
// ReSharper disable once InconsistentNaming private static async Task PutObject(string region, int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType) { var bucketName = $"testing-{Guid.NewGuid()}"; var key = testBlobKeyAsFolder == false? Guid.NewGuid().ToString() : $"{Guid.NewGuid()}/folder/testKey"; var uploadProgress = new UploadProgress(); var maxUploadPutObjectInBytesSetter = ExpressionHelper.CreateFieldSetter <RavenAwsS3Client, int>("MaxUploadPutObjectSizeInBytes"); var minOnePartUploadSizeLimitInBytesSetter = ExpressionHelper.CreateFieldSetter <RavenAwsS3Client, int>("MinOnePartUploadSizeLimitInBytes"); using (var client = new RavenAwsS3Client(AwsAccessKey, AwsSecretKey, region, bucketName, uploadProgress)) { maxUploadPutObjectInBytesSetter(client, 10 * 1024 * 1024); // 10MB minOnePartUploadSizeLimitInBytesSetter(client, 7 * 1024 * 1024); // 7MB // make sure that the bucket doesn't exist await client.DeleteBucket(); try { await client.PutBucket(); var value1 = Guid.NewGuid().ToString(); var value2 = Guid.NewGuid().ToString(); var sb = new StringBuilder(); for (var i = 0; i < sizeInMB * 1024 * 1024; i++) { sb.Append("a"); } long streamLength; using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { streamLength = memoryStream.Length; await client.PutObject(key, memoryStream, new Dictionary <string, string> { { "property1", value1 }, { "property2", value2 } }); } var @object = await client.GetObject(key); Assert.NotNull(@object); using (var reader = new StreamReader(@object.Data)) Assert.Equal(sb.ToString(), reader.ReadToEnd()); var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1")); var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2")); Assert.Equal(value1, @object.Metadata[property1]); Assert.Equal(value2, @object.Metadata[property2]); Assert.Equal(UploadState.Done, uploadProgress.UploadState); Assert.Equal(uploadType, uploadProgress.UploadType); Assert.Equal(streamLength, uploadProgress.TotalInBytes); Assert.Equal(streamLength, uploadProgress.UploadedInBytes); } finally { await client.DeleteObject(key); await client.DeleteBucket(); } } }
protected async Task incremental_and_full_check_last_file_for_backup_internal() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-1" }, "users/1"); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 */6 * * *", }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus backupStatus = null; var value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: 1, timeout: 30_000); Assert.True(1 == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-2" }, "users/2"); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); string lastFileToRestore; using (var client = new RavenAwsS3Client(defaultS3Settings)) { var fullBackupPath = $"{defaultS3Settings.RemoteFolderName}/{backupStatus.FolderName}"; lastFileToRestore = (await client.ListObjectsAsync(fullBackupPath, string.Empty, false)).FileInfoDetails.Last().FullPath; } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "user-3" }, "users/3"); await session.SaveChangesAsync(); } lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(backupStatus.FolderName); using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { Settings = subfolderS3Settings, DatabaseName = databaseName, LastFileNameToRestore = lastFileToRestore })) { using (var session = store.OpenSession(databaseName)) { var users = session.Load <User>("users/1"); Assert.NotNull(users); users = session.Load <User>("users/2"); Assert.NotNull(users); users = session.Load <User>("users/3"); Assert.Null(users); } } } }
// ReSharper disable once InconsistentNaming private async Task PutObject(int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType) { var settings = GetS3Settings(); var blobs = GenerateBlobNames(settings, 1, out _); Assert.Equal(1, blobs.Count); var key = $"{blobs[0]}"; if (testBlobKeyAsFolder) { key += "/"; } var progress = new Progress(); using (var client = new RavenAwsS3Client(settings, DefaultConfiguration, progress)) { client.MaxUploadPutObject = new Sparrow.Size(10, SizeUnit.Megabytes); client.MinOnePartUploadSizeLimit = new Sparrow.Size(7, SizeUnit.Megabytes); var value1 = Guid.NewGuid().ToString(); var value2 = Guid.NewGuid().ToString(); var sb = new StringBuilder(); for (var i = 0; i < sizeInMB * 1024 * 1024; i++) { sb.Append("a"); } long streamLength; using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { streamLength = memoryStream.Length; client.PutObject(key, memoryStream, new Dictionary <string, string> { { "property1", value1 }, { "property2", value2 } }); } var @object = await client.GetObjectAsync(key); Assert.NotNull(@object); using (var reader = new StreamReader(@object.Data)) Assert.Equal(sb.ToString(), reader.ReadToEnd()); var property1 = @object.Metadata.Keys.Single(x => x.Contains("property1")); var property2 = @object.Metadata.Keys.Single(x => x.Contains("property2")); Assert.Equal(value1, @object.Metadata[property1]); Assert.Equal(value2, @object.Metadata[property2]); Assert.Equal(UploadState.Done, progress.UploadProgress.UploadState); Assert.Equal(uploadType, progress.UploadProgress.UploadType); Assert.Equal(streamLength, progress.UploadProgress.TotalInBytes); Assert.Equal(streamLength, progress.UploadProgress.UploadedInBytes); } }