Beispiel #1
0
        public void PutBlob()
        {
            var containerName = "testContainer";
            var blobKey       = "testKey";

            using (var client = new RavenAzureClient("devstoreaccount1", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="))
            {
                client.PutContainer(containerName);
                client.PutBlob(containerName, blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
                {
                    { "property1", "value1" },
                    { "property2", "value2" }
                });
                var blob = client.GetBlob(containerName, blobKey);
                Assert.NotNull(blob);

                using (var reader = new StreamReader(blob.Data))
                    Assert.Equal("123", reader.ReadToEnd());

                var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
                var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));

                Assert.Equal("value1", blob.Metadata[property1]);
                Assert.Equal("value2", blob.Metadata[property2]);
            }
        }
Beispiel #2
0
        public void CanRemoveBlobsInBatch()
        {
            var containerName = Guid.NewGuid().ToString();
            var blobKey       = Guid.NewGuid().ToString();

            using var client = new RavenAzureClient(GetAzureSettings(containerName));
            var blobs = new List <string>();

            try
            {
                client.DeleteContainer();
                client.PutContainer();

                for (int i = 0; i < 10; i++)
                {
                    var key    = $"{blobKey}/northwind_{i}.ravendump";
                    var tmpArr = new byte[3];
                    new Random().NextBytes(tmpArr);
                    client.PutBlob(key, new MemoryStream(tmpArr), new Dictionary <string, string> {
                        { $"property_{i}", $"value_{i}" }
                    });

                    var blob = client.GetBlob(key);
                    Assert.NotNull(blob);

                    blobs.Add(key);
                }

                client.DeleteMultipleBlobs(blobs);
            }
            finally
            {
                client.DeleteContainer();
            }
        }
Beispiel #3
0
        public async Task can_delete_backups_by_date_azure(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental)
        {
            await Locker.WaitAsync();

            var containerName = Guid.NewGuid().ToString();

            using var client = new RavenAzureClient(new AzureSettings { AccountName = Azure.AzureAccountName, AccountKey = Azure.AzureAccountKey, StorageContainer = containerName });

            try
            {
                client.DeleteContainer();
                client.PutContainer();

                BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true;

                await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate,
                                             (configuration, databaseName) =>
                {
                    configuration.AzureSettings = GetAzureSettings(containerName, databaseName);
                },
                                             async databaseName =>
                {
                    using var c = new RavenAzureClient(GetAzureSettings(containerName, databaseName));
                    var folders = await c.ListBlobsAsync($"{c.RemoteFolderName}/", delimiter: "/", listFolders: true);
                    return(folders.ListBlob.Count());
                }, timeout : 120000, checkIncremental);
            }
            finally
            {
                BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false;
                client.DeleteContainer();
                Locker.Release();
            }
        }
Beispiel #4
0
        public void put_blob()
        {
            var containerName = Guid.NewGuid().ToString();
            var blobKey       = Guid.NewGuid().ToString();

            using (var client = new RavenAzureClient(GenerateAzureSettings(containerName)))
            {
                try
                {
                    client.DeleteContainer();
                    client.PutContainer();

                    client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
                    {
                        { "property1", "value1" },
                        { "property2", "value2" }
                    });
                    var blob = client.GetBlob(blobKey);
                    Assert.NotNull(blob);

                    using (var reader = new StreamReader(blob.Data))
                        Assert.Equal("123", reader.ReadToEnd());

                    var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
                    var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));

                    Assert.Equal("value1", blob.Metadata[property1]);
                    Assert.Equal("value2", blob.Metadata[property2]);
                }
                finally
                {
                    client.DeleteContainer();
                }
            }
        }
Beispiel #5
0
        public async Task PutBlobIntoFolder()
        {
            var containerName = "testContainer";
            var blobKey       = "folder1/folder2/testKey";

            using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
            {
                await client.PutContainer();

                await client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
                {
                    { "property1", "value1" },
                    { "property2", "value2" }
                });

                var blob = await client.GetBlob(blobKey);

                Assert.NotNull(blob);

                using (var reader = new StreamReader(blob.Data))
                    Assert.Equal("123", reader.ReadToEnd());

                var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
                var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));

                Assert.Equal("value1", blob.Metadata[property1]);
                Assert.Equal("value2", blob.Metadata[property2]);
            }
        }
        private async Task UploadToAzure(string exportPath, string fileName, bool isFullExport)
        {
            if (_azureStorageAccount == Constants.DataCouldNotBeDecrypted ||
                _azureStorageKey == Constants.DataCouldNotBeDecrypted)
            {
                throw new InvalidOperationException("Could not decrypt the Azure access settings, if you are running on IIS, make sure that load user profile is set to true.");
            }

            using (var client = new RavenAzureClient(_azureStorageAccount, _azureStorageKey, _configuration.AzureStorageContainer))
            {
                await client.PutContainer();

                using (var fileStream = File.OpenRead(exportPath))
                {
                    var key = CombinePathAndKey(_configuration.AzureRemoteFolderName, fileName);
                    await client.PutBlob(key, fileStream, new Dictionary <string, string>
                    {
                        { "Description", GetArchiveDescription(isFullExport) }
                    });

                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info($"Successfully uploaded export {fileName} to Azure container {_configuration.AzureStorageContainer}, with key {key}");
                    }
                }
            }
        }
Beispiel #7
0
        private void UploadToAzure(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
        {
            if (azureStorageAccount == Constants.DataCouldNotBeDecrypted ||
                azureStorageKey == Constants.DataCouldNotBeDecrypted)
            {
                throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true.");
            }
            using (var client = new RavenAzureClient(azureStorageAccount, azureStorageKey, true))
            {
                client.PutContainer(localExportConfigs.AzureStorageContainer);
                using (var fileStream = File.OpenRead(backupPath))
                {
                    var key = Path.GetFileName(backupPath);
                    client.PutBlob(localExportConfigs.AzureStorageContainer, key, fileStream, new Dictionary <string, string>
                    {
                        { "Description", GetArchiveDescription(isFullBackup) }
                    });

                    logger.Info(string.Format(
                                    "Successfully uploaded backup {0} to Azure container {1}, with key {2}",
                                    Path.GetFileName(backupPath),
                                    localExportConfigs.AzureStorageContainer,
                                    key));
                }
            }
        }
Beispiel #8
0
        public void list_blobs_()
        {
            var containerName = "mycontainer";
            var blobKey1      = $"{Guid.NewGuid()}/folder/testKey";
            var blobKey2      = $"{Guid.NewGuid()}/folder/testKey";

            using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
            {
                try
                {
                    client.DeleteContainer();
                    client.PutContainer();

                    var path = NewDataPath(forceCreateDir: true);

                    var filePath1 = Path.Combine(path, Guid.NewGuid().ToString()) + ".txt";
                    var filePath2 = Path.Combine(path, Guid.NewGuid().ToString()) + ".txt";

                    File.WriteAllText(filePath1, "abc", Encoding.UTF8);
                    File.WriteAllText(filePath2, "def", Encoding.UTF8);
                    using (var file1 = File.Open(filePath1, FileMode.Open))
                        using (var file2 = File.Open(filePath2, FileMode.Open))
                        {
                            client.PutBlob(blobKey1, file1, new Dictionary <string, string>());
                            client.PutBlob(blobKey2, file2, new Dictionary <string, string>());
                        }

                    // Assert.Equal((await client.ListBlobs(containerName)).Count, 2);
                }
                finally
                {
                    client.DeleteContainer();
                }
            }
        }
Beispiel #9
0
 private void InitContainer(AzureSettings azureSettings)
 {
     using (var client = new RavenAzureClient(azureSettings))
     {
         client.DeleteContainer();
         client.PutContainer();
     }
 }
Beispiel #10
0
            public AzureClientHolder(AzureSettings setting, Progress progress = null, [CallerMemberName] string caller = null)
            {
                Assert.False(string.IsNullOrEmpty(setting.StorageContainer), "string.IsNullOrEmpty(setting.StorageContainer)");
                Settings = setting;

                // keep only alphanumeric characters
                Settings.RemoteFolderName = _remoteFolder = GetRemoteFolder(caller);
                Client = RavenAzureClient.Create(Settings, DefaultConfiguration, progress);
            }
Beispiel #11
0
            public AzureClientHolder(AzureSettings setting, Progress progress = null, [CallerMemberName] string caller = null)
            {
                Assert.False(string.IsNullOrEmpty(setting.StorageContainer), "string.IsNullOrEmpty(setting.StorageContainer)");

                // keep only alphanumeric characters
                caller            = caller == null ? string.Empty : string.Concat(caller.Where(char.IsLetterOrDigit));
                _remoteFolderName = setting.RemoteFolderName = $"{caller}{Guid.NewGuid()}";
                Client            = new RavenAzureClient(setting, progress);
            }
Beispiel #12
0
        private void DeleteFromAzure(AzureSettings settings)
        {
            using (var client = RavenAzureClient.Create(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
            {
                var key = CombinePathAndKey(settings.RemoteFolderName);
                client.DeleteBlobs(new List <string> {
                    key
                });

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"{ReportDeletion(AzureName)} container: {settings.StorageContainer}, with key: {key}");
                }
            }
        }
Beispiel #13
0
        public void can_get_container_not_found()
        {
            var containerName = Guid.NewGuid().ToString();

            using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
            {
                var containerNames = client.GetContainerNames(500);
                Assert.False(containerNames.Exists(x => x.Equals(containerName)));

                var e = Assert.Throws <ContainerNotFoundException>(() => client.TestConnection());
                Assert.Equal($"Container '{containerName}' wasn't found!", e.Message);

                containerNames = client.GetContainerNames(500);
                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
            }
        }
Beispiel #14
0
        public void RemoveNonExistingBlobsInBatchShouldThrow()
        {
            var containerName = Guid.NewGuid().ToString();
            var blobKey       = Guid.NewGuid().ToString();

            using var client = new RavenAzureClient(GenerateAzureSettings(containerName));
            var blobs = new List <string>();

            try
            {
                client.DeleteContainer();
                client.PutContainer();

                // put blob
                var k      = $"{blobKey}/northwind_322.ravendump";
                var tmpArr = new byte[3];
                new Random().NextBytes(tmpArr);
                client.PutBlob(k, new MemoryStream(tmpArr), new Dictionary <string, string>
                {
                    { "Nice", "NotNice" }
                });

                var blob = client.GetBlob(k);
                Assert.NotNull(blob);
                blobs.Add(k);

                for (int i = 0; i < 10; i++)
                {
                    blobs.Add($"{blobKey}/northwind_{i}.ravendump");
                }

                try
                {
                    client.DeleteMultipleBlobs(blobs);
                }
                catch (Exception e)
                {
                    Assert.Equal(typeof(InvalidOperationException), e.GetType());
                    Assert.True(e.Message.StartsWith($"Failed to delete {blobs.Count - 1} blobs from container: {containerName}. Successfully deleted 1 blob. Reason: The specified blob does not exist."));
                }
            }
            finally
            {
                client.DeleteContainer();
            }
        }
Beispiel #15
0
        public void can_get_and_delete_container()
        {
            var containerName = Guid.NewGuid().ToString();

            using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
            {
                var containerNames = client.GetContainerNames(500);
                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
                client.PutContainer();

                containerNames = client.GetContainerNames(500);
                Assert.True(containerNames.Exists(x => x.Equals(containerName)));

                client.DeleteContainer();

                containerNames = client.GetContainerNames(500);
                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
            }
        }
Beispiel #16
0
        private void UploadToAzure(AzureSettings settings, Stream stream, Progress progress)
        {
            using (var client = RavenAzureClient.Create(settings, _settings.Configuration, progress, TaskCancelToken.Token))
            {
                var key = CombinePathAndKey(settings.RemoteFolderName);
                client.PutBlob(key, stream, new Dictionary <string, string>
                {
                    { "Description", GetArchiveDescription() }
                });

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"{ReportSuccess(AzureName)} container: {settings.StorageContainer}, with key: {key}");
                }

                var runner = new AzureRetentionPolicyRunner(_retentionPolicyParameters, client);
                runner.Execute();
            }
        }
Beispiel #17
0
        public async Task can_get_container_not_found()
        {
            var containerName = Guid.NewGuid().ToString();

            using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
            {
                var containerNames = await client.GetContainerNames(500);

                Assert.False(containerNames.Exists(x => x.Equals(containerName)));

                var e = await Assert.ThrowsAsync <ContainerNotFoundException>(async() => await client.TestConnection());

                Assert.Equal($"Container '{containerName}' not found!", e.Message);

                containerNames = await client.GetContainerNames(500);

                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
            }
        }
Beispiel #18
0
        private void UploadToAzure(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
        {
            using (var client = new RavenAzureClient(azureStorageAccount, azureStorageKey, true))
            {
                client.PutContainer(localExportConfigs.AzureStorageContainer);
                using (var fileStream = File.OpenRead(backupPath))
                {
                    var key = Path.GetFileName(backupPath);
                    client.PutBlob(localExportConfigs.AzureStorageContainer, key, fileStream, new Dictionary <string, string>
                    {
                        { "Description", GetArchiveDescription(isFullBackup) }
                    });

                    logger.Info(string.Format(
                                    "Successfully uploaded backup {0} to Azure container {1}, with key {2}",
                                    Path.GetFileName(backupPath),
                                    localExportConfigs.AzureStorageContainer,
                                    key));
                }
            }
        }
Beispiel #19
0
        public async Task can_get_and_delete_container()
        {
            var containerName = Guid.NewGuid().ToString();

            using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
            {
                var containerNames = await client.GetContainerNames(500);

                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
                await client.PutContainer();

                containerNames = await client.GetContainerNames(500);

                Assert.True(containerNames.Exists(x => x.Equals(containerName)));

                await client.DeleteContainer();

                containerNames = await client.GetContainerNames(500);

                Assert.False(containerNames.Exists(x => x.Equals(containerName)));
            }
        }
Beispiel #20
0
        public async Task put_blob_100MB()
        {
            var containerName = "testContainer";
            var blobKey       = "testKey";

            using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
            {
                await client.PutContainer();

                var sb = new StringBuilder();
                for (var i = 0; i < 100 * 1024 * 1024; i++)
                {
                    sb.Append("a");
                }

                await client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())),
                                     new Dictionary <string, string>
                {
                    { "property1", "value1" },
                    { "property2", "value2" },
                    { "property3", "value3" }
                });

                var blob = await client.GetBlob(blobKey);

                Assert.NotNull(blob);

                using (var reader = new StreamReader(blob.Data))
                    Assert.Equal(sb.ToString(), reader.ReadToEnd());

                var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
                var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
                var property3 = blob.Metadata.Keys.Single(x => x.Contains("property3"));

                Assert.Equal("value1", blob.Metadata[property1]);
                Assert.Equal("value2", blob.Metadata[property2]);
                Assert.Equal("value3", blob.Metadata[property3]);
            }
        }
Beispiel #21
0
        private static async Task DeleteObjects(AzureSettings azureSettings, string prefix, string delimiter, bool listFolder = false)
        {
            if (azureSettings == null)
            {
                return;
            }

            try
            {
                using (var client = RavenAzureClient.Create(azureSettings, DefaultBackupConfiguration))
                {
                    var result = await client.ListBlobsAsync(prefix, delimiter, listFolder);

                    List <string> filesToDelete;

                    if (listFolder == false)
                    {
                        filesToDelete = result.List.Select(b => b.Name).ToList();
                    }
                    else
                    {
                        filesToDelete = await ListAllFilesInFolders(client, result.List);
                    }

                    if (filesToDelete.Count == 0)
                    {
                        return;
                    }

                    client.DeleteBlobs(filesToDelete);
                }
            }
            catch (Exception)
            {
                // ignored
            }
        }
Beispiel #22
0
        private async Task UploadToAzure(
            AzureSettings settings,
            Stream stream,
            string folderName,
            string fileName,
            Progress progress,
            string archiveDecription)
        {
            using (var client = new RavenAzureClient(settings.AccountName, settings.AccountKey,
                                                     settings.StorageContainer, progress, TaskCancelToken.Token))
            {
                var key = CombinePathAndKey(settings.RemoteFolderName, folderName, fileName);
                await client.PutBlob(key, stream, new Dictionary <string, string>
                {
                    { "Description", archiveDecription }
                });

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Successfully uploaded backup file '{fileName}' " +
                                 $"to Azure container: {settings.StorageContainer}, with key: {key}");
                }
            }
        }
        public async Task TestPeriodicBackupCredentials()
        {
            var type = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");

            if (Enum.TryParse(type, out PeriodicBackupConnectionType connectionType) == false)
            {
                throw new ArgumentException($"Unknown backup connection: {type}");
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                DynamicJsonValue result;
                try
                {
                    var connectionInfo = await context.ReadForMemoryAsync(RequestBodyStream(), "test-connection");

                    switch (connectionType)
                    {
                    case PeriodicBackupConnectionType.S3:
                        var s3Settings = JsonDeserializationClient.S3Settings(connectionInfo);
                        using (var awsClient = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
                        {
                            awsClient.TestConnection();
                        }
                        break;

                    case PeriodicBackupConnectionType.Glacier:
                        var glacierSettings = JsonDeserializationClient.GlacierSettings(connectionInfo);
                        using (var glacierClient = new RavenAwsGlacierClient(glacierSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
                        {
                            glacierClient.TestConnection();
                        }
                        break;

                    case PeriodicBackupConnectionType.Azure:
                        var azureSettings = JsonDeserializationClient.AzureSettings(connectionInfo);
                        using (var azureClient = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
                        {
                            azureClient.TestConnection();
                        }
                        break;

                    case PeriodicBackupConnectionType.GoogleCloud:
                        var googleCloudSettings = JsonDeserializationClient.GoogleCloudSettings(connectionInfo);
                        using (var googleCloudClient = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
                        {
                            await googleCloudClient.TestConnection();
                        }
                        break;

                    case PeriodicBackupConnectionType.FTP:
                        var ftpSettings = JsonDeserializationClient.FtpSettings(connectionInfo);
                        using (var ftpClient = new RavenFtpClient(ftpSettings))
                        {
                            ftpClient.TestConnection();
                        }
                        break;

                    case PeriodicBackupConnectionType.Local:
                    case PeriodicBackupConnectionType.None:
                    default:
                        throw new ArgumentOutOfRangeException();
                    }

                    result = new DynamicJsonValue
                    {
                        [nameof(NodeConnectionTestResult.Success)] = true,
                    };
                }
                catch (Exception e)
                {
                    result = new DynamicJsonValue
                    {
                        [nameof(NodeConnectionTestResult.Success)] = false,
                        [nameof(NodeConnectionTestResult.Error)]   = e.ToString()
                    };
                }

                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    context.Write(writer, result);
                }
            }
        }
Beispiel #24
0
        // ReSharper disable once InconsistentNaming
        private async Task PutBlob(int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType)
        {
            var containerName = Guid.NewGuid().ToString();
            var blobKey       = testBlobKeyAsFolder == false?
                                Guid.NewGuid().ToString() :
                                    $"{Guid.NewGuid()}/folder/testKey";

            var uploadProgress = new UploadProgress();

            using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, uploadProgress, isTest: true))
            {
                try
                {
                    await client.DeleteContainer();

                    await client.PutContainer();

                    var sb = new StringBuilder();
                    for (var i = 0; i < sizeInMB * 1024 * 1024; i++)
                    {
                        sb.Append("a");
                    }

                    var value1 = Guid.NewGuid().ToString();
                    var value2 = Guid.NewGuid().ToString();
                    var value3 = Guid.NewGuid().ToString();

                    long streamLength;
                    using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
                    {
                        streamLength = memoryStream.Length;
                        await client.PutBlob(blobKey, memoryStream,
                                             new Dictionary <string, string>
                        {
                            { "property1", value1 },
                            { "property2", value2 },
                            { "property3", value3 }
                        });
                    }

                    var blob = await client.GetBlob(blobKey);

                    Assert.NotNull(blob);

                    using (var reader = new StreamReader(blob.Data))
                        Assert.Equal(sb.ToString(), reader.ReadToEnd());

                    var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
                    var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
                    var property3 = blob.Metadata.Keys.Single(x => x.Contains("property3"));

                    Assert.Equal(value1, blob.Metadata[property1]);
                    Assert.Equal(value2, blob.Metadata[property2]);
                    Assert.Equal(value3, blob.Metadata[property3]);

                    Assert.Equal(UploadState.Done, uploadProgress.UploadState);
                    Assert.Equal(uploadType, uploadProgress.UploadType);
                    Assert.Equal(streamLength, uploadProgress.TotalInBytes);
                    Assert.Equal(streamLength, uploadProgress.UploadedInBytes);
                }
                finally
                {
                    await client.DeleteContainer();
                }
            }
        }
 public AzureRestorePoints(BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, AzureSettings azureSettings) : base(sortedList, context)
 {
     _client = RavenAzureClient.Create(azureSettings, configuration);
 }
Beispiel #26
0
        public async Task CanModifyPartitionColumnName()
        {
            var settings = GetAzureSettings();

            try
            {
                using (var store = GetDocumentStore())
                {
                    const string partitionColumn = "order_date";

                    var baseline = new DateTime(2020, 1, 1);

                    using (var session = store.OpenAsyncSession())
                    {
                        for (int i = 0; i < 31; i++)
                        {
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i}",
                                OrderedAt = baseline.AddDays(i),
                                ShipVia   = $"shippers/{i}",
                                Company   = $"companies/{i}"
                            });
                        }

                        for (int i = 0; i < 28; i++)
                        {
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i + 31}",
                                OrderedAt = baseline.AddMonths(1).AddDays(i),
                                ShipVia   = $"shippers/{i + 31}",
                                Company   = $"companies/{i + 31}"
                            });
                        }

                        await session.SaveChangesAsync();
                    }

                    var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);

                    var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);

loadToOrders(partitionBy(['order_date', key]),
    {
        Company : this.Company,
        ShipVia : this.ShipVia
    })
";
                    var connectionStringName = $"{store.Database} to Azure";

                    var configuration = new OlapEtlConfiguration
                    {
                        Name = "olap-azure-test",
                        ConnectionStringName = connectionStringName,
                        RunFrequency         = LocalTests.DefaultFrequency,
                        Transforms           =
                        {
                            new Transformation
                            {
                                Name        = "MonthlyOrders",
                                Collections = new List <string>{
                                    "Orders"
                                },
                                Script = script
                            }
                        }
                    };

                    SetupAzureEtl(store, settings, configuration);

                    etlDone.Wait(TimeSpan.FromMinutes(1));

                    using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
                    {
                        var prefix       = $"{settings.RemoteFolderName}/{CollectionName}";
                        var cloudObjects = await client.ListBlobsAsync(prefix, string.Empty, false);

                        var list = cloudObjects.List.ToList();

                        Assert.Equal(2, list.Count);
                        Assert.Contains($"{partitionColumn}=2020-01-01", list[0].Name);
                        Assert.Contains($"{partitionColumn}=2020-02-01", list[1].Name);
                    }
                }
            }

            finally
            {
                await DeleteObjects(settings);
            }
        }
Beispiel #27
0
        public async Task SimpleTransformation_NoPartition()
        {
            var settings = GetAzureSettings();

            try
            {
                using (var store = GetDocumentStore())
                {
                    var baseline = new DateTime(2020, 1, 1).ToUniversalTime();

                    using (var session = store.OpenAsyncSession())
                    {
                        for (int i = 0; i < 100; i++)
                        {
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i}",
                                OrderedAt = baseline.AddDays(i),
                                ShipVia   = $"shippers/{i}",
                                Company   = $"companies/{i}"
                            });
                        }

                        await session.SaveChangesAsync();
                    }

                    var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);

                    var script = @"
loadToOrders(noPartition(),
    {
        OrderDate : this.OrderedAt
        Company : this.Company,
        ShipVia : this.ShipVia
    });
";
                    SetupAzureEtl(store, script, settings);

                    etlDone.Wait(TimeSpan.FromMinutes(1));

                    using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
                    {
                        var prefix = $"{settings.RemoteFolderName}/{CollectionName}";

                        var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : string.Empty, listFolders : false);

                        var list = cloudObjects.List.ToList();

                        Assert.Equal(1, list.Count);

                        var blob = await client.GetBlobAsync(list[0].Name);

                        await using var ms = new MemoryStream();
                        blob.Data.CopyTo(ms);

                        using (var parquetReader = new ParquetReader(ms))
                        {
                            Assert.Equal(1, parquetReader.RowGroupCount);

                            var expectedFields = new[] { "OrderDate", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };

                            Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);

                            using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
                            foreach (var field in parquetReader.Schema.Fields)
                            {
                                Assert.True(field.Name.In(expectedFields));

                                var data = rowGroupReader.ReadColumn((DataField)field).Data;
                                Assert.True(data.Length == 100);

                                if (field.Name == ParquetTransformedItems.LastModifiedColumn)
                                {
                                    continue;
                                }

                                var count = 0;
                                foreach (var val in data)
                                {
                                    if (field.Name == "OrderDate")
                                    {
                                        var expectedDto = new DateTimeOffset(DateTime.SpecifyKind(baseline.AddDays(count), DateTimeKind.Utc));
                                        Assert.Equal(expectedDto, val);
                                    }

                                    else
                                    {
                                        var expected = field.Name switch
                                        {
                                            ParquetTransformedItems.DefaultIdColumn => $"orders/{count}",
                                            "Company" => $"companies/{count}",
                                            "ShipVia" => $"shippers/{count}",
                                            _ => null
                                        };

                                        Assert.Equal(expected, val);
                                    }

                                    count++;
                                }
                            }
                        }
                    }
                }
            }
            finally
            {
                await DeleteObjects(settings);
            }
        }
        public async Task GetFolderPathOptions()
        {
            PeriodicBackupConnectionType connectionType;
            var type = GetStringValuesQueryString("type", false).FirstOrDefault();

            if (type == null)
            {
                //Backward compatibility
                connectionType = PeriodicBackupConnectionType.Local;
            }
            else if (Enum.TryParse(type, out connectionType) == false)
            {
                throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                var folderPathOptions = new FolderPathOptions();
                ;
                switch (connectionType)
                {
                case PeriodicBackupConnectionType.Local:
                    var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
                    var path           = GetStringQueryString("path", required: false);
                    folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
                    break;

                case PeriodicBackupConnectionType.S3:
                    var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && json == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var s3Settings = JsonDeserializationServer.S3Settings(json);
                    if (s3Settings == null)
                    {
                        throw new BadRequestException("No S3Settings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
                        string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
                        string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
                        string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
                    {
                        break;
                    }

                    using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
                    {
                        // fetching only the first 64 results for the auto complete
                        var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);

                        if (folders != null)
                        {
                            foreach (var folder in folders.FileInfoDetails)
                            {
                                var fullPath = folder.FullPath;
                                if (string.IsNullOrWhiteSpace(fullPath))
                                {
                                    continue;
                                }

                                folderPathOptions.List.Add(fullPath);
                            }
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.Azure:
                    var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
                    if (azureSettings == null)
                    {
                        throw new BadRequestException("No AzureSettings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
                        string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
                        string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
                    {
                        break;
                    }

                    using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
                    {
                        var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));

                        foreach (var folder in folders.List)
                        {
                            var fullPath = folder.Name;
                            if (string.IsNullOrWhiteSpace(fullPath))
                            {
                                continue;
                            }

                            folderPathOptions.List.Add(fullPath);
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.GoogleCloud:
                    var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");

                    if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
                    {
                        throw new BadRequestException("No JSON was posted.");
                    }

                    var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
                    if (googleCloudSettings == null)
                    {
                        throw new BadRequestException("No AzureSettings were found.");
                    }

                    if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
                        string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
                    {
                        break;
                    }

                    using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
                    {
                        var folders             = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
                        var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;

                        foreach (var folder in folders)
                        {
                            const char separator = '/';
                            var        splitted  = folder.Name.Split(separator);
                            var        result    = string.Join(separator, splitted.Take(requestedPathLength)) + separator;

                            if (string.IsNullOrWhiteSpace(result))
                            {
                                continue;
                            }

                            folderPathOptions.List.Add(result);
                        }
                    }
                    break;

                case PeriodicBackupConnectionType.FTP:
                case PeriodicBackupConnectionType.Glacier:
                    throw new NotSupportedException();

                default:
                    throw new ArgumentOutOfRangeException();
                }

                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    context.Write(writer, new DynamicJsonValue
                    {
                        [nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
                    });
                }
            }
        }
Beispiel #29
0
        public async Task SimpleTransformation_MultiplePartitions()
        {
            var settings = GetAzureSettings();
            var prefix   = $"{settings.RemoteFolderName}/{CollectionName}/";

            try
            {
                using (var store = GetDocumentStore())
                {
                    var baseline = DateTime.SpecifyKind(new DateTime(2020, 1, 1), DateTimeKind.Utc);

                    using (var session = store.OpenAsyncSession())
                    {
                        const int total = 31 + 28; // days in January + days in February

                        for (int i = 0; i < total; i++)
                        {
                            var orderedAt = baseline.AddDays(i);
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i}",
                                OrderedAt = orderedAt,
                                RequireAt = orderedAt.AddDays(7),
                                ShipVia   = $"shippers/{i}",
                                Company   = $"companies/{i}"
                            });
                        }

                        for (int i = 1; i <= 37; i++)
                        {
                            var index     = i + total;
                            var orderedAt = baseline.AddYears(1).AddMonths(1).AddDays(i);
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{index}",
                                OrderedAt = orderedAt,
                                RequireAt = orderedAt.AddDays(7),
                                ShipVia   = $"shippers/{index}",
                                Company   = $"companies/{index}"
                            });
                        }

                        await session.SaveChangesAsync();
                    }

                    var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0 && statistics.LoadErrors == 0);

                    var script = @"
var orderDate = new Date(this.OrderedAt);

loadToOrders(partitionBy(
    ['year', orderDate.getFullYear()],
    ['month', orderDate.getMonth() + 1]
),
    {
        Company : this.Company,
        ShipVia : this.ShipVia,
        RequireAt : this.RequireAt
    });
";
                    SetupAzureEtl(store, script, settings);

                    etlDone.Wait(TimeSpan.FromMinutes(1));

                    var expectedFields = new[] { "RequireAt", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };

                    using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
                    {
                        var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : "/", listFolders : true);

                        var list = cloudObjects.List.ToList();

                        Assert.Equal(2, list.Count);
                        Assert.Contains("Orders/year=2020/", list[0].Name);
                        Assert.Contains("Orders/year=2021/", list[1].Name);

                        for (var index = 1; index <= list.Count; index++)
                        {
                            var folder          = list[index - 1];
                            var objectsInFolder = await client.ListBlobsAsync(prefix : folder.Name, delimiter : "/", listFolders : true);

                            var objects = objectsInFolder.List.ToList();
                            Assert.Equal(2, objects.Count);
                            Assert.Contains($"month={index}/", objects[0].Name);
                            Assert.Contains($"month={index + 1}/", objects[1].Name);
                        }

                        var files = await ListAllFilesInFolders(client, list);

                        Assert.Equal(4, files.Count);

                        foreach (var filePath in files)
                        {
                            var blob = await client.GetBlobAsync(filePath);

                            await using var ms = new MemoryStream();
                            blob.Data.CopyTo(ms);

                            using (var parquetReader = new ParquetReader(ms))
                            {
                                Assert.Equal(1, parquetReader.RowGroupCount);
                                Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);

                                using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
                                foreach (var field in parquetReader.Schema.Fields)
                                {
                                    Assert.True(field.Name.In(expectedFields));
                                    var data = rowGroupReader.ReadColumn((DataField)field).Data;

                                    Assert.True(data.Length == 31 || data.Length == 28 || data.Length == 27 || data.Length == 10);
                                    if (field.Name != "RequireAt")
                                    {
                                        continue;
                                    }

                                    var count = data.Length switch
                                    {
                                        31 => 0,
                                        28 => 31,
                                        27 => 365 + 33,
                                        10 => 365 + 33 + 27,
                                        _ => throw new ArgumentOutOfRangeException()
                                    };

                                    foreach (var val in data)
                                    {
                                        var expectedOrderDate = new DateTimeOffset(DateTime.SpecifyKind(baseline.AddDays(count++), DateTimeKind.Utc));
                                        var expected          = expectedOrderDate.AddDays(7);
                                        Assert.Equal(expected, val);
                                    }
                                }
                            }
                        }
                    }
                }
            }
            finally
            {
                await DeleteObjects(settings, prefix, delimiter : "/", listFolder : true);
            }
        }
Beispiel #30
0
        public async Task CanUseCustomPrefix()
        {
            var settings = GetAzureSettings();

            try
            {
                using (var store = GetDocumentStore())
                {
                    var baseline = new DateTime(2020, 1, 1);

                    using (var session = store.OpenAsyncSession())
                    {
                        for (int i = 0; i < 31; i++)
                        {
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i}",
                                OrderedAt = baseline.AddDays(i),
                                ShipVia   = $"shippers/{i}",
                                Company   = $"companies/{i}"
                            });
                        }

                        for (int i = 0; i < 28; i++)
                        {
                            await session.StoreAsync(new Order
                            {
                                Id        = $"orders/{i + 31}",
                                OrderedAt = baseline.AddMonths(1).AddDays(i),
                                ShipVia   = $"shippers/{i + 31}",
                                Company   = $"companies/{i + 31}"
                            });
                        }

                        await session.SaveChangesAsync();
                    }

                    var          script          = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth() + 1;

loadToOrders(partitionBy(['year', year], ['month', month], ['source', $customPartitionValue]),
{
    Company : this.Company,
    ShipVia : this.ShipVia
});
";
                    const string customPartition = "shop-16";
                    var          etlDone         = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
                    SetupAzureEtl(store, script, settings, customPartition);

                    etlDone.Wait(TimeSpan.FromMinutes(1));

                    using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
                    {
                        var prefix       = $"{settings.RemoteFolderName}/{CollectionName}";
                        var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : string.Empty, listFolders : false);

                        var list = cloudObjects.List.ToList();

                        Assert.Equal(2, list.Count);
                        Assert.Contains($"/Orders/year=2020/month=1/source={customPartition}/", list[0].Name);
                        Assert.Contains($"/Orders/year=2020/month=2/source={customPartition}/", list[1].Name);
                    }
                }
            }

            finally
            {
                await DeleteObjects(settings, prefix : $"{settings.RemoteFolderName}/{CollectionName}", delimiter : string.Empty);
            }
        }