public void PutBlob()
{
var containerName = "testContainer";
var blobKey = "testKey";
using (var client = new RavenAzureClient("devstoreaccount1", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="))
{
client.PutContainer(containerName);
client.PutBlob(containerName, blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
{
{ "property1", "value1" },
{ "property2", "value2" }
});
var blob = client.GetBlob(containerName, blobKey);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal("123", reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal("value1", blob.Metadata[property1]);
Assert.Equal("value2", blob.Metadata[property2]);
}
}
public void CanRemoveBlobsInBatch()
{
var containerName = Guid.NewGuid().ToString();
var blobKey = Guid.NewGuid().ToString();
using var client = new RavenAzureClient(GetAzureSettings(containerName));
var blobs = new List <string>();
try
{
client.DeleteContainer();
client.PutContainer();
for (int i = 0; i < 10; i++)
{
var key = $"{blobKey}/northwind_{i}.ravendump";
var tmpArr = new byte[3];
new Random().NextBytes(tmpArr);
client.PutBlob(key, new MemoryStream(tmpArr), new Dictionary <string, string> {
{ $"property_{i}", $"value_{i}" }
});
var blob = client.GetBlob(key);
Assert.NotNull(blob);
blobs.Add(key);
}
client.DeleteMultipleBlobs(blobs);
}
finally
{
client.DeleteContainer();
}
}
public async Task can_delete_backups_by_date_azure(int backupAgeInSeconds, int numberOfBackupsToCreate, bool checkIncremental)
{
await Locker.WaitAsync();
var containerName = Guid.NewGuid().ToString();
using var client = new RavenAzureClient(new AzureSettings { AccountName = Azure.AzureAccountName, AccountKey = Azure.AzureAccountKey, StorageContainer = containerName });
try
{
client.DeleteContainer();
client.PutContainer();
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = true;
await CanDeleteBackupsByDate(backupAgeInSeconds, numberOfBackupsToCreate,
(configuration, databaseName) =>
{
configuration.AzureSettings = GetAzureSettings(containerName, databaseName);
},
async databaseName =>
{
using var c = new RavenAzureClient(GetAzureSettings(containerName, databaseName));
var folders = await c.ListBlobsAsync($"{c.RemoteFolderName}/", delimiter: "/", listFolders: true);
return(folders.ListBlob.Count());
}, timeout : 120000, checkIncremental);
}
finally
{
BackupConfigurationHelper.SkipMinimumBackupAgeToKeepValidation = false;
client.DeleteContainer();
Locker.Release();
}
}
public void put_blob()
{
var containerName = Guid.NewGuid().ToString();
var blobKey = Guid.NewGuid().ToString();
using (var client = new RavenAzureClient(GenerateAzureSettings(containerName)))
{
try
{
client.DeleteContainer();
client.PutContainer();
client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
{
{ "property1", "value1" },
{ "property2", "value2" }
});
var blob = client.GetBlob(blobKey);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal("123", reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal("value1", blob.Metadata[property1]);
Assert.Equal("value2", blob.Metadata[property2]);
}
finally
{
client.DeleteContainer();
}
}
}
public async Task PutBlobIntoFolder()
{
var containerName = "testContainer";
var blobKey = "folder1/folder2/testKey";
using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
{
await client.PutContainer();
await client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes("123")), new Dictionary <string, string>
{
{ "property1", "value1" },
{ "property2", "value2" }
});
var blob = await client.GetBlob(blobKey);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal("123", reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal("value1", blob.Metadata[property1]);
Assert.Equal("value2", blob.Metadata[property2]);
}
}
private async Task UploadToAzure(string exportPath, string fileName, bool isFullExport)
{
if (_azureStorageAccount == Constants.DataCouldNotBeDecrypted ||
_azureStorageKey == Constants.DataCouldNotBeDecrypted)
{
throw new InvalidOperationException("Could not decrypt the Azure access settings, if you are running on IIS, make sure that load user profile is set to true.");
}
using (var client = new RavenAzureClient(_azureStorageAccount, _azureStorageKey, _configuration.AzureStorageContainer))
{
await client.PutContainer();
using (var fileStream = File.OpenRead(exportPath))
{
var key = CombinePathAndKey(_configuration.AzureRemoteFolderName, fileName);
await client.PutBlob(key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullExport) }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"Successfully uploaded export {fileName} to Azure container {_configuration.AzureStorageContainer}, with key {key}");
}
}
}
}
private void UploadToAzure(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
{
if (azureStorageAccount == Constants.DataCouldNotBeDecrypted ||
azureStorageKey == Constants.DataCouldNotBeDecrypted)
{
throw new InvalidOperationException("Could not decrypt the AWS access settings, if you are running on IIS, make sure that load user profile is set to true.");
}
using (var client = new RavenAzureClient(azureStorageAccount, azureStorageKey, true))
{
client.PutContainer(localExportConfigs.AzureStorageContainer);
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
client.PutBlob(localExportConfigs.AzureStorageContainer, key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullBackup) }
});
logger.Info(string.Format(
"Successfully uploaded backup {0} to Azure container {1}, with key {2}",
Path.GetFileName(backupPath),
localExportConfigs.AzureStorageContainer,
key));
}
}
}
public void list_blobs_()
{
var containerName = "mycontainer";
var blobKey1 = $"{Guid.NewGuid()}/folder/testKey";
var blobKey2 = $"{Guid.NewGuid()}/folder/testKey";
using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
{
try
{
client.DeleteContainer();
client.PutContainer();
var path = NewDataPath(forceCreateDir: true);
var filePath1 = Path.Combine(path, Guid.NewGuid().ToString()) + ".txt";
var filePath2 = Path.Combine(path, Guid.NewGuid().ToString()) + ".txt";
File.WriteAllText(filePath1, "abc", Encoding.UTF8);
File.WriteAllText(filePath2, "def", Encoding.UTF8);
using (var file1 = File.Open(filePath1, FileMode.Open))
using (var file2 = File.Open(filePath2, FileMode.Open))
{
client.PutBlob(blobKey1, file1, new Dictionary <string, string>());
client.PutBlob(blobKey2, file2, new Dictionary <string, string>());
}
// Assert.Equal((await client.ListBlobs(containerName)).Count, 2);
}
finally
{
client.DeleteContainer();
}
}
}
private void InitContainer(AzureSettings azureSettings)
{
using (var client = new RavenAzureClient(azureSettings))
{
client.DeleteContainer();
client.PutContainer();
}
}
public AzureClientHolder(AzureSettings setting, Progress progress = null, [CallerMemberName] string caller = null)
{
Assert.False(string.IsNullOrEmpty(setting.StorageContainer), "string.IsNullOrEmpty(setting.StorageContainer)");
Settings = setting;
// keep only alphanumeric characters
Settings.RemoteFolderName = _remoteFolder = GetRemoteFolder(caller);
Client = RavenAzureClient.Create(Settings, DefaultConfiguration, progress);
}
public AzureClientHolder(AzureSettings setting, Progress progress = null, [CallerMemberName] string caller = null)
{
Assert.False(string.IsNullOrEmpty(setting.StorageContainer), "string.IsNullOrEmpty(setting.StorageContainer)");
// keep only alphanumeric characters
caller = caller == null ? string.Empty : string.Concat(caller.Where(char.IsLetterOrDigit));
_remoteFolderName = setting.RemoteFolderName = $"{caller}{Guid.NewGuid()}";
Client = new RavenAzureClient(setting, progress);
}
private void DeleteFromAzure(AzureSettings settings)
{
using (var client = RavenAzureClient.Create(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.DeleteBlobs(new List <string> {
key
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportDeletion(AzureName)} container: {settings.StorageContainer}, with key: {key}");
}
}
}
public void can_get_container_not_found()
{
var containerName = Guid.NewGuid().ToString();
using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
{
var containerNames = client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
var e = Assert.Throws <ContainerNotFoundException>(() => client.TestConnection());
Assert.Equal($"Container '{containerName}' wasn't found!", e.Message);
containerNames = client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
}
}
public void RemoveNonExistingBlobsInBatchShouldThrow()
{
var containerName = Guid.NewGuid().ToString();
var blobKey = Guid.NewGuid().ToString();
using var client = new RavenAzureClient(GenerateAzureSettings(containerName));
var blobs = new List <string>();
try
{
client.DeleteContainer();
client.PutContainer();
// put blob
var k = $"{blobKey}/northwind_322.ravendump";
var tmpArr = new byte[3];
new Random().NextBytes(tmpArr);
client.PutBlob(k, new MemoryStream(tmpArr), new Dictionary <string, string>
{
{ "Nice", "NotNice" }
});
var blob = client.GetBlob(k);
Assert.NotNull(blob);
blobs.Add(k);
for (int i = 0; i < 10; i++)
{
blobs.Add($"{blobKey}/northwind_{i}.ravendump");
}
try
{
client.DeleteMultipleBlobs(blobs);
}
catch (Exception e)
{
Assert.Equal(typeof(InvalidOperationException), e.GetType());
Assert.True(e.Message.StartsWith($"Failed to delete {blobs.Count - 1} blobs from container: {containerName}. Successfully deleted 1 blob. Reason: The specified blob does not exist."));
}
}
finally
{
client.DeleteContainer();
}
}
public void can_get_and_delete_container()
{
var containerName = Guid.NewGuid().ToString();
using (var client = new RavenAzureClient(Azure.GetAzureSettings(containerName)))
{
var containerNames = client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
client.PutContainer();
containerNames = client.GetContainerNames(500);
Assert.True(containerNames.Exists(x => x.Equals(containerName)));
client.DeleteContainer();
containerNames = client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
}
}
private void UploadToAzure(AzureSettings settings, Stream stream, Progress progress)
{
using (var client = RavenAzureClient.Create(settings, _settings.Configuration, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.PutBlob(key, stream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription() }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportSuccess(AzureName)} container: {settings.StorageContainer}, with key: {key}");
}
var runner = new AzureRetentionPolicyRunner(_retentionPolicyParameters, client);
runner.Execute();
}
}
public async Task can_get_container_not_found()
{
var containerName = Guid.NewGuid().ToString();
using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
{
var containerNames = await client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
var e = await Assert.ThrowsAsync <ContainerNotFoundException>(async() => await client.TestConnection());
Assert.Equal($"Container '{containerName}' not found!", e.Message);
containerNames = await client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
}
}
private void UploadToAzure(string backupPath, PeriodicExportSetup localExportConfigs, bool isFullBackup)
{
using (var client = new RavenAzureClient(azureStorageAccount, azureStorageKey, true))
{
client.PutContainer(localExportConfigs.AzureStorageContainer);
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
client.PutBlob(localExportConfigs.AzureStorageContainer, key, fileStream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription(isFullBackup) }
});
logger.Info(string.Format(
"Successfully uploaded backup {0} to Azure container {1}, with key {2}",
Path.GetFileName(backupPath),
localExportConfigs.AzureStorageContainer,
key));
}
}
}
public async Task can_get_and_delete_container()
{
var containerName = Guid.NewGuid().ToString();
using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
{
var containerNames = await client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
await client.PutContainer();
containerNames = await client.GetContainerNames(500);
Assert.True(containerNames.Exists(x => x.Equals(containerName)));
await client.DeleteContainer();
containerNames = await client.GetContainerNames(500);
Assert.False(containerNames.Exists(x => x.Equals(containerName)));
}
}
public async Task put_blob_100MB()
{
var containerName = "testContainer";
var blobKey = "testKey";
using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, isTest: true))
{
await client.PutContainer();
var sb = new StringBuilder();
for (var i = 0; i < 100 * 1024 * 1024; i++)
{
sb.Append("a");
}
await client.PutBlob(blobKey, new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())),
new Dictionary <string, string>
{
{ "property1", "value1" },
{ "property2", "value2" },
{ "property3", "value3" }
});
var blob = await client.GetBlob(blobKey);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal(sb.ToString(), reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
var property3 = blob.Metadata.Keys.Single(x => x.Contains("property3"));
Assert.Equal("value1", blob.Metadata[property1]);
Assert.Equal("value2", blob.Metadata[property2]);
Assert.Equal("value3", blob.Metadata[property3]);
}
}
private static async Task DeleteObjects(AzureSettings azureSettings, string prefix, string delimiter, bool listFolder = false)
{
if (azureSettings == null)
{
return;
}
try
{
using (var client = RavenAzureClient.Create(azureSettings, DefaultBackupConfiguration))
{
var result = await client.ListBlobsAsync(prefix, delimiter, listFolder);
List <string> filesToDelete;
if (listFolder == false)
{
filesToDelete = result.List.Select(b => b.Name).ToList();
}
else
{
filesToDelete = await ListAllFilesInFolders(client, result.List);
}
if (filesToDelete.Count == 0)
{
return;
}
client.DeleteBlobs(filesToDelete);
}
}
catch (Exception)
{
// ignored
}
}
private async Task UploadToAzure(
AzureSettings settings,
Stream stream,
string folderName,
string fileName,
Progress progress,
string archiveDecription)
{
using (var client = new RavenAzureClient(settings.AccountName, settings.AccountKey,
settings.StorageContainer, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName, folderName, fileName);
await client.PutBlob(key, stream, new Dictionary <string, string>
{
{ "Description", archiveDecription }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"Successfully uploaded backup file '{fileName}' " +
$"to Azure container: {settings.StorageContainer}, with key: {key}");
}
}
}
public async Task TestPeriodicBackupCredentials()
{
var type = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");
if (Enum.TryParse(type, out PeriodicBackupConnectionType connectionType) == false)
{
throw new ArgumentException($"Unknown backup connection: {type}");
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
DynamicJsonValue result;
try
{
var connectionInfo = await context.ReadForMemoryAsync(RequestBodyStream(), "test-connection");
switch (connectionType)
{
case PeriodicBackupConnectionType.S3:
var s3Settings = JsonDeserializationClient.S3Settings(connectionInfo);
using (var awsClient = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
awsClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Glacier:
var glacierSettings = JsonDeserializationClient.GlacierSettings(connectionInfo);
using (var glacierClient = new RavenAwsGlacierClient(glacierSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
glacierClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Azure:
var azureSettings = JsonDeserializationClient.AzureSettings(connectionInfo);
using (var azureClient = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
azureClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudSettings = JsonDeserializationClient.GoogleCloudSettings(connectionInfo);
using (var googleCloudClient = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
await googleCloudClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.FTP:
var ftpSettings = JsonDeserializationClient.FtpSettings(connectionInfo);
using (var ftpClient = new RavenFtpClient(ftpSettings))
{
ftpClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Local:
case PeriodicBackupConnectionType.None:
default:
throw new ArgumentOutOfRangeException();
}
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = true,
};
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = e.ToString()
};
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
}
// ReSharper disable once InconsistentNaming
private async Task PutBlob(int sizeInMB, bool testBlobKeyAsFolder, UploadType uploadType)
{
var containerName = Guid.NewGuid().ToString();
var blobKey = testBlobKeyAsFolder == false?
Guid.NewGuid().ToString() :
$"{Guid.NewGuid()}/folder/testKey";
var uploadProgress = new UploadProgress();
using (var client = new RavenAzureClient(AzureAccountName, AzureAccountKey, containerName, uploadProgress, isTest: true))
{
try
{
await client.DeleteContainer();
await client.PutContainer();
var sb = new StringBuilder();
for (var i = 0; i < sizeInMB * 1024 * 1024; i++)
{
sb.Append("a");
}
var value1 = Guid.NewGuid().ToString();
var value2 = Guid.NewGuid().ToString();
var value3 = Guid.NewGuid().ToString();
long streamLength;
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
streamLength = memoryStream.Length;
await client.PutBlob(blobKey, memoryStream,
new Dictionary <string, string>
{
{ "property1", value1 },
{ "property2", value2 },
{ "property3", value3 }
});
}
var blob = await client.GetBlob(blobKey);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal(sb.ToString(), reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
var property3 = blob.Metadata.Keys.Single(x => x.Contains("property3"));
Assert.Equal(value1, blob.Metadata[property1]);
Assert.Equal(value2, blob.Metadata[property2]);
Assert.Equal(value3, blob.Metadata[property3]);
Assert.Equal(UploadState.Done, uploadProgress.UploadState);
Assert.Equal(uploadType, uploadProgress.UploadType);
Assert.Equal(streamLength, uploadProgress.TotalInBytes);
Assert.Equal(streamLength, uploadProgress.UploadedInBytes);
}
finally
{
await client.DeleteContainer();
}
}
}
public AzureRestorePoints(BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, AzureSettings azureSettings) : base(sortedList, context)
{
_client = RavenAzureClient.Create(azureSettings, configuration);
}
public async Task CanModifyPartitionColumnName()
{
var settings = GetAzureSettings();
try
{
using (var store = GetDocumentStore())
{
const string partitionColumn = "order_date";
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
loadToOrders(partitionBy(['order_date', key]),
{
Company : this.Company,
ShipVia : this.ShipVia
})
";
var connectionStringName = $"{store.Database} to Azure";
var configuration = new OlapEtlConfiguration
{
Name = "olap-azure-test",
ConnectionStringName = connectionStringName,
RunFrequency = LocalTests.DefaultFrequency,
Transforms =
{
new Transformation
{
Name = "MonthlyOrders",
Collections = new List <string>{
"Orders"
},
Script = script
}
}
};
SetupAzureEtl(store, settings, configuration);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListBlobsAsync(prefix, string.Empty, false);
var list = cloudObjects.List.ToList();
Assert.Equal(2, list.Count);
Assert.Contains($"{partitionColumn}=2020-01-01", list[0].Name);
Assert.Contains($"{partitionColumn}=2020-02-01", list[1].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task SimpleTransformation_NoPartition()
{
var settings = GetAzureSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1).ToUniversalTime();
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 100; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
loadToOrders(noPartition(),
{
OrderDate : this.OrderedAt
Company : this.Company,
ShipVia : this.ShipVia
});
";
SetupAzureEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : string.Empty, listFolders : false);
var list = cloudObjects.List.ToList();
Assert.Equal(1, list.Count);
var blob = await client.GetBlobAsync(list[0].Name);
await using var ms = new MemoryStream();
blob.Data.CopyTo(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
var expectedFields = new[] { "OrderDate", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 100);
if (field.Name == ParquetTransformedItems.LastModifiedColumn)
{
continue;
}
var count = 0;
foreach (var val in data)
{
if (field.Name == "OrderDate")
{
var expectedDto = new DateTimeOffset(DateTime.SpecifyKind(baseline.AddDays(count), DateTimeKind.Utc));
Assert.Equal(expectedDto, val);
}
else
{
var expected = field.Name switch
{
ParquetTransformedItems.DefaultIdColumn => $"orders/{count}",
"Company" => $"companies/{count}",
"ShipVia" => $"shippers/{count}",
_ => null
};
Assert.Equal(expected, val);
}
count++;
}
}
}
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var folderPathOptions = new FolderPathOptions();
;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
break;
}
using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);
if (folders != null)
{
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
}
break;
case PeriodicBackupConnectionType.Azure:
var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
if (azureSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
{
break;
}
using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));
foreach (var folder in folders.List)
{
var fullPath = folder.Name;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
if (googleCloudSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
{
break;
}
using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;
foreach (var folder in folders)
{
const char separator = '/';
var splitted = folder.Name.Split(separator);
var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator;
if (string.IsNullOrWhiteSpace(result))
{
continue;
}
folderPathOptions.List.Add(result);
}
}
break;
case PeriodicBackupConnectionType.FTP:
case PeriodicBackupConnectionType.Glacier:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
public async Task SimpleTransformation_MultiplePartitions()
{
var settings = GetAzureSettings();
var prefix = $"{settings.RemoteFolderName}/{CollectionName}/";
try
{
using (var store = GetDocumentStore())
{
var baseline = DateTime.SpecifyKind(new DateTime(2020, 1, 1), DateTimeKind.Utc);
using (var session = store.OpenAsyncSession())
{
const int total = 31 + 28; // days in January + days in February
for (int i = 0; i < total; i++)
{
var orderedAt = baseline.AddDays(i);
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 1; i <= 37; i++)
{
var index = i + total;
var orderedAt = baseline.AddYears(1).AddMonths(1).AddDays(i);
await session.StoreAsync(new Order
{
Id = $"orders/{index}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
ShipVia = $"shippers/{index}",
Company = $"companies/{index}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0 && statistics.LoadErrors == 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
loadToOrders(partitionBy(
['year', orderDate.getFullYear()],
['month', orderDate.getMonth() + 1]
),
{
Company : this.Company,
ShipVia : this.ShipVia,
RequireAt : this.RequireAt
});
";
SetupAzureEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
var expectedFields = new[] { "RequireAt", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
{
var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : "/", listFolders : true);
var list = cloudObjects.List.ToList();
Assert.Equal(2, list.Count);
Assert.Contains("Orders/year=2020/", list[0].Name);
Assert.Contains("Orders/year=2021/", list[1].Name);
for (var index = 1; index <= list.Count; index++)
{
var folder = list[index - 1];
var objectsInFolder = await client.ListBlobsAsync(prefix : folder.Name, delimiter : "/", listFolders : true);
var objects = objectsInFolder.List.ToList();
Assert.Equal(2, objects.Count);
Assert.Contains($"month={index}/", objects[0].Name);
Assert.Contains($"month={index + 1}/", objects[1].Name);
}
var files = await ListAllFilesInFolders(client, list);
Assert.Equal(4, files.Count);
foreach (var filePath in files)
{
var blob = await client.GetBlobAsync(filePath);
await using var ms = new MemoryStream();
blob.Data.CopyTo(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 31 || data.Length == 28 || data.Length == 27 || data.Length == 10);
if (field.Name != "RequireAt")
{
continue;
}
var count = data.Length switch
{
31 => 0,
28 => 31,
27 => 365 + 33,
10 => 365 + 33 + 27,
_ => throw new ArgumentOutOfRangeException()
};
foreach (var val in data)
{
var expectedOrderDate = new DateTimeOffset(DateTime.SpecifyKind(baseline.AddDays(count++), DateTimeKind.Utc));
var expected = expectedOrderDate.AddDays(7);
Assert.Equal(expected, val);
}
}
}
}
}
}
}
finally
{
await DeleteObjects(settings, prefix, delimiter : "/", listFolder : true);
}
}
public async Task CanUseCustomPrefix()
{
var settings = GetAzureSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth() + 1;
loadToOrders(partitionBy(['year', year], ['month', month], ['source', $customPartitionValue]),
{
Company : this.Company,
ShipVia : this.ShipVia
});
";
const string customPartition = "shop-16";
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
SetupAzureEtl(store, script, settings, customPartition);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = RavenAzureClient.Create(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListBlobsAsync(prefix, delimiter : string.Empty, listFolders : false);
var list = cloudObjects.List.ToList();
Assert.Equal(2, list.Count);
Assert.Contains($"/Orders/year=2020/month=1/source={customPartition}/", list[0].Name);
Assert.Contains($"/Orders/year=2020/month=2/source={customPartition}/", list[1].Name);
}
}
}
finally
{
await DeleteObjects(settings, prefix : $"{settings.RemoteFolderName}/{CollectionName}", delimiter : string.Empty);
}
}
