public async Task download_objects()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFactAttribute.GoogleCloudSettings))
{
try
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("123"))
);
var stream = client.DownloadObject(fileName);
using (var sr = new StreamReader(stream))
{
Assert.Equal("123", sr.ReadToEnd());
}
}
finally
{
await client.DeleteObjectAsync(fileName);
}
}
}
public async Task list_objects()
{
var file1 = "file1.txt";
var file2 = "folder1/file2.txt";
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
try
{
// Upload some files
var content = Encoding.UTF8.GetBytes("hello, world");
await client.UploadObjectAsync(file1, new MemoryStream(content));
await client.UploadObjectAsync(file2, new MemoryStream(content));
var objects = await client.ListObjectsAsync();
Assert.Contains(objects, o => o.Name == file1);
Assert.Contains(objects, o => o.Name == file2);
}
finally
{
await client.DeleteObjectAsync(file1);
await client.DeleteObjectAsync(file2);
}
}
}
public async Task upload_object_with_metadata()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
try
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("456")),
new Dictionary <string, string>
{
{ "key1", "value1" },
{ "key2", "value2" }
});
var obj = await client.GetObjectAsync(fileName);
Assert.Equal("value1", obj.Metadata["key1"]);
Assert.Equal("value2", obj.Metadata["key2"]);
}
finally
{
await client.DeleteObjectAsync(fileName);
}
}
}
public async Task download_objects()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
try
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("123"))
);
var file = new MemoryStream();
await client.DownloadObjectAsync(fileName, file);
Assert.Equal("123", Encoding.ASCII.GetString(file.ToArray()));
}
finally
{
await client.DeleteObjectAsync(fileName);
}
}
}
public override void Dispose()
{
base.Dispose();
var settings = GetGoogleCloudSettings();
if (settings == null)
{
return;
}
try
{
using (var client = new RavenGoogleCloudClient(settings))
{
var cloudObjects = client.ListObjectsAsync(settings.RemoteFolderName).GetAwaiter().GetResult();
foreach (var cloudObject in cloudObjects)
{
try
{
client.DeleteObjectAsync(cloudObject.Name).GetAwaiter().GetResult();
}
catch (Exception)
{
// ignored
}
}
}
}
catch (Exception)
{
// ignored
}
}
private async Task <bool> IsFileNameInCloudObjects(RavenGoogleCloudClient client, string fileName)
{
var cloudObjects = await client.ListObjectsAsync();
var containsFileName = cloudObjects?.Any(x => x.Name == fileName) ?? false;
return(containsFileName);
}
public void list_buckets()
{
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
var buckets = client.ListBuckets();
foreach (var b in buckets)
{
Assert.NotNull(b.Name);
}
}
}
public void list_buckets()
{
using (var client = new RavenGoogleCloudClient(GoogleCloudFactAttribute.GoogleCloudSettings, DefaultConfiguration))
{
var buckets = client.ListBuckets();
foreach (var b in buckets)
{
Assert.NotNull(b.Name);
}
}
}
private void DeleteFromGoogleCloud(GoogleCloudSettings settings)
{
using (var client = new RavenGoogleCloudClient(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.DeleteObject(key);
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportDeletion(GoogleCloudName)} storage bucket: {settings.BucketName}");
}
}
}
public async Task delete_objects()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("123"))
);
await client.DeleteObjectAsync(fileName);
Assert.Empty(await client.ListObjectsAsync());
}
}
private void UploadToGoogleCloud(GoogleCloudSettings settings, Stream stream, Progress progress)
{
using (var client = new RavenGoogleCloudClient(settings, _settings.Configuration, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.UploadObject(key, stream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription() }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportSuccess(GoogleCloudName)} storage bucket: {settings.BucketName}");
}
var runner = new GoogleCloudRetentionPolicyRunner(_retentionPolicyParameters, client);
runner.Execute();
}
}
public async Task uploading_objects()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFact.GoogleCloudSettings))
{
try
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("123"))
);
}
finally
{
await client.DeleteObjectAsync(fileName);
}
}
}
public async Task delete_objects()
{
var fileName = Guid.NewGuid().ToString();
using (var client = new RavenGoogleCloudClient(GoogleCloudFactAttribute.GoogleCloudSettings))
{
await client.UploadObjectAsync(
fileName,
new MemoryStream(Encoding.UTF8.GetBytes("123"))
);
var fileExists = await WaitForValueAsync(() => IsFileNameInCloudObjects(client, fileName), true);
Assert.True(fileExists);
await client.DeleteObjectAsync(fileName);
fileExists = await WaitForValueAsync(() => IsFileNameInCloudObjects(client, fileName), false);
Assert.False(fileExists);
}
}
private static async Task DeleteObjects(GoogleCloudSettings settings)
{
if (settings == null)
{
return;
}
try
{
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var all = await client.ListObjectsAsync(prefix : settings.RemoteFolderName);
foreach (var obj in all)
{
await client.DeleteObjectAsync(obj.Name);
}
}
}
catch (Exception)
{
// ignored
}
}
public RestoreFromGoogleCloud(ServerStore serverStore, RestoreFromGoogleCloudConfiguration restoreFromConfiguration, string nodeTag, OperationCancelToken operationCancelToken) : base(serverStore, restoreFromConfiguration, nodeTag, operationCancelToken)
{
_client = new RavenGoogleCloudClient(restoreFromConfiguration.Settings, serverStore.Configuration.Backup);
_remoteFolderName = restoreFromConfiguration.Settings.RemoteFolderName;
}
public GoogleCloudRestorePoints(Config.Categories.BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, GoogleCloudSettings client) : base(sortedList, context)
{
_configuration = configuration;
_client = new RavenGoogleCloudClient(client, configuration);
}
public GoogleCloudRetentionPolicyRunner(RetentionPolicyBaseParameters parameters, RavenGoogleCloudClient client)
: base(parameters)
{
_client = client;
}
public async Task CanLoadToMultipleTables()
{
const string salesTableName = "Sales";
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
var orderedAt = baseline.AddDays(i);
var lines = new List <OrderLine>();
for (int j = 1; j <= 5; j++)
{
lines.Add(new OrderLine
{
Quantity = j * 10,
PricePerUnit = i + j,
Product = $"Products/{j}"
});
}
var o = new Order
{
Id = $"orders/{i}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
Company = $"companies/{i}",
Lines = lines
};
await session.StoreAsync(o);
}
baseline = baseline.AddMonths(1);
for (int i = 0; i < 28; i++)
{
var orderedAt = baseline.AddDays(i);
var lines = new List <OrderLine>();
for (int j = 1; j <= 5; j++)
{
lines.Add(new OrderLine
{
Quantity = j * 10,
PricePerUnit = i + j,
Product = $"Products/{j}"
});
}
var o = new Order
{
Id = $"orders/{i + 31}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
Company = $"companies/{i}",
Lines = lines
};
await session.StoreAsync(o);
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderData = {
Company : this.Company,
RequireAt : new Date(this.RequireAt),
ItemsCount: this.Lines.length,
TotalCost: 0
};
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
for (var i = 0; i < this.Lines.length; i++) {
var line = this.Lines[i];
orderData.TotalCost += (line.PricePerUnit * line.Quantity);
// load to 'sales' table
loadToSales(partitionBy(key), {
Qty: line.Quantity,
Product: line.Product,
Cost: line.PricePerUnit
});
}
// load to 'orders' table
loadToOrders(partitionBy(key), orderData);
";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(2, cloudObjects.Count);
Assert.Contains("2020-01-01", cloudObjects[0].Name);
Assert.Contains("2020-02-01", cloudObjects[1].Name);
var fullPath = cloudObjects[0].Name;
var stream = client.DownloadObject(fullPath);
var ms = new MemoryStream();
await stream.CopyToAsync(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
var expectedFields = new[] { "Company", "RequireAt", "ItemsCount", "TotalCost", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 31);
}
}
}
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{salesTableName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(2, cloudObjects.Count);
Assert.Contains("2020-01-01", cloudObjects[0].Name);
Assert.Contains("2020-02-01", cloudObjects[1].Name);
var fullPath = cloudObjects[1].Name;
var stream = client.DownloadObject(fullPath);
var ms = new MemoryStream();
await stream.CopyToAsync(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
var expectedFields = new[] { "Qty", "Product", "Cost", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 28 * 5);
}
}
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task SimpleTransformation()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 1; i <= 10; i++)
{
var o = new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
Company = $"companies/{i}"
};
await session.StoreAsync(o);
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
loadToOrders(partitionBy(key),
{
Company : this.Company,
ShipVia : this.ShipVia
})
";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(1, cloudObjects.Count);
var fullPath = cloudObjects[0].Name;
var stream = client.DownloadObject(fullPath);
var ms = new MemoryStream();
stream.CopyTo(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
var expectedFields = new[] { "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 10);
if (field.Name == ParquetTransformedItems.LastModifiedColumn)
{
continue;
}
var count = 1;
foreach (var val in data)
{
switch (field.Name)
{
case ParquetTransformedItems.DefaultIdColumn:
Assert.Equal($"orders/{count}", val);
break;
case "Company":
Assert.Equal($"companies/{count}", val);
break;
}
count++;
}
}
}
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task CanModifyPartitionColumnName()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
const string partitionColumn = "order_date";
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
loadToOrders(partitionBy(['order_date', key]),
{
Company : this.Company,
ShipVia : this.ShipVia
})
";
var connectionStringName = $"{store.Database} to GCP";
var configuration = new OlapEtlConfiguration
{
Name = "olap-gcp-test",
ConnectionStringName = connectionStringName,
RunFrequency = LocalTests.DefaultFrequency,
Transforms =
{
new Transformation
{
Name = "MonthlyOrders",
Collections = new List <string>{
"Orders"
},
Script = script
}
}
};
SetupGoogleCloudOlapEtl(store, settings, configuration);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(2, cloudObjects.Count);
Assert.Contains($"{partitionColumn}=2020-01-01", cloudObjects[0].Name);
Assert.Contains($"{partitionColumn}=2020-02-01", cloudObjects[1].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task CanUploadToGoogleCloud()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
loadToOrders(partitionBy(key),
{
Company : this.Company,
ShipVia : this.ShipVia
})
";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(2, cloudObjects.Count);
Assert.Contains("2020-01-01", cloudObjects[0].Name);
Assert.Contains("2020-02-01", cloudObjects[1].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task SimpleTransformation_NoPartition()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1).ToUniversalTime();
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 100; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
loadToOrders(noPartition(),
{
OrderDate : this.OrderedAt
Company : this.Company,
ShipVia : this.ShipVia
});
";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(1, cloudObjects.Count);
var stream = client.DownloadObject(cloudObjects[0].Name);
var ms = new MemoryStream();
stream.CopyTo(ms);
using (var parquetReader = new ParquetReader(ms))
{
Assert.Equal(1, parquetReader.RowGroupCount);
var expectedFields = new[] { "OrderDate", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
Assert.Equal(expectedFields.Length, parquetReader.Schema.Fields.Count);
using var rowGroupReader = parquetReader.OpenRowGroupReader(0);
foreach (var field in parquetReader.Schema.Fields)
{
Assert.True(field.Name.In(expectedFields));
var data = rowGroupReader.ReadColumn((DataField)field).Data;
Assert.True(data.Length == 100);
if (field.Name == ParquetTransformedItems.LastModifiedColumn)
{
continue;
}
var count = 0;
foreach (var val in data)
{
if (field.Name == "OrderDate")
{
var expectedDto = new DateTimeOffset(DateTime.SpecifyKind(baseline.AddDays(count), DateTimeKind.Utc));
Assert.Equal(expectedDto, val);
}
else
{
var expected = field.Name switch
{
ParquetTransformedItems.DefaultIdColumn => $"orders/{count}",
"Company" => $"companies/{count}",
"ShipVia" => $"shippers/{count}",
_ => null
};
Assert.Equal(expected, val);
}
count++;
}
}
}
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task SimpleTransformation_MultiplePartitions()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = DateTime.SpecifyKind(new DateTime(2020, 1, 1), DateTimeKind.Utc);
using (var session = store.OpenAsyncSession())
{
const int total = 31 + 28; // days in January + days in February
for (int i = 0; i < total; i++)
{
var orderedAt = baseline.AddDays(i);
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 1; i <= 37; i++)
{
var index = i + total;
var orderedAt = baseline.AddYears(1).AddMonths(1).AddDays(i);
await session.StoreAsync(new Order
{
Id = $"orders/{index}",
OrderedAt = orderedAt,
RequireAt = orderedAt.AddDays(7),
ShipVia = $"shippers/{index}",
Company = $"companies/{index}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
loadToOrders(partitionBy(
['year', orderDate.getFullYear()],
['month', orderDate.getMonth() + 1]
),
{
Company : this.Company,
ShipVia : this.ShipVia,
RequireAt : this.RequireAt
});
";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
var expectedFields = new[] { "RequireAt", "ShipVia", "Company", ParquetTransformedItems.DefaultIdColumn, ParquetTransformedItems.LastModifiedColumn };
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}/";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(4, cloudObjects.Count);
Assert.Contains("Orders/year=2020/month=1", cloudObjects[0].Name);
Assert.Contains("Orders/year=2020/month=2", cloudObjects[1].Name);
Assert.Contains("Orders/year=2021/month=2", cloudObjects[2].Name);
Assert.Contains("Orders/year=2021/month=3", cloudObjects[3].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task CanPartitionByCustomDataFieldViaScript()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth() + 1;
loadToOrders(partitionBy(['year', year], ['month', month], ['source', $customPartitionValue]),
{
Company : this.Company,
ShipVia : this.ShipVia
});
";
const string customPartition = "shop-16";
SetupGoogleCloudOlapEtl(store, script, settings, customPartition: customPartition);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}/";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(2, cloudObjects.Count);
Assert.Contains($"/Orders/year=2020/month=1/source={customPartition}/", cloudObjects[0].Name);
Assert.Contains($"/Orders/year=2020/month=2/source={customPartition}/", cloudObjects[1].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task CanHandleSpecialCharsInFolderPath()
{
var settings = GetGoogleCloudSettings();
try
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
var today = DateTime.Today;
await session.StoreAsync(new Order
{
OrderedAt = today,
Lines = new List <OrderLine>
{
new OrderLine
{
ProductName = "Wimmers gute Semmelknödel",
PricePerUnit = 12
},
new OrderLine
{
ProductName = "Guaraná Fantástica",
PricePerUnit = 42
},
new OrderLine
{
ProductName = "Thüringer Rostbratwurst",
PricePerUnit = 19
}
}
});
await session.StoreAsync(new Order
{
OrderedAt = today.AddYears(-1),
Lines = new List <OrderLine>
{
new OrderLine
{
ProductName = "Uncle Bob's Cajon Sauce",
PricePerUnit = 11
},
new OrderLine
{
ProductName = "Côte de Blaye",
PricePerUnit = 25
}
}
});
await session.StoreAsync(new Order
{
OrderedAt = today.AddYears(-2),
Lines = new List <OrderLine>
{
new OrderLine
{
ProductName = "גבינה צהובה",
PricePerUnit = 20
},
new OrderLine
{
ProductName = "במבה",
PricePerUnit = 7
}
}
});
await session.SaveChangesAsync();
}
var etlDone = WaitForEtl(store, (n, statistics) => statistics.LoadSuccesses != 0);
var script = @"
for (var i = 0; i < this.Lines.length; i++){
var line = this.Lines[i];
loadToOrders(partitionBy(['product-name', line.ProductName]), {
PricePerUnit: line.PricePerUnit,
OrderedAt: this.OrderedAt
})}";
SetupGoogleCloudOlapEtl(store, script, settings);
etlDone.Wait(TimeSpan.FromMinutes(1));
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var prefix = $"{settings.RemoteFolderName}/{CollectionName}";
var cloudObjects = await client.ListObjectsAsync(prefix);
Assert.Equal(7, cloudObjects.Count);
Assert.Contains("Côte de Blaye", cloudObjects[0].Name);
Assert.Contains("Guaraná Fantástica", cloudObjects[1].Name);
Assert.Contains("Thüringer Rostbratwurst", cloudObjects[2].Name);
Assert.Contains("Uncle Bob's Cajon Sauce", cloudObjects[3].Name);
Assert.Contains("Wimmers gute Semmelknödel", cloudObjects[4].Name);
Assert.Contains("במבה", cloudObjects[5].Name);
Assert.Contains("גבינה צהובה", cloudObjects[6].Name);
}
}
}
finally
{
await DeleteObjects(settings);
}
}
public async Task TestPeriodicBackupCredentials()
{
var type = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");
if (Enum.TryParse(type, out PeriodicBackupConnectionType connectionType) == false)
{
throw new ArgumentException($"Unknown backup connection: {type}");
}
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
DynamicJsonValue result;
try
{
var connectionInfo = await context.ReadForMemoryAsync(RequestBodyStream(), "test-connection");
switch (connectionType)
{
case PeriodicBackupConnectionType.S3:
var s3Settings = JsonDeserializationClient.S3Settings(connectionInfo);
using (var awsClient = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
awsClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Glacier:
var glacierSettings = JsonDeserializationClient.GlacierSettings(connectionInfo);
using (var glacierClient = new RavenAwsGlacierClient(glacierSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
glacierClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Azure:
var azureSettings = JsonDeserializationClient.AzureSettings(connectionInfo);
using (var azureClient = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
azureClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudSettings = JsonDeserializationClient.GoogleCloudSettings(connectionInfo);
using (var googleCloudClient = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup, cancellationToken: ServerStore.ServerShutdown))
{
await googleCloudClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.FTP:
var ftpSettings = JsonDeserializationClient.FtpSettings(connectionInfo);
using (var ftpClient = new RavenFtpClient(ftpSettings))
{
ftpClient.TestConnection();
}
break;
case PeriodicBackupConnectionType.Local:
case PeriodicBackupConnectionType.None:
default:
throw new ArgumentOutOfRangeException();
}
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = true,
};
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = e.ToString()
};
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var folderPathOptions = new FolderPathOptions();
;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
break;
}
using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);
if (folders != null)
{
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
}
break;
case PeriodicBackupConnectionType.Azure:
var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
if (azureSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
{
break;
}
using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));
foreach (var folder in folders.List)
{
var fullPath = folder.Name;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
if (googleCloudSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
{
break;
}
using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;
foreach (var folder in folders)
{
const char separator = '/';
var splitted = folder.Name.Split(separator);
var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator;
if (string.IsNullOrWhiteSpace(result))
{
continue;
}
folderPathOptions.List.Add(result);
}
}
break;
case PeriodicBackupConnectionType.FTP:
case PeriodicBackupConnectionType.Glacier:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
public GoogleCloudRestorePoints(SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, GoogleCloudSettings client) : base(sortedList, context)
{
_client = new RavenGoogleCloudClient(client);
}
