public async Task PostScriptTest()
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var testConfig = await context.ReadForMemoryAsync(RequestBodyStream(), "ola-etl-test");
var testScript = JsonDeserializationServer.TestOlapEtlScript(testConfig);
using (OlapEtl.TestScript(testScript, Database, ServerStore, context, out var testResult))
{
var result = (OlapEtlTestScriptResult)testResult;
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(result);
writer.WriteObject(context.ReadObject(djv, "olap-etl-test"));
}
}
}
}
public async Task ShouldNotReuseCustomPartitionFromPreviousTestRun()
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var database = await GetDatabase(store.Database);
var configuration = new OlapEtlConfiguration
{
Name = "simulate",
Transforms =
{
new Transformation
{
Collections ={ "Orders" },
Name = "MonthlyOrders",
Script =
@"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
loadToOrders(partitionBy(['order_date', key], ['location', $customPartitionValue]),
{
Company : this.Company,
ShipVia : this.ShipVia
});
"
}
},
CustomPartitionValue = "USA"
};
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (OlapEtl.TestScript(new TestOlapEtlScript
{
DocumentId = "orders/1",
Configuration = configuration
}, database, database.ServerStore, context, out var testResult))
{
var result = (OlapEtlTestScriptResult)testResult;
Assert.Equal(1, result.ItemsByPartition.Count);
Assert.Equal(4, result.ItemsByPartition[0].Columns.Count);
Assert.Equal("Orders/order_date=2020-01-01-00-00/location=USA", result.ItemsByPartition[0].Key);
}
}
configuration.CustomPartitionValue = null;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (OlapEtl.TestScript(new TestOlapEtlScript
{
DocumentId = "orders/1",
Configuration = configuration
}, database, database.ServerStore, context, out var testResult))
{
var result = (OlapEtlTestScriptResult)testResult;
Assert.Equal(1, result.ItemsByPartition.Count);
Assert.Equal(4, result.ItemsByPartition[0].Columns.Count);
Assert.Equal("Orders/order_date=2020-01-01-00-00/location=undefined", result.ItemsByPartition[0].Key);
}
}
}
}
public async Task CanTestOlapEtlScript()
{
using (var store = GetDocumentStore())
{
var baseline = new DateTime(2020, 1, 1);
using (var session = store.OpenAsyncSession())
{
for (int i = 0; i < 31; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i}",
OrderedAt = baseline.AddDays(i),
ShipVia = $"shippers/{i}",
Company = $"companies/{i}"
});
}
for (int i = 0; i < 28; i++)
{
await session.StoreAsync(new Order
{
Id = $"orders/{i + 31}",
OrderedAt = baseline.AddMonths(1).AddDays(i),
ShipVia = $"shippers/{i + 31}",
Company = $"companies/{i + 31}"
});
}
await session.SaveChangesAsync();
}
var database = await GetDatabase(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (OlapEtl.TestScript(new TestOlapEtlScript
{
DocumentId = "orders/1",
Configuration = new OlapEtlConfiguration
{
Name = "simulate",
Transforms =
{
new Transformation
{
Collections ={ "Orders" },
Name = "MonthlyOrders",
Script =
@"
var orderDate = new Date(this.OrderedAt);
var year = orderDate.getFullYear();
var month = orderDate.getMonth();
var key = new Date(year, month);
output('test output')
loadToOrders(partitionBy(['order_date', key]),
{
Company : this.Company,
ShipVia : this.ShipVia
});
"
}
}
}
}, database, database.ServerStore, context, out var testResult))
{
var result = (OlapEtlTestScriptResult)testResult;
Assert.Equal(0, result.TransformationErrors.Count);
Assert.Equal(1, result.ItemsByPartition.Count);
Assert.Equal(4, result.ItemsByPartition[0].Columns.Count);
var companyColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == "Company");
Assert.Equal("companies/1", companyColumn.Values[0]);
var shipViaColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == "ShipVia");
Assert.Equal("shippers/1", shipViaColumn.Values[0]);
var idColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == ParquetTransformedItems.DefaultIdColumn);
Assert.Equal("orders/1", idColumn.Values[0]);
var lastModifiedColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == ParquetTransformedItems.LastModifiedColumn);
Assert.NotNull(lastModifiedColumn.Values[0]);
Assert.Equal("Orders/order_date=2020-01-01-00-00", result.ItemsByPartition[0].Key);
Assert.Equal("test output", result.DebugOutput[0]);
}
}
}
}
