public async Task PostScriptTest() { using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var testConfig = await context.ReadForMemoryAsync(RequestBodyStream(), "ola-etl-test"); var testScript = JsonDeserializationServer.TestOlapEtlScript(testConfig); using (OlapEtl.TestScript(testScript, Database, ServerStore, context, out var testResult)) { var result = (OlapEtlTestScriptResult)testResult; await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(result); writer.WriteObject(context.ReadObject(djv, "olap-etl-test")); } } } }
public async Task ShouldNotReuseCustomPartitionFromPreviousTestRun() { using (var store = GetDocumentStore()) { var baseline = new DateTime(2020, 1, 1); using (var session = store.OpenAsyncSession()) { for (int i = 0; i < 31; i++) { await session.StoreAsync(new Order { Id = $"orders/{i}", OrderedAt = baseline.AddDays(i), ShipVia = $"shippers/{i}", Company = $"companies/{i}" }); } for (int i = 0; i < 28; i++) { await session.StoreAsync(new Order { Id = $"orders/{i + 31}", OrderedAt = baseline.AddMonths(1).AddDays(i), ShipVia = $"shippers/{i + 31}", Company = $"companies/{i + 31}" }); } await session.SaveChangesAsync(); } var database = await GetDatabase(store.Database); var configuration = new OlapEtlConfiguration { Name = "simulate", Transforms = { new Transformation { Collections ={ "Orders" }, Name = "MonthlyOrders", Script = @" var orderDate = new Date(this.OrderedAt); var year = orderDate.getFullYear(); var month = orderDate.getMonth(); var key = new Date(year, month); loadToOrders(partitionBy(['order_date', key], ['location', $customPartitionValue]), { Company : this.Company, ShipVia : this.ShipVia }); " } }, CustomPartitionValue = "USA" }; using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (OlapEtl.TestScript(new TestOlapEtlScript { DocumentId = "orders/1", Configuration = configuration }, database, database.ServerStore, context, out var testResult)) { var result = (OlapEtlTestScriptResult)testResult; Assert.Equal(1, result.ItemsByPartition.Count); Assert.Equal(4, result.ItemsByPartition[0].Columns.Count); Assert.Equal("Orders/order_date=2020-01-01-00-00/location=USA", result.ItemsByPartition[0].Key); } } configuration.CustomPartitionValue = null; using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (OlapEtl.TestScript(new TestOlapEtlScript { DocumentId = "orders/1", Configuration = configuration }, database, database.ServerStore, context, out var testResult)) { var result = (OlapEtlTestScriptResult)testResult; Assert.Equal(1, result.ItemsByPartition.Count); Assert.Equal(4, result.ItemsByPartition[0].Columns.Count); Assert.Equal("Orders/order_date=2020-01-01-00-00/location=undefined", result.ItemsByPartition[0].Key); } } } }
public async Task CanTestOlapEtlScript() { using (var store = GetDocumentStore()) { var baseline = new DateTime(2020, 1, 1); using (var session = store.OpenAsyncSession()) { for (int i = 0; i < 31; i++) { await session.StoreAsync(new Order { Id = $"orders/{i}", OrderedAt = baseline.AddDays(i), ShipVia = $"shippers/{i}", Company = $"companies/{i}" }); } for (int i = 0; i < 28; i++) { await session.StoreAsync(new Order { Id = $"orders/{i + 31}", OrderedAt = baseline.AddMonths(1).AddDays(i), ShipVia = $"shippers/{i + 31}", Company = $"companies/{i + 31}" }); } await session.SaveChangesAsync(); } var database = await GetDatabase(store.Database); using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (OlapEtl.TestScript(new TestOlapEtlScript { DocumentId = "orders/1", Configuration = new OlapEtlConfiguration { Name = "simulate", Transforms = { new Transformation { Collections ={ "Orders" }, Name = "MonthlyOrders", Script = @" var orderDate = new Date(this.OrderedAt); var year = orderDate.getFullYear(); var month = orderDate.getMonth(); var key = new Date(year, month); output('test output') loadToOrders(partitionBy(['order_date', key]), { Company : this.Company, ShipVia : this.ShipVia }); " } } } }, database, database.ServerStore, context, out var testResult)) { var result = (OlapEtlTestScriptResult)testResult; Assert.Equal(0, result.TransformationErrors.Count); Assert.Equal(1, result.ItemsByPartition.Count); Assert.Equal(4, result.ItemsByPartition[0].Columns.Count); var companyColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == "Company"); Assert.Equal("companies/1", companyColumn.Values[0]); var shipViaColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == "ShipVia"); Assert.Equal("shippers/1", shipViaColumn.Values[0]); var idColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == ParquetTransformedItems.DefaultIdColumn); Assert.Equal("orders/1", idColumn.Values[0]); var lastModifiedColumn = result.ItemsByPartition[0].Columns.First(x => x.Name == ParquetTransformedItems.LastModifiedColumn); Assert.NotNull(lastModifiedColumn.Values[0]); Assert.Equal("Orders/order_date=2020-01-01-00-00", result.ItemsByPartition[0].Key); Assert.Equal("test output", result.DebugOutput[0]); } } } }