public async Task PostScriptTest()
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var testConfig = await context.ReadForMemoryAsync(RequestBodyStream(), "TestRavenEtlScript");
var testScript = JsonDeserializationServer.TestRavenEtlScript(testConfig);
var result = (RavenEtlTestScriptResult)RavenEtl.TestScript(testScript, Database, ServerStore, context);
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
var defaultConventions = new DocumentConventions();
var djv = new DynamicJsonValue()
{
[nameof(result.Commands)] = new DynamicJsonArray(result.Commands.Select(x => x.ToJson(defaultConventions, context))),
[nameof(result.TransformationErrors)] = new DynamicJsonArray(result.TransformationErrors.Select(x => x.ToJson())),
[nameof(result.DebugOutput)] = new DynamicJsonArray(result.DebugOutput)
};
writer.WriteObject(context.ReadObject(djv, "et/raven/test"));
}
}
}
public async Task CanTestEmptyScript()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Order());
await session.SaveChangesAsync();
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var result = (RavenEtlTestScriptResult)RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "orders/1-A",
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections =
{
"Orders"
},
Name = "OrdersAndLines",
Script = null
}
}
}
}, database, database.ServerStore, context);
Assert.Equal(0, result.TransformationErrors.Count);
Assert.Equal(1, result.Commands.Count);
Assert.IsType(typeof(PutCommandDataWithBlittableJson), result.Commands[0]);
Assert.Empty(result.DebugOutput);
}
}
}
}
public void ShouldThrowIfTestingOnDocumentBelongingToDifferentCollection()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Order());
session.SaveChanges();
}
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var ex = Assert.Throws <InvalidOperationException>(() => RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "orders/1-A",
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections =
{
"DifferentCollection"
},
Name = "test",
Script =
@"
loadToDifferentCollection(this);"
}
}
}
}, database, database.ServerStore, context));
Assert.Contains(
"Document 'orders/1-A' belongs to Orders collection while tested ETL script works on the following collections: DifferentCollection",
ex.Message);
}
}
}
public void CanTestScriptSpecifiedOnMultipleCollections()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Order());
session.SaveChanges();
}
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "orders/1-A",
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections =
{
"Orders",
"SpecialOrders"
},
Name = "test",
Script =
@"
loadToOrders(this);"
}
}
}
}, database, database.ServerStore, context);
}
}
}
public async Task CanTestDeletion()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Order
{
Lines = new List <OrderLine>
{
new OrderLine {
PricePerUnit = 3, Product = "Milk", Quantity = 3
},
new OrderLine {
PricePerUnit = 4, Product = "Bear", Quantity = 2
},
}
});
await session.SaveChangesAsync();
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "orders/1-A",
IsDelete = true,
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections ={ "Orders" },
Name = "OrdersAndLines",
Script =
@"
var orderData = {
Id: id(this),
LinesCount: this.Lines.length,
TotalCost: 0
};
for (var i = 0; i < this.Lines.length; i++) {
var line = this.Lines[i];
orderData.TotalCost += line.PricePerUnit * line.Quantity;
loadToOrderLines({
OrderId: id(this),
Qty: line.Quantity,
Product: line.Product,
Cost: line.PricePerUnit
});
}
loadToOrders(orderData);
"
}
}
}
}, database, database.ServerStore, context, out var testResult))
{
var result = (RavenEtlTestScriptResult)testResult;
Assert.Equal(0, result.TransformationErrors.Count);
Assert.Equal(2, result.Commands.Count);
Assert.IsType(typeof(DeletePrefixedCommandData), result.Commands[0]);
Assert.IsType(typeof(DeleteCommandData), result.Commands[1]);
}
}
}
using (var session = store.OpenAsyncSession())
{
Assert.NotNull(session.Query <Order>("orders/1-A"));
}
}
}
public async Task CanOutputInDeleteBehaviorFunction()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User()
{
Name = "Joe"
}, "users/1");
await session.SaveChangesAsync();
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "users/1",
IsDelete = true,
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections ={ "Users" },
Name = "Users",
Script =
@"
loadToUsers(this);
function deleteDocumentsOfUsersBehavior(docId) {
output('document: ' + docId);
return false;
}
"
}
}
}
}, database, database.ServerStore, context, out var testResult))
{
var result = (RavenEtlTestScriptResult)testResult;
Assert.Equal(0, result.TransformationErrors.Count);
Assert.Equal(0, result.Commands.Count);
Assert.Equal("document: users/1", result.DebugOutput[0]);
}
}
}
using (var session = store.OpenAsyncSession())
{
Assert.NotNull(session.Query <Order>("users/1"));
}
}
}
private IEnumerable <EtlProcess> GetRelevantProcesses <T, TConnectionString>(List <T> configurations, HashSet <string> uniqueNames) where T : EtlConfiguration <TConnectionString> where TConnectionString : ConnectionString
{
foreach (var config in configurations)
{
SqlEtlConfiguration sqlConfig = null;
RavenEtlConfiguration ravenConfig = null;
var connectionStringNotFound = false;
switch (config.EtlType)
{
case EtlType.Raven:
ravenConfig = config as RavenEtlConfiguration;
if (_databaseRecord.RavenConnectionStrings.TryGetValue(config.ConnectionStringName, out var ravenConnection))
{
ravenConfig.Initialize(ravenConnection);
}
else
{
connectionStringNotFound = true;
}
break;
case EtlType.Sql:
sqlConfig = config as SqlEtlConfiguration;
if (_databaseRecord.SqlConnectionStrings.TryGetValue(config.ConnectionStringName, out var sqlConnection))
{
sqlConfig.Initialize(sqlConnection);
}
else
{
connectionStringNotFound = true;
}
break;
default:
ThrownUnknownEtlConfiguration(config.GetType());
break;
}
if (connectionStringNotFound)
{
LogConfigurationError(config,
new List <string>
{
$"Connection string named '{config.ConnectionStringName}' was not found for {config.EtlType} ETL"
});
continue;
}
if (config.Disabled)
{
continue;
}
if (ValidateConfiguration(config, uniqueNames) == false)
{
continue;
}
var processState = GetProcessState(config.Transforms, _database, config.Name);
var whoseTaskIsIt = _database.WhoseTaskIsIt(_databaseRecord.Topology, config, processState);
if (whoseTaskIsIt != _serverStore.NodeTag)
{
continue;
}
foreach (var transform in config.Transforms)
{
if (transform.Disabled)
{
continue;
}
EtlProcess process = null;
if (sqlConfig != null)
{
process = new SqlEtl(transform, sqlConfig, _database, _serverStore);
}
if (ravenConfig != null)
{
process = new RavenEtl(transform, ravenConfig, _database, _serverStore);
}
yield return(process);
}
}
}
public async Task CanTestScript()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Order
{
Lines = new List <OrderLine>
{
new OrderLine {
PricePerUnit = 3, Product = "Milk", Quantity = 3
},
new OrderLine {
PricePerUnit = 4, Product = "Bear", Quantity = 2
},
}
});
await session.SaveChangesAsync();
var database = GetDatabase(store.Database).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var result = (RavenEtlTestScriptResult)RavenEtl.TestScript(new TestRavenEtlScript
{
DocumentId = "orders/1-A",
Configuration = new RavenEtlConfiguration()
{
Name = "simulate",
Transforms =
{
new Transformation()
{
Collections =
{
"Orders"
},
Name = "OrdersAndLines",
Script =
@"
var orderData = {
Id: id(this),
LinesCount: this.Lines.length,
TotalCost: 0
};
for (var i = 0; i < this.Lines.length; i++) {
var line = this.Lines[i];
orderData.TotalCost += line.PricePerUnit * line.Quantity;
loadToOrderLines({
OrderId: id(this),
Qty: line.Quantity,
Product: line.Product,
Cost: line.PricePerUnit
});
}
output('test output');
loadToOrders(orderData);"
}
}
}
}, database, database.ServerStore, context);
Assert.Equal(0, result.TransformationErrors.Count);
Assert.Equal(4, result.Commands.Count);
Assert.Equal(1, result.Commands.OfType <DeletePrefixedCommandData>().Count());
Assert.Equal(3, result.Commands.OfType <PutCommandDataWithBlittableJson>().Count());
Assert.Equal("test output", result.DebugOutput[0]);
}
}
}
}