[InlineData(100000)] //should run in ~ 900ms
public async Task MappingPerformanceFunctions(int rows)
{
var data = Helpers.CreateLargeTable(rows);
var transformMapping = new TransformMapping();
var columnMappings = new List <TransformFunction>();
for (var i = 0; i < data.FieldCount; i++)
{
var newTransformFunction = new TransformFunction(
new Func <object, object>((value) => value),
new TableColumn[] { new TableColumn(data.GetName(i)) },
new TableColumn(data.GetName(i)),
null);
columnMappings.Add(newTransformFunction);
}
transformMapping.PassThroughColumns = false;
transformMapping.Functions = columnMappings;
transformMapping.SetInTransform(data);
var count = 0;
while (await transformMapping.ReadAsync())
{
count++;
}
Assert.Equal(rows, count);
output.WriteLine(transformMapping.PerformanceSummary());
}
[InlineData(100000)] //should run in ~ 250ms
public async Task MappingPerformanceColumnPairs(int rows)
{
var data = Helpers.CreateLargeTable(rows);
var transformMapping = new TransformMapping();
var columnMappings = new List <ColumnPair>();
for (var i = 0; i < data.FieldCount; i++)
{
columnMappings.Add(new ColumnPair(new TableColumn(data.GetName(i))));
}
transformMapping.PassThroughColumns = false;
transformMapping.MapFields = columnMappings;
transformMapping.SetInTransform(data);
var count = 0;
while (await transformMapping.ReadAsync())
{
count++;
}
Assert.Equal(rows, count);
output.WriteLine(transformMapping.PerformanceSummary());
}
// TODO: Move this?
// Checks MatchType and PrefabVariantIndex
private bool TransformMappingMatch(TransformMapping transformMapping, Entity inventoryOwner, Entity entityBecomingInventory)
{
// TODO: Throw error if indexes are invalid
if (transformMapping.thisPrefabVariantIndex >= 0 &&
transformMapping.thisPrefabVariantIndex < prefabVariants.Count &&
entityBecomingInventory.prefabVariantIndex != transformMapping.thisPrefabVariantIndex)
{
return(false);
}
if (transformMapping.ownerPrefabVariantIndex >= 0 &&
transformMapping.ownerPrefabVariantIndex < inventoryOwner.entityType.prefabVariants.Count &&
inventoryOwner.prefabVariantIndex != transformMapping.ownerPrefabVariantIndex)
{
return(false);
}
switch (transformMapping.ownerMatchType)
{
case InputCondition.MatchType.TypeGroup:
if (transformMapping.typeGroupMatch == null ||
!transformMapping.typeGroupMatch.InGroup(inventoryOwner.entityType))
{
return(false);
}
break;
case InputCondition.MatchType.TypeCategory:
if (transformMapping.typeCategoryMatch == null ||
!transformMapping.typeCategoryMatch.IsCategoryOrDescendantOf(inventoryOwner.entityType))
{
return(false);
}
break;
case InputCondition.MatchType.EntityType:
if (transformMapping.entityTypeMatch == null ||
transformMapping.entityTypeMatch != inventoryOwner.entityType)
{
return(false);
}
break;
}
return(true);
}
[InlineData(100000)] //should run in ~ 250ms
public async Task MappingPerformancePassthrough(int rows)
{
var data = Helpers.CreateLargeTable(rows);
var transformMapping = new TransformMapping
{
PassThroughColumns = true
};
transformMapping.SetInTransform(data);
var count = 0;
while (await transformMapping.ReadAsync())
{
count++;
}
Assert.Equal(rows, count);
output.WriteLine(transformMapping.PerformanceSummary());
}
/// <summary>
/// Perfromance tests should run in around 1 minute.
/// </summary>
/// <param name="connection"></param>
public async Task PerformanceTransformWriter(Connection connection, string databaseName, long rows)
{
await connection.CreateDatabase(databaseName, CancellationToken.None);
//create a table that utilizes every available datatype.
var table = new Table("LargeTable" + (DataSets.counter++));
table.Columns.Add(
new TableColumn("SurrogateKey", ETypeCode.Int32, TableColumn.EDeltaType.SurrogateKey)
{
IsIncrementalUpdate = true
});
table.Columns.Add(new TableColumn("UpdateTest", ETypeCode.Int32));
foreach (ETypeCode typeCode in Enum.GetValues(typeof(ETypeCode)))
{
if (typeCode == ETypeCode.Binary && connection.CanUseBinary)
{
continue;
}
if (typeCode != ETypeCode.Binary)
{
continue;
}
table.Columns.Add(new TableColumn()
{
Name = "column" + typeCode,
DataType = typeCode,
MaxLength = 50,
DeltaType = TableColumn.EDeltaType.TrackingField
});
}
//create the table
await connection.CreateTable(table, true, CancellationToken.None);
//add rows.
var buffer = 0;
for (var i = 0; i < rows; i++)
{
var row = new object[table.Columns.Count];
row[0] = i;
row[1] = 0;
//load the rows with random values.
for (var j = 2; j < table.Columns.Count; j++)
{
var dataType = DataType.GetType(table.Columns[j].DataType);
if (i % 2 == 0)
{
row[j] = connection.GetConnectionMaxValue(table.Columns[j].DataType, 20);
}
else
{
row[j] = connection.GetConnectionMinValue(table.Columns[j].DataType);
}
}
table.Data.Add(row);
buffer++;
if (buffer >= 5000 || rows == i + 1)
{
//start a datawriter and insert the test data
await connection.DataWriterStart(table);
await connection.ExecuteInsertBulk(table, new ReaderMemory(table), CancellationToken.None);
table.Data.Clear();
buffer = 0;
}
}
var targetTable = table.Copy();
targetTable.AddAuditColumns();
targetTable.Name = "TargetTable";
await connection.CreateTable(targetTable, false, CancellationToken.None);
var targetTransform = connection.GetTransformReader(targetTable);
//count rows using reader
var transform = connection.GetTransformReader(table);
transform = new TransformMapping(transform, true, null, null);
transform = new TransformValidation(transform, null, false);
transform = new TransformDelta(transform, targetTransform, TransformDelta.EUpdateStrategy.Reload, 1, false);
var writer = new TransformWriter();
var writerResult = new TransformWriterResult();
await connection.InitializeAudit(writerResult, 0, "Datalink", 1, 2, "Test", 1, "Source", 2, "Target",
TransformWriterResult.ETriggerMethod.Manual, "Test", CancellationToken.None);
Assert.NotNull(writerResult);
var result = await writer.WriteAllRecords(writerResult, transform, targetTable, connection, null, null,
null, null, CancellationToken.None);
Assert.Equal(rows, writerResult.RowsCreated);
//check the audit table loaded correctly.
var auditTable = await connection.GetTransformWriterResults(0, null, "Datalink", writerResult.AuditKey, null, true,
false, false, null, 1, 2, false, CancellationToken.None);
Assert.Equal(writerResult.RowsCreated, auditTable[0].RowsCreated);
Assert.Equal(rows - 1, Convert.ToInt64(auditTable[0].MaxIncrementalValue));
}
public async Task Mappings()
{
var Source = Helpers.CreateSortedTestData();
var transformMapping = new TransformMapping();
var Mappings = new List <TransformFunction>();
//Mappings.Add(new Function("CustomFunction", false, "test", "return StringColumn + number.ToString();", null, ETypeCode.String,
// new dexih.functions.Parameter[] {
// new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, "StringColumn" ),
// new dexih.functions.Parameter("number", ETypeCode.Int32, false, 123)
// }, null));
var transformFunction = new TransformFunction(
new Func <string, int, string>((StringColumn, number) => StringColumn + number.ToString()),
new TableColumn[] { new TableColumn("StringColumn"), new TableColumn("number", ETypeCode.Int32) },
new TableColumn("CustomFunction"),
null)
{
Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("number", ETypeCode.Int32, false, 123)
}
};
Mappings.Add(transformFunction);
transformFunction = Functions.GetFunction("dexih.functions.BuiltIn.MapFunctions", "Substring").GetTransformFunction();
transformFunction.TargetColumn = new TableColumn("Substring");
transformFunction.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("name", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("start", ETypeCode.Int32, false, 1),
new dexih.functions.Parameter("start", ETypeCode.Int32, false, 3)
};
Mappings.Add(transformFunction);
var MappingColumn = new List <ColumnPair>
{
new ColumnPair(new TableColumn("DateColumn", ETypeCode.DateTime), new TableColumn("DateColumn", ETypeCode.DateTime))
};
transformMapping = new TransformMapping(Source, false, MappingColumn, Mappings);
Assert.Equal(3, transformMapping.FieldCount);
var count = 0;
while (await transformMapping.ReadAsync() == true)
{
count = count + 1;
Assert.Equal("value" + count.ToString().PadLeft(2, '0') + "123", transformMapping["CustomFunction"]);
Assert.Equal("alu", transformMapping["Substring"]);
Assert.Equal((DateTime)Convert.ToDateTime("2015-01-" + count.ToString()), (DateTime)transformMapping["DateColumn"]);
}
Assert.Equal(10, count);
//test the getschematable table function.
//DataReaderAdapter SchemaTable = TransformMapping.GetSchemaTable();
//Assert.Equal("DateColumn", SchemaTable.Rows[0]["ColumnName"]);
//Assert.Equal("CustomFunction", SchemaTable.Rows[1]["ColumnName"]);
//Assert.Equal("Substring", SchemaTable.Rows[2]["ColumnName"]);
}
public async Task Filters()
{
var table = Helpers.CreateSortedTestData();
//set a filter that filters all
var conditions = new List <TransformFunction>();
var function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsEqual").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, new TableColumn("StringColumn"), isArray: true),
new dexih.functions.Parameter("Compare", ETypeCode.String, false, "junk", isArray: true)
};
conditions.Add(function);
var transformFilter = new TransformFilter(table, conditions, null);
await transformFilter.Open(0, null, CancellationToken.None);
Assert.Equal(5, transformFilter.FieldCount);
var count = 0;
while (await transformFilter.ReadAsync())
{
count = count + 1;
}
Assert.Equal(0, count);
//set a filter than filters to 1 row.
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsEqual").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, new TableColumn("StringColumn"), isArray: true),
new dexih.functions.Parameter("Compare", ETypeCode.String, false, "value03", isArray: true)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
transformFilter.Reset();
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
if (count == 1)
{
Assert.Equal(3, transformFilter["IntColumn"]);
}
}
Assert.Equal(1, count);
// use the "IN" function to filter 3 rows.
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsIn").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("Value", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value03", isArray: true),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value05", isArray: true),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value07", isArray: true)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
table.Reset();
transformFilter.SetInTransform(table);
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
}
Assert.Equal(3, count);
// create a mapping, and use the filter after the calculation.
var mappings = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.MapFunctions", "Substring").GetTransformFunction();
function.TargetColumn = new TableColumn("Substring");
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("name", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("start", ETypeCode.Int32, false, 5),
new dexih.functions.Parameter("end", ETypeCode.Int32, false, 50)
};
mappings.Add(function);
table.Reset();
var transformMapping = new TransformMapping(table, false, null, mappings);
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "LessThan").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("Substring", ETypeCode.Int32, true, null, new TableColumn("Substring")),
new dexih.functions.Parameter("Compare", ETypeCode.Int32, false, 5)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
transformFilter.SetInTransform(transformMapping);
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
}
Assert.Equal(4, count);
}
