public static ReaderMemory CreateTestData()
{
Table table = new Table("test" + (counter++).ToString(), 0,
new TableColumn("StringColumn", ETypeCode.String),
new TableColumn("IntColumn", ETypeCode.Int32, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DecimalColumn", ETypeCode.Decimal),
new TableColumn("DateColumn", ETypeCode.DateTime),
new TableColumn("GuidColumn", ETypeCode.Guid)
);
table.Data.Add(new object[] { "value1", 1, 1.1, Convert.ToDateTime("2015/01/01"), Guid.NewGuid() });
table.Data.Add(new object[] { "value2", 2, 2.1, Convert.ToDateTime("2015/01/02"), Guid.NewGuid() });
table.Data.Add(new object[] { "value3", 3, 3.1, Convert.ToDateTime("2015/01/03"), Guid.NewGuid() });
table.Data.Add(new object[] { "value4", 4, 4.1, Convert.ToDateTime("2015/01/04"), Guid.NewGuid() });
table.Data.Add(new object[] { "value5", 5, 5.1, Convert.ToDateTime("2015/01/05"), Guid.NewGuid() });
table.Data.Add(new object[] { "value6", 6, 6.1, Convert.ToDateTime("2015/01/06"), Guid.NewGuid() });
table.Data.Add(new object[] { "value7", 7, 7.1, Convert.ToDateTime("2015/01/07"), Guid.NewGuid() });
table.Data.Add(new object[] { "value8", 8, 8.1, Convert.ToDateTime("2015/01/08"), Guid.NewGuid() });
table.Data.Add(new object[] { "value9", 9, 9.1, Convert.ToDateTime("2015/01/09"), Guid.NewGuid() });
table.Data.Add(new object[] { "value10", 10, 10.1, Convert.ToDateTime("2015/01/10"), Guid.NewGuid() });
ReaderMemory Adapter = new ReaderMemory(table);
Adapter.Reset();
return(Adapter);
}
public static ReaderMemory CreateSortedJoinData()
{
var table = new Table("Join", 0);
table.AddColumn("StringColumn", ETypeCode.String);
table.AddColumn("IntColumn", ETypeCode.Int32);
table.AddColumn("LookupValue", ETypeCode.String);
table.AddRow("value01", 1, "lookup1");
table.AddRow("value02", 2, "lookup2");
table.AddRow("value03", 3, "lookup3");
table.AddRow("value04", 4, "lookup4");
table.AddRow("value05", 5, "lookup5");
table.AddRow("value06", 6, "lookup6");
table.AddRow("value07", 7, "lookup7");
table.AddRow("value08", 8, "lookup8");
table.AddRow("value09", 9, "lookup9");
var adapter = new ReaderMemory(table, new List <Sort>()
{
new Sort("StringColumn")
});
adapter.Reset();
return(adapter);
}
public static ReaderMemory CreateProfileTestData()
{
var table = new Table("test", 0,
new TableColumn("StringColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("IntColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DecimalColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DateColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("NullsBlanksColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("ZerosColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("MaxLengthColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("MaxValueColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DistinctValuesColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("PatternsColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey)
);
table.Data.Add(new object[] { "value01", 1, 1.1, Convert.ToDateTime("2015/01/01"), DBNull.Value, 0, "a", 1, "value1", "12345" });
table.Data.Add(new object[] { "value02", 2, 2.1, Convert.ToDateTime("2015/01/02"), null, 0, "ab", 3, "value2", "1234a" });
table.Data.Add(new object[] { "value03", 3, 3.1, Convert.ToDateTime("2015/01/03"), "", 1, "abcd", 4, "value1", "54321" });
table.Data.Add(new object[] { "value04", 4, 4.1, Convert.ToDateTime("2015/01/04"), "", 2, "abcde", 1, "value3", "AB123" });
table.Data.Add(new object[] { "value05", 5, 5.1, Convert.ToDateTime("2015/01/05"), "not null", -1, "abc", 3, "value1", "ZA123" });
table.Data.Add(new object[] { "value06", 6, 6.1, Convert.ToDateTime("2015/01/06"), "not null", "not zero", "abc", 3.5, "value1", "AB123" });
table.Data.Add(new object[] { "value07", 7, 7.1, Convert.ToDateTime("2015/01/07"), "not null", 2, "ab", 4.1, "value1", "12345" });
table.Data.Add(new object[] { "value08", 8, 8.1, Convert.ToDateTime("2015/01/08"), "not null", 2, "ab", 1, "value2", "32122" });
table.Data.Add(new object[] { "value09", 9, 9.1, Convert.ToDateTime("2015/01/09"), "not null", 2, "ab", 1, "value1", "1234a" });
table.Data.Add(new object[] { "value10", 10, 10.1, Convert.ToDateTime("2015/01/10"), "not null", 2.1, "ab", -1, "value1", "12335" });
var Adapter = new ReaderMemory(table, new List <Sort>()
{
new Sort("StringColumn")
});
Adapter.Reset();
return(Adapter);
}
public static ReaderMemory CreateUnSortedTestData()
{
var table = new Table("test", 0,
new TableColumn("StringColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("IntColumn", ETypeCode.Int32, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DecimalColumn", ETypeCode.Decimal, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DateColumn", ETypeCode.DateTime, TableColumn.EDeltaType.NaturalKey),
new TableColumn("SortColumn", ETypeCode.Int32),
new TableColumn("GroupColumn", ETypeCode.String)
);
table.AddRow("value01", 1, 1.1, Convert.ToDateTime("2015/01/01"), 10, "Odd");
table.AddRow("value10", 10, 10.1, Convert.ToDateTime("2015/01/10"), 1, "Even");
table.AddRow("value03", 3, 3.1, Convert.ToDateTime("2015/01/03"), 8, "Odd");
table.AddRow("value02", 2, 2.1, Convert.ToDateTime("2015/01/02"), 9, "Even");
table.AddRow("value08", 8, 8.1, Convert.ToDateTime("2015/01/08"), 3, "Even");
table.AddRow("value04", 4, 4.1, Convert.ToDateTime("2015/01/04"), 7, "Even");
table.AddRow("value07", 7, 7.1, Convert.ToDateTime("2015/01/07"), 4, "Odd");
table.AddRow("value05", 5, 5.1, Convert.ToDateTime("2015/01/05"), 6, "Odd");
table.AddRow("value06", 6, 6.1, Convert.ToDateTime("2015/01/06"), 5, "Even");
table.AddRow("value09", 9, 9.1, Convert.ToDateTime("2015/01/09"), 2, "Odd");
var adapter = new ReaderMemory(table, null);
adapter.Reset();
return(adapter);
}
public static ReaderMemory CreateValidationTestData()
{
var table = new Table("test", 0,
new TableColumn("StringColumn", ETypeCode.String, TableColumn.EDeltaType.NaturalKey),
new TableColumn("IntColumn", ETypeCode.Int32, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DecimalColumn", ETypeCode.Decimal, TableColumn.EDeltaType.NaturalKey),
new TableColumn("DateColumn", ETypeCode.DateTime, TableColumn.EDeltaType.NaturalKey),
new TableColumn("SortColumn", ETypeCode.Int32, TableColumn.EDeltaType.TrackingField),
new TableColumn("RejectReason", ETypeCode.Int32, TableColumn.EDeltaType.RejectedReason)
);
table.AddRow("value01", 1, 1.1, Convert.ToDateTime("2015/01/01"), 10, "");
table.AddRow("value02", 2, 2.1, Convert.ToDateTime("2015/01/02"), 9, "");
table.AddRow("value03", 3, 3.1, Convert.ToDateTime("2015/01/03"), 8, "");
table.AddRow("value04", 4, 4.1, Convert.ToDateTime("2015/01/04"), 7, "");
table.AddRow("value05", 5, 5.1, Convert.ToDateTime("2015/01/05"), 6, "");
table.AddRow("value06", 6, 6.1, Convert.ToDateTime("2015/01/06"), 5, "");
table.AddRow("value07", 7, 7.1, Convert.ToDateTime("2015/01/07"), 4, "");
table.AddRow("value08", 8, 8.1, Convert.ToDateTime("2015/01/08"), 3, "");
table.AddRow("value09", 9, 9.1, Convert.ToDateTime("2015/01/09"), 2, "");
table.AddRow("value10", 10, 10.1, Convert.ToDateTime("2015/01/10"), 1, "");
var adapter = new ReaderMemory(table, new List <Sort>()
{
new Sort("StringColumn")
});
adapter.Reset();
return(adapter);
}
public static ReaderMemory CreateDuplicatesJoinData()
{
var table = new Table("Join", 0);
table.AddColumn("StringColumn", ETypeCode.String);
table.AddColumn("IntColumn", ETypeCode.Int32);
table.AddColumn("LookupValue", ETypeCode.String);
table.AddColumn("IsValid", ETypeCode.Boolean, TableColumn.EDeltaType.IsCurrentField);
table.AddRow("value09", 9, "lookup9", true);
table.AddRow("value06", 6, "lookup6", true);
table.AddRow("value01", 1, "lookup1", true);
table.AddRow("value05", 5, "lookup5", true);
table.AddRow("value02", 2, "lookup2", true);
table.AddRow("value04", 4, "lookup4a", false);
table.AddRow("value04", 4, "lookup4", true);
table.AddRow("value03", 3, "lookup3", true);
table.AddRow("value07", 7, "lookup7", true);
table.AddRow("value08", 8, "lookup8", true);
var adapter = new ReaderMemory(table);
adapter.Reset();
return(adapter);
}
public async Task TransformDeltaPerformance(int rows, TransformDelta.EUpdateStrategy updateStrategy)
{
var source = Helpers.CreateLargeTable(rows);
var targetTable = source.CacheTable.Copy();
targetTable.AddAuditColumns();
Transform target = new ReaderMemory(targetTable);
var transformDelta = new TransformDelta(source, target, updateStrategy, 1, false);
var count = 0;
while (await transformDelta.ReadAsync())
{
count++;
}
//appendupdate and appenddelete will merge all rows in to one.
if (updateStrategy == TransformDelta.EUpdateStrategy.AppendUpdate ||
updateStrategy == TransformDelta.EUpdateStrategy.AppendUpdateDelete)
{
Assert.True(count == 1);
}
//reload has one extract row which is the truncate row.
else if (updateStrategy == TransformDelta.EUpdateStrategy.Reload)
{
Assert.True(count == rows + 1);
}
else
{
Assert.True(count == rows);
}
WriteTransformPerformance(transformDelta);
////write result to a memory table
//ConnectionMemory memoryConnection = new ConnectionMemory();
//TransformWriter writer = new TransformWriter();
//writer.WriteAllRecords(transformDelta, Target.CacheTable, memoryConnection, null, null).Wait();
//Target = memoryConnection.ExecuteReader(Target.CacheTable, null).Result.Value;
//Assert.True(transformDelta.RowsCreated == 100000);
}
public async Task RunDeltaTest_reload()
{
var source = Helpers.CreateSortedTestData();
var targetTable = source.CacheTable.Copy();
targetTable.AddAuditColumns();
var target = new ReaderMemory(targetTable);
//run a reload.
var transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.Reload, 0, false);
await transformDelta.ReadAsync();
Assert.True((char)transformDelta["Operation"] == 'T');
var count = 0;
while (await transformDelta.ReadAsync())
{
Assert.True((char)transformDelta["Operation"] == 'C');
Assert.True((long)transformDelta["SurrogateKey"] == count + 1);
Assert.True((int)transformDelta["IntColumn"] == count + 1);
count++;
}
Assert.True(count == 10);
//run an append. (only difference from reload is no truncate record at start.
transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.Append, 0, false);
transformDelta.Reset();
count = 0;
while (await transformDelta.ReadAsync())
{
Assert.True((char)transformDelta["Operation"] == 'C');
Assert.True((long)transformDelta["SurrogateKey"] == count + 1);
Assert.True((int)transformDelta["IntColumn"] == count + 1);
count++;
}
Assert.True(count == 10);
}
public async Task RunDeltaTest_update()
{
var source = Helpers.CreateUnSortedTestData();
source.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
var targetTable = source.CacheTable.Copy();
targetTable.AddAuditColumns();
Transform target = new ReaderMemory(targetTable);
//run an update load with nothing in the target, which will result in 10 rows created.
var transformDelta =
new TransformDelta(source, target, TransformDelta.EUpdateStrategy.AppendUpdate, 0, false);
transformDelta.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
var count = 0;
while (await transformDelta.ReadAsync())
{
Assert.True((char)transformDelta["Operation"] == 'C');
Assert.True((long)transformDelta["SurrogateKey"] == count + 1);
Assert.True((int)transformDelta["IntColumn"] == count + 1);
count++;
}
Assert.Equal(10, count);
transformDelta.SetRowNumber(0);
//write result to a memory table
var memoryConnection = new ConnectionMemory();
var writer = new TransformWriter();
var result = new TransformWriterResult();
result.SetProperties(0, 10, "DataLink", 1, 2, "Test", 1, "Source", 2, "Target", null, null,
TransformWriterResult.ETriggerMethod.Manual, "Test");
var writeResult = await writer.WriteAllRecords(result, transformDelta, target.CacheTable, memoryConnection,
CancellationToken.None);
Assert.True(writeResult);
target = memoryConnection
.GetTransformReader(target.CacheTable); // new ReaderMemory(target.CacheTable, null);
target.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
//Set the target pointer back to the start and rerun. Now 10 rows should be ignored.
source.SetRowNumber(0);
target.SetRowNumber(0);
//run an append. (only difference from reload is no truncate record at start.
transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.AppendUpdate, 0, false);
count = 0;
while (await transformDelta.ReadAsync())
{
count++;
}
Assert.Equal(10, transformDelta.TotalRowsIgnored);
Assert.Equal(0, count);
//change 3 rows. (first, middle, last) to similate target table data changes
target.CacheTable.Data[0][4] = 100;
target.CacheTable.Data[5][4] = 200;
target.CacheTable.Data[9][4] = 300;
//add a duplicate in the source
var row = new object[target.CacheTable.Columns.Count];
target.CacheTable.Data[9].CopyTo(row, 0);
target.CacheTable.Data.Add(row);
transformDelta.Reset();
count = 0;
while (await transformDelta.ReadAsync())
{
count++;
Assert.True((char)transformDelta["Operation"] == 'U');
}
Assert.True(count == 3);
//delete rows from the target, which should trigger two creates.
target.CacheTable.Data.RemoveAt(1);
target.CacheTable.Data.RemoveAt(7);
transformDelta.Reset();
count = 0;
var rowsCreated = 0;
var rowsUpdated = 0;
while (await transformDelta.ReadAsync())
{
rowsCreated += (char)transformDelta["Operation"] == 'C' ? 1 : 0;
rowsUpdated += (char)transformDelta["Operation"] == 'U' ? 1 : 0;
count++;
}
Assert.Equal(2, rowsCreated);
Assert.Equal(3, rowsUpdated);
Assert.Equal(5, count);
//delete rows from the source, which should not cause any change as delete detection is not on.
source.CacheTable.Data.RemoveAt(9);
source.CacheTable.Data.RemoveAt(0); //this is the row that was updated, so update now = 2
transformDelta.Reset();
count = 0;
rowsCreated = 0;
rowsUpdated = 0;
while (await transformDelta.ReadAsync())
{
rowsCreated += (char)transformDelta["Operation"] == 'C' ? 1 : 0;
rowsUpdated += (char)transformDelta["Operation"] == 'U' ? 1 : 0;
count++;
}
Assert.True(rowsCreated == 1);
Assert.True(rowsUpdated == 2);
Assert.True(count == 3);
}
public async Task RunDeltaTest_updatePreserve()
{
var source = Helpers.CreateUnSortedTestData();
source.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
var targetTable = source.CacheTable.Copy();
targetTable.AddAuditColumns();
long surrrogateKey = 0;
Transform target = new ReaderMemory(targetTable);
target.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
//run an update load with nothing in the target.
var transformDelta = new TransformDelta(source, target,
TransformDelta.EUpdateStrategy.AppendUpdateDeletePreserve, surrrogateKey, false);
transformDelta.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
var count = 0;
while (await transformDelta.ReadAsync())
{
Assert.True((char)transformDelta["Operation"] == 'C');
Assert.True((long)transformDelta["SurrogateKey"] == count + 1);
Assert.True((int)transformDelta["IntColumn"] == count + 1);
count++;
}
Assert.True(count == 10);
surrrogateKey = transformDelta.SurrogateKey;
transformDelta.SetRowNumber(0);
//write result to a memory table
var memoryConnection = new ConnectionMemory();
var writer = new TransformWriter();
var result = new TransformWriterResult();
result.SetProperties(0, 1, "DataLink", 1, 2, "Test", 1, "Source", 2, "Target", null, null,
TransformWriterResult.ETriggerMethod.Manual, "Test");
await writer.WriteAllRecords(result, transformDelta, target.CacheTable, memoryConnection,
CancellationToken.None);
target = memoryConnection.GetTransformReader(target.CacheTable);
target.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
//run an append. (only difference from reload is no truncate record at start.
transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.AppendUpdatePreserve,
surrrogateKey, false);
count = 0;
while (await transformDelta.ReadAsync())
{
count++;
}
//change 3 rows. (first, middle, last)
target.CacheTable.Data[0][4] = 100;
target.CacheTable.Data[5][4] = 200;
target.CacheTable.Data[9][4] = 300;
//add a duplicate in the source
var row = new object[target.CacheTable.Columns.Count];
target.CacheTable.Data[9].CopyTo(row, 0);
target.CacheTable.Data.Add(row);
transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.AppendUpdatePreserve,
surrrogateKey, false);
transformDelta.SetCacheMethod(Transform.ECacheMethod.PreLoadCache);
count = 0;
var rowsCreated = 0;
while (await transformDelta.ReadAsync())
{
rowsCreated += (char)transformDelta["Operation"] == 'C' ? 1 : 0;
count++;
}
Assert.Equal(3, rowsCreated);
Assert.Equal(3, transformDelta.TotalRowsPreserved);
Assert.Equal(6, count);
//run the delta again. this should ignore all 10 records.
transformDelta.SetRowNumber(0);
result = new TransformWriterResult();
result.SetProperties(0, 1, "DataLink", 30, 40, "Test", 1, "Source", 2, "Target", null, null,
TransformWriterResult.ETriggerMethod.Manual, "Test");
await writer.WriteAllRecords(result, transformDelta, target.CacheTable, memoryConnection,
CancellationToken.None);
target = memoryConnection.GetTransformReader(target.CacheTable);
transformDelta = new TransformDelta(source, target, TransformDelta.EUpdateStrategy.AppendUpdatePreserve,
surrrogateKey, false);
count = 0;
while (await transformDelta.ReadAsync())
{
count++;
}
Assert.Equal(10, transformDelta.TotalRowsIgnored);
Assert.Equal(0, count);
}
/// <summary>Parse whole SM-Bios into SmBios.Data.BiosData structure</summary>
/// <exception cref="T:System.IO.IOException">An I/O error occurred while opening the bios.</exception>
public virtual BiosData ReadBios()
{
var result = new BiosData();
while (BaseStream.Position < BaseStream.Length)
{
var header = ReadHeader();
switch (header.Type)
{
case Const.DMI_TYPE_END:
return(result);
case Const.DMI_TYPE_BIOS:
if (!Flags.HasFlag(TableFlags.Bios))
{
break;
}
var readerBios = new ReaderBios <TableBios>(this, header);
var tableBios = readerBios.Result;
if (result.Bios == null)
{
result.Bios = new List <TableBios>();
}
result.Bios.Add(tableBios);
break;
case Const.DMI_TYPE_BASEBOARD:
if (!Flags.HasFlag(TableFlags.BaseBoard))
{
break;
}
var readerBaseboard = new ReaderBaseboard <TableBaseboard>(this, header);
var tableBaseboard = readerBaseboard.Result;
if (result.BaseBoard == null)
{
result.BaseBoard = new List <TableBaseboard>();
}
result.BaseBoard.Add(tableBaseboard);
break;
case Const.DMI_TYPE_MEMORY:
if (!Flags.HasFlag(TableFlags.MemoryDevice))
{
break;
}
var readerMemory = new ReaderMemory <TableMemoryDevice>(this, header);
var tableMemory = readerMemory.Result;
if (result.Memory == null)
{
result.Memory = new List <TableMemoryDevice>();
}
result.Memory.Add(tableMemory);
break;
case Const.DMI_TYPE_PHYSMEM:
if (!Flags.HasFlag(TableFlags.PhysMemory))
{
break;
}
var readerPhysicalMemory = new ReaderPhysicalMemory <TablePhysicalMemory>(this, header);
var tablePhysicalMemory = readerPhysicalMemory.Result;
if (result.PhyMemory == null)
{
result.PhyMemory = new List <TablePhysicalMemory>();
}
result.PhyMemory.Add(tablePhysicalMemory);
break;
case Const.DMI_TYPE_PROCESSOR:
if (!Flags.HasFlag(TableFlags.Processor))
{
break;
}
var readerProcessor = new ReaderProcessor <TableProcessor>(this, header);
var tableProcessor = readerProcessor.Result;
if (result.Processor == null)
{
result.Processor = new List <TableProcessor>();
}
result.Processor.Add(tableProcessor);
break;
}
BaseStream.Seek(Idx, SeekOrigin.Begin);
}
return(result);
}
/// <summary>
/// Runs the datalink test and returns the results
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="DatalinkTestRunException"></exception>
public async Task <List <TestResult> > Run(CancellationToken cancellationToken)
{
try
{
var ct = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token,
cancellationToken);
var token = ct.Token;
token.ThrowIfCancellationRequested();
var tempTargetTableKey = -10000;
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Started, null, null);
var passed = 0;
var failed = 0;
foreach (var step in _datalinkTest.DexihDatalinkTestSteps.OrderBy(c => c.Position).Where(c => c.IsValid))
{
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatalinkTestRunException($"The datalink test {_datalinkTest.Name} failed as the datalink with the key {step.DatalinkKey} could not be found.");
}
UpdateProgress(1);
// prepare all the relevant tables
foreach (var testTable in step.DexihDatalinkTestTables)
{
await PrepareTestTable(testTable, token);
}
UpdateProgress(2);
datalink.AuditConnectionKey = _datalinkTest.AuditConnectionKey;
var dexihTargetConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.TargetConnectionKey);
ICollection <DexihTable> targetTables;
// add a target table to store the data when the datalink doesn't have one.
if (!datalink.DexihDatalinkTargets.Any())
{
var target = new DexihDatalinkTarget()
{
TableKey = tempTargetTableKey--,
};
datalink.DexihDatalinkTargets.Add(target);
datalink.UpdateStrategy = EUpdateStrategy.Reload;
datalink.LoadStrategy = TransformWriterTarget.ETransformWriterMethod.Bulk;
// var targetTable = datalink.GetOutputTable();
var table = new DexihTable()
{
Key = target.TableKey,
DexihTableColumns = datalink.GetOutputTable().DexihDatalinkColumns
.Select(c => c.CloneProperties <DexihTableColumn>()).ToArray()
};
// dexihTargetConnection.DexihTables.Add(table);
targetTables = new List <DexihTable> {
table
};
}
else
{
targetTables = datalink.DexihDatalinkTargets.Select(c => _hub.GetTableFromKey(c.TableKey)).ToList();
}
if (targetTables.Count > 1)
{
throw new DatalinkTestRunException("Currently datalink tests can only be used with datalinks containing no more than one target table.");
}
foreach (var table in targetTables)
{
table.ConnectionKey = dexihTargetConnection.Key;
table.Name = step.TargetTableName;
table.Schema = step.TargetSchema;
}
UpdateProgress(50);
// run the datalink
var datalinkRun = new DatalinkRun(_transformSettings, _logger, WriterResult.AuditKey, datalink, _hub, null, _transformWriterOptions, _alertQueue, _alertEmails);
datalinkRun.WriterTarget.WriterResult.AuditType = "DatalinkTestStep";
datalinkRun.WriterTarget.WriterResult.ReferenceKey = step.Key;
// await datalinkRun.Initialize(cancellationToken);
await datalinkRun.Build(token);
await datalinkRun.Run(token);
UpdateProgress(70);
foreach (var table in targetTables)
{
var testResult = new TestResult()
{
Name = step.Name,
StartDate = DateTime.Now,
TestStepKey = step.Key
};
var dexihExpectedConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.ExpectedConnectionKey);
var dexihExpectedTable = table.CloneProperties();
dexihExpectedTable.ConnectionKey = dexihExpectedConnection.Key;
dexihExpectedTable.Name = step.ExpectedTableName;
dexihExpectedTable.Schema = step.ExpectedSchema;
var expectedConnection = dexihExpectedConnection.GetConnection(_transformSettings);
var expectedTable = dexihExpectedTable.GetTable(_hub, expectedConnection, _transformSettings);
var expectedTransform = expectedConnection.GetTransformReader(expectedTable);
var targetConnection = dexihTargetConnection.GetConnection(_transformSettings);
var targetTable = table.GetTable(_hub, targetConnection, _transformSettings);
var targetTransform = targetConnection.GetTransformReader(targetTable);
// the error table is used to store any rows which do not match.
var dexihErrorConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == step.ErrorConnectionKey);
Connection errorConnection = null;
Table errorTable = null;
if (dexihErrorConnection != null)
{
var dexihErrorTable = table.CloneProperties();
dexihErrorTable.ConnectionKey = dexihErrorConnection.Key;
dexihErrorTable.Name = step.ErrorTableName;
dexihErrorTable.Schema = step.ErrorSchema;
errorConnection = dexihErrorConnection.GetConnection(_transformSettings);
errorTable = dexihErrorTable.GetTable(_hub, errorConnection, _transformSettings);
foreach (var column in errorTable.Columns)
{
column.DeltaType = EDeltaType.NonTrackingField;
}
errorTable.Columns.Add(new TableColumn("error_audit_key", ETypeCode.Int64,
EDeltaType.CreateAuditKey));
errorTable.Columns.Add(new TableColumn("error_operation", ETypeCode.CharArray,
EDeltaType.DatabaseOperation)
{
MaxLength = 1
});
errorTable.Columns.Add(new TableColumn("mismatch_reason", ETypeCode.String,
EDeltaType.UpdateReason)
{
AllowDbNull = true
});
}
// use the delta transform to compare expected and target tables.
await using var delta = new TransformDelta(targetTransform, expectedTransform,
EUpdateStrategy.AppendUpdateDelete, 0, false, true);
await delta.Open(0, null, token);
testResult.RowsMismatching = 0;
testResult.RowsMissingFromSource = 0;
testResult.RowsMissingFromTarget = 0;
var operationColumn = delta.CacheTable.Columns.GetOrdinal(EDeltaType.DatabaseOperation);
var updateReasonColumn = delta.CacheTable.Columns.GetOrdinal(EDeltaType.UpdateReason);
var errorCache = new TableCache();
// loop through the delta. any rows which don't match on source/target should filter through, others will be ignored.
while (await delta.ReadAsync(token))
{
testResult.TestPassed = false;
switch (delta[operationColumn])
{
case 'C':
testResult.RowsMissingFromTarget++;
break;
case 'U':
testResult.RowsMismatching++;
break;
case 'D':
testResult.RowsMissingFromSource++;
break;
}
datalinkRun.WriterTarget.WriterResult.Failed++;
WriterResult.Failed++;
WriterResult.IncrementRowsCreated();
if (errorTable != null && errorCache.Count < MaxErrorRows)
{
var row = new object[errorTable.Columns.Count];
for (var i = 0; i < errorTable.Columns.Count; i++)
{
var column = errorTable[i];
switch (column.DeltaType)
{
case EDeltaType.CreateAuditKey:
row[i] = datalinkRun.WriterTarget.WriterResult.AuditKey;
break;
case EDeltaType.DatabaseOperation:
row[i] = delta[operationColumn];
break;
case EDeltaType.UpdateReason:
row[i] = delta[updateReasonColumn];
break;
default:
row[i] = delta[column.Name];
break;
}
}
errorCache.Add(row);
}
}
if (errorCache.Count > 0)
{
errorTable.Data = errorCache;
var createReader = new ReaderMemory(errorTable);
if (!await errorConnection.TableExists(errorTable, cancellationToken))
{
await errorConnection.CreateTable(errorTable, false, cancellationToken);
}
await errorConnection.ExecuteInsertBulk(errorTable, createReader, cancellationToken);
}
WriterResult.RowsIgnored += delta.TotalRowsIgnored;
WriterResult.RowsPreserved += delta.TotalRowsPreserved;
if (testResult.TestPassed == false)
{
failed++;
}
else
{
passed++;
}
if (datalinkRun.WriterTarget.WriterResult.RunStatus == TransformWriterResult.ERunStatus.Finished)
{
if (testResult.TestPassed)
{
datalinkRun.WriterTarget.WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Passed, "Datalink test passed", null);
}
else
{
datalinkRun.WriterTarget.WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Failed, $"Datalink test failed, {testResult.RowsMissingFromSource} rows missing from expected, {testResult.RowsMissingFromTarget} rows missing from actual, {testResult.RowsMismatching} rows with mismatching columns.", null);
}
}
TestResults.Add(testResult);
}
}
if (WriterResult.Failed > 0)
{
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Failed, $"{passed} tests passed, {failed} test failed.", null);
}
else
{
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Passed, $"{passed} tests passed.", null);
}
await WriterResult.CompleteDatabaseWrites();
return(TestResults);
}
catch (Exception ex)
{
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Abended, ex.Message, ex);
return(TestResults);
}
}
