private async Task <(Transform transform, string name)> GetDatalinkTransform(long key, long?datalinkTransformKey, InputColumn[] inputColumns, InputParameters inputParameters, SelectQuery selectQuery, CancellationToken cancellationToken)
{
var dbDatalink =
Cache.Hub.DexihDatalinks.SingleOrDefault(c => c.Key == key);
if (dbDatalink == null)
{
throw new DownloadDataException(
$"The datalink with key {key} could not be found in the cache.");
}
dbDatalink.UpdateParameters(inputParameters);
var transformWriterOptions = new TransformWriterOptions()
{
PreviewMode = true,
};
var transformSettings = new TransformSettings()
{
HubVariables = TransformSettings.HubVariables,
InputParameters = dbDatalink.Parameters.ToArray(),
RemoteSettings = TransformSettings.RemoteSettings,
ClientFactory = TransformSettings.ClientFactory
};
var transformManager = new TransformsManager(transformSettings);
//Get the last Transform that will load the target table.
var runPlan = transformManager.CreateRunPlan(Cache.Hub, dbDatalink,
inputColumns, datalinkTransformKey, null,
transformWriterOptions);
var transform = runPlan.sourceTransform;
var openReturn = await transform.Open(0, selectQuery, cancellationToken);
if (!openReturn)
{
throw new DownloadDataException(
$"The datalink {dbDatalink.Name} failed to open for reading.");
}
// transform.SetCacheMethod(ECacheMethod.DemandCache);
transform.SetEncryptionMethod(EEncryptionMethod.MaskSecureFields, "");
return(transform, dbDatalink.Name);
}
/// <summary>
/// Runs copies the datalink's current source/target tables and uses them as the basis for future test.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <bool> RunSnapshot(CancellationToken cancellationToken)
{
try
{
var ct = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token,
cancellationToken);
var token = ct.Token;
token.ThrowIfCancellationRequested();
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Started, null, null);
var percent = 0;
foreach (var step in _datalinkTest.DexihDatalinkTestSteps.OrderBy(c => c.Position).Where(c => c.IsValid))
{
percent += 100 / _datalinkTest.DexihDatalinkTestSteps.Count;
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatalinkTestRunException(
$"The datalink test {_datalinkTest.Name} failed as the datalink with the key {step.DatalinkKey} could not be found.");
}
// copy the expected results
if (datalink.LoadStrategy == TransformWriterTarget.ETransformWriterMethod.Bulk)
{
foreach (var target in datalink.DexihDatalinkTargets)
{
var dbDatalinkTargetTable = _hub.GetTableFromKey(target.TableKey);
if (dbDatalinkTargetTable == null)
{
throw new DatalinkTestRunException($"The table with key {target.TableKey} could not be found.");
}
var dbDatalinkTargetConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == dbDatalinkTargetTable.ConnectionKey);
if (dbDatalinkTargetConnection == null)
{
throw new DatalinkTestRunException($"The connection for the table {dbDatalinkTargetTable.Name} with the connection key {dbDatalinkTargetTable.Key} could not be found.");
}
var targetConnection = dbDatalinkTargetConnection.GetConnection(_transformSettings);
var targetTable = dbDatalinkTargetTable.GetTable(_hub, targetConnection, _transformSettings);
var dbExpectedConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == step.ExpectedConnectionKey);
if (dbExpectedConnection == null)
{
throw new DatalinkTestRunException($"The connection for the step {step.Name} with the connection key {step.ExpectedConnectionKey} could not be found.");
}
var dbExpectedTable = dbDatalinkTargetTable.CloneProperties();
dbExpectedTable.Name = step.ExpectedTableName;
dbExpectedTable.Schema = step.ExpectedSchema;
var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings);
var expectedTable = dbExpectedTable.GetTable(_hub, targetConnection, _transformSettings);
UpdateProgress(percent);
await expectedConnection.CreateTable(expectedTable, true, token);
await using (var targetReader = targetConnection.GetTransformReader(targetTable))
await using (var targetReader2 = new ReaderConvertDataTypes(expectedConnection, targetReader))
{
await targetReader2.Open(0, null, token);
await expectedConnection.ExecuteInsertBulk(expectedTable, targetReader2, token);
WriterResult.IncrementRowsCreated(targetReader2.TransformRows);
WriterResult.IncrementRowsReadPrimary(targetReader2.TotalRowsReadPrimary);
}
}
}
else
{
// if there is no target table, then copy the outputs of the datalink.
var dbExpectedConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.ExpectedConnectionKey);
var dbExpectedTable = datalink.GetOutputTable();
var expectedTable = dbExpectedTable.GetTable(null, null);
expectedTable.Name = step.ExpectedTableName;
expectedTable.Schema = step.ExpectedSchema;
var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings);
var transformOperations = new TransformsManager(_transformSettings);
var runPlan =
transformOperations.CreateRunPlan(_hub, datalink, null, null, null, _transformWriterOptions);
UpdateProgress(percent);
await expectedConnection.CreateTable(expectedTable, true, cancellationToken);
await using (var transform = runPlan.sourceTransform)
await using (var transform2 = new ReaderConvertDataTypes(expectedConnection, transform))
{
await transform2.Open(0, null, cancellationToken);
await expectedConnection.ExecuteInsertBulk(expectedTable, transform2, cancellationToken);
WriterResult.IncrementRowsCreated(transform2.TransformRows);
WriterResult.IncrementRowsReadPrimary(transform2.TotalRowsReadPrimary);
}
}
foreach (var testTable in step.DexihDatalinkTestTables)
{
var dbDatalinkTable = _hub.GetTableFromKey(testTable.TableKey);
var dbDatalinkConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == dbDatalinkTable.ConnectionKey);
var datalinkConnection = dbDatalinkConnection.GetConnection(_transformSettings);
var datalinkTable = dbDatalinkTable.GetTable(_hub, datalinkConnection, _transformSettings);
var dbSourceConnection =
_hub.DexihConnections.Single(c => c.IsValid && c.Key == testTable.SourceConnectionKey);
var dbSourceTable = dbDatalinkTable.CloneProperties();
dbSourceTable.Name = testTable.SourceTableName;
dbSourceTable.Schema = testTable.SourceSchema;
var testConnection = dbSourceConnection.GetConnection(_transformSettings);
var testTable1 = dbSourceTable.GetTable(_hub, testConnection, _transformSettings);
UpdateProgress(percent);
await testConnection.CreateTable(testTable1, true, cancellationToken);
await using (var datalinkReader = datalinkConnection.GetTransformReader(datalinkTable))
await using (var datalinkReader2 = new ReaderConvertDataTypes(testConnection, datalinkReader))
{
await datalinkReader2.Open(0, null, cancellationToken);
await testConnection.ExecuteInsertBulk(testTable1, datalinkReader2, cancellationToken);
WriterResult.IncrementRowsCreated(datalinkReader2.TransformRows);
WriterResult.IncrementRowsReadPrimary(datalinkReader2.TotalRowsReadPrimary);
}
}
}
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Finished, "Finished", null);
await WriterResult.CompleteDatabaseWrites();
return(true);
}
catch (Exception ex)
{
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Abended, ex.Message, ex);
return(false);
}
}
