/// <summary>
/// Runs copies the datalink's current source/target tables and uses them as the basis for future test.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <bool> RunSnapshot(CancellationToken cancellationToken)
{
try
{
var ct = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token,
cancellationToken);
var token = ct.Token;
token.ThrowIfCancellationRequested();
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Started, null, null);
var percent = 0;
foreach (var step in _datalinkTest.DexihDatalinkTestSteps.OrderBy(c => c.Position).Where(c => c.IsValid))
{
percent += 100 / _datalinkTest.DexihDatalinkTestSteps.Count;
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatalinkTestRunException(
$"The datalink test {_datalinkTest.Name} failed as the datalink with the key {step.DatalinkKey} could not be found.");
}
// copy the expected results
if (datalink.LoadStrategy == TransformWriterTarget.ETransformWriterMethod.Bulk)
{
foreach (var target in datalink.DexihDatalinkTargets)
{
var dbDatalinkTargetTable = _hub.GetTableFromKey(target.TableKey);
if (dbDatalinkTargetTable == null)
{
throw new DatalinkTestRunException($"The table with key {target.TableKey} could not be found.");
}
var dbDatalinkTargetConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == dbDatalinkTargetTable.ConnectionKey);
if (dbDatalinkTargetConnection == null)
{
throw new DatalinkTestRunException($"The connection for the table {dbDatalinkTargetTable.Name} with the connection key {dbDatalinkTargetTable.Key} could not be found.");
}
var targetConnection = dbDatalinkTargetConnection.GetConnection(_transformSettings);
var targetTable = dbDatalinkTargetTable.GetTable(_hub, targetConnection, _transformSettings);
var dbExpectedConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == step.ExpectedConnectionKey);
if (dbExpectedConnection == null)
{
throw new DatalinkTestRunException($"The connection for the step {step.Name} with the connection key {step.ExpectedConnectionKey} could not be found.");
}
var dbExpectedTable = dbDatalinkTargetTable.CloneProperties();
dbExpectedTable.Name = step.ExpectedTableName;
dbExpectedTable.Schema = step.ExpectedSchema;
var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings);
var expectedTable = dbExpectedTable.GetTable(_hub, targetConnection, _transformSettings);
UpdateProgress(percent);
await expectedConnection.CreateTable(expectedTable, true, token);
await using (var targetReader = targetConnection.GetTransformReader(targetTable))
await using (var targetReader2 = new ReaderConvertDataTypes(expectedConnection, targetReader))
{
await targetReader2.Open(0, null, token);
await expectedConnection.ExecuteInsertBulk(expectedTable, targetReader2, token);
WriterResult.IncrementRowsCreated(targetReader2.TransformRows);
WriterResult.IncrementRowsReadPrimary(targetReader2.TotalRowsReadPrimary);
}
}
}
else
{
// if there is no target table, then copy the outputs of the datalink.
var dbExpectedConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.ExpectedConnectionKey);
var dbExpectedTable = datalink.GetOutputTable();
var expectedTable = dbExpectedTable.GetTable(null, null);
expectedTable.Name = step.ExpectedTableName;
expectedTable.Schema = step.ExpectedSchema;
var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings);
var transformOperations = new TransformsManager(_transformSettings);
var runPlan =
transformOperations.CreateRunPlan(_hub, datalink, null, null, null, _transformWriterOptions);
UpdateProgress(percent);
await expectedConnection.CreateTable(expectedTable, true, cancellationToken);
await using (var transform = runPlan.sourceTransform)
await using (var transform2 = new ReaderConvertDataTypes(expectedConnection, transform))
{
await transform2.Open(0, null, cancellationToken);
await expectedConnection.ExecuteInsertBulk(expectedTable, transform2, cancellationToken);
WriterResult.IncrementRowsCreated(transform2.TransformRows);
WriterResult.IncrementRowsReadPrimary(transform2.TotalRowsReadPrimary);
}
}
foreach (var testTable in step.DexihDatalinkTestTables)
{
var dbDatalinkTable = _hub.GetTableFromKey(testTable.TableKey);
var dbDatalinkConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == dbDatalinkTable.ConnectionKey);
var datalinkConnection = dbDatalinkConnection.GetConnection(_transformSettings);
var datalinkTable = dbDatalinkTable.GetTable(_hub, datalinkConnection, _transformSettings);
var dbSourceConnection =
_hub.DexihConnections.Single(c => c.IsValid && c.Key == testTable.SourceConnectionKey);
var dbSourceTable = dbDatalinkTable.CloneProperties();
dbSourceTable.Name = testTable.SourceTableName;
dbSourceTable.Schema = testTable.SourceSchema;
var testConnection = dbSourceConnection.GetConnection(_transformSettings);
var testTable1 = dbSourceTable.GetTable(_hub, testConnection, _transformSettings);
UpdateProgress(percent);
await testConnection.CreateTable(testTable1, true, cancellationToken);
await using (var datalinkReader = datalinkConnection.GetTransformReader(datalinkTable))
await using (var datalinkReader2 = new ReaderConvertDataTypes(testConnection, datalinkReader))
{
await datalinkReader2.Open(0, null, cancellationToken);
await testConnection.ExecuteInsertBulk(testTable1, datalinkReader2, cancellationToken);
WriterResult.IncrementRowsCreated(datalinkReader2.TransformRows);
WriterResult.IncrementRowsReadPrimary(datalinkReader2.TotalRowsReadPrimary);
}
}
}
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Finished, "Finished", null);
await WriterResult.CompleteDatabaseWrites();
return(true);
}
catch (Exception ex)
{
WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Abended, ex.Message, ex);
return(false);
}
}
public async Task <bool> Run(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
WriterResult.StartTime = DateTime.Now;
WriterResult.LastUpdateTime = DateTime.Now;
var runStatus = WriterResult.SetRunStatus(ERunStatus.Started, null, null);
if (!runStatus)
{
throw new DatajobRunException($"Failed to set status");
}
if (_transformWriterOptions.TargetAction == TransformWriterOptions.ETargetAction.Truncate)
{
runStatus = WriterResult.SetRunStatus(ERunStatus.Started, "Truncating tables...", null);
var targetTables = new HashSet <DexihTable>();
foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid))
{
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatajobRunException($"The datalink in the step {step.Name} with the datalink key {step.DatalinkKey} cound not be found.");
}
foreach (var target in datalink.DexihDatalinkTargets)
{
if (target.TableKey > 0)
{
var table = _hub.GetTableFromKey(target.TableKey);
if (table != null)
{
targetTables.Add(table);
}
}
}
}
// this loops through attempting to truncate tables one by one.
// is repeats when there are failures to accomodate for some table having foriegn keys
var atLeastOneSuccess = true;
var atLeastOneFail = true;
while (atLeastOneSuccess && atLeastOneFail)
{
atLeastOneSuccess = false;
atLeastOneFail = false;
var newTagetTables = new HashSet <DexihTable>();
foreach (var dbTable in targetTables)
{
var dbConnection = _hub.DexihConnections.SingleOrDefault(c => c.Key == dbTable.ConnectionKey);
if (dbConnection == null)
{
throw new DatajobRunException($"The connection for the table {dbTable.Name} with the connection key {dbTable.ConnectionKey} could not be found.");
}
var connection = dbConnection.GetConnection(_transformSettings);
var table = dbTable.GetTable(_hub, connection, _transformSettings);
try
{
await connection.TruncateTable(table, cancellationToken);
atLeastOneSuccess = true;
}
catch
{
atLeastOneFail = true;
newTagetTables.Add(dbTable);
}
}
targetTables = newTagetTables;
}
if (targetTables.Count > 0)
{
var message = $"The job failed as the following tables could not be truncated: {string.Join(", ", targetTables.Select(c => c.Name))}";
WriterResult.SetRunStatus(ERunStatus.Abended, message, null);
return(false);
}
}
var inputParameters = new InputParameters();
foreach (var parameter in Datajob.Parameters)
{
inputParameters.Add(new InputParameter()
{
Name = parameter.Name, Value = parameter.Value, Rank = parameter.Rank
});
}
//start all jobs async
foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid))
{
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatajobRunException($"The step {step.Name} contains a datalink with the key {step.DatalinkKey} which can not be found.");
}
foreach (var parameter in step.Parameters)
{
parameter.Value = inputParameters.SetParameters(parameter.Value, parameter.Rank);
}
var transformSettings = new TransformSettings
{
HubVariables = _transformSettings.HubVariables,
RemoteSettings = _transformSettings.RemoteSettings,
InputParameters = step.Parameters.ToArray <InputParameterBase>(),
ClientFactory = _transformSettings.ClientFactory
};
var inputColumns = step.DexihDatalinkStepColumns.Select(c => c.ToInputColumn()).ToArray();
var datalinkRun = new DatalinkRun(transformSettings, _logger, WriterResult.AuditKey, datalink, _hub, inputColumns, _transformWriterOptions, _alertQueue, _alertEmails)
{
DatalinkStepKey = step.Key
};
DatalinkSteps.Add(datalinkRun);
//start datalinks that have no dependencies.
if (step.DexihDatalinkDependencies == null || step.DexihDatalinkDependencies.Count == 0)
{
StartDatalink(datalinkRun);
}
}
WriterResult.SetRunStatus(ERunStatus.Running, null, null);
await WaitUntilFinished();
return(true);
}
catch (OperationCanceledException)
{
Cancel();
await WaitUntilFinished();
WriterResult.SetRunStatus(ERunStatus.Cancelled, "Datajob was cancelled", null);
throw new DatajobRunException($"The datajob {Datajob.Name} was cancelled.");
}
catch (Exception ex)
{
Cancel();
await WaitUntilFinished();
var message = $"The job {Datajob.Name} failed. {ex.Message}";
WriterResult?.SetRunStatus(ERunStatus.Abended, message, ex);
throw new DatajobRunException(message, ex);
}
finally
{
await WriterResult.CompleteDatabaseWrites();
OnFinish?.Invoke(this);
}
}
