/// <summary> /// Runs copies the datalink's current source/target tables and uses them as the basis for future test. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <bool> RunSnapshot(CancellationToken cancellationToken) { try { var ct = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token, cancellationToken); var token = ct.Token; token.ThrowIfCancellationRequested(); WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Started, null, null); var percent = 0; foreach (var step in _datalinkTest.DexihDatalinkTestSteps.OrderBy(c => c.Position).Where(c => c.IsValid)) { percent += 100 / _datalinkTest.DexihDatalinkTestSteps.Count; var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey); if (datalink == null) { throw new DatalinkTestRunException( $"The datalink test {_datalinkTest.Name} failed as the datalink with the key {step.DatalinkKey} could not be found."); } // copy the expected results if (datalink.LoadStrategy == TransformWriterTarget.ETransformWriterMethod.Bulk) { foreach (var target in datalink.DexihDatalinkTargets) { var dbDatalinkTargetTable = _hub.GetTableFromKey(target.TableKey); if (dbDatalinkTargetTable == null) { throw new DatalinkTestRunException($"The table with key {target.TableKey} could not be found."); } var dbDatalinkTargetConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == dbDatalinkTargetTable.ConnectionKey); if (dbDatalinkTargetConnection == null) { throw new DatalinkTestRunException($"The connection for the table {dbDatalinkTargetTable.Name} with the connection key {dbDatalinkTargetTable.Key} could not be found."); } var targetConnection = dbDatalinkTargetConnection.GetConnection(_transformSettings); var targetTable = dbDatalinkTargetTable.GetTable(_hub, targetConnection, _transformSettings); var dbExpectedConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == step.ExpectedConnectionKey); if (dbExpectedConnection == null) { throw new DatalinkTestRunException($"The connection for the step {step.Name} with the connection key {step.ExpectedConnectionKey} could not be found."); } var dbExpectedTable = dbDatalinkTargetTable.CloneProperties(); dbExpectedTable.Name = step.ExpectedTableName; dbExpectedTable.Schema = step.ExpectedSchema; var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings); var expectedTable = dbExpectedTable.GetTable(_hub, targetConnection, _transformSettings); UpdateProgress(percent); await expectedConnection.CreateTable(expectedTable, true, token); await using (var targetReader = targetConnection.GetTransformReader(targetTable)) await using (var targetReader2 = new ReaderConvertDataTypes(expectedConnection, targetReader)) { await targetReader2.Open(0, null, token); await expectedConnection.ExecuteInsertBulk(expectedTable, targetReader2, token); WriterResult.IncrementRowsCreated(targetReader2.TransformRows); WriterResult.IncrementRowsReadPrimary(targetReader2.TotalRowsReadPrimary); } } } else { // if there is no target table, then copy the outputs of the datalink. var dbExpectedConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.ExpectedConnectionKey); var dbExpectedTable = datalink.GetOutputTable(); var expectedTable = dbExpectedTable.GetTable(null, null); expectedTable.Name = step.ExpectedTableName; expectedTable.Schema = step.ExpectedSchema; var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings); var transformOperations = new TransformsManager(_transformSettings); var runPlan = transformOperations.CreateRunPlan(_hub, datalink, null, null, null, _transformWriterOptions); UpdateProgress(percent); await expectedConnection.CreateTable(expectedTable, true, cancellationToken); await using (var transform = runPlan.sourceTransform) await using (var transform2 = new ReaderConvertDataTypes(expectedConnection, transform)) { await transform2.Open(0, null, cancellationToken); await expectedConnection.ExecuteInsertBulk(expectedTable, transform2, cancellationToken); WriterResult.IncrementRowsCreated(transform2.TransformRows); WriterResult.IncrementRowsReadPrimary(transform2.TotalRowsReadPrimary); } } foreach (var testTable in step.DexihDatalinkTestTables) { var dbDatalinkTable = _hub.GetTableFromKey(testTable.TableKey); var dbDatalinkConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == dbDatalinkTable.ConnectionKey); var datalinkConnection = dbDatalinkConnection.GetConnection(_transformSettings); var datalinkTable = dbDatalinkTable.GetTable(_hub, datalinkConnection, _transformSettings); var dbSourceConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == testTable.SourceConnectionKey); var dbSourceTable = dbDatalinkTable.CloneProperties(); dbSourceTable.Name = testTable.SourceTableName; dbSourceTable.Schema = testTable.SourceSchema; var testConnection = dbSourceConnection.GetConnection(_transformSettings); var testTable1 = dbSourceTable.GetTable(_hub, testConnection, _transformSettings); UpdateProgress(percent); await testConnection.CreateTable(testTable1, true, cancellationToken); await using (var datalinkReader = datalinkConnection.GetTransformReader(datalinkTable)) await using (var datalinkReader2 = new ReaderConvertDataTypes(testConnection, datalinkReader)) { await datalinkReader2.Open(0, null, cancellationToken); await testConnection.ExecuteInsertBulk(testTable1, datalinkReader2, cancellationToken); WriterResult.IncrementRowsCreated(datalinkReader2.TransformRows); WriterResult.IncrementRowsReadPrimary(datalinkReader2.TotalRowsReadPrimary); } } } WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Finished, "Finished", null); await WriterResult.CompleteDatabaseWrites(); return(true); } catch (Exception ex) { WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Abended, ex.Message, ex); return(false); } }
public async Task <bool> Run(CancellationToken cancellationToken) { try { cancellationToken.ThrowIfCancellationRequested(); WriterResult.StartTime = DateTime.Now; WriterResult.LastUpdateTime = DateTime.Now; var runStatus = WriterResult.SetRunStatus(ERunStatus.Started, null, null); if (!runStatus) { throw new DatajobRunException($"Failed to set status"); } if (_transformWriterOptions.TargetAction == TransformWriterOptions.ETargetAction.Truncate) { runStatus = WriterResult.SetRunStatus(ERunStatus.Started, "Truncating tables...", null); var targetTables = new HashSet <DexihTable>(); foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid)) { var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey); if (datalink == null) { throw new DatajobRunException($"The datalink in the step {step.Name} with the datalink key {step.DatalinkKey} cound not be found."); } foreach (var target in datalink.DexihDatalinkTargets) { if (target.TableKey > 0) { var table = _hub.GetTableFromKey(target.TableKey); if (table != null) { targetTables.Add(table); } } } } // this loops through attempting to truncate tables one by one. // is repeats when there are failures to accomodate for some table having foriegn keys var atLeastOneSuccess = true; var atLeastOneFail = true; while (atLeastOneSuccess && atLeastOneFail) { atLeastOneSuccess = false; atLeastOneFail = false; var newTagetTables = new HashSet <DexihTable>(); foreach (var dbTable in targetTables) { var dbConnection = _hub.DexihConnections.SingleOrDefault(c => c.Key == dbTable.ConnectionKey); if (dbConnection == null) { throw new DatajobRunException($"The connection for the table {dbTable.Name} with the connection key {dbTable.ConnectionKey} could not be found."); } var connection = dbConnection.GetConnection(_transformSettings); var table = dbTable.GetTable(_hub, connection, _transformSettings); try { await connection.TruncateTable(table, cancellationToken); atLeastOneSuccess = true; } catch { atLeastOneFail = true; newTagetTables.Add(dbTable); } } targetTables = newTagetTables; } if (targetTables.Count > 0) { var message = $"The job failed as the following tables could not be truncated: {string.Join(", ", targetTables.Select(c => c.Name))}"; WriterResult.SetRunStatus(ERunStatus.Abended, message, null); return(false); } } var inputParameters = new InputParameters(); foreach (var parameter in Datajob.Parameters) { inputParameters.Add(new InputParameter() { Name = parameter.Name, Value = parameter.Value, Rank = parameter.Rank }); } //start all jobs async foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid)) { var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey); if (datalink == null) { throw new DatajobRunException($"The step {step.Name} contains a datalink with the key {step.DatalinkKey} which can not be found."); } foreach (var parameter in step.Parameters) { parameter.Value = inputParameters.SetParameters(parameter.Value, parameter.Rank); } var transformSettings = new TransformSettings { HubVariables = _transformSettings.HubVariables, RemoteSettings = _transformSettings.RemoteSettings, InputParameters = step.Parameters.ToArray <InputParameterBase>(), ClientFactory = _transformSettings.ClientFactory }; var inputColumns = step.DexihDatalinkStepColumns.Select(c => c.ToInputColumn()).ToArray(); var datalinkRun = new DatalinkRun(transformSettings, _logger, WriterResult.AuditKey, datalink, _hub, inputColumns, _transformWriterOptions, _alertQueue, _alertEmails) { DatalinkStepKey = step.Key }; DatalinkSteps.Add(datalinkRun); //start datalinks that have no dependencies. if (step.DexihDatalinkDependencies == null || step.DexihDatalinkDependencies.Count == 0) { StartDatalink(datalinkRun); } } WriterResult.SetRunStatus(ERunStatus.Running, null, null); await WaitUntilFinished(); return(true); } catch (OperationCanceledException) { Cancel(); await WaitUntilFinished(); WriterResult.SetRunStatus(ERunStatus.Cancelled, "Datajob was cancelled", null); throw new DatajobRunException($"The datajob {Datajob.Name} was cancelled."); } catch (Exception ex) { Cancel(); await WaitUntilFinished(); var message = $"The job {Datajob.Name} failed. {ex.Message}"; WriterResult?.SetRunStatus(ERunStatus.Abended, message, ex); throw new DatajobRunException(message, ex); } finally { await WriterResult.CompleteDatabaseWrites(); OnFinish?.Invoke(this); } }