private async Task <(Transform transform, string name)> GetDatalinkTransform(long key, long?datalinkTransformKey, InputColumn[] inputColumns, InputParameters inputParameters, SelectQuery selectQuery, CancellationToken cancellationToken) { var dbDatalink = Cache.Hub.DexihDatalinks.SingleOrDefault(c => c.Key == key); if (dbDatalink == null) { throw new DownloadDataException( $"The datalink with key {key} could not be found in the cache."); } dbDatalink.UpdateParameters(inputParameters); var transformWriterOptions = new TransformWriterOptions() { PreviewMode = true, }; var transformSettings = new TransformSettings() { HubVariables = TransformSettings.HubVariables, InputParameters = dbDatalink.Parameters.ToArray(), RemoteSettings = TransformSettings.RemoteSettings, ClientFactory = TransformSettings.ClientFactory }; var transformManager = new TransformsManager(transformSettings); //Get the last Transform that will load the target table. var runPlan = transformManager.CreateRunPlan(Cache.Hub, dbDatalink, inputColumns, datalinkTransformKey, null, transformWriterOptions); var transform = runPlan.sourceTransform; var openReturn = await transform.Open(0, selectQuery, cancellationToken); if (!openReturn) { throw new DownloadDataException( $"The datalink {dbDatalink.Name} failed to open for reading."); } // transform.SetCacheMethod(ECacheMethod.DemandCache); transform.SetEncryptionMethod(EEncryptionMethod.MaskSecureFields, ""); return(transform, dbDatalink.Name); }
/// <summary> /// Runs copies the datalink's current source/target tables and uses them as the basis for future test. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <bool> RunSnapshot(CancellationToken cancellationToken) { try { var ct = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token, cancellationToken); var token = ct.Token; token.ThrowIfCancellationRequested(); WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Started, null, null); var percent = 0; foreach (var step in _datalinkTest.DexihDatalinkTestSteps.OrderBy(c => c.Position).Where(c => c.IsValid)) { percent += 100 / _datalinkTest.DexihDatalinkTestSteps.Count; var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey); if (datalink == null) { throw new DatalinkTestRunException( $"The datalink test {_datalinkTest.Name} failed as the datalink with the key {step.DatalinkKey} could not be found."); } // copy the expected results if (datalink.LoadStrategy == TransformWriterTarget.ETransformWriterMethod.Bulk) { foreach (var target in datalink.DexihDatalinkTargets) { var dbDatalinkTargetTable = _hub.GetTableFromKey(target.TableKey); if (dbDatalinkTargetTable == null) { throw new DatalinkTestRunException($"The table with key {target.TableKey} could not be found."); } var dbDatalinkTargetConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == dbDatalinkTargetTable.ConnectionKey); if (dbDatalinkTargetConnection == null) { throw new DatalinkTestRunException($"The connection for the table {dbDatalinkTargetTable.Name} with the connection key {dbDatalinkTargetTable.Key} could not be found."); } var targetConnection = dbDatalinkTargetConnection.GetConnection(_transformSettings); var targetTable = dbDatalinkTargetTable.GetTable(_hub, targetConnection, _transformSettings); var dbExpectedConnection = _hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == step.ExpectedConnectionKey); if (dbExpectedConnection == null) { throw new DatalinkTestRunException($"The connection for the step {step.Name} with the connection key {step.ExpectedConnectionKey} could not be found."); } var dbExpectedTable = dbDatalinkTargetTable.CloneProperties(); dbExpectedTable.Name = step.ExpectedTableName; dbExpectedTable.Schema = step.ExpectedSchema; var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings); var expectedTable = dbExpectedTable.GetTable(_hub, targetConnection, _transformSettings); UpdateProgress(percent); await expectedConnection.CreateTable(expectedTable, true, token); await using (var targetReader = targetConnection.GetTransformReader(targetTable)) await using (var targetReader2 = new ReaderConvertDataTypes(expectedConnection, targetReader)) { await targetReader2.Open(0, null, token); await expectedConnection.ExecuteInsertBulk(expectedTable, targetReader2, token); WriterResult.IncrementRowsCreated(targetReader2.TransformRows); WriterResult.IncrementRowsReadPrimary(targetReader2.TotalRowsReadPrimary); } } } else { // if there is no target table, then copy the outputs of the datalink. var dbExpectedConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == step.ExpectedConnectionKey); var dbExpectedTable = datalink.GetOutputTable(); var expectedTable = dbExpectedTable.GetTable(null, null); expectedTable.Name = step.ExpectedTableName; expectedTable.Schema = step.ExpectedSchema; var expectedConnection = dbExpectedConnection.GetConnection(_transformSettings); var transformOperations = new TransformsManager(_transformSettings); var runPlan = transformOperations.CreateRunPlan(_hub, datalink, null, null, null, _transformWriterOptions); UpdateProgress(percent); await expectedConnection.CreateTable(expectedTable, true, cancellationToken); await using (var transform = runPlan.sourceTransform) await using (var transform2 = new ReaderConvertDataTypes(expectedConnection, transform)) { await transform2.Open(0, null, cancellationToken); await expectedConnection.ExecuteInsertBulk(expectedTable, transform2, cancellationToken); WriterResult.IncrementRowsCreated(transform2.TransformRows); WriterResult.IncrementRowsReadPrimary(transform2.TotalRowsReadPrimary); } } foreach (var testTable in step.DexihDatalinkTestTables) { var dbDatalinkTable = _hub.GetTableFromKey(testTable.TableKey); var dbDatalinkConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == dbDatalinkTable.ConnectionKey); var datalinkConnection = dbDatalinkConnection.GetConnection(_transformSettings); var datalinkTable = dbDatalinkTable.GetTable(_hub, datalinkConnection, _transformSettings); var dbSourceConnection = _hub.DexihConnections.Single(c => c.IsValid && c.Key == testTable.SourceConnectionKey); var dbSourceTable = dbDatalinkTable.CloneProperties(); dbSourceTable.Name = testTable.SourceTableName; dbSourceTable.Schema = testTable.SourceSchema; var testConnection = dbSourceConnection.GetConnection(_transformSettings); var testTable1 = dbSourceTable.GetTable(_hub, testConnection, _transformSettings); UpdateProgress(percent); await testConnection.CreateTable(testTable1, true, cancellationToken); await using (var datalinkReader = datalinkConnection.GetTransformReader(datalinkTable)) await using (var datalinkReader2 = new ReaderConvertDataTypes(testConnection, datalinkReader)) { await datalinkReader2.Open(0, null, cancellationToken); await testConnection.ExecuteInsertBulk(testTable1, datalinkReader2, cancellationToken); WriterResult.IncrementRowsCreated(datalinkReader2.TransformRows); WriterResult.IncrementRowsReadPrimary(datalinkReader2.TotalRowsReadPrimary); } } } WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Finished, "Finished", null); await WriterResult.CompleteDatabaseWrites(); return(true); } catch (Exception ex) { WriterResult.SetRunStatus(TransformWriterResult.ERunStatus.Abended, ex.Message, ex); return(false); } }