public void Start() { CSVSource sourceOrderData = new CSVSource("src/DataFlow/DemoData.csv"); sourceOrderData.Configuration.Delimiter = ";"; RowTransformation <string[], Order> transIntoObject = new RowTransformation <string[], Order>(CSVIntoObject); DBSource <Customer> sourceCustomerData = new DBSource <Customer>(CustomerTableDef); LookupCustomerKey lookupCustKeyClass = new LookupCustomerKey(); Lookup <Order, Order, Customer> lookupCustomerKey = new Lookup <Order, Order, Customer>( lookupCustKeyClass.FindKey, sourceCustomerData, lookupCustKeyClass.LookupData); Multicast <Order> multiCast = new Multicast <Order>(); DBDestination <Order> destOrderTable = new DBDestination <Order>(OrderDataTableDef); BlockTransformation <Order> blockOrders = new BlockTransformation <Order>(BlockTransformOrders); DBDestination <Rating> destRating = new DBDestination <Rating>(CustomerRatingTableDef); RowTransformation <Order, Rating> transOrderIntoCust = new RowTransformation <Order, Rating>(OrderIntoRating); VoidDestination <Order> destSink = new VoidDestination <Order>(); sourceOrderData.LinkTo(transIntoObject); transIntoObject.LinkTo(lookupCustomerKey); lookupCustomerKey.LinkTo(multiCast); multiCast.LinkTo(destOrderTable); multiCast.LinkTo(blockOrders); blockOrders.LinkTo(transOrderIntoCust, ord => ord.Rating != null); blockOrders.LinkTo(destSink, ord => ord.Rating == null); transOrderIntoCust.LinkTo(destRating); sourceOrderData.ExecuteAsync(); destOrderTable.Wait(); destRating.Wait(); }
public void UpdateOnHashMatch() { //Arrange CreateSourceTable("dbo.HashMatchSource"); CreateDestinationTable("dbo.HashMatchDestination"); //Act DbSource <string[]> source = new DbSource <string[]>(ConnectionSource, "dbo.HashMatchSource"); RowTransformation <string[]> trans = new RowTransformation <string[]>( row => { Array.Resize(ref row, row.Length + 1); row[row.Length - 1] = HashHelper.Encrypt_Char40(String.Join("", row)); return(row); }); List <string[]> allEntriesInDestination = new List <string[]>(); LookupTransformation <string[], string[]> lookup = new LookupTransformation <string[], string[]> ( new DbSource <string[]>(ConnectionDestination, "dbo.HashMatchDestination"), row => { var matchingIdEntry = allEntriesInDestination.Where(destRow => destRow[0] == row[0]).FirstOrDefault(); if (matchingIdEntry == null) { row = null; } else if (matchingIdEntry[matchingIdEntry.Length - 1] != row[row.Length - 1]) { SqlTask.ExecuteNonQuery(ConnectionDestination, "update entry with different hashcode", $@"UPDATE dbo.HashMatchDestination SET name = '{ row[1] }', age = '{ row[2] }', hashcode = '{ row[3] }' WHERE id = { row[0] } "); } return(row); }, allEntriesInDestination ); VoidDestination <string[]> voidDest = new VoidDestination <string[]>(); source.LinkTo(trans); trans.LinkTo(lookup); lookup.LinkTo(voidDest); source.Execute(); voidDest.Wait(); //Assert Assert.Equal(1, RowCountTask.Count(ConnectionDestination, $"dbo.HashMatchDestination", $"id = 1 AND name='Bugs' AND age = 12 AND hashcode = '{HashHelper.Encrypt_Char40("1Bugs12")}'")); Assert.Equal(1, RowCountTask.Count(ConnectionDestination, $"dbo.HashMatchDestination", $"id = 2 AND name='Coyote' AND age = 8 AND hashcode = '{HashHelper.Encrypt_Char40("2Coyote8")}'")); }
public void TestDuplicateCheckInRowTrans() { CreateLogTablesTask.CreateLog(); DataFlow.LoggingThresholdRows = 2; CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv"); source.Configuration.Delimiter = ";"; source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim; source.Configuration.MissingFieldFound = null; List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input => { if (IDs.Contains(input.ID)) { input.IsDuplicate = true; } else { IDs.Add(input.ID); } return(input); }); var multicast = new Multicast <Poco>(); var dest = new DBDestination <Poco>("dbo.Staging"); TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>() { new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true), new TableColumn("ID", "INT", allowNulls: false), new TableColumn("Value", "NVARCHAR(100)", allowNulls: false), new TableColumn("Name", "NVARCHAR(100)", allowNulls: false) }); stagingTable.CreateTable(); var trash = new VoidDestination <Poco>(); source.LinkTo(rowTrans); rowTrans.LinkTo(multicast); multicast.LinkTo(dest, input => input.IsDuplicate == false); multicast.LinkTo(trash, input => input.IsDuplicate == true); source.Execute(); dest.Wait(); trash.Wait(); }
public IDataFlowLinkSource <TConvert> LinkTo <TConvert>(IDataFlowLinkTarget <TOutput> target, Predicate <TOutput> rowsToKeep, Predicate <TOutput> rowsIntoVoid) { SourceBlock.LinkTo(target.TargetBlock, rowsToKeep); target.AddPredecessorCompletion(SourceBlock.Completion); if (!DisableLogging) { NLogger.Debug(CallingTask.TaskName + $" was linked to (with predicate): {((ITask)target).TaskName}!", CallingTask.TaskType, "LOG", CallingTask.TaskHash, ControlFlow.ControlFlow.STAGE, ControlFlow.ControlFlow.CurrentLoadProcess?.Id); } VoidDestination <TOutput> voidTarget = new VoidDestination <TOutput>(); SourceBlock.LinkTo <TOutput>(voidTarget.TargetBlock, rowsIntoVoid); voidTarget.AddPredecessorCompletion(SourceBlock.Completion); if (!DisableLogging) { NLogger.Debug(CallingTask.TaskName + $" was also linked to: VoidDestination to ignore certain rows!", CallingTask.TaskType, "LOG", CallingTask.TaskHash, ControlFlow.ControlFlow.STAGE, ControlFlow.ControlFlow.CurrentLoadProcess?.Id); } return(target as IDataFlowLinkSource <TConvert>); }
public void DuplicateCheckInRowTrans() { //Arrange CsvSource <Poco> source = CreateDuplicateCsvSource("res/UseCases/DuplicateCheck.csv"); List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source //Act RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input => { if (IDs.Contains(input.ID)) { input.IsDuplicate = true; } else { IDs.Add(input.ID); } return(input); }); Multicast <Poco> multicast = new Multicast <Poco>(); DbDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck"); VoidDestination <Poco> trash = new VoidDestination <Poco>(); source.LinkTo(rowTrans); rowTrans.LinkTo(multicast); multicast.LinkTo(dest, input => input.IsDuplicate == false); multicast.LinkTo(trash, input => input.IsDuplicate == true); source.Execute(); dest.Wait(); trash.Wait(); //Assert AssertDataWithoutDuplicates(); }