public void ProcessPipeline_RawOutputMultiItems_ItemsMerged() { //arrange _config.InputDataKind = KindOfTextData.Keyword; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLinesMF()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String var orchestrator = new EtlOrchestrator(_config); //act var counts = orchestrator.ExecuteAsync().Result; //assert counts.RowsRead.Should().Be(5); counts.ClustersRead.Should().Be(5); counts.ClustersWritten.Should().Be(5); counts.RowsWritten.Should().Be(5); _resultingLines.Should().HaveCount(5); _resultingLines[0].Should().Be("XYZMary123"); _resultingLines[1].Should().Be("ABCDXYZ00883 Mary223"); _resultingLines[2].Should().Be("ABCDXYZ00883Susan 323"); _resultingLines[3].Should().Be("ABCDXYZ00883 Mary423 "); //trailing spaces respected _resultingLines[4].Should().Be(string.Empty); //EOF, no value }
public EtlOrchestrator_tests_ArbitraryOutput() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String _config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //single record clusters _config.AllowOnTheFlyInputFields = true; _config.TransformerType = TransformerType.ClusterFilter; _config.ClusterFilterPredicate = c => true; // no transformations, data passed as is _config.OutputDataKind = KindOfTextData.Arbitrary; _config.ArbitraryOutputDefs = new string[] { "Record type is {RECTYPE},", " name is {NAME}", " and number is {NUM}.", " Void item here." }; _config.SetOutputConsumer(l => { if (l != null) { _resultingLines.Add(l); } }); _resultingLines = new List <string>(); }
public ClusterMarker_tests() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; var sn = 0; //closure to facilitate SourceNo calculation (note that GetStringTupleSupplier calls the sourceNoEval function (provided as parameter below) exactly once per iteration) _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).GetStringTupleSupplier(() => sn++ < 3 ? 1 : 2)); //first 3 - source 1, rest - source 2 _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; // The DeferTransformation.Indefinitely setting below prevents Data Conveyer from linking clusteringBlock to transformingBlock. // Without it, transformingBlock could randomly "steal" clusters from _resultsExtractor, which would cause randowm test failures. _config.DeferTransformation = DeferTransformation.Indefinitely; _config.AllowOnTheFlyInputFields = true; _config.OutputConsumer = (t, gc) => { }; //throwaway consumer //prepare extraction of the results from the pipeline _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); // AsserterOutput tuple: Item1=Ext, Item2=Header, Item3=Formatter, Item4=ExcFormatter string ShowRec(ICluster c, int idx) => idx >= c.Count ? string.Empty : c[idx]?["NUM"] +"{" + c[idx].Count + "}"; // e.g. 223{4} means record with 4 items and item NUM = 223 AsserterOutput asserterOutputToCsv = (".csv", "ClstrNo,NoOfRecs,Rec1,Rec2,Rec3,Rec4,Rec5,Rec6,Rec7,Rec8,Rec9,Rec10", c => $"{ c.ClstrNo },{ c.Count },{ ShowRec(c,0) },{ ShowRec(c,1) },{ ShowRec(c,2) },{ ShowRec(c,3) },{ ShowRec(c,4) },{ ShowRec(c,5) },{ ShowRec(c,6) },{ ShowRec(c,7) },{ ShowRec(c,8) },{ ShowRec(c,9) }", ex => ex.ToString().Split("\r\n").Select(l => "\"" + l.Replace('"', '\'') + "\"") ); _traceableAsserter = new TraceableAsserter <KeyValCluster>("ClusterTestFailures\\", asserterOutputToCsv); }
private readonly List <string> _resultingLines; //container of the test results public EtlOrchestrator_tests_ProcessResult() { _fatalLogMsgs = new List <Tuple <LogEntrySeverity, string, LogEntry> >(); var mockFatalLogger = new Mock <ILogger>(); //records Fatal messages to _fatalLogMsgs (plus title box) mockFatalLogger.Setup(l => l.LoggingThreshold).Returns(LogEntrySeverity.Fatal); mockFatalLogger.Setup(l => l.Log(It.IsAny <LogEntry>())) .Callback((LogEntry e) => { if (e.Severity <= LogEntrySeverity.Fatal) { _fatalLogMsgs.Add(Tuple.Create(e.Severity, e.MessageOnDemand(), e)); } }); _config = new OrchestratorConfig(mockFatalLogger.Object) { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.TransformerType = TransformerType.ClusterFilter; _config.OutputConsumer = (t, gc) => _resultingLines.Add(t?.Item1.Text); // place the lines on the list to be tested/asserted //prepare extraction of the results from the pipeline _resultingLines = new List <string>(); }
public void ProcessPipeline_SimpleConfig_CorrectData() { //arrange _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.HeadersInFirstInputRow = true; var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsExtractor); //act _ = orchestrator.ExecuteAsync(); _resultsExtractor.Completion.Wait(); //assert var resultingClusters = _resultingClusters.ToList(); resultingClusters.Should().HaveCount(4); //4 single-record clusters ((IntakeProvider)(new PrivateAccessor(orchestrator)).GetField("_intakeProvider")).FieldsInUse.Should().HaveCount(12); var kvRec = resultingClusters[0][0]; kvRec["ID"].Should().Be("1"); kvRec["Country"].Should().Be("Papua New Guinea"); kvRec["IATA/FAA"].Should().Be("GKA"); kvRec["Latitude"].Should().Be("-6.081689"); kvRec["Offset from UTC"].Should().Be("10"); kvRec["Timezone"].Should().Be("Pacific/Port_Moresby"); kvRec = resultingClusters[3][0]; kvRec[0].Should().Be("4"); kvRec[1].Should().Be("Nadzab"); kvRec["ICAO"].Should().Be("AYNZ"); kvRec[11].Should().Be("Pacific/Port_Moresby"); kvRec[12].Should().BeNull(); }
public void ProcessIntake_RoundRobin_CorrectSourceNumbers() { //arrange var sn = 0; //closure to facilitate SourceNo calculation (note that GetStringTupleSupplier calls the sourceNoEval function (provided as parameter below) exactly once per iteration) _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).GetStringTupleSupplier(() => sn++ % 3 + 1)); //assign sourceNo in a round-robin fashion: 1,2,3,1,2,3,1,2,3 var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsExtractor); //act _ = orchestrator.ExecuteAsync(); _resultsExtractor.Completion.Wait(); //assert var resultingClusters = _resultingClusters.ToList(); resultingClusters.Should().HaveCount(2); resultingClusters[0].Count.Should().Be(5); resultingClusters[0].StartRecNo.Should().Be(1); resultingClusters[0].StartSourceNo.Should().Be(1); resultingClusters[0][0].RecNo.Should().Be(1); resultingClusters[0][0].SourceNo.Should().Be(1); resultingClusters[0][1].RecNo.Should().Be(2); resultingClusters[0][1].SourceNo.Should().Be(2); resultingClusters[0][2].RecNo.Should().Be(3); resultingClusters[0][2].SourceNo.Should().Be(3); resultingClusters[0][3].RecNo.Should().Be(4); resultingClusters[0][3].SourceNo.Should().Be(1); resultingClusters[0][4].RecNo.Should().Be(5); resultingClusters[0][4].SourceNo.Should().Be(2); resultingClusters[1].Count.Should().Be(4); resultingClusters[1].StartRecNo.Should().Be(6); resultingClusters[1].StartSourceNo.Should().Be(3); resultingClusters[1][0].RecNo.Should().Be(6); resultingClusters[1][0].SourceNo.Should().Be(3); resultingClusters[1][1].RecNo.Should().Be(7); resultingClusters[1][1].SourceNo.Should().Be(1); resultingClusters[1][2].RecNo.Should().Be(8); resultingClusters[1][2].SourceNo.Should().Be(2); resultingClusters[1][3].RecNo.Should().Be(9); resultingClusters[1][3].SourceNo.Should().Be(3); }
public void ProcessPipeline_TrimValues_CorrectData() { //arrange _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.HeadersInFirstInputRow = false; _config.RetainQuotes = false; _config.TrimInputValues = true; _config.InputFields = "Year|4,Month|4,Data|12"; var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsExtractor); //act _ = orchestrator.ExecuteAsync(); _resultsExtractor.Completion.Wait(); //assert var resultingClusters = _resultingClusters.ToList(); resultingClusters.Should().HaveCount(10); ((IntakeProvider)(new PrivateAccessor(orchestrator)).GetField("_intakeProvider")).FieldsInUse.Count.Should().Be(3); var kvRec = resultingClusters[0][0]; kvRec.Keys[0].Should().Be("Year"); kvRec[0].Should().Be("1966"); kvRec["Year"].Should().Be("1966"); kvRec.Keys[1].Should().Be("Month"); kvRec[1].Should().Be("11"); kvRec["Month"].Should().Be("11"); kvRec.GetItem(2).Key.Should().Be("Data"); kvRec[2].Should().Be("34943905"); kvRec["Data"].Should().Be("34943905"); kvRec = resultingClusters[9][0]; kvRec.Keys[0].Should().Be("Year"); kvRec[0].Should().Be("1967"); kvRec["Year"].Should().Be("1967"); kvRec.GetItem(1).Key.Should().Be("Month"); kvRec[1].Should().Be("08"); kvRec["Month"].Should().Be("08"); kvRec.GetItem(2).Key.Should().Be("Data"); kvRec[2].Should().Be("5308387"); kvRec["Data"].Should().Be("5308387"); resultingClusters[6][0]["Data"].Should().Be("279438"); //quotes are stripped }
private readonly ConcurrentQueue <IReadOnlyDictionary <string, object> > _traceBinHistory; //will contain results to verify public EtlOrchestrator_tests_TraceBin() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.X12 }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); //no type definitions (everything string) _traceBinHistory = new ConcurrentQueue <IReadOnlyDictionary <string, object> >(); }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public SimpleIntakeSupplier_tests() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Raw }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.DeferTransformation = DeferTransformation.Indefinitely; //to prevent linking clusteringBlock to transformingBlock (which could steal clusters from results extractor) _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); _resultingClusters = new ConcurrentQueue <KeyValCluster>(); }
private readonly ActionBlock <KeyValRecord> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_GlobalCache() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Delimited }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.TextOutputConsumer = l => { }; //throwaway consumer //prepare extraction of the results from the pipeline _resultingRecords = new List <KeyValRecord>(); _resultsExtractor = new ActionBlock <KeyValRecord>(c => _resultingRecords.Add(c)); }
public EtlOrchestrator_tests_FieldsToUse() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Flat }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.HeadersInFirstInputRow = true; _config.InputFields = "InFld1|4,InFld2|4,InFld3|12"; _config.TransformerType = TransformerType.ClusterFilter; _config.ClusterFilterPredicate = c => true; // no transformations, data passed as is _config.SetOutputConsumer((string l) => { }); //throwaay consumer, these tests do not evaluate output }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_KwIntake() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.AllowOnTheFlyInputFields = true; _config.DeferTransformation = DeferTransformation.Indefinitely; //to prevent linking clusteringBlock to transformingBlock (which could steal clusters from results extractor) _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); }
public EtlOrchestrator_tests_PropertyBin() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Delimited }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.HeadersInFirstInputRow = true; _config.ConcurrencyLevel = 3; _config.TransformerType = TransformerType.Clusterbound; _pbSnapshots = new Dictionary <string, IDictionary <string, object> >(); _locker = new object(); }
public void ProcessPipeline_X12OutputKwInputSegmentAdded_CorrectData() { //arrange _config.InputDataKind = KindOfTextData.Keyword; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLinesKW()).StringSupplier); _config.InputKeyPrefix = "@p"; _config.TransformerType = TransformerType.Clusterbound; _config.ClusterboundTransformer = c => { if ((string)c[0]["RECTYPE"] == "XYZ") // add NM1 segment after XYZ, i.e. first rec { c.AddRecord(c[0].CreateEmptyX12Segment("NM1", 4)); c[c.Count - 1][3] = "Smith"; c[c.Count - 1]["Elem004"] = "Lucie"; } return(c); }; var orchestrator = new EtlOrchestrator(_config); //act var counts = orchestrator.ExecuteAsync().Result; //assert counts.RowsRead.Should().Be(5); counts.ClustersRead.Should().Be(5); counts.ClustersWritten.Should().Be(5); counts.RowsWritten.Should().Be(6); // 5 + 1 (NM1 added to 1st cluster) _resultingLines.Count.Should().Be(6); _resultingLines[0].Should().Be("XYZ*Mary*123"); _resultingLines[1].Should().Be("NM1***Smith*Lucie"); _resultingLines[2].Should().Be("ABCD*XYZ00883*Mary*223"); _resultingLines[3].Should().Be("ABCD*XYZ00883*Susan *323"); _resultingLines[4].Should().Be("ABCD*XYZ00883*Mary*423"); _resultingLines[5].Should().Be("GE"); }
public void ProcessIntake_SimpleSupplier_SourceNoAlways1() { //arrange _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).ExternalTupleSupplier); var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsExtractor); //act _ = orchestrator.ExecuteAsync(); _resultsExtractor.Completion.Wait(); //assert _resultingClusters.Should().HaveCount(8); // 8 single record clusters ValidateResultingRecords(_resultingClusters.SelectMany(c => c.Records).ToList()); }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_ArbitraryIntake() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Arbitrary }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); //no type definitions (everything string) _config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //each record is its own cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.ArbitraryInputDefs = new string[] { "Segment ^[^*]*", @"ISA06 (?<=^ISA\*([^*]*\*){5})([^*]*)", @"N301 (?<=^N3\*)([^*]*)" }; _config.DeferTransformation = DeferTransformation.Indefinitely; //to prevent linking clusteringBlock to transformingBlock (which could steal clusters from results extractor) _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); }
ConcurrentQueue <Tuple <ExternalLine, int> > _resultingLines; //Item2=targetNo public EtlOrchestrator_tests_Router() { _config = new OrchestratorConfig(); _config.InputDataKind = KindOfTextData.Keyword; var sn = 0; //closure to facilitate SourceNo calculation (note that GetStringTupleSupplier calls the sourceNoEval function (provided as parameter below) exactly once per iteration) _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).GetStringTupleSupplier(() => sn++ % 3 + 1)); //assign sourceNo in a round-robin fashion: 1,2,3,1,2,3,1,2,3 _config.InputKeyPrefix = "@p"; //no type definitions (everything string) _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.SetOutputConsumer(tpl => _resultingLines.Enqueue(tpl)); //Item1=ExternalLine/Xrecord, Item2=targetNo _resultingLines = new ConcurrentQueue <Tuple <ExternalLine, int> >(); }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_FieldsInUse() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; //no type definitions (everything string) _config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //single record cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.DeferTransformation = DeferTransformation.Indefinitely; //to prevent linking clusteringBlock to transformingBlock (which could steal clusters from results extractor) _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); }
public EtlOrchestrator_tests_X12Output() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.X12 }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //single record clusters _config.AllowOnTheFlyInputFields = true; // to allow fields to showing trailing spaces (should matter for KW, but not X12) _config.TransformerType = TransformerType.ClusterFilter; _config.ClusterFilterPredicate = c => true; // no transformations, data passed as is _config.OutputDataKind = KindOfTextData.X12; _config.SetOutputConsumer(l => { if (l != null) { _resultingLines.Add(l); } }); _resultingLines = new List <string>(); }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_HeadAndFootClusters() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; //no type definitions (everything string) _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.DeferTransformation = DeferTransformation.Indefinitely; //to prevent linking clusteringBlock to transformingBlock (which could steal clusters from results extractor) _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); }
private readonly ActionBlock <KeyValCluster> _resultsExtractor; //block to load results to container public EtlOrchestrator_tests_ClusterFilterTransformer() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.TransformerType = TransformerType.ClusterFilter; _config.DeferOutput = DeferOutput.Indefinitely; //so that Output won't steal resulting clusters _resultingClusters = new ConcurrentQueue <KeyValCluster>(); _resultsExtractor = new ActionBlock <KeyValCluster>(c => _resultingClusters.Enqueue(c)); }
public EtlOrchestrator_tests_KwOutput() { _config = new OrchestratorConfig { InputDataKind = KindOfTextData.Keyword }; _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier); _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster _config.AllowOnTheFlyInputFields = true; _config.TransformerType = TransformerType.ClusterFilter; _config.ClusterFilterPredicate = c => true; // no transformations, data passed as is _config.OutputDataKind = KindOfTextData.Keyword; _config.SetOutputConsumer(l => _resultingLines.Add(l)); _resultingLines = new List <string>(); }
public void ProcessPipelineCsvIntake_TrickyHeaderRow_CorrectFldNames() { //arrange _config.InputDataKind = KindOfTextData.Delimited; _config.SetIntakeSupplier(new IntakeSupplierProvider(_csvIntakeLines()).StringSupplier); _config.HeadersInFirstInputRow = true; _config.AllowOnTheFlyInputFields = true; var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsDiscarder); //act _ = orchestrator.ExecuteAsync(); _resultsDiscarder.Completion.Wait(); //assert var fldNames = ((IntakeProvider)(new PrivateAccessor(orchestrator)).GetField("_intakeProvider")).FieldsInUse; fldNames.Count.Should().Be(12); fldNames[0].Should().Be("Fld001"); fldNames[1].Should().Be("Col002"); fldNames[2].Should().Be("Fld003"); fldNames[3].Should().Be("Fld007"); fldNames[4].Should().Be("Fld006"); fldNames[5].Should().Be("Fld008"); fldNames[6].Should().Be("Fld009"); fldNames[7].Should().Be("Fld010"); fldNames[8].Should().Be("Fld011"); fldNames[9].Should().Be("Fld012"); fldNames[10].Should().Be("Fld013"); fldNames[11].Should().Be("Fld014"); }
public void ProcessPipeline_SimpleConfigPipeDelimited_CorrectData() { //arrange _config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines().Select(l => l.Replace(',', '|'))).StringSupplier); _config.InputFieldSeparator = '|'; _config.RetainQuotes = false; _config.InputKeyPrefix = "@p"; _config.ExcludeItemsMissingPrefix = false; _config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem; _config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster _config.MarkerStartsCluster = true; //predicate matches the first record in cluster var orchestrator = TestUtilities.GetTestOrchestrator(_config, "_clusteringBlock", _resultsExtractor); //act _ = orchestrator.ExecuteAsync(); _resultsExtractor.Completion.Wait(); //assert var resultingClusters = _resultingClusters.ToList(); resultingClusters.Should().HaveCount(2); resultingClusters[0].Count.Should().Be(5); resultingClusters[1].Count.Should().Be(7); var kvRec = resultingClusters[0][0]; kvRec.Count.Should().Be(4); kvRec["RECTYPE"].Should().Be("XYZ"); kvRec["BadKey"].Should().BeNull(); kvRec["ABCD_ID"].Should().Be("XYZ00883"); kvRec["NAME"].Should().Be("Mary| Ann"); kvRec["@NUM"].Should().Be("123"); //note the prefix mismatch kvRec = resultingClusters[0][1]; kvRec.Count.Should().Be(4); kvRec["RECTYPE"].Should().Be("ABCD"); kvRec["ABCD_ID"].Should().Be("XYZ00883"); kvRec["NAME"].Should().Be("Mary| Ann"); kvRec = resultingClusters[1][0]; kvRec["RECTYPE"].Should().Be("XYZ"); kvRec["ABCD_ID"].Should().Be(" XYZ00883"); //unquoted with leading whitespace kvRec["@NUM"].Should().Be("123"); kvRec = resultingClusters[1][1]; kvRec.Count.Should().Be(4); kvRec["RECTYPE"].Should().Be("ABCD"); kvRec["ABCD_ID"].Should().Be(" XYZ00883"); kvRec["NAME"].Should().Be("Mary| Ann"); kvRec = resultingClusters[1][5]; kvRec.Count.Should().Be(1); kvRec["RECTYPE"].Should().BeNull(); kvRec["EOF"].Should().Be(string.Empty); //note the prefix mismatch kvRec = resultingClusters[1][6]; kvRec.Count.Should().Be(1); kvRec["RECTYPE"].Should().BeNull(); kvRec["EOF"].Should().BeNull(); //note the prefix mismatch }