internal DataProcessor(Func <Task <string> > intakeSupplierAsync, Action <string> outputConsumer, Action <int> progressHandler)
{
var config = new OrchestratorConfig()
{
ReportProgress = true,
ProgressInterval = 10,
ProgressChangedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
progressHandler(e.RecCnt);
}
},
PhaseFinishedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
progressHandler(e.RecCnt);
}
},
InputDataKind = KindOfTextData.Delimited,
InputFields = "PlaneDescription,IataCode,IcaoCode",
AsyncIntake = true,
TransformerType = TransformerType.Universal,
UniversalTransformer = FilterAndReorganizeFields,
AllowTransformToAlterFields = true,
OutputDataKind = KindOfTextData.Flat,
OutputFields = "IataCode|4,Hyphen|2,PlaneDescription|70",
ExcludeExtraneousFields = true
};
config.SetAsyncIntakeSupplier(intakeSupplierAsync);
config.SetOutputConsumer(outputConsumer);
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
public EtlOrchestrator_tests_ArbitraryOutput()
{
_config = new OrchestratorConfig
{
InputDataKind = KindOfTextData.Keyword
};
_config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier);
_config.RetainQuotes = false;
_config.InputKeyPrefix = "@p";
_config.ExcludeItemsMissingPrefix = false;
_config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem;
_config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String
_config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //single record clusters
_config.AllowOnTheFlyInputFields = true;
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c => true; // no transformations, data passed as is
_config.OutputDataKind = KindOfTextData.Arbitrary;
_config.ArbitraryOutputDefs = new string[] {
"Record type is {RECTYPE},",
" name is {NAME}",
" and number is {NUM}.",
" Void item here."
};
_config.SetOutputConsumer(l => { if (l != null)
{
_resultingLines.Add(l);
}
});
_resultingLines = new List <string>();
}
public void processEntirePipeline_SimpleConsumer_TargetAlways1()
{
//arrange
_config.InputDataKind = KindOfTextData.Raw;
_config.IntakeSupplier = _inLine; //has SourceNo assigned in a round-robin fashion
//default (no) transformation
_config.RouterType = RouterType.SourceToTarget;
_config.OutputDataKind = KindOfTextData.Raw;
_config.SetOutputConsumer(l => _resultingLines.Add(l));
var orchestrator = new EtlOrchestrator(_config);
//act
var counts = orchestrator.ExecuteAsync().Result;
//assert
counts.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
counts.RowsRead.Should().Be(8);
counts.ClustersRead.Should().Be(8);
counts.RowsWritten.Should().Be(8);
counts.ClustersWritten.Should().Be(8);
_resultingLines.Count.Should().Be(9); //includes end-of-data mark
_resultingLines[0].Should().Be("Line 01");
_resultingLines[1].Should().Be("Line 02");
_resultingLines[2].Should().Be("Line 03");
_resultingLines[3].Should().Be("Line 04");
_resultingLines[4].Should().Be("Line 05");
_resultingLines[5].Should().Be("Line 06");
_resultingLines[6].Should().Be("Line 07");
_resultingLines[7].Should().Be("Line 08");
_resultingLines[8].Should().BeNull();
}
public EtlOrchestrator_tests_FieldsToUse()
{
_config = new OrchestratorConfig
{
InputDataKind = KindOfTextData.Flat
};
_config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier);
_config.HeadersInFirstInputRow = true;
_config.InputFields = "InFld1|4,InFld2|4,InFld3|12";
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c => true; // no transformations, data passed as is
_config.SetOutputConsumer((string l) => { }); //throwaay consumer, these tests do not evaluate output
}
public void ProcessPipeline_PassUnchangedClustersEntirePipeline_SameDataAsInput()
{
//The same test as the previous one, except that the entire pipeline is executed.
// It verifies what the output at the end of the pipeline.
//arrange
var outputLines = new ConcurrentQueue <string>(); //output lines are set here by output consumer
_config.ClusterboundTransformer = clstr => clstr; // pass the same cluster to output
_config.DeferOutput = DeferOutput.Auto; //restore normal setting, so that entire pipeline is processed (nothing goes to _resultingClusters, but ...
_config.SetOutputConsumer(l => outputLines.Enqueue(l)); // ... final output is sent to outputLines).
_config.OutputDataKind = KindOfTextData.Keyword;
_config.OutputKeyPrefix = "@p"; //we want to match output to input
var orchestrator = new EtlOrchestrator(_config);
//act
var counts = orchestrator.ExecuteAsync().Result;
//assert
counts.RowsRead.Should().Be(10);
counts.ClustersRead.Should().Be(2);
counts.RowsWritten.Should().Be(10);
counts.ClustersWritten.Should().Be(2);
_resultingClusters.Should().BeEmpty(); //nothing was redirected to _resultingClusters in this test
var outLines = outputLines.ToList();
outLines.Should().HaveCount(11); //incl. EOD mark
outLines[0].Should().Be("@pRECTYPE=XYZ,@pNAME=Mary,@pNUM=123");
outLines[1].Should().Be("@pRECTYPE=ABCD,@pABCD_ID=XYZ00883,@pNAME=Mary,@pNUM=223");
outLines[8].Should().Be("@pRECTYPE=ABCD,@pABCD_ID=XYZ00883,@pNAME=Cindy,@pNUM=923");
outLines[9].Should().Be("@pEOF"); //prefix added here as well
outLines[10].Should().BeNull(); //EOD mark
}
ConcurrentQueue <Tuple <ExternalLine, int> > _resultingLines; //Item2=targetNo
public EtlOrchestrator_tests_Router()
{
_config = new OrchestratorConfig();
_config.InputDataKind = KindOfTextData.Keyword;
var sn = 0; //closure to facilitate SourceNo calculation (note that GetStringTupleSupplier calls the sourceNoEval function (provided as parameter below) exactly once per iteration)
_config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).GetStringTupleSupplier(() => sn++ % 3 + 1)); //assign sourceNo in a round-robin fashion: 1,2,3,1,2,3,1,2,3
_config.InputKeyPrefix = "@p";
//no type definitions (everything string)
_config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster
_config.MarkerStartsCluster = true; //predicate matches the first record in cluster
_config.AllowOnTheFlyInputFields = true;
_config.SetOutputConsumer(tpl => _resultingLines.Enqueue(tpl)); //Item1=ExternalLine/Xrecord, Item2=targetNo
_resultingLines = new ConcurrentQueue <Tuple <ExternalLine, int> >();
}
public EtlOrchestrator_tests_X12Output()
{
_config = new OrchestratorConfig
{
InputDataKind = KindOfTextData.X12
};
_config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier);
_config.ClusterMarker = (rec, prevRec, recCnt) => { return(true); }; //single record clusters
_config.AllowOnTheFlyInputFields = true; // to allow fields to showing trailing spaces (should matter for KW, but not X12)
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c => true; // no transformations, data passed as is
_config.OutputDataKind = KindOfTextData.X12;
_config.SetOutputConsumer(l => { if (l != null)
{
_resultingLines.Add(l);
}
});
_resultingLines = new List <string>();
}
public EtlOrchestrator_tests_KwOutput()
{
_config = new OrchestratorConfig
{
InputDataKind = KindOfTextData.Keyword
};
_config.SetIntakeSupplier(new IntakeSupplierProvider(_intakeLines()).StringSupplier);
_config.RetainQuotes = false;
_config.InputKeyPrefix = "@p";
_config.ExcludeItemsMissingPrefix = false;
_config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem;
_config.TypeDefiner = key => key == "NUM" ? new ItemDef(ItemType.Int, null) : new ItemDef(ItemType.String, null); //NUM Int, everything else String
_config.ClusterMarker = (rec, prevRec, recCnt) => { return((string)rec["RECTYPE"] == "XYZ"); }; //records having @pRECTYPE=XYZ denote start of the cluster
_config.MarkerStartsCluster = true; //predicate matches the first record in cluster
_config.AllowOnTheFlyInputFields = true;
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c => true; // no transformations, data passed as is
_config.OutputDataKind = KindOfTextData.Keyword;
_config.SetOutputConsumer(l => _resultingLines.Add(l));
_resultingLines = new List <string>();
}
public void produceExtLineOfTypeXrecord_SimpleSettings_CorrectData()
{
//arrange
_config.InputDataKind = KindOfTextData.Keyword;
_config.IntakeSupplier = _inLine;
_config.RetainQuotes = false;
_config.InputKeyPrefix = "@p";
_config.ExcludeItemsMissingPrefix = false;
_config.ActionOnDuplicateKey = ActionOnDuplicateKey.IgnoreItem;
_config.MarkerStartsCluster = true; //predicate matches the first record in cluster
_config.AllowOnTheFlyInputFields = true;
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c => true; // no transformations, data passed as is
_config.OutputDataKind = KindOfTextData.XML;
_config.SetOutputConsumer((IEnumerable <Tuple <string, object> > r) => { _resultingLines.Add(r.ToExternalLine()); }); //note that type of lambda argument needs to be defined (disambiguated)
var orchestrator = new EtlOrchestrator(_config);
//act
var counts = orchestrator.ExecuteAsync().Result;
//assert
counts.RowsRead.Should().Be(10);
counts.ClustersRead.Should().Be(10);
counts.ClustersWritten.Should().Be(10);
counts.RowsWritten.Should().Be(10);
_resultingLines.Count.Should().Be(11); //10 records + EOD mark
var xRec = _resultingLines[0];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(3);
var xItm = xRec.Items[0];
xItm.Item1.Should().Be("RECTYPE");
xItm.Item2.Should().Be("XYZ");
xItm = xRec.Items[1];
xItm.Item1.Should().Be("NAME");
xItm.Item2.Should().Be("Mary");
xItm = xRec.Items[2];
xItm.Item1.Should().Be("NUM");
xItm.Item2.Should().Be("123");
xRec = _resultingLines[2];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(4);
xItm = xRec.Items[0];
xItm.Item1.Should().Be("RECTYPE");
xItm.Item2.Should().Be("ABCD");
xItm = xRec.Items[1];
xItm.Item1.Should().Be("ABCD_ID");
xItm.Item2.Should().Be("XYZ00883");
xItm = xRec.Items[2];
xItm.Item1.Should().Be("NAME");
xItm.Item2.Should().Be("Susan");
xItm = xRec.Items[3];
xItm.Item1.Should().Be("NUM");
xItm.Item2.Should().Be("323");
xRec = _resultingLines[3];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(4);
xItm = xRec.Items[0];
xItm.Item1.Should().Be("RECTYPE");
xItm.Item2.Should().Be("ABCD");
xItm = xRec.Items[1];
xItm.Item1.Should().Be("ABCD_ID");
xItm.Item2.Should().Be("XYZ00883");
xItm = xRec.Items[2];
xItm.Item1.Should().Be("NAME");
xItm.Item2.Should().Be("Mary,Ann"); //commas OK
xItm = xRec.Items[3];
xItm.Item1.Should().Be("NUM");
xItm.Item2.Should().Be("423");
xRec = _resultingLines[6];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(4);
xItm = xRec.Items[0];
xItm.Item1.Should().Be("RECTYPE");
xItm.Item2.Should().Be("ABCD");
xItm = xRec.Items[1];
xItm.Item1.Should().Be("ABCD_ID");
xItm.Item2.Should().Be("XYZ00883");
xItm = xRec.Items[2];
xItm.Item1.Should().Be("NAME");
xItm.Item2.Should().Be("Joan");
xItm = xRec.Items[3];
xItm.Item1.Should().Be("NUM");
xItm.Item2.Should().Be("723");
xRec = _resultingLines[8];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(4);
xItm = xRec.Items[0];
xItm.Item1.Should().Be("ABCD_ID");
xItm.Item2.Should().Be(string.Empty);
xItm = xRec.Items[1];
xItm.Item1.Should().Be("NAME");
xItm.Item2.Should().Be("Cindy");
xItm = xRec.Items[2];
xItm.Item1.Should().Be("NUM");
xItm.Item2.Should().Be("923");
xItm = xRec.Items[3];
xItm.Item1.Should().Be("RECTYPE");
xItm.Item2.Should().Be("ABCD");
xRec = _resultingLines[9];
xRec.Should().BeOfType <Xrecord>();
xRec.Items.Count.Should().Be(1);
xItm = xRec.Items[0];
xItm.Item1.Should().Be("EOF");
xItm.Item2.Should().BeNull(); //note that this is null even for XML - this is b/c EOF field exists - it was parsed on intake as string with a null value (IOW, it is not a void item)
xRec = _resultingLines[10];
xRec.Should().BeNull(); //EOD mark
}