public void FilterWhenFilterReturnsFalse() { FilterRule filterRule = new FilterRule() { Name = "Rule1", Expression = "source['a'] == 3" , IsEnabled = true }; var expressionEvaluator = new TestExpressionEvaluator(); expressionEvaluator.Evaluations["source['a'] == 3"] = () => false; var headerList = CsvParser.GetParts("a,b,c"); var row1 = CsvParser.Parse("1,2,3", headerList); Filterer filterer = new Filterer(new List <FilterRule>() { filterRule }, expressionEvaluator); var rowCtx = new SourceContext() { Source = row1 }; filterer.Filter(rowCtx); Assert.IsTrue(rowCtx.IsDeleted); }
public void FilterWhenExpressionThrows() { FilterRule filterRule = new FilterRule() { Name = "Rule1", Expression = "source['a'] == 3" , IsEnabled = true }; var expressionEvaluator = new TestExpressionEvaluator(); expressionEvaluator.Evaluations["source['a'] == 3"] = () => throw new Exception("Some error"); var headerList = CsvParser.GetParts("a,b,c"); var row1 = CsvParser.Parse("1,2,3", headerList); Filterer filterer = new Filterer(new List <FilterRule>() { filterRule }, expressionEvaluator); var rowCtx = new SourceContext() { Source = row1 }; filterer.Filter(rowCtx); Assert.IsFalse(rowCtx.IsDeleted); Assert.AreEqual("Some error", rowCtx.SourceRowErrors[0].Exception.Message); }
public void SimpleMapping() { TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator(); List <TransformationMap> maps = new List <TransformationMap>(); maps.Add(new TransformationMap() { KeyExpression = "Col0", MappingType = MappingType.Field, Output = "Key1" }); Transformer transformer = new Transformer(maps, expressionEvaluator); var header = "Col0,Col1,Col2"; var row1 = "1,2,3"; var row2 = "4,5,6"; var row3 = "7,8,9"; var headerList = CsvParser.GetParts(header); List <Dictionary <string, object> > list = new List <Dictionary <string, object> >(); list.Add(CsvParser.Parse(row1, headerList)); list.Add(CsvParser.Parse(row2, headerList)); list.Add(CsvParser.Parse(row3, headerList)); var rows = list.Select(a => new SourceContext() { Source = a }).ToList(); foreach (var sourceContext in rows) { transformer.Transform(sourceContext); } Assert.AreEqual("1", rows[0].Destination["Key1"]); Assert.AreEqual("4", rows[1].Destination["Key1"]); Assert.AreEqual("7", rows[2].Destination["Key1"]); }
public void TransformationMappingWhenExpressionThrows() { TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator(); expressionEvaluator.Evaluations["source['Col0']+'AAA'"] = () => throw new Exception("Some error"); List <TransformationMap> maps = new List <TransformationMap>(); maps.Add(new TransformationMap() { Expression = "source['Col0']+'AAA'", MappingType = MappingType.Expression, Output = "Key1" }); Transformer transformer = new Transformer(maps, expressionEvaluator); var header = "Col0,Col1,Col2"; var row1 = "1,2,3"; var row2 = "4,5,6"; var row3 = "7,8,9"; var headerList = CsvParser.GetParts(header); List <Dictionary <string, object> > list = new List <Dictionary <string, object> >(); list.Add(CsvParser.Parse(row1, headerList)); list.Add(CsvParser.Parse(row2, headerList)); list.Add(CsvParser.Parse(row3, headerList)); var rows = list.Select(a => new SourceContext() { Source = a }).ToList(); foreach (var sourceContext in rows) { expressionEvaluator.EnrichContext("source['Col0']", sourceContext.Source["Col0"]); transformer.Transform(sourceContext); } Assert.True(rows[0].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error")); Assert.True(rows[1].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error")); Assert.True(rows[2].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error")); Assert.IsNull(rows[0].Destination["Key1"]); Assert.IsNull(rows[1].Destination["Key1"]); Assert.IsNull(rows[2].Destination["Key1"]); }
public void TransformationMapping() { TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator(); expressionEvaluator.Evaluations["source['Col0']+'AAA'"] = () => expressionEvaluator.Variables["source['Col0']"] + "AAA"; List <TransformationMap> maps = new List <TransformationMap>(); maps.Add(new TransformationMap() { Expression = "source['Col0']+'AAA'", MappingType = MappingType.Expression, Output = "Key1" }); Transformer transformer = new Transformer(maps, expressionEvaluator); var header = "Col0,Col1,Col2"; var row1 = "1,2,3"; var row2 = "4,5,6"; var row3 = "7,8,9"; var headerList = CsvParser.GetParts(header); List <Dictionary <string, object> > list = new List <Dictionary <string, object> >(); list.Add(CsvParser.Parse(row1, headerList)); list.Add(CsvParser.Parse(row2, headerList)); list.Add(CsvParser.Parse(row3, headerList)); var rows = list.Select(a => new SourceContext() { Source = a }).ToList(); foreach (var sourceContext in rows) { expressionEvaluator.EnrichContext("source['Col0']", sourceContext.Source["Col0"]); transformer.Transform(sourceContext); } Assert.AreEqual("1AAA", rows[0].Destination["Key1"]); Assert.AreEqual("4AAA", rows[1].Destination["Key1"]); Assert.AreEqual("7AAA", rows[2].Destination["Key1"]); }
public void SimpleMappingWhenSourceIsEmptyAndTransformOptionIsTrue() { TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator(); List <TransformationMap> maps = new List <TransformationMap>(); maps.Add(new TransformationMap() { KeyExpression = "Col0", MappingType = MappingType.Field, Output = "Key1" , TransformOnlyIfColumnExists = true }); Transformer transformer = new Transformer(maps, expressionEvaluator); var header = "Col0,Col1,Col2"; var row1 = "NULL,2,3"; var row2 = "4,5,6"; var row3 = ",8,9"; var headerList = CsvParser.GetParts(header); List <Dictionary <string, object> > list = new List <Dictionary <string, object> >(); list.Add(CsvParser.Parse(row1, headerList)); list.Add(CsvParser.Parse(row2, headerList)); list.Add(CsvParser.Parse(row3, headerList)); var rows = list.Select(a => new SourceContext() { Source = a }).ToList(); rows[0].ListOfSchemaMissedColumns.Add("Col0"); rows[2].ListOfSchemaMissedColumns.Add("Col0"); foreach (var sourceContext in rows) { transformer.Transform(sourceContext); } Assert.IsFalse(rows[0].Destination.ContainsKey("Key1")); Assert.AreEqual("4", rows[1].Destination["Key1"]); Assert.IsFalse(rows[2].Destination.ContainsKey("Key1")); }
public void SimpleMappingWithMissedSourceKey() { TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator(); List <TransformationMap> maps = new List <TransformationMap>(); maps.Add(new TransformationMap() { KeyExpression = "WrongCol", MappingType = MappingType.Field, Output = "Key1" }); Transformer transformer = new Transformer(maps, expressionEvaluator); var header = "Col0,Col1,Col2"; var row1 = "1,2,3"; var row2 = "4,5,6"; var row3 = "4,8,9"; var headerList = CsvParser.GetParts(header); List <Dictionary <string, object> > list = new List <Dictionary <string, object> >(); list.Add(CsvParser.Parse(row1, headerList)); list.Add(CsvParser.Parse(row2, headerList)); list.Add(CsvParser.Parse(row3, headerList)); var rows = list.Select(a => new SourceContext() { Source = a }).ToList(); foreach (var sourceContext in rows) { transformer.Transform(sourceContext); } Assert.True(rows[0].DestinationCellErrors["Key1"][0].Message.Contains("missed")); Assert.True(rows[1].DestinationCellErrors["Key1"][0].Message.Contains("missed")); Assert.True(rows[2].DestinationCellErrors["Key1"][0].Message.Contains("missed")); }
public void CheckDependenciesTest() { var expressionEvaluator = new TestExpressionEvaluator(); //expressionEvaluator.Evaluations["source['Col1'] != ccc"] = // () => (string) expressionEvaluator.Variables["source['Col1']"] != "ccc"; expressionEvaluator.Evaluations["'0'"] = () => { return("0"); }; var globalPath = AppDomain.CurrentDomain.SetupInformation.ApplicationBase; var filePath = "TestData\\test.csv"; filePath = Path.Combine(globalPath, filePath); //using (StreamWriter streamWriter = new StreamWriter(filePath)) //{ // streamWriter.WriteLine("Field1,Field2,Field3"); // for (int i = 0; i < 1000000; i++) // { // streamWriter.WriteLine($"line#{i},{i},{(decimal)((decimal)12 / (decimal)(i+1))}"); // } //} IEtlNodeBuilder etlNodeBuilder = new EtlNodeBuilder(); etlNodeBuilder.AddExpressionEvaluator(expressionEvaluator); etlNodeBuilder.AddStart(); etlNodeBuilder.AddDataReader(filePath); etlNodeBuilder.AddCleanup(new List <CleanupRule>() { new CleanupRule() { IsEnabled = true, Action = CleanupAction.Replace, ColumnName = "Field3", Condition = CleanupCondition.Equal, ConditionArgument = "NaN", Expression = "'0'" } }); etlNodeBuilder.AddSchema(new List <DataSourceSchemaItem>() { new DataSourceSchemaItem() { Alias = "Col1", ColumnName = "Field1", Type = ValueType.String }, new DataSourceSchemaItem() { Alias = "Col2", ColumnName = "Field2", Type = ValueType.Int }, new DataSourceSchemaItem() { Alias = "Col3", ColumnName = "Field3", Type = ValueType.Decimal } }); //etlNodeBuilder.AddFilter(new List<FilterRule>() //{ // new FilterRule() // { // Expression = "source['Col1'] != ccc", IsEnabled = true, Name = "One rule" // } //}); etlNodeBuilder.AddDataWriter(Path.Combine(globalPath, "result.csv")); var result = etlNodeBuilder.Get(); //var filterNode = result.NodeDefinitionProvider.GetByName("Filter"); var schemaNode = result.NodeDefinitionProvider.GetByName("Schema"); var cleanupNode = result.NodeDefinitionProvider.GetByName("Cleanup"); var readerNode = result.NodeDefinitionProvider.GetByName("Reader"); var startNode = result.NodeDefinitionProvider.GetByName("Start"); var action0 = result.NodeDefinitionProvider.GetByName("Action0"); Assert.AreEqual("Schema", action0.DependsOn[0]); Assert.AreEqual("Cleanup", schemaNode.DependsOn[0]); Assert.AreEqual("Reader", cleanupNode.DependsOn[0]); Assert.AreEqual("Start", readerNode.DependsOn[0]); Assert.AreEqual(0, startNode.DependsOn.Count); //var runner = new DataNodeRunner<SourceContext>(result.NodeDefinitionProvider, result.Registry); //runner.Run(); }