public void FilterWhenFilterReturnsFalse()
{
FilterRule filterRule = new FilterRule()
{
Name = "Rule1",
Expression = "source['a'] == 3"
,
IsEnabled = true
};
var expressionEvaluator = new TestExpressionEvaluator();
expressionEvaluator.Evaluations["source['a'] == 3"] = () => false;
var headerList = CsvParser.GetParts("a,b,c");
var row1 = CsvParser.Parse("1,2,3", headerList);
Filterer filterer = new Filterer(new List <FilterRule>()
{
filterRule
}, expressionEvaluator);
var rowCtx = new SourceContext()
{
Source = row1
};
filterer.Filter(rowCtx);
Assert.IsTrue(rowCtx.IsDeleted);
}
public void FilterWhenExpressionThrows()
{
FilterRule filterRule = new FilterRule()
{
Name = "Rule1",
Expression = "source['a'] == 3"
,
IsEnabled = true
};
var expressionEvaluator = new TestExpressionEvaluator();
expressionEvaluator.Evaluations["source['a'] == 3"] = () => throw new Exception("Some error");
var headerList = CsvParser.GetParts("a,b,c");
var row1 = CsvParser.Parse("1,2,3", headerList);
Filterer filterer = new Filterer(new List <FilterRule>()
{
filterRule
}, expressionEvaluator);
var rowCtx = new SourceContext()
{
Source = row1
};
filterer.Filter(rowCtx);
Assert.IsFalse(rowCtx.IsDeleted);
Assert.AreEqual("Some error", rowCtx.SourceRowErrors[0].Exception.Message);
}
public void SimpleMapping()
{
TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator();
List <TransformationMap> maps = new List <TransformationMap>();
maps.Add(new TransformationMap()
{
KeyExpression = "Col0", MappingType = MappingType.Field, Output = "Key1"
});
Transformer transformer = new Transformer(maps, expressionEvaluator);
var header = "Col0,Col1,Col2";
var row1 = "1,2,3";
var row2 = "4,5,6";
var row3 = "7,8,9";
var headerList = CsvParser.GetParts(header);
List <Dictionary <string, object> > list = new List <Dictionary <string, object> >();
list.Add(CsvParser.Parse(row1, headerList));
list.Add(CsvParser.Parse(row2, headerList));
list.Add(CsvParser.Parse(row3, headerList));
var rows = list.Select(a => new SourceContext()
{
Source = a
}).ToList();
foreach (var sourceContext in rows)
{
transformer.Transform(sourceContext);
}
Assert.AreEqual("1", rows[0].Destination["Key1"]);
Assert.AreEqual("4", rows[1].Destination["Key1"]);
Assert.AreEqual("7", rows[2].Destination["Key1"]);
}
public void TransformationMappingWhenExpressionThrows()
{
TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator();
expressionEvaluator.Evaluations["source['Col0']+'AAA'"] =
() => throw new Exception("Some error");
List <TransformationMap> maps = new List <TransformationMap>();
maps.Add(new TransformationMap()
{
Expression = "source['Col0']+'AAA'",
MappingType = MappingType.Expression,
Output = "Key1"
});
Transformer transformer = new Transformer(maps, expressionEvaluator);
var header = "Col0,Col1,Col2";
var row1 = "1,2,3";
var row2 = "4,5,6";
var row3 = "7,8,9";
var headerList = CsvParser.GetParts(header);
List <Dictionary <string, object> > list = new List <Dictionary <string, object> >();
list.Add(CsvParser.Parse(row1, headerList));
list.Add(CsvParser.Parse(row2, headerList));
list.Add(CsvParser.Parse(row3, headerList));
var rows = list.Select(a => new SourceContext()
{
Source = a
}).ToList();
foreach (var sourceContext in rows)
{
expressionEvaluator.EnrichContext("source['Col0']", sourceContext.Source["Col0"]);
transformer.Transform(sourceContext);
}
Assert.True(rows[0].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error"));
Assert.True(rows[1].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error"));
Assert.True(rows[2].DestinationCellErrors["Key1"][0].Exception.Message.Contains("Some error"));
Assert.IsNull(rows[0].Destination["Key1"]);
Assert.IsNull(rows[1].Destination["Key1"]);
Assert.IsNull(rows[2].Destination["Key1"]);
}
public void TransformationMapping()
{
TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator();
expressionEvaluator.Evaluations["source['Col0']+'AAA'"] =
() => expressionEvaluator.Variables["source['Col0']"] + "AAA";
List <TransformationMap> maps = new List <TransformationMap>();
maps.Add(new TransformationMap()
{
Expression = "source['Col0']+'AAA'",
MappingType = MappingType.Expression,
Output = "Key1"
});
Transformer transformer = new Transformer(maps, expressionEvaluator);
var header = "Col0,Col1,Col2";
var row1 = "1,2,3";
var row2 = "4,5,6";
var row3 = "7,8,9";
var headerList = CsvParser.GetParts(header);
List <Dictionary <string, object> > list = new List <Dictionary <string, object> >();
list.Add(CsvParser.Parse(row1, headerList));
list.Add(CsvParser.Parse(row2, headerList));
list.Add(CsvParser.Parse(row3, headerList));
var rows = list.Select(a => new SourceContext()
{
Source = a
}).ToList();
foreach (var sourceContext in rows)
{
expressionEvaluator.EnrichContext("source['Col0']", sourceContext.Source["Col0"]);
transformer.Transform(sourceContext);
}
Assert.AreEqual("1AAA", rows[0].Destination["Key1"]);
Assert.AreEqual("4AAA", rows[1].Destination["Key1"]);
Assert.AreEqual("7AAA", rows[2].Destination["Key1"]);
}
public void SimpleMappingWhenSourceIsEmptyAndTransformOptionIsTrue()
{
TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator();
List <TransformationMap> maps = new List <TransformationMap>();
maps.Add(new TransformationMap()
{
KeyExpression = "Col0",
MappingType = MappingType.Field,
Output = "Key1"
, TransformOnlyIfColumnExists = true
});
Transformer transformer = new Transformer(maps, expressionEvaluator);
var header = "Col0,Col1,Col2";
var row1 = "NULL,2,3";
var row2 = "4,5,6";
var row3 = ",8,9";
var headerList = CsvParser.GetParts(header);
List <Dictionary <string, object> > list = new List <Dictionary <string, object> >();
list.Add(CsvParser.Parse(row1, headerList));
list.Add(CsvParser.Parse(row2, headerList));
list.Add(CsvParser.Parse(row3, headerList));
var rows = list.Select(a => new SourceContext()
{
Source = a
}).ToList();
rows[0].ListOfSchemaMissedColumns.Add("Col0");
rows[2].ListOfSchemaMissedColumns.Add("Col0");
foreach (var sourceContext in rows)
{
transformer.Transform(sourceContext);
}
Assert.IsFalse(rows[0].Destination.ContainsKey("Key1"));
Assert.AreEqual("4", rows[1].Destination["Key1"]);
Assert.IsFalse(rows[2].Destination.ContainsKey("Key1"));
}
public void SimpleMappingWithMissedSourceKey()
{
TestExpressionEvaluator expressionEvaluator = new TestExpressionEvaluator();
List <TransformationMap> maps = new List <TransformationMap>();
maps.Add(new TransformationMap()
{
KeyExpression = "WrongCol",
MappingType = MappingType.Field,
Output = "Key1"
});
Transformer transformer = new Transformer(maps, expressionEvaluator);
var header = "Col0,Col1,Col2";
var row1 = "1,2,3";
var row2 = "4,5,6";
var row3 = "4,8,9";
var headerList = CsvParser.GetParts(header);
List <Dictionary <string, object> > list = new List <Dictionary <string, object> >();
list.Add(CsvParser.Parse(row1, headerList));
list.Add(CsvParser.Parse(row2, headerList));
list.Add(CsvParser.Parse(row3, headerList));
var rows = list.Select(a => new SourceContext()
{
Source = a
}).ToList();
foreach (var sourceContext in rows)
{
transformer.Transform(sourceContext);
}
Assert.True(rows[0].DestinationCellErrors["Key1"][0].Message.Contains("missed"));
Assert.True(rows[1].DestinationCellErrors["Key1"][0].Message.Contains("missed"));
Assert.True(rows[2].DestinationCellErrors["Key1"][0].Message.Contains("missed"));
}
public void CheckDependenciesTest()
{
var expressionEvaluator = new TestExpressionEvaluator();
//expressionEvaluator.Evaluations["source['Col1'] != ccc"] =
// () => (string) expressionEvaluator.Variables["source['Col1']"] != "ccc";
expressionEvaluator.Evaluations["'0'"] =
() =>
{
return("0");
};
var globalPath = AppDomain.CurrentDomain.SetupInformation.ApplicationBase;
var filePath = "TestData\\test.csv";
filePath = Path.Combine(globalPath, filePath);
//using (StreamWriter streamWriter = new StreamWriter(filePath))
//{
// streamWriter.WriteLine("Field1,Field2,Field3");
// for (int i = 0; i < 1000000; i++)
// {
// streamWriter.WriteLine($"line#{i},{i},{(decimal)((decimal)12 / (decimal)(i+1))}");
// }
//}
IEtlNodeBuilder etlNodeBuilder = new EtlNodeBuilder();
etlNodeBuilder.AddExpressionEvaluator(expressionEvaluator);
etlNodeBuilder.AddStart();
etlNodeBuilder.AddDataReader(filePath);
etlNodeBuilder.AddCleanup(new List <CleanupRule>()
{
new CleanupRule()
{
IsEnabled = true, Action = CleanupAction.Replace, ColumnName = "Field3", Condition = CleanupCondition.Equal, ConditionArgument = "NaN", Expression = "'0'"
}
});
etlNodeBuilder.AddSchema(new List <DataSourceSchemaItem>()
{
new DataSourceSchemaItem()
{
Alias = "Col1", ColumnName = "Field1", Type = ValueType.String
},
new DataSourceSchemaItem()
{
Alias = "Col2", ColumnName = "Field2", Type = ValueType.Int
},
new DataSourceSchemaItem()
{
Alias = "Col3", ColumnName = "Field3", Type = ValueType.Decimal
}
});
//etlNodeBuilder.AddFilter(new List<FilterRule>()
//{
// new FilterRule()
// {
// Expression = "source['Col1'] != ccc", IsEnabled = true, Name = "One rule"
// }
//});
etlNodeBuilder.AddDataWriter(Path.Combine(globalPath, "result.csv"));
var result = etlNodeBuilder.Get();
//var filterNode = result.NodeDefinitionProvider.GetByName("Filter");
var schemaNode = result.NodeDefinitionProvider.GetByName("Schema");
var cleanupNode = result.NodeDefinitionProvider.GetByName("Cleanup");
var readerNode = result.NodeDefinitionProvider.GetByName("Reader");
var startNode = result.NodeDefinitionProvider.GetByName("Start");
var action0 = result.NodeDefinitionProvider.GetByName("Action0");
Assert.AreEqual("Schema", action0.DependsOn[0]);
Assert.AreEqual("Cleanup", schemaNode.DependsOn[0]);
Assert.AreEqual("Reader", cleanupNode.DependsOn[0]);
Assert.AreEqual("Start", readerNode.DependsOn[0]);
Assert.AreEqual(0, startNode.DependsOn.Count);
//var runner = new DataNodeRunner<SourceContext>(result.NodeDefinitionProvider, result.Registry);
//runner.Run();
}
