public static void Import(string csv)
{
var csvMapper = new CsvMapper();
//var csv = File.ReadAllText(@"D:\Dev\git\FatDividends\_documentation\D&E\earning_cba.csv");
var mappings = new Dictionary<string, int>
{
{"StockCode", 0},
{"Year", 1},
{"Period", 2},
{"ReportedDate", 3},
{"NPAT", 4},
{"Margin", 5},
{"CashFlow", 6},
{"EPS", 7},
{"DPS", 8},
{"ROE", 9}
};
var earnings = csvMapper.MapCsvTo<StockEarning>(mappings, csv, true);
using (var service = new EarningService())
{
service.Add(earnings);
}
}
public void MapByFilePathCustomMappings()
{
// arrange
var mapper = new CsvMapper<CurryOrder>();
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
// act
mapper.OnItemDataBound += this.CurryOrder_OnItemDataBound;
var result = mapper.Map(this.CurryOrderFilePath);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(3, result.Count);
var lastRow = result.LastOrDefault();
Assert.IsNotNull(lastRow);
Assert.AreEqual("Korma", lastRow.Curry);
Assert.AreEqual(2, lastRow.Quantity);
Assert.AreEqual(12.5M, lastRow.Price);
Assert.AreEqual(25M, lastRow.Total);
}
public void MapByFilePathParseValues()
{
// arrange
var mapper = new CsvMapper <CurryOrder>();
var mappings = new NameValueCollection();
mappings.Add("Curry", "0");
mappings.Add("Quantity", "1");
mappings.Add("Price", "2");
mappings.Add("Total", "3");
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
// act
var result = mapper.Map(this.CurryOrderFilePath, mappings);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(3, result.Count);
var lastRow = result.LastOrDefault();
Assert.IsNotNull(lastRow);
Assert.AreEqual("Korma", lastRow.Curry);
Assert.AreEqual(2, lastRow.Quantity);
Assert.AreEqual(12.5M, lastRow.Price);
Assert.AreEqual(25M, lastRow.Total);
}
public string[] GetHeaders(string filePath, char delimiter)
{
using (var csvProcessor = new CsvMapper <TranslationLabel>(filePath, delimiter))
{
return(csvProcessor.GetHeaders());
}
}
public void MapByFilePathCustomMappings()
{
// arrange
var mapper = new CsvMapper <CurryOrder>();
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
// act
mapper.OnItemDataBound += this.CurryOrder_OnItemDataBound;
var result = mapper.Map(this.CurryOrderFilePath);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(3, result.Count);
var lastRow = result.LastOrDefault();
Assert.IsNotNull(lastRow);
Assert.AreEqual("Korma", lastRow.Curry);
Assert.AreEqual(2, lastRow.Quantity);
Assert.AreEqual(12.5M, lastRow.Price);
Assert.AreEqual(25M, lastRow.Total);
}
private void ButtonLoadData_Click(object sender, RoutedEventArgs e)
{
if (!UpDownLineSkip.Value.HasValue || !UpDownClassCount.Value.HasValue)
{
return;
}
Mapper = new CsvMapper
{
Normalized = true,
Separator = TextBoxColumnSeparator.Text[0],
DecimalSeparator =
CheckBoxDecimalSeparator.IsChecked != null && CheckBoxDecimalSeparator.IsChecked.Value ? '.' : ',',
SkipRows = (int)UpDownLineSkip.Value.Value
};
var openFileDialog = new OpenFileDialog
{
Filter = "csv file (*.csv)|*.csv",
Multiselect = false
};
if (openFileDialog.ShowDialog() != true)
{
return;
}
var filename = openFileDialog.FileName;
Data = Mapper.ReadDataFromFile(filename, Convert.ToInt32(UpDownClassCount.Value.Value));
DataGridHelper.SetTableData(DataGridSet, new TableData(Data));
ButtonStart.IsEnabled = true;
}
public CsvReader(Stream stream, bool isHeaderIncluded, char delimeter, CsvMapper mapper)
{
this.Mapper = mapper;
LineEnumerator = RowReader(stream, delimeter).GetEnumerator();
if (isHeaderIncluded)
{
LineEnumerator.MoveNext();
mapper.ColumnNames = LineEnumerator.Current;
}
}
public void Load_NullConfig_Throws()
{
// arrange
var parser = new CsvParser("Foo,Bar,Baz");
var mapper = new CsvMapper(parser, _outStream);
// act
// assert
Assert.Throws <ArgumentNullException>(() => mapper.Load((IDictionary <string, IncomingFormat>)null));
}
public void GetHeadersByFilePath()
{
// arrange
var mapper = new CsvMapper<TranslationLabel>();
// act
var result = mapper.GetHeaders(this.TranslationLabelFilePath);
// assert
Assert.IsNotNull(result);
Assert.AreEqual(6, result.Length);
Assert.AreEqual("Italian", result[4]);
}
public void GetHeadersByFilePath()
{
// arrange
var mapper = new CsvMapper <TranslationLabel>();
// act
var result = mapper.GetHeaders(this.TranslationLabelFilePath);
// assert
Assert.IsNotNull(result);
Assert.AreEqual(6, result.Length);
Assert.AreEqual("Italian", result[4]);
}
public void MappingWithRowLimit2()
{
var mapper = new CsvMapper <TestClass>(new MappingOptions
{
RowsLimit = 100,
HasHeader = true
});
var items = mapper.Map(TestCsvPath);
Assert.IsNotNull(items, "Result is null");
Assert.IsTrue(items.Any(), "Items empty");
items.SimpleMappingTrace(TestCsvPath);
}
public void SimpleMappingByPath()
{
var mapper = new CsvMapper <TestClass>(new MappingOptions
{
MappingMode = MappingMode.ByNumber,
HasHeader = false
});
var items = mapper.Map(TestCsvPath);
Assert.IsNotNull(items, "Result is null");
Assert.IsTrue(items.Any(), "Items empty");
items.SimpleMappingTrace(TestCsvPath);
}
public void map_should_throw_missing_mapping_exception_for_not_mapings()
{
//Given
var csvMapper = new CsvMapper <Employee>();
var csvLines = new List <string> {
"john",
"mary,[email protected]"
};
//When
Action action = () => csvMapper.Map(csvLines);
//Then
action.Should().Throw <MissingMappingException>();
}
public async Task Remap_Writes_Stream()
{
// arrange
var config = GetTestConfig();
var parser = GetTestParser().UseHeaders();
var mapper = new CsvMapper(parser, _outStream);
mapper.Load(config);
// act
await mapper.RemapAsync(',');
// assert
Assert.NotEqual(0, _outStream.Length);
}
public void SimpleMappingByStream()
{
var mapper = new CsvMapper <TestClass>(new MappingOptions {
});
using (var fs = new FileStream(TestCsvPath, FileMode.Open))
{
var items = mapper.Map(fs);
Assert.IsNotNull(items, "Result is null");
Assert.IsTrue(items.Any(), "Result empty");
items.SimpleMappingTrace(TestCsvPath);
}
}
public void create_map_should_add_mapping()
{
//Given
var csvMapper = new CsvMapper <Employee>();
//when
csvMapper.CreateMap(0, e => e.Name);
//then
csvMapper.IndexPropertyMappings.Should().NotBeNullOrEmpty();
var mappedProperty = csvMapper.IndexPropertyMappings.First();
mappedProperty.Index.Should().Be(0);
mappedProperty.AttributeName.Should().Be("Name");
}
public void MappingWithRowLimit()
{
try
{
var mapper = new CsvMapper <TestClass>(new MappingOptions {
RowsLimit = 99
});
var items = mapper.Map(TestCsvPath);
}
catch (TableMappingException ex)
{
Trace.WriteLine($"{ex.Message}; Row {ex.Row}");
return;
}
Assert.Fail($"{nameof(TableMappingException)} has not been thrown");
}
public void SimpleMappingByPath()
{
var mapper = new CsvMapper <TestClass>(
new MappingOptions
{
MappingMode = MappingMode.ByNumber,
SuppressConvertTypeErrors = false,
EnableValidation = true,
HasHeader = true
});
var items = mapper.Map(TestCsvPath);
Assert.IsNotNull(items, "Result is null");
Assert.IsTrue(items.Any(), "Items empty");
items.SimpleMappingTrace(TestCsvPath);
}
public MappingResponse Map(string filePath, char delimiter, NameValueCollection mappings)
{
using (var mapper = new CsvMapper <TranslationLabel>(filePath, delimiter))
{
this.mappings = mappings;
mapper.OnItemDataBound += CsvProcessor_OnItemDataBound;
var translationLabels = mapper.MapToClass().Skip(1).ToList();
var response = new MappingResponse
{
Results = ReflectionUtility.GetDataTableByReflection <TranslationLabel>(translationLabels),
Log = mapper.Log,
SqlFilePath = this.CreateSql(translationLabels)
};
return(response);
}
}
public void map_should_throw_exception_for_invalid_data()
{
//Given
var csvMapper = new CsvMapper <Employee>();
csvMapper.CreateMap(0, e => e.Name);
csvMapper.CreateMap(1, e => e.Email);
var csvLines = new List <string> {
"john",
"mary,[email protected]"
};
//When
Action action = () => csvMapper.Map(csvLines);
//then
action.Should().Throw <Exception>();
}
public void MappingWithValidation()
{
var mapper = new CsvMapper <ValidationTestClass>(new MappingOptions {
EnableValidation = true
});
try
{
var items = mapper.Map(ValidationTestCsvPath);
}
catch (TableMappingException ex)
{
Trace.WriteLine($"{ex.Message}; Row {ex.Row}");
return;
}
Assert.Fail($"{nameof(TableMappingException)} has not been thrown");
}
private void cSVImportierenToolStripMenuItem_Click(object sender, EventArgs e)
{
// Datei öffnen
OpenFileDialog ofd = new OpenFileDialog();
if (ofd.ShowDialog() != DialogResult.OK) return;
IList<Judoka> jlst = Judoka.List();
CsvMapper<Judoka> jcsv = new CsvMapper<Judoka>(ofd.FileName);
jcsv.IgnoreErrors = true;
List<Judoka> jclst = jcsv.List();
foreach (Judoka perJ in jlst)
{
String perJHash = perJ.GetNameHash(Judoka.HashType.None);
foreach (Judoka csvJ in jclst)
{
if (perJHash == csvJ.GetNameHash(Judoka.HashType.None))
{
Console.WriteLine("Persistiert: " + perJ.ToString() + " [" + perJ.Id + "]");
csvJ.Id = perJ.Id;
}
}
}
foreach (Judoka csvJ in jclst)
{
if (csvJ.Id == 0)
{
Console.WriteLine("Nicht Persistent: " + csvJ.ToString());
if (csvJ.Geschlecht == null || csvJ.Verein == null || csvJ.Gürtel == null)
{
MessageBox.Show("Judoka hat Geschlecht, Verein oder Gürtel NULL");
continue;
}
else
{
csvJ.Save();
}
}
}
}
public async Task Remap_IgnoresUnmapped()
{
// arrange
var config = GetTestConfig();
var parser = GetTestParser().UseHeaders();
var mapper = new CsvMapper(parser, _outStream);
mapper.Load(config);
var outStream = new MemoryStream();
// act
await mapper.RemapAsync(',');
outStream.Seek(0, SeekOrigin.Begin);
var asText = Encoding.UTF8.GetString(outStream.ToArray());
// assert
Assert.DoesNotContain("bat", asText);
}
public void MappingWithSuppressConvertTypeErrors()
{
try
{
var mapper = new CsvMapper <TestClass>(new MappingOptions {
SuppressConvertTypeErrors = false
});
var items = mapper.Map(SuppressConvertTypeErrorsTestCsvPath);
Assert.IsNotNull(items, "Result is null");
Assert.IsTrue(items.Any(), "Items empty");
items.SimpleMappingTrace(TestCsvPath);
}
catch (TableMappingException ex)
{
Trace.WriteLine($"{ex.Message}; Row {ex.Row}");
return;
}
Assert.Fail($"{nameof(TableMappingException)} has not been thrown");
}
public IEnumerable <T> Parse <T>(string fileName) where T : class, new()
{
var lines = File.ReadLines(fileName);
CsvMapper <T> mapper = null;
if (typeof(T) == typeof(Employee))
{
mapper = new EmployeeMapper() as CsvMapper <T>;
}
if (typeof(T) == typeof(Department))
{
mapper = new DepartmentMapper() as CsvMapper <T>;
}
if (mapper == null)
{
throw new MissingMappingException();
}
return(mapper.Map(lines));
}
public void map_should_map_csv_data()
{
//Given
var csvMapper = new CsvMapper <Employee>();
csvMapper.CreateMap(0, e => e.Name);
csvMapper.CreateMap(1, e => e.Email);
var csvLines = new List <string> {
"john,[email protected]",
"mary,[email protected]"
};
//When
var output = csvMapper.Map(csvLines);
//Then
output.Should().NotBeNullOrEmpty();
output.Count().Should().Be(2);
output.First().Name.Should().Be("john");
output.First().Email.Should().Be("*****@*****.**");
}
private void ValidateFile(string file, InputData input)
{
var header = System.IO.File.ReadLines(file).First();
//Validate file is good by validating header record.
var listOfRequiredWords = new List <string>()
{
CsvMapper.GetByDefaultName(CsvMapper.Year),
CsvMapper.GetByDefaultName(CsvMapper.Month),
};
listOfRequiredWords.AddRange(input.DisplayNameToCsvHeader.Values);
listOfRequiredWords.AddRange(input.DisplayNameOfCapeMeasureToHeader.Values);
foreach (var item in listOfRequiredWords)
{
if (!header.Contains(item))
{
throw new InvalidDataException($"Header of CSV (the first line of the file) does not contain '{item}'. Header: '{header}'");
}
}
}
public static void Import(string csv)
{
var csvMapper = new CsvMapper();
var mappings = new Dictionary<string, int>
{
{"StockCode", 0},
{"ExDate", 1},
{"Amount", 2},
{"Franked", 3},
{"FrankingCredit", 4},
{"PayableDate", 5}
};
var stockDividends = csvMapper.MapCsvTo<StockDividend>(mappings, csv, true).ToList();
UpdateClosingPrices(stockDividends);
using (var service = new DividendService())
{
service.Add(stockDividends);
}
}
public virtual MappingResponse Map(List <string> filePaths, NameValueCollection mappings)
{
var rows = new List <T>();
var log = new List <LogEntry>();
foreach (var filePath in filePaths)
{
var mapper = new CsvMapper <T>();
rows.AddRange(mapper.Map(filePath, mappings).ToList());
log.AddRange(mapper.Log);
}
this.Rows = rows;
var response = new MappingResponse
{
ResultsAsTable = ReflectionUtility.GetDataTableByReflection <T>(rows),
Log = log
};
return(response);
}
public async Task Remap_MutatorWasRun()
{
// arrange
var config = GetTestConfig();
var parser = GetTestParser().UseHeaders();
var mapper = new CsvMapper(parser, _outStream);
mapper.Load(config);
// act
await mapper.RemapAsync(',');
// assert
_outStream.Seek(0, SeekOrigin.Begin);
var asText = Encoding.UTF8.GetString(_outStream.ToArray());
var lines = asText.Split(Environment.NewLine);
var firstRecord = lines[1];
var fields = firstRecord.Split(',');
Assert.Equal("unknown", fields[0]);
}
public void MapByFilePath()
{
// arrange
var mapper = new CsvMapper<TranslationLabel>();
var mappings = this.GetTranslationLabelMappings();
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.TranslationLabelFilePath);
mapper.GetHeaders(this.TranslationLabelFilePath);
mapper.GetHeaders(this.TranslationLabelFilePath);
// act
var result = mapper.Map(this.TranslationLabelFilePath, mappings);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(22, result.Count);
var firstRow = result.FirstOrDefault();
Assert.IsNotNull(firstRow);
Assert.AreEqual("InfinitiMonthlyPayment", firstRow.LabelName);
}
public IEnumerable <ReturnData> GetData(InputData input)
{
var file = input.FilePath;
ValidateFile(file, input);
using (var reader = new StreamReader(file))
using (var csv = new CsvReader(reader))
{
var records = new List <ReturnData>();
csv.Read();
csv.ReadHeader();
int row = 0;
while (csv.Read())
{
var record = new ReturnData
{
Year = csv.Get <int?>(CsvMapper.GetByDefaultName(CsvMapper.Year)),
Month = csv.Get <int?>(CsvMapper.GetByDefaultName(CsvMapper.Month)),
Row = row++,
NameToPercentageReturn = new Dictionary <string, decimal?>(),
};
foreach (var item in input.DisplayNameToCsvHeader)
{
record.NameToPercentageReturn.Add(item.Key, csv.Get(item.Value).ToNumber());
}
foreach (var item in input.DisplayNameOfCapeMeasureToHeader)
{
record.NameToCape.Add(item.Key, csv.Get(item.Value).ToNumber());
}
records.Add(record);
}
return(records);
}
}
public async Task Remap_Writes_TransformedFields()
{
// arrange
var config = GetTestConfig();
var parser = GetTestParser().UseHeaders();
var mapper = new CsvMapper(parser, _outStream);
mapper.Load(config);
// act
await mapper.RemapAsync(',');
// assert
_outStream.Seek(0, SeekOrigin.Begin);
var asText = Encoding.UTF8.GetString(_outStream.ToArray());
var firstLine = asText.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries)[0];
var fields = firstLine.Split(',');
Assert.Equal("foo-1", fields[0]);
Assert.Equal("bar-1", fields[1]);
Assert.Equal("baz-1", fields[2]);
}
public void MapByFilePath()
{
// arrange
var mapper = new CsvMapper <TranslationLabel>();
var mappings = this.GetTranslationLabelMappings();
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.TranslationLabelFilePath);
mapper.GetHeaders(this.TranslationLabelFilePath);
mapper.GetHeaders(this.TranslationLabelFilePath);
// act
var result = mapper.Map(this.TranslationLabelFilePath, mappings);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(22, result.Count);
var firstRow = result.FirstOrDefault();
Assert.IsNotNull(firstRow);
Assert.AreEqual("InfinitiMonthlyPayment", firstRow.LabelName);
}
public void Test2([Values(500)] int iterations, [Values(100)] int columns)
{
var testDir = Path.GetDirectoryName(Path.GetDirectoryName(TestContext.CurrentContext.TestDirectory));
var dataPath = Path.Combine(testDir ?? throw new NullReferenceException(), "..", "Data", "data.csv");
var mapper = new CsvMapper();
var data = mapper.ReadDataFromFile(dataPath, 1);
var correlation = Correlation.GetCorrelation(data);
data = data.RemoveLeastRelevantColumn(correlation, columns);
var dataSet = data.SplitData(1, 2, 0);
var trainingData = dataSet[DataType.Training];
var testingData = dataSet[DataType.Testing];
var neuralNetwork = new NeuralNetwork.NeuralNetwork {
MinWeight = -5, MaxWeight = 5
};
var activationFunction = new SigmoidFunction(-1);
neuralNetwork.ActivationFunction = activationFunction;
neuralNetwork.AddLayer(new Layer(10, columns));
neuralNetwork.AddLayer(new Layer(1, 10));
var genetic = new GeneticAlgorithm.GeneticAlgorithm(100, neuralNetwork.GetConnectionCount())
{
Selection = new TournamentSelection(100, 10),
Crossover = new TwoPointCrossover(0.65, 100),
Mutation = new FlipBitMutation(0.05)
};
var keepLooping = true;
for (var it = 0; it < iterations && keepLooping; it++)
{
foreach (var chromosome in genetic.Population)
{
neuralNetwork.SetWeights(chromosome.Genome);
Parallel.For(0, trainingData.Attributes.Count, i =>
{
var res = neuralNetwork.GetResult(trainingData.Attributes[i]);
lock (chromosome)
{
chromosome.Fitness += Math.Abs(res[0] - trainingData.ObjectClass[i]);
}
});
}
keepLooping = !genetic.Population.Any(chromosome => chromosome.Fitness < 5.0);
Console.WriteLine($@"iteration: {it} | " +
$@"best: {genetic.Population.MinBy(chromosome => chromosome.Fitness).First().Fitness}");
if (keepLooping && it < iterations - 1)
{
genetic.NextGeneration();
}
}
var best = genetic.Population.MinBy(chromosome => chromosome.Fitness).First();
neuralNetwork.SetWeights(best.Genome);
var correct = testingData.Attributes
.AsParallel()
.Where((t, i) => Math.Abs(neuralNetwork.GetResult(t)[0] - testingData.ObjectClass[i]) < 0.3)
.Count();
Console.WriteLine(
$@"Population: {100}, iterations: {iterations} - fitness: {best.Fitness:0.0} " +
$@"| correctly recognized: {correct}/{testingData.Attributes.Count}");
}
public void MapByFilePathParseValues()
{
// arrange
var mapper = new CsvMapper<CurryOrder>();
var mappings = new NameValueCollection();
mappings.Add("Curry", "0");
mappings.Add("Quantity", "1");
mappings.Add("Price", "2");
mappings.Add("Total", "3");
// make sure calling GetHeaders does not affect the result
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
mapper.GetHeaders(this.CurryOrderFilePath);
// act
var result = mapper.Map(this.CurryOrderFilePath, mappings);
// assert
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
Assert.AreEqual(3, result.Count);
var lastRow = result.LastOrDefault();
Assert.IsNotNull(lastRow);
Assert.AreEqual("Korma", lastRow.Curry);
Assert.AreEqual(2, lastRow.Quantity);
Assert.AreEqual(12.5M, lastRow.Price);
Assert.AreEqual(25M, lastRow.Total);
}
public string[] GetHeaders(string filePath)
{
var csvProcessor = new CsvMapper <T>();
return(csvProcessor.GetHeaders(filePath));
}
