public void TestWritingSimpleRecord()
{
var records = new List<IReferenceRecord>();
var record = new ReferenceRecord();
const string Key = @"Name";
const string Value = @"Alexanderplatz";
record.Properties.Add(Key, Value);
records.Add(record);
var indexWriter = new IndexFileWriter();
var indexFile = new FileInfo(Path.GetTempFileName());
indexWriter.WriteFile(indexFile, records);
var indexReader = new IndexFileReader();
var writtenRecords = indexReader.ReadFile(indexFile);
var writtenRecord = Enumerable.First<IReferenceRecord>(writtenRecords);
Assert.AreEqual(record.Properties.Count, writtenRecord.Properties.Count, @"The number of properties do not match!");
Assert.IsTrue(writtenRecord.Properties.ContainsKey(Key), @"There is no property having a key equal to 'Name'!");
Assert.AreEqual(record.Properties[Key], writtenRecord.Properties[Key], @"The property values do not match!");
}
/// <summary>
/// Reads a CSV file using the specified delimiter.
/// </summary>
/// <param name="fileInfo">The CSV file to read.</param>
/// <param name="readHeader"><c>true</c> if the first row contains the header.</param>
/// <param name="delimiter">The delimiter for the values.</param>
/// <returns>A reference dataset.</returns>
public ReferenceDataset ReadFile(FileInfo fileInfo, bool readHeader, char delimiter)
{
var dataset = new ReferenceDataset();
var header = new List<string>();
using (var reader = new StreamReader(fileInfo.OpenRead()))
{
string line;
const int ChunkSize = 1000000;
while (null != (line = reader.ReadLine()))
{
var splittedLine = line.Split(delimiter);
if (!header.Any())
{
// Read the header
if (readHeader)
{
foreach (var headerValue in splittedLine)
{
header.Add(headerValue);
}
continue;
}
else
{
for (var index = 0; index < splittedLine.Length; index++)
{
header.Add(string.Format(@"FIELD{0}", index + 1));
}
}
}
if (header.Count == splittedLine.Length)
{
try
{
// Read the values
IReferenceRecord record = new ReferenceRecord();
for (var index = 0; index < header.Count; index++)
{
var key = header[index];
var value = splittedLine[index];
if (null != ValueFactory)
{
value = ValueFactory.CreateValue(value);
}
if (null != RecordBuilder)
{
record = RecordBuilder.BuildRecord(record, key, value);
}
else if (!record.Properties.ContainsKey(key))
{
record.Properties.Add(key, value);
}
}
// Add the record
dataset.Records.Add(record);
if (0 == dataset.Records.Count % ChunkSize)
{
_logger.Info(@"{0} records created.", dataset.Records.Count);
}
}
catch (Exception ex)
{
_logger.Error(@"Error during record creation! '{0}'", ex.Message);
}
}
}
}
return dataset;
}
/// <summary>
/// Streams all records of a CSV file using the specified delimiter.
/// </summary>
/// <param name="fileInfo">The CSV file to read.</param>
/// <param name="readHeader"><c>true</c> if the first row contains the header.</param>
/// <param name="delimiter">The delimiter for the values.</param>
/// <returns>The records of this file.</returns>
public IEnumerable<IReferenceRecord> StreamFile(FileInfo fileInfo, bool readHeader, char delimiter)
{
var header = new List<string>();
using (var reader = new StreamReader(fileInfo.OpenRead()))
{
string line;
const int ChunkSize = 1000000;
while (null != (line = reader.ReadLine()))
{
var splittedLine = line.Split(delimiter);
if (!header.Any())
{
// Read the header
if (readHeader)
{
foreach (var headerValue in splittedLine)
{
header.Add(headerValue);
}
continue;
}
else
{
for (var index = 0; index < splittedLine.Length; index++)
{
header.Add(string.Format(@"FIELD{0}", index + 1));
}
}
}
if (header.Count == splittedLine.Length)
{
// Read the values
IReferenceRecord record = new ReferenceRecord();
for (var index = 0; index < header.Count; index++)
{
var key = header[index];
var value = splittedLine[index];
if (null != ValueFactory)
{
value = ValueFactory.CreateValue(value);
}
if (null != RecordBuilder)
{
record = RecordBuilder.BuildRecord(record, key, value);
}
else if (!record.Properties.ContainsKey(key))
{
record.Properties.Add(key, value);
}
}
// Stream the record
yield return record;
}
}
}
}
