/// <summary>
/// Gets data from files and maintains order of values in files using a Dictionary.
/// </summary>
/// <param name="files"></param>
/// <returns></returns>
private List <OutputRow> GetOrderedFileDataDictionary(List <string> files)
{
Dictionary <DateTime, List <double?> > keyValues = new Dictionary <DateTime, List <double?> >();
foreach (var file in files)
{
OutputRow.Headers.Add(file);
var values = new DbfReader().ReadValues(file);
foreach (var value in values)
{
var item = new OutputRow
{
Timestamp = value.Timestamp,
Values = new List <double?>(new double?[files.Count])
};
if (keyValues.ContainsKey(item.Timestamp))
{
keyValues[item.Timestamp][OutputRow.Headers.Count - 1] = value.Value;
}
else
{
item.Values[OutputRow.Headers.Count - 1] = value.Value;
keyValues.Add(value.Timestamp, item.Values);
}
}
}
return(keyValues.GroupBy(x => x.Key).OrderBy(x => x.Key).Select(x => new OutputRow
{
Timestamp = x.Key,
Values = x.SelectMany(y => y.Value).ToList(),
}).ToList());
}
/// <summary>
/// Gets data from files and maintains order of values in files using BinarySearch.
/// </summary>
/// <param name="files"></param>
/// <returns></returns>
private List <OutputRow> GetOrderedFileDataBinarySearch(List <string> files)
{
List <OutputRow> outputs = new List <OutputRow>();
foreach (var file in files)
{
OutputRow.Headers.Add(file);
var values = new DbfReader().ReadValues(file);
foreach (var value in values)
{
var item = new OutputRow
{
Timestamp = value.Timestamp,
Values = new List <double?>(new double?[files.Count])
};
var index = outputs.BinarySearch(item);
if (index < 0)
{
item.Values[OutputRow.Headers.Count - 1] = value.Value;
outputs.Insert(~index, item);
}
else
{
outputs[index].Values[OutputRow.Headers.Count - 1] = value.Value;
}
}
}
return(outputs.OrderBy(x => x.Timestamp).ToList());
}
// Since the new value is at the end, but we are using columns, we need to add null values for the previous columns
private void SetValuesList(OutputRow newRow, double value)
{
for (int i = 0; i < OutputRow.Headers.Count - 1; i++)
{
newRow.Values.Add(null);
}
newRow.Values.Add(value);
}
private bool ProcessFile(FileInfo file, List <OutputRow> outputs, DbfReader dbfReader, List <string> warnings, ref string errorMessage)
{
List <DbfReader.ValueRow> valueRows = null;
try
{
valueRows = dbfReader.ReadValues(file.FullName);
}
catch (Exception ex)
{
errorMessage = "Error reading values: " + ex.Message;
return(false);
}
OutputRow.Headers.Add(file.DirectoryName);
AddNullValuesAtEnd(outputs);
foreach (var row in valueRows)
{
var newRow = new OutputRow()
{
Timestamp = row.Timestamp,
};
var index = outputs.BinarySearch(newRow, new CompareRowTimestamp());
if (index < 0) // Not found
{
outputs.Insert(~index, newRow);
SetValuesList(newRow, row.Value);
}
else
{
var outputRow = outputs[index];
if (outputRow.Values.Count < OutputRow.Headers.Count)
{
outputRow.Values.Add(row.Value);
}
else
{
if (row.Value != outputRow.Values[outputRow.Values.Count - 1])
{
/* It could be the case that the same timestamp was duplicate in the file.
* Here we are creating a list of warnings to deal with this situation, but we might as well throw an exeception
* or return false and stop the recursive processing - it was a judgement call.
* If we knew for sure that the timestamp is a db primary key, that is never duplicated we wouldn't need this verification */
if (outputRow.Values[outputRow.Values.Count - 1] != null)
{
warnings.Add("Value overriten. Path=" + file.Directory + "; Timestamp=" + row.Timestamp +
"; old value = " + outputRow.Values[outputRow.Values.Count - 1] + "; new value=" + row.Value);
}
outputRow.Values[outputRow.Values.Count - 1] = row.Value;
}
}
}
}
return(true);
}