private static TripRecordFactory ImportLookupTable(string DataFile, string LookupFile)
{
TripRecordFactory context = new TripRecordFactory(DataFile);
if (Path.GetExtension(LookupFile) != '.' + Constants.LookupTableExtension)
{
throw new Exception(string.Format("Looking for Index Map '.{0}' file", Constants.LookupTableExtension));
}
BinaryReader <LookupRow> reader = new BinaryReader <LookupRow>(LookupFile);
//Build Lookup Table
foreach (var row in reader.ReadAll())
{
List <long> ret;
if (!context.LookupTable.TryGetValue(row.Key, out ret))
{
ret = new List <long>();
context.LookupTable[row.Key] = ret;
}
ret.Add(row.Value);
}
reader.Close();
return(context);
}
private static TripRecordFactory CsvToBinaryFileWithLookupTable <T>(string[] sources, string destination, char delimiter)
where T : TripRecordModel, new()
{
BinaryWriter <TripRecord> writer = new BinaryWriter <TripRecord>(destination);
TripRecordFactory context = new TripRecordFactory(destination);
T model = new T();
foreach (var source in sources)
{
Console.WriteLine(">Processing {0}<", source);
using (StreamReader sreader = new StreamReader(source))
{
TripRecord record = new TripRecord();
int rowcount = 0;
int currentrow = 0;
//Count Rows
while (!sreader.EndOfStream)
{
sreader.ReadLine();
rowcount++;
}
Console.WriteLine("Row Count = {0}", rowcount);
Timer timer = new Timer(1000);
var action = new ElapsedEventHandler((s, e) =>
{
Console.CursorLeft = 0;
Console.Write("Proccesed {0}/{1} Rows - {2:P2}\t", currentrow, rowcount, (float)currentrow / (float)rowcount);
});
timer.Elapsed += action;
timer.Start();
Dictionary <string, string> row = new Dictionary <string, string>();
sreader.BaseStream.Position = 0; // Reset stream
string[] header = sreader.ReadLine().ToLower().Split(delimiter);
while (!sreader.EndOfStream)
{
row.Clear();
// Read entire row as string and tokenize
string[] tokens = sreader.ReadLine().Split(delimiter).Take(header.Length).ToArray();
for (int i = 0; i < tokens.Length; i++)
{
row.Add(header[i], tokens[i]);
}
//Parse tokens
record = model.ParseTokens(row);
//Write data structure to file
writer.Write(record);
//Index the data structure
DateTime key = new DateTime(record.TimeStamp.Year, record.TimeStamp.Month, record.TimeStamp.Day);
List <long> ret;
if (!context.LookupTable.TryGetValue(key, out ret))
{
ret = new List <long>();
context.LookupTable[key] = ret;
}
ret.Add(record.ID);
currentrow++;
}
action.Invoke(null, null);
Console.WriteLine();
timer.Stop();
timer.Close();
}
writer.Flush();
}
writer.Close();
//Save Lookup table to file
string path = Path.GetDirectoryName(destination);
string name = Path.GetFileNameWithoutExtension(destination);
string filename = Path.Combine(path, string.Format("{0}.{1}", name, Constants.LookupTableExtension));
Console.WriteLine(">Storing Lookup Table<");
BinaryWriter <LookupRow> stream = new BinaryWriter <LookupRow>(filename);
foreach (var entry in context.LookupTable)
{
foreach (var value in entry.Value)
{
LookupRow row = new LookupRow(entry.Key, value);
stream.Write(row);
}
stream.Flush();
}
stream.Close();
Console.WriteLine("Lookup Table Successfully stored at {0}", filename);
return(context);
}
