public void UniqueConstrainShouldKeepOutDuplicateRecordsTest()
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = 1;
eventrow1.EventParam = 1;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow();
eventrow2.Timestamp = eventrow1.Timestamp;
eventrow2.SignalID = eventrow1.SignalID;
eventrow2.EventCode = eventrow1.EventCode;
eventrow2.EventParam = eventrow1.EventParam;
elTable.AddController_Event_LogRow(eventrow1);
try
{
elTable.AddController_Event_LogRow(eventrow2);
}
catch
{
}
Assert.IsTrue(elTable.Count == 1);
}
private static MOE.Common.Data.MOE.Controller_Event_LogDataTable CreateDataTableForImport()
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
return(elTable);
}
private static void SaveToDB(XmlDocument xml, string SignalId)
{
System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings;
string destTable = appSettings["DestinationTableName"];
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = new DateTime();
int EventCode = 0;
int EventParam = 0;
DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime);
int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode);
int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate)
{
elTable.AddController_Event_LogRow(eventrow);
}
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable);
}
private MOE.Common.Data.MOE.Controller_Event_LogDataTable CreateTableObject(bool hasUniqueConstraint)
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
if (hasUniqueConstraint)
{
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
}
return(elTable);
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
List <string> dirList = new List <string>();
string cwd = appSettings["ASC3LogsPath"];
foreach (string s in Directory.GetDirectories(cwd))
{
dirList.Add(s);
}
SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList);
ParallelOptions optionsMain = new ParallelOptions {
MaxDegreeOfParallelism = Convert.ToInt32(appSettings["MaxThreadsMain"])
};
Parallel.ForEach(sp, optionsMain, dir =>
{
var toDelete = new ConcurrentBag <string>();
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
if (Convert.ToBoolean(appSettings["WriteToConsole"]))
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
string signalId;
string[] fileNames;
GetFileNamesAndSignalId(dir, out signalId, out fileNames);
foreach (var fileName in fileNames)
{
try
{
MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(fileName, signalId,
mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]));
toDelete.Add(fileName);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateDataTableForImport();
AddEventsToImportTable(mergedEventsTable, elTable);
mergedEventsTable.Dispose();
BulkImportRecordsAndDeleteFiles(appSettings, toDelete, elTable);
});
}
public void CopyFromBlockingCollectionIntoConstrainedTableMustNotFailTest()
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = 1;
eventrow1.EventParam = 1;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow();
eventrow2.Timestamp = eventrow1.Timestamp;
eventrow2.SignalID = eventrow1.SignalID;
eventrow2.EventCode = eventrow1.EventCode;
eventrow2.EventParam = eventrow1.EventParam;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow3 = elTable.NewController_Event_LogRow();
eventrow3.Timestamp = DateTime.Now;
eventrow3.SignalID = "103";
eventrow3.EventCode = 3;
eventrow3.EventParam = 3;
mergedEventsTable.Add(eventrow1);
mergedEventsTable.Add(eventrow2);
mergedEventsTable.Add(eventrow3);
//mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch { }
}
Assert.IsTrue(elTable.Count == 2);
}
private static void SaveToDB(XmlDocument xml, string SignalId)
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value);
int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value);
int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
elTable.AddController_Event_LogRow(eventrow);
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.Signal.BulktoDB(elTable, Options);
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = FileByFileASC3Decoder.Properties.Settings.Default.ASC3LogsPath;
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_") || f.Length < 367)
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = FileByFileASC3Decoder.Properties.Settings.Default.SPMConnectionString;
string destTable = FileByFileASC3Decoder.Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole, true, 0, FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile,
FileByFileASC3Decoder.Properties.Settings.Default.EarliestAcceptableDate, 5000, 30);
if (EventsTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile)
{
try
{
File.Delete(s);
}
catch { }
}
}
}
}
catch { }
}
}
}
//subroutine to write the decoded log to the database.
//this is where most of the work is done.
//The only way we match signalid to the collected logs is by the directory name.
//static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA)
private void SaveEvents()
{
int insertErrorCount = 0;
//int duplicateLineCount = 0;
int insertedLinecount = 0;
double errorRatio = 0;
//bool fileHasBeenRead = false;
DateTime startTime = new DateTime();
DateTime endTime = new DateTime();
TimeSpan elapsedTime = new TimeSpan();
string CWD = Properties.Settings.Default.PeekDatPath;
List<string> dirList = new List<string>();
ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>();
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth);
if(Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(s);
}
}
var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(dirList.AsEnumerable(), options, dir =>
//Parallel.ForEach(dirList.AsEnumerable(), dir =>
//foreach (string dir in dirList)
{
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit[strsplit.Count() - 2];
string sigid = dirname;
Console.WriteLine("Starting signal " + sigid);
var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file =>
//Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file =>
//foreach (string file in Directory.GetFiles(dir, "*.csv"))
{
bool fileHasBeenRead = false;
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"]
});
elTable.Constraints.Add(custUnique);
startTime = DateTime.Now;
foreach (string line in File.ReadAllLines(file))
{
//Every other line is blank. We only care about the lines that have data, and
//every data line has a comma
if (line.Contains(','))
{
//the first five lines or so are header information. They need to be filtered out.
if (line.Contains(",ATC"))
{
//even if there is nothing but header in the file, we want it marked as being read
//so we can delete is later.
fileHasBeenRead = true;
}
else if (line.Contains(".dat"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",IP Address:"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",MAC Address"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Data Log Beginning"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Phases in use"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Binary"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Start Of"))
{
fileHasBeenRead = true;
}
else
{
//split the line on commas and assign each split to a var
string[] lineSplit = line.Split(new char[] { ',' });
DateTime timeStamp = new DateTime();
int eventCode = 0;
int eventParam = 0;
bool lineError = false;
//it might happen that the character on the line are not quite right.
//the Try/catch stuff is an attempt to deal with that.
try
{
timeStamp = Convert.ToDateTime(lineSplit[0]);
}
catch
{
Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventCode = Convert.ToInt32(lineSplit[1]);
}
catch
{
Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventParam = Convert.ToInt32(lineSplit[2]);
}
catch
{
Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]);
lineError = true;
}
//If there were no errors on the line, then put the line into the bulk queue
if (!lineError)
{
try
{
elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
//If it gets this far, the file has been opened
fileHasBeenRead = true;
}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("NEXT LINE");
}
}
// Array.Clear(lines, 0, lines.Count());
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("$$$ Entire file has been read $$$");
}
//Do the Math to find out if the error ratio is intolerably high before deleting the file
if (insertErrorCount > 0)
{
errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount));
}
else
{
errorRatio = 0;
}
if (file.Length == 0)
{
fileHasBeenRead = true;
}
endTime = DateTime.Now;
//the Signal class has a static method to insert the tableinto the DB. We are using that.
string connectionString = Properties.Settings.Default.SPM.ToString();
MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName);
elapsedTime = endTime - startTime;
if (Properties.Settings.Default.DeleteFiles)
{
try{
Directory.Delete(dir);
}
catch
{
}
string d = Properties.Settings.Default.PeekDatPath;
foreach (string f in Directory.GetFiles(d, "*.dat"))
{
File.Delete(f);
}
//try
//{
// //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead))
// //{
// try
// {
// File.Delete(file);
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} Deleted", file);
// }
// }
// catch (SystemException sysex)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file);
// }
// Thread.Sleep(100);
// try
// {
// File.Delete(file);
// }
// catch (SystemException sysex2)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file);
// }
// }
// }
// try
// {
// // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage);
// }
// catch (SqlException ex)
// {
// Console.WriteLine(ex);
// }
// // }
// //else
// //{
// // if (Properties.Settings.Default.WriteToConsole)
// // {
// // Console.WriteLine("Too many insertion errors to delete {0}", file);
// // }
// //}
//}
//catch (Exception ex)
//{
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while deleting file {1}", ex, file);
// }
// FilesToDelete.Enqueue(file);
//}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of file Loop%%%");
Thread.Sleep(100);
}
}
);
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of DIRECTORY Loop%%%");
Thread.Sleep(100);
}
//CleanUpFiles(FilesToDelete);
//});
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("###Start of Queue Build Hit###");
}
}
);
}
public void DoesCopyTakeLongerThanForEach()
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
for (int i = 0; i < 100000; i++)
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = rnd.Next(1, 256);
eventrow1.EventParam = rnd.Next(1, 256);
mergedEventsTable.Add(eventrow1);
}
DateTime startCopy = DateTime.Now;
try
{
mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The copy method didn't work");
}
elTable.Clear();
DateTime endCopy = DateTime.Now;
DateTime startForEach = DateTime.Now;
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The foreach method didn't work");
}
DateTime endForEach = DateTime.Now;
var copyDiff = (endCopy - startCopy);
var forEachDiff = (endForEach - startForEach);
Assert.IsTrue(forEachDiff < copyDiff);
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string cwd = appSettings["ASC3LogsPath"];
bool writeToConsole = Convert.ToBoolean(appSettings["WriteToConsole"]);
bool deleteFile = Convert.ToBoolean(appSettings["DeleteFile"]);
bool writeToCsv = Convert.ToBoolean(appSettings["WriteToCsv"]);
foreach (string s in Directory.GetDirectories(cwd))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (writeToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_"))
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(s, sigid, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]));
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable eventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(eventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
string destTable = appSettings["DestinationTableNAme"];
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
writeToConsole, true, 0, deleteFile, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), 5000, 30);
if (eventsTable.Count > 0)
{
if (MOE.Common.Business.SignalFtp.BulktoDb(eventsTable, Options) && deleteFile)
{
try
{
File.Delete(s);
}
catch
{
}
}
}
if (writeToCsv)
{
FileInfo fileInfo = new FileInfo(s);
StringBuilder sb = new StringBuilder();
IEnumerable <string> columnNames = eventsTable.Columns.Cast <DataColumn>().
Select(column => column.ColumnName);
sb.AppendLine(string.Join(",", columnNames));
foreach (MOE.Common.Data.MOE.Controller_Event_LogRow row in eventsTable.Rows)
{
List <string> fields = new List <string>();
fields.Add(row.SignalID);
fields.Add(row.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff"));
fields.Add(row.EventCode.ToString());
fields.Add(row.EventParam.ToString());
sb.AppendLine(string.Join(",", fields));
}
var csvFilePath = fileInfo.FullName.TrimEnd(new char[] { '.', 'd', 'a', 't' }) + ".csv";
File.WriteAllText(csvFilePath, sb.ToString());
}
}
}
catch { }
}
}
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = Properties.Settings.Default.ASC3LogsPath;
string CSV = Properties.Settings.Default.CSVOutPAth;
//var tableCollection = new BlockingCollection<DataTable>();
//DataTable mergedEventsTable = new DataTable();
ParallelOptions options;
if (!Properties.Settings.Default.forceNonParallel)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = -1
};
}
else
{
if (Properties.Settings.Default.MaxThreads < 2)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = 1
};
}
else
{
options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
}
}
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList);
//foreach (string dir in dirList)
ParallelOptions optionsMain = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain
};
Parallel.ForEach(sp, optionsMain, dir =>
{
var ToDelete = new ConcurrentBag <string>();
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
//SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat"));
//Parallel.ForEach(sp2, options, s =>
foreach (var s in Directory.GetFiles(dir, "*.dat"))
{
try
{
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
ToDelete.Add(s);
}
catch { }
}
//);
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
//mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch { }
}
mergedEventsTable.Dispose();
string connectionString = Properties.Settings.Default.SPMConnectionString;
string destTable = Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
if (elTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile)
{
DeleteFiles(ToDelete);
}
//string filename = sigid.ToString();
//filename += "_";
//filename += DateTime.Now.Month.ToString();
//filename += "_";
//filename += DateTime.Now.Day.ToString();
//filename += "_";
//filename += DateTime.Now.Year.ToString();
//filename += "_";
//filename += DateTime.Now.Hour.ToString();
//filename += "_";
//filename += DateTime.Now.Minute.ToString();
//filename += "_";
//filename += DateTime.Now.Second.ToString();
//filename += ".csv";
//SaveAsCSV(EventsTable, Path.Combine(CSV, filename));
//if (Properties.Settings.Default.DeleteFile)
//{
// DeleteFiles(ToDelete);
//}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in ToDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
);
}
private static void BulkImportRecordsAndDeleteFiles(NameValueCollection appSettings, ConcurrentBag <string> toDelete, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable)
{
string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
string destTable = appSettings["DestinationTableNAme"];
MOE.Common.Business.BulkCopyOptions options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Convert.ToBoolean(appSettings["WriteToConsole"]),
Convert.ToBoolean(appSettings["forceNonParallel"]),
Convert.ToInt32(appSettings["MaxThreads"]),
Convert.ToBoolean(appSettings["DeleteFile"]),
Convert.ToDateTime(appSettings["EarliestAcceptableDate"]),
Convert.ToInt32(appSettings["BulkCopyBatchSize"]),
Convert.ToInt32(appSettings["BulkCopyTimeOut"]));
if (elTable.Count > 0)
{
if (MOE.Common.Business.SignalFtp.BulktoDb(elTable, options, destTable) && Convert.ToBoolean(appSettings["DeleteFile"]))
{
DeleteFiles(toDelete);
}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in toDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
private static void AddEventsToImportTable(BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> mergedEventsTable, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable)
{
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r.SignalID, r.Timestamp, r.EventCode,
r.EventParam);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
//static public MOE.Common.Data.MOE.Controller_Event_LogDataTable DecodeASC3File(string FileName, string SignalId)
static public void DecodeASC3File(string FileName, string SignalId, BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> RowBag)
{
DateTime EarliestAcceptableDate = Properties.Settings.Default.EarliestAcceptableDate;
System.Text.Encoding encoding = System.Text.Encoding.ASCII;
using (BinaryReader BR = new BinaryReader(File.Open(FileName, FileMode.Open), encoding))
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
if (BR.BaseStream.Position + 21 < BR.BaseStream.Length)
{
//Find the start Date
String dateString = "";
for (int i = 1; i < 21; i++)
{
Char c = BR.ReadChar();
dateString += c;
}
//Console.WriteLine(dateString);
DateTime StartTime = new DateTime();
if (DateTime.TryParse(dateString, out StartTime) && (BR.BaseStream.Position < BR.BaseStream.Length))
{
//find line feed characters, that should take us to the end of the header.
// First line break is after Version
// Second LF is after FileName
// Third LF is after Interseciton number, which isn't used as far as I can tell
// Fourth LF is after IP address
// Fifth is after MAC Address
// Sixth is after "Controller data log beginning:," and then the date
// Seven is after "Phases in use:," and then the list of phases, seperated by commas
int i = 0;
while (i < 7 && (BR.BaseStream.Position < BR.BaseStream.Length))
{
Char c = BR.ReadChar();
//Console.WriteLine(c.ToString());
if (c == '\n')
{
i++;
}
}
//The first record alwasy seems to be missing a timestamp. Econolite assumes the first even occures at the same time
// the second event occurs. I am going to tag the first event with secondevet.timestamp - 1/10 second
int firstEventCode = new Int32();
int firstEventParam = new Int32();
if (BR.BaseStream.Position + sizeof(char) < BR.BaseStream.Length)
{
firstEventCode = Convert.ToInt32(BR.ReadChar());
}
if (BR.BaseStream.Position + sizeof(char) < BR.BaseStream.Length)
{
firstEventParam = Convert.ToInt32(BR.ReadChar());
}
bool firstEventEntered = false;
//MOE.Common.Business.ControllerEvent firstEvent = new MOE.Common.Business.ControllerEvent(SignalID, StartTime, firstEventCode, firstEventParam);
//After that, we can probably start reading
while ((BR.BaseStream.Position + (sizeof(byte) * 4)) <= BR.BaseStream.Length) //we need ot make sure we are more that 4 characters from the end
{
DateTime EventTime = new DateTime();
int EventCode = new Int32();
int EventParam = new Int32();
//MOE.Common.Business.ControllerEvent controllerEvent = null;
for (int EventPart = 1; EventPart < 4; EventPart++)
{
//getting the time offset
if (EventPart == 1)
{
byte[] rawoffset = new byte[2];
//char[] offset = new char[2];
rawoffset = BR.ReadBytes(2);
Array.Reverse(rawoffset);
int offset = BitConverter.ToInt16(rawoffset, 0);
double tenths = Convert.ToDouble(offset) / 10;
EventTime = StartTime.AddSeconds(tenths);
}
//getting the EventCode
if (EventPart == 2)
{
//Char EventCodeChar = BR.ReadChar();
//EventCode = Convert.ToInt32(BR.ReadChar());
EventCode = Convert.ToInt32(BR.ReadByte());
}
if (EventPart == 3)
{
//Char EventParamChar = BR.ReadChar();
//EventParam = Convert.ToInt32(BR.ReadChar());
EventParam = Convert.ToInt32(BR.ReadByte());
}
}
//controllerEvent = new MOE.Common.Business.ControllerEvent(SignalID, EventTime, EventCode, EventParam);
if ((EventTime) <= DateTime.Now && (EventTime > EarliestAcceptableDate))
{
if (!firstEventEntered)
{
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime.AddMilliseconds(-1);
eventrow.SignalID = SignalId;
eventrow.EventCode = firstEventCode;
eventrow.EventParam = firstEventParam;
if (!RowBag.Contains(eventrow))
{
RowBag.Add(eventrow);
}
//elTable.AddController_Event_LogRow(SignalID, EventTime.AddMilliseconds(-1), firstEventCode, firstEventParam);
}
catch
{
}
firstEventEntered = true;
}
else
{
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
if (!RowBag.Contains(eventrow))
{
RowBag.Add(eventrow);
}
//elTable.AddController_Event_LogRow(Convert.ToInt32(SignalID), EventTime, EventCode, EventParam);
}
catch
{
}
}
}
}
}
//this is what we do when the datestring doesn't parse
else
{
// return false;
}
}
//this is what we do when the datestring doesn't parse
else
{
//return false;
}
//return true;
}
}
