public void UniqueConstrainShouldKeepOutDuplicateRecordsTest()
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = 1;
eventrow1.EventParam = 1;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow();
eventrow2.Timestamp = eventrow1.Timestamp;
eventrow2.SignalID = eventrow1.SignalID;
eventrow2.EventCode = eventrow1.EventCode;
eventrow2.EventParam = eventrow1.EventParam;
elTable.AddController_Event_LogRow(eventrow1);
try
{
elTable.AddController_Event_LogRow(eventrow2);
}
catch
{
}
Assert.IsTrue(elTable.Count == 1);
}
private static void SaveToDB(XmlDocument xml, string SignalId)
{
System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings;
string destTable = appSettings["DestinationTableName"];
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = new DateTime();
int EventCode = 0;
int EventParam = 0;
DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime);
int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode);
int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate)
{
elTable.AddController_Event_LogRow(eventrow);
}
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable);
}
private static void AddEventsToImportTable(BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> mergedEventsTable, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable)
{
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r.SignalID, r.Timestamp, r.EventCode,
r.EventParam);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
public void CopyFromBlockingCollectionIntoConstrainedTableMustNotFailTest()
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = 1;
eventrow1.EventParam = 1;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow();
eventrow2.Timestamp = eventrow1.Timestamp;
eventrow2.SignalID = eventrow1.SignalID;
eventrow2.EventCode = eventrow1.EventCode;
eventrow2.EventParam = eventrow1.EventParam;
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow3 = elTable.NewController_Event_LogRow();
eventrow3.Timestamp = DateTime.Now;
eventrow3.SignalID = "103";
eventrow3.EventCode = 3;
eventrow3.EventParam = 3;
mergedEventsTable.Add(eventrow1);
mergedEventsTable.Add(eventrow2);
mergedEventsTable.Add(eventrow3);
//mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch { }
}
Assert.IsTrue(elTable.Count == 2);
}
private static void SaveToDB(XmlDocument xml, string SignalId)
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value);
int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value);
int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
elTable.AddController_Event_LogRow(eventrow);
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.Signal.BulktoDB(elTable, Options);
}
//subroutine to write the decoded log to the database.
//this is where most of the work is done.
//The only way we match signalid to the collected logs is by the directory name.
//static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA)
private void SaveEvents()
{
int insertErrorCount = 0;
//int duplicateLineCount = 0;
int insertedLinecount = 0;
double errorRatio = 0;
//bool fileHasBeenRead = false;
DateTime startTime = new DateTime();
DateTime endTime = new DateTime();
TimeSpan elapsedTime = new TimeSpan();
string CWD = Properties.Settings.Default.PeekDatPath;
List<string> dirList = new List<string>();
ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>();
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth);
if(Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(s);
}
}
var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(dirList.AsEnumerable(), options, dir =>
//Parallel.ForEach(dirList.AsEnumerable(), dir =>
//foreach (string dir in dirList)
{
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit[strsplit.Count() - 2];
string sigid = dirname;
Console.WriteLine("Starting signal " + sigid);
var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file =>
//Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file =>
//foreach (string file in Directory.GetFiles(dir, "*.csv"))
{
bool fileHasBeenRead = false;
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"]
});
elTable.Constraints.Add(custUnique);
startTime = DateTime.Now;
foreach (string line in File.ReadAllLines(file))
{
//Every other line is blank. We only care about the lines that have data, and
//every data line has a comma
if (line.Contains(','))
{
//the first five lines or so are header information. They need to be filtered out.
if (line.Contains(",ATC"))
{
//even if there is nothing but header in the file, we want it marked as being read
//so we can delete is later.
fileHasBeenRead = true;
}
else if (line.Contains(".dat"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",IP Address:"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",MAC Address"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Data Log Beginning"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Phases in use"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Binary"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Start Of"))
{
fileHasBeenRead = true;
}
else
{
//split the line on commas and assign each split to a var
string[] lineSplit = line.Split(new char[] { ',' });
DateTime timeStamp = new DateTime();
int eventCode = 0;
int eventParam = 0;
bool lineError = false;
//it might happen that the character on the line are not quite right.
//the Try/catch stuff is an attempt to deal with that.
try
{
timeStamp = Convert.ToDateTime(lineSplit[0]);
}
catch
{
Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventCode = Convert.ToInt32(lineSplit[1]);
}
catch
{
Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventParam = Convert.ToInt32(lineSplit[2]);
}
catch
{
Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]);
lineError = true;
}
//If there were no errors on the line, then put the line into the bulk queue
if (!lineError)
{
try
{
elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
//If it gets this far, the file has been opened
fileHasBeenRead = true;
}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("NEXT LINE");
}
}
// Array.Clear(lines, 0, lines.Count());
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("$$$ Entire file has been read $$$");
}
//Do the Math to find out if the error ratio is intolerably high before deleting the file
if (insertErrorCount > 0)
{
errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount));
}
else
{
errorRatio = 0;
}
if (file.Length == 0)
{
fileHasBeenRead = true;
}
endTime = DateTime.Now;
//the Signal class has a static method to insert the tableinto the DB. We are using that.
string connectionString = Properties.Settings.Default.SPM.ToString();
MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName);
elapsedTime = endTime - startTime;
if (Properties.Settings.Default.DeleteFiles)
{
try{
Directory.Delete(dir);
}
catch
{
}
string d = Properties.Settings.Default.PeekDatPath;
foreach (string f in Directory.GetFiles(d, "*.dat"))
{
File.Delete(f);
}
//try
//{
// //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead))
// //{
// try
// {
// File.Delete(file);
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} Deleted", file);
// }
// }
// catch (SystemException sysex)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file);
// }
// Thread.Sleep(100);
// try
// {
// File.Delete(file);
// }
// catch (SystemException sysex2)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file);
// }
// }
// }
// try
// {
// // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage);
// }
// catch (SqlException ex)
// {
// Console.WriteLine(ex);
// }
// // }
// //else
// //{
// // if (Properties.Settings.Default.WriteToConsole)
// // {
// // Console.WriteLine("Too many insertion errors to delete {0}", file);
// // }
// //}
//}
//catch (Exception ex)
//{
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while deleting file {1}", ex, file);
// }
// FilesToDelete.Enqueue(file);
//}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of file Loop%%%");
Thread.Sleep(100);
}
}
);
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of DIRECTORY Loop%%%");
Thread.Sleep(100);
}
//CleanUpFiles(FilesToDelete);
//});
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("###Start of Queue Build Hit###");
}
}
);
}
public void DoesCopyTakeLongerThanForEach()
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
for (int i = 0; i < 100000; i++)
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = rnd.Next(1, 256);
eventrow1.EventParam = rnd.Next(1, 256);
mergedEventsTable.Add(eventrow1);
}
DateTime startCopy = DateTime.Now;
try
{
mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The copy method didn't work");
}
elTable.Clear();
DateTime endCopy = DateTime.Now;
DateTime startForEach = DateTime.Now;
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The foreach method didn't work");
}
DateTime endForEach = DateTime.Now;
var copyDiff = (endCopy - startCopy);
var forEachDiff = (endForEach - startForEach);
Assert.IsTrue(forEachDiff < copyDiff);
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = Properties.Settings.Default.ASC3LogsPath;
string CSV = Properties.Settings.Default.CSVOutPAth;
//var tableCollection = new BlockingCollection<DataTable>();
//DataTable mergedEventsTable = new DataTable();
ParallelOptions options;
if (!Properties.Settings.Default.forceNonParallel)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = -1
};
}
else
{
if (Properties.Settings.Default.MaxThreads < 2)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = 1
};
}
else
{
options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
}
}
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList);
//foreach (string dir in dirList)
ParallelOptions optionsMain = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain
};
Parallel.ForEach(sp, optionsMain, dir =>
{
var ToDelete = new ConcurrentBag <string>();
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
//SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat"));
//Parallel.ForEach(sp2, options, s =>
foreach (var s in Directory.GetFiles(dir, "*.dat"))
{
try
{
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
ToDelete.Add(s);
}
catch { }
}
//);
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
//mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch { }
}
mergedEventsTable.Dispose();
string connectionString = Properties.Settings.Default.SPMConnectionString;
string destTable = Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
if (elTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile)
{
DeleteFiles(ToDelete);
}
//string filename = sigid.ToString();
//filename += "_";
//filename += DateTime.Now.Month.ToString();
//filename += "_";
//filename += DateTime.Now.Day.ToString();
//filename += "_";
//filename += DateTime.Now.Year.ToString();
//filename += "_";
//filename += DateTime.Now.Hour.ToString();
//filename += "_";
//filename += DateTime.Now.Minute.ToString();
//filename += "_";
//filename += DateTime.Now.Second.ToString();
//filename += ".csv";
//SaveAsCSV(EventsTable, Path.Combine(CSV, filename));
//if (Properties.Settings.Default.DeleteFile)
//{
// DeleteFiles(ToDelete);
//}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in ToDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
);
}