public void UniqueConstrainShouldKeepOutDuplicateRecordsTest() { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = 1; eventrow1.EventParam = 1; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow(); eventrow2.Timestamp = eventrow1.Timestamp; eventrow2.SignalID = eventrow1.SignalID; eventrow2.EventCode = eventrow1.EventCode; eventrow2.EventParam = eventrow1.EventParam; elTable.AddController_Event_LogRow(eventrow1); try { elTable.AddController_Event_LogRow(eventrow2); } catch { } Assert.IsTrue(elTable.Count == 1); }
private static void SaveToDB(XmlDocument xml, string SignalId) { System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings; string destTable = appSettings["DestinationTableName"]; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = new DateTime(); int EventCode = 0; int EventParam = 0; DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime); int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode); int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate) { elTable.AddController_Event_LogRow(eventrow); } } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable); }
private static void AddEventsToImportTable(BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> mergedEventsTable, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable) { foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r.SignalID, r.Timestamp, r.EventCode, r.EventParam); } catch (Exception ex) { Console.WriteLine(ex.Message); } } }
public void CopyFromBlockingCollectionIntoConstrainedTableMustNotFailTest() { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = 1; eventrow1.EventParam = 1; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow(); eventrow2.Timestamp = eventrow1.Timestamp; eventrow2.SignalID = eventrow1.SignalID; eventrow2.EventCode = eventrow1.EventCode; eventrow2.EventParam = eventrow1.EventParam; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow3 = elTable.NewController_Event_LogRow(); eventrow3.Timestamp = DateTime.Now; eventrow3.SignalID = "103"; eventrow3.EventCode = 3; eventrow3.EventParam = 3; mergedEventsTable.Add(eventrow1); mergedEventsTable.Add(eventrow2); mergedEventsTable.Add(eventrow3); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } Assert.IsTrue(elTable.Count == 2); }
private static void SaveToDB(XmlDocument xml, string SignalId) { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value); int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value); int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; elTable.AddController_Event_LogRow(eventrow); } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.Signal.BulktoDB(elTable, Options); }
//subroutine to write the decoded log to the database. //this is where most of the work is done. //The only way we match signalid to the collected logs is by the directory name. //static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA) private void SaveEvents() { int insertErrorCount = 0; //int duplicateLineCount = 0; int insertedLinecount = 0; double errorRatio = 0; //bool fileHasBeenRead = false; DateTime startTime = new DateTime(); DateTime endTime = new DateTime(); TimeSpan elapsedTime = new TimeSpan(); string CWD = Properties.Settings.Default.PeekDatPath; List<string> dirList = new List<string>(); ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>(); foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth); if(Properties.Settings.Default.WriteToConsole) { Console.WriteLine(s); } } var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(dirList.AsEnumerable(), options, dir => //Parallel.ForEach(dirList.AsEnumerable(), dir => //foreach (string dir in dirList) { if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine(dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit[strsplit.Count() - 2]; string sigid = dirname; Console.WriteLine("Starting signal " + sigid); var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file => //Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file => //foreach (string file in Directory.GetFiles(dir, "*.csv")) { bool fileHasBeenRead = false; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); startTime = DateTime.Now; foreach (string line in File.ReadAllLines(file)) { //Every other line is blank. We only care about the lines that have data, and //every data line has a comma if (line.Contains(',')) { //the first five lines or so are header information. They need to be filtered out. if (line.Contains(",ATC")) { //even if there is nothing but header in the file, we want it marked as being read //so we can delete is later. fileHasBeenRead = true; } else if (line.Contains(".dat")) { fileHasBeenRead = true; } else if (line.Contains(",IP Address:")) { fileHasBeenRead = true; } else if (line.Contains(",MAC Address")) { fileHasBeenRead = true; } else if (line.Contains(",Data Log Beginning")) { fileHasBeenRead = true; } else if (line.Contains(",Phases in use")) { fileHasBeenRead = true; } else if (line.Contains(",Binary")) { fileHasBeenRead = true; } else if (line.Contains(",Start Of")) { fileHasBeenRead = true; } else { //split the line on commas and assign each split to a var string[] lineSplit = line.Split(new char[] { ',' }); DateTime timeStamp = new DateTime(); int eventCode = 0; int eventParam = 0; bool lineError = false; //it might happen that the character on the line are not quite right. //the Try/catch stuff is an attempt to deal with that. try { timeStamp = Convert.ToDateTime(lineSplit[0]); } catch { Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]); lineError = true; } try { eventCode = Convert.ToInt32(lineSplit[1]); } catch { Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]); lineError = true; } try { eventParam = Convert.ToInt32(lineSplit[2]); } catch { Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]); lineError = true; } //If there were no errors on the line, then put the line into the bulk queue if (!lineError) { try { elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } //If it gets this far, the file has been opened fileHasBeenRead = true; } } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("NEXT LINE"); } } // Array.Clear(lines, 0, lines.Count()); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("$$$ Entire file has been read $$$"); } //Do the Math to find out if the error ratio is intolerably high before deleting the file if (insertErrorCount > 0) { errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount)); } else { errorRatio = 0; } if (file.Length == 0) { fileHasBeenRead = true; } endTime = DateTime.Now; //the Signal class has a static method to insert the tableinto the DB. We are using that. string connectionString = Properties.Settings.Default.SPM.ToString(); MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName); elapsedTime = endTime - startTime; if (Properties.Settings.Default.DeleteFiles) { try{ Directory.Delete(dir); } catch { } string d = Properties.Settings.Default.PeekDatPath; foreach (string f in Directory.GetFiles(d, "*.dat")) { File.Delete(f); } //try //{ // //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead)) // //{ // try // { // File.Delete(file); // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} Deleted", file); // } // } // catch (SystemException sysex) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file); // } // Thread.Sleep(100); // try // { // File.Delete(file); // } // catch (SystemException sysex2) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file); // } // } // } // try // { // // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage); // } // catch (SqlException ex) // { // Console.WriteLine(ex); // } // // } // //else // //{ // // if (Properties.Settings.Default.WriteToConsole) // // { // // Console.WriteLine("Too many insertion errors to delete {0}", file); // // } // //} //} //catch (Exception ex) //{ // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while deleting file {1}", ex, file); // } // FilesToDelete.Enqueue(file); //} } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of file Loop%%%"); Thread.Sleep(100); } } ); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of DIRECTORY Loop%%%"); Thread.Sleep(100); } //CleanUpFiles(FilesToDelete); //}); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("###Start of Queue Build Hit###"); } } ); }
public void DoesCopyTakeLongerThanForEach() { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); for (int i = 0; i < 100000; i++) { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = rnd.Next(1, 256); eventrow1.EventParam = rnd.Next(1, 256); mergedEventsTable.Add(eventrow1); } DateTime startCopy = DateTime.Now; try { mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); } catch (Exception ex) { Console.WriteLine(ex.Message); } if (elTable.Count != mergedEventsTable.Count) { Assert.Fail("The copy method didn't work"); } elTable.Clear(); DateTime endCopy = DateTime.Now; DateTime startForEach = DateTime.Now; foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch (Exception ex) { Console.WriteLine(ex.Message); } } if (elTable.Count != mergedEventsTable.Count) { Assert.Fail("The foreach method didn't work"); } DateTime endForEach = DateTime.Now; var copyDiff = (endCopy - startCopy); var forEachDiff = (endForEach - startForEach); Assert.IsTrue(forEachDiff < copyDiff); }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = Properties.Settings.Default.ASC3LogsPath; string CSV = Properties.Settings.Default.CSVOutPAth; //var tableCollection = new BlockingCollection<DataTable>(); //DataTable mergedEventsTable = new DataTable(); ParallelOptions options; if (!Properties.Settings.Default.forceNonParallel) { options = new ParallelOptions { MaxDegreeOfParallelism = -1 }; } else { if (Properties.Settings.Default.MaxThreads < 2) { options = new ParallelOptions { MaxDegreeOfParallelism = 1 }; } else { options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; } } foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); //foreach (string dir in dirList) ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain }; Parallel.ForEach(sp, optionsMain, dir => { var ToDelete = new ConcurrentBag <string>(); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); //SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat")); //Parallel.ForEach(sp2, options, s => foreach (var s in Directory.GetFiles(dir, "*.dat")) { try { MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); ToDelete.Add(s); } catch { } } //); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } mergedEventsTable.Dispose(); string connectionString = Properties.Settings.Default.SPMConnectionString; string destTable = Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); if (elTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile) { DeleteFiles(ToDelete); } //string filename = sigid.ToString(); //filename += "_"; //filename += DateTime.Now.Month.ToString(); //filename += "_"; //filename += DateTime.Now.Day.ToString(); //filename += "_"; //filename += DateTime.Now.Year.ToString(); //filename += "_"; //filename += DateTime.Now.Hour.ToString(); //filename += "_"; //filename += DateTime.Now.Minute.ToString(); //filename += "_"; //filename += DateTime.Now.Second.ToString(); //filename += ".csv"; //SaveAsCSV(EventsTable, Path.Combine(CSV, filename)); //if (Properties.Settings.Default.DeleteFile) //{ // DeleteFiles(ToDelete); //} } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in ToDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } } ); }