public void UniqueConstrainShouldKeepOutDuplicateRecordsTest() { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = 1; eventrow1.EventParam = 1; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow(); eventrow2.Timestamp = eventrow1.Timestamp; eventrow2.SignalID = eventrow1.SignalID; eventrow2.EventCode = eventrow1.EventCode; eventrow2.EventParam = eventrow1.EventParam; elTable.AddController_Event_LogRow(eventrow1); try { elTable.AddController_Event_LogRow(eventrow2); } catch { } Assert.IsTrue(elTable.Count == 1); }
private static MOE.Common.Data.MOE.Controller_Event_LogDataTable CreateDataTableForImport() { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); return(elTable); }
private static void SaveToDB(XmlDocument xml, string SignalId) { System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings; string destTable = appSettings["DestinationTableName"]; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = new DateTime(); int EventCode = 0; int EventParam = 0; DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime); int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode); int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate) { elTable.AddController_Event_LogRow(eventrow); } } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable); }
private MOE.Common.Data.MOE.Controller_Event_LogDataTable CreateTableObject(bool hasUniqueConstraint) { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); if (hasUniqueConstraint) { UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); } return(elTable); }
static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; List <string> dirList = new List <string>(); string cwd = appSettings["ASC3LogsPath"]; foreach (string s in Directory.GetDirectories(cwd)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(appSettings["MaxThreadsMain"]) }; Parallel.ForEach(sp, optionsMain, dir => { var toDelete = new ConcurrentBag <string>(); var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); if (Convert.ToBoolean(appSettings["WriteToConsole"])) { Console.WriteLine("-----------------------------Starting Signal " + dir); } string signalId; string[] fileNames; GetFileNamesAndSignalId(dir, out signalId, out fileNames); foreach (var fileName in fileNames) { try { MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(fileName, signalId, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"])); toDelete.Add(fileName); } catch (Exception ex) { Console.WriteLine(ex.Message); } } MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateDataTableForImport(); AddEventsToImportTable(mergedEventsTable, elTable); mergedEventsTable.Dispose(); BulkImportRecordsAndDeleteFiles(appSettings, toDelete, elTable); }); }
public void CopyFromBlockingCollectionIntoConstrainedTableMustNotFailTest() { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = 1; eventrow1.EventParam = 1; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow2 = elTable.NewController_Event_LogRow(); eventrow2.Timestamp = eventrow1.Timestamp; eventrow2.SignalID = eventrow1.SignalID; eventrow2.EventCode = eventrow1.EventCode; eventrow2.EventParam = eventrow1.EventParam; MOE.Common.Data.MOE.Controller_Event_LogRow eventrow3 = elTable.NewController_Event_LogRow(); eventrow3.Timestamp = DateTime.Now; eventrow3.SignalID = "103"; eventrow3.EventCode = 3; eventrow3.EventParam = 3; mergedEventsTable.Add(eventrow1); mergedEventsTable.Add(eventrow2); mergedEventsTable.Add(eventrow3); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } Assert.IsTrue(elTable.Count == 2); }
private static void SaveToDB(XmlDocument xml, string SignalId) { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value); int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value); int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; elTable.AddController_Event_LogRow(eventrow); } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.Signal.BulktoDB(elTable, Options); }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = FileByFileASC3Decoder.Properties.Settings.Default.ASC3LogsPath; foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } foreach (string dir in dirList) { if (FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an string //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var files = (Directory.GetFiles(dir, "*.dat")); foreach (string s in files) { try { FileInfo f = new FileInfo(s); if (f.Name.Contains("INT") || f.Name.Contains("_1970_") || f.Length < 367) { try { File.Delete(s); } catch { } continue; } } catch { } try { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable()) { mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges); mergedEventsTable.Dispose(); string connectionString = FileByFileASC3Decoder.Properties.Settings.Default.SPMConnectionString; string destTable = FileByFileASC3Decoder.Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole, true, 0, FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile, FileByFileASC3Decoder.Properties.Settings.Default.EarliestAcceptableDate, 5000, 30); if (EventsTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile) { try { File.Delete(s); } catch { } } } } } catch { } } } }
//subroutine to write the decoded log to the database. //this is where most of the work is done. //The only way we match signalid to the collected logs is by the directory name. //static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA) private void SaveEvents() { int insertErrorCount = 0; //int duplicateLineCount = 0; int insertedLinecount = 0; double errorRatio = 0; //bool fileHasBeenRead = false; DateTime startTime = new DateTime(); DateTime endTime = new DateTime(); TimeSpan elapsedTime = new TimeSpan(); string CWD = Properties.Settings.Default.PeekDatPath; List<string> dirList = new List<string>(); ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>(); foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth); if(Properties.Settings.Default.WriteToConsole) { Console.WriteLine(s); } } var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(dirList.AsEnumerable(), options, dir => //Parallel.ForEach(dirList.AsEnumerable(), dir => //foreach (string dir in dirList) { if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine(dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit[strsplit.Count() - 2]; string sigid = dirname; Console.WriteLine("Starting signal " + sigid); var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file => //Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file => //foreach (string file in Directory.GetFiles(dir, "*.csv")) { bool fileHasBeenRead = false; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); startTime = DateTime.Now; foreach (string line in File.ReadAllLines(file)) { //Every other line is blank. We only care about the lines that have data, and //every data line has a comma if (line.Contains(',')) { //the first five lines or so are header information. They need to be filtered out. if (line.Contains(",ATC")) { //even if there is nothing but header in the file, we want it marked as being read //so we can delete is later. fileHasBeenRead = true; } else if (line.Contains(".dat")) { fileHasBeenRead = true; } else if (line.Contains(",IP Address:")) { fileHasBeenRead = true; } else if (line.Contains(",MAC Address")) { fileHasBeenRead = true; } else if (line.Contains(",Data Log Beginning")) { fileHasBeenRead = true; } else if (line.Contains(",Phases in use")) { fileHasBeenRead = true; } else if (line.Contains(",Binary")) { fileHasBeenRead = true; } else if (line.Contains(",Start Of")) { fileHasBeenRead = true; } else { //split the line on commas and assign each split to a var string[] lineSplit = line.Split(new char[] { ',' }); DateTime timeStamp = new DateTime(); int eventCode = 0; int eventParam = 0; bool lineError = false; //it might happen that the character on the line are not quite right. //the Try/catch stuff is an attempt to deal with that. try { timeStamp = Convert.ToDateTime(lineSplit[0]); } catch { Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]); lineError = true; } try { eventCode = Convert.ToInt32(lineSplit[1]); } catch { Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]); lineError = true; } try { eventParam = Convert.ToInt32(lineSplit[2]); } catch { Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]); lineError = true; } //If there were no errors on the line, then put the line into the bulk queue if (!lineError) { try { elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } //If it gets this far, the file has been opened fileHasBeenRead = true; } } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("NEXT LINE"); } } // Array.Clear(lines, 0, lines.Count()); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("$$$ Entire file has been read $$$"); } //Do the Math to find out if the error ratio is intolerably high before deleting the file if (insertErrorCount > 0) { errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount)); } else { errorRatio = 0; } if (file.Length == 0) { fileHasBeenRead = true; } endTime = DateTime.Now; //the Signal class has a static method to insert the tableinto the DB. We are using that. string connectionString = Properties.Settings.Default.SPM.ToString(); MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName); elapsedTime = endTime - startTime; if (Properties.Settings.Default.DeleteFiles) { try{ Directory.Delete(dir); } catch { } string d = Properties.Settings.Default.PeekDatPath; foreach (string f in Directory.GetFiles(d, "*.dat")) { File.Delete(f); } //try //{ // //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead)) // //{ // try // { // File.Delete(file); // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} Deleted", file); // } // } // catch (SystemException sysex) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file); // } // Thread.Sleep(100); // try // { // File.Delete(file); // } // catch (SystemException sysex2) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file); // } // } // } // try // { // // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage); // } // catch (SqlException ex) // { // Console.WriteLine(ex); // } // // } // //else // //{ // // if (Properties.Settings.Default.WriteToConsole) // // { // // Console.WriteLine("Too many insertion errors to delete {0}", file); // // } // //} //} //catch (Exception ex) //{ // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while deleting file {1}", ex, file); // } // FilesToDelete.Enqueue(file); //} } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of file Loop%%%"); Thread.Sleep(100); } } ); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of DIRECTORY Loop%%%"); Thread.Sleep(100); } //CleanUpFiles(FilesToDelete); //}); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("###Start of Queue Build Hit###"); } } ); }
public void DoesCopyTakeLongerThanForEach() { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true); for (int i = 0; i < 100000; i++) { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow(); eventrow1.Timestamp = DateTime.Now; eventrow1.SignalID = "101"; eventrow1.EventCode = rnd.Next(1, 256); eventrow1.EventParam = rnd.Next(1, 256); mergedEventsTable.Add(eventrow1); } DateTime startCopy = DateTime.Now; try { mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); } catch (Exception ex) { Console.WriteLine(ex.Message); } if (elTable.Count != mergedEventsTable.Count) { Assert.Fail("The copy method didn't work"); } elTable.Clear(); DateTime endCopy = DateTime.Now; DateTime startForEach = DateTime.Now; foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch (Exception ex) { Console.WriteLine(ex.Message); } } if (elTable.Count != mergedEventsTable.Count) { Assert.Fail("The foreach method didn't work"); } DateTime endForEach = DateTime.Now; var copyDiff = (endCopy - startCopy); var forEachDiff = (endForEach - startForEach); Assert.IsTrue(forEachDiff < copyDiff); }
static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string cwd = appSettings["ASC3LogsPath"]; bool writeToConsole = Convert.ToBoolean(appSettings["WriteToConsole"]); bool deleteFile = Convert.ToBoolean(appSettings["DeleteFile"]); bool writeToCsv = Convert.ToBoolean(appSettings["WriteToCsv"]); foreach (string s in Directory.GetDirectories(cwd)) { dirList.Add(s); } foreach (string dir in dirList) { if (writeToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an string //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var files = (Directory.GetFiles(dir, "*.dat")); foreach (string s in files) { try { FileInfo f = new FileInfo(s); if (f.Name.Contains("INT") || f.Name.Contains("_1970_")) { try { File.Delete(s); } catch { } continue; } } catch { } try { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(s, sigid, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"])); using (MOE.Common.Data.MOE.Controller_Event_LogDataTable eventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable()) { mergedEventsTable.CopyToDataTable(eventsTable, LoadOption.PreserveChanges); mergedEventsTable.Dispose(); string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; string destTable = appSettings["DestinationTableNAme"]; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, writeToConsole, true, 0, deleteFile, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), 5000, 30); if (eventsTable.Count > 0) { if (MOE.Common.Business.SignalFtp.BulktoDb(eventsTable, Options) && deleteFile) { try { File.Delete(s); } catch { } } } if (writeToCsv) { FileInfo fileInfo = new FileInfo(s); StringBuilder sb = new StringBuilder(); IEnumerable <string> columnNames = eventsTable.Columns.Cast <DataColumn>(). Select(column => column.ColumnName); sb.AppendLine(string.Join(",", columnNames)); foreach (MOE.Common.Data.MOE.Controller_Event_LogRow row in eventsTable.Rows) { List <string> fields = new List <string>(); fields.Add(row.SignalID); fields.Add(row.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff")); fields.Add(row.EventCode.ToString()); fields.Add(row.EventParam.ToString()); sb.AppendLine(string.Join(",", fields)); } var csvFilePath = fileInfo.FullName.TrimEnd(new char[] { '.', 'd', 'a', 't' }) + ".csv"; File.WriteAllText(csvFilePath, sb.ToString()); } } } catch { } } } }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = Properties.Settings.Default.ASC3LogsPath; string CSV = Properties.Settings.Default.CSVOutPAth; //var tableCollection = new BlockingCollection<DataTable>(); //DataTable mergedEventsTable = new DataTable(); ParallelOptions options; if (!Properties.Settings.Default.forceNonParallel) { options = new ParallelOptions { MaxDegreeOfParallelism = -1 }; } else { if (Properties.Settings.Default.MaxThreads < 2) { options = new ParallelOptions { MaxDegreeOfParallelism = 1 }; } else { options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; } } foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); //foreach (string dir in dirList) ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain }; Parallel.ForEach(sp, optionsMain, dir => { var ToDelete = new ConcurrentBag <string>(); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); //SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat")); //Parallel.ForEach(sp2, options, s => foreach (var s in Directory.GetFiles(dir, "*.dat")) { try { MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); ToDelete.Add(s); } catch { } } //); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } mergedEventsTable.Dispose(); string connectionString = Properties.Settings.Default.SPMConnectionString; string destTable = Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); if (elTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile) { DeleteFiles(ToDelete); } //string filename = sigid.ToString(); //filename += "_"; //filename += DateTime.Now.Month.ToString(); //filename += "_"; //filename += DateTime.Now.Day.ToString(); //filename += "_"; //filename += DateTime.Now.Year.ToString(); //filename += "_"; //filename += DateTime.Now.Hour.ToString(); //filename += "_"; //filename += DateTime.Now.Minute.ToString(); //filename += "_"; //filename += DateTime.Now.Second.ToString(); //filename += ".csv"; //SaveAsCSV(EventsTable, Path.Combine(CSV, filename)); //if (Properties.Settings.Default.DeleteFile) //{ // DeleteFiles(ToDelete); //} } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in ToDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } } ); }
private static void BulkImportRecordsAndDeleteFiles(NameValueCollection appSettings, ConcurrentBag <string> toDelete, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable) { string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; string destTable = appSettings["DestinationTableNAme"]; MOE.Common.Business.BulkCopyOptions options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Convert.ToBoolean(appSettings["WriteToConsole"]), Convert.ToBoolean(appSettings["forceNonParallel"]), Convert.ToInt32(appSettings["MaxThreads"]), Convert.ToBoolean(appSettings["DeleteFile"]), Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), Convert.ToInt32(appSettings["BulkCopyBatchSize"]), Convert.ToInt32(appSettings["BulkCopyTimeOut"])); if (elTable.Count > 0) { if (MOE.Common.Business.SignalFtp.BulktoDb(elTable, options, destTable) && Convert.ToBoolean(appSettings["DeleteFile"])) { DeleteFiles(toDelete); } } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in toDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } }
private static void AddEventsToImportTable(BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> mergedEventsTable, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable) { foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r.SignalID, r.Timestamp, r.EventCode, r.EventParam); } catch (Exception ex) { Console.WriteLine(ex.Message); } } }
//static public MOE.Common.Data.MOE.Controller_Event_LogDataTable DecodeASC3File(string FileName, string SignalId) static public void DecodeASC3File(string FileName, string SignalId, BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow> RowBag) { DateTime EarliestAcceptableDate = Properties.Settings.Default.EarliestAcceptableDate; System.Text.Encoding encoding = System.Text.Encoding.ASCII; using (BinaryReader BR = new BinaryReader(File.Open(FileName, FileMode.Open), encoding)) { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); if (BR.BaseStream.Position + 21 < BR.BaseStream.Length) { //Find the start Date String dateString = ""; for (int i = 1; i < 21; i++) { Char c = BR.ReadChar(); dateString += c; } //Console.WriteLine(dateString); DateTime StartTime = new DateTime(); if (DateTime.TryParse(dateString, out StartTime) && (BR.BaseStream.Position < BR.BaseStream.Length)) { //find line feed characters, that should take us to the end of the header. // First line break is after Version // Second LF is after FileName // Third LF is after Interseciton number, which isn't used as far as I can tell // Fourth LF is after IP address // Fifth is after MAC Address // Sixth is after "Controller data log beginning:," and then the date // Seven is after "Phases in use:," and then the list of phases, seperated by commas int i = 0; while (i < 7 && (BR.BaseStream.Position < BR.BaseStream.Length)) { Char c = BR.ReadChar(); //Console.WriteLine(c.ToString()); if (c == '\n') { i++; } } //The first record alwasy seems to be missing a timestamp. Econolite assumes the first even occures at the same time // the second event occurs. I am going to tag the first event with secondevet.timestamp - 1/10 second int firstEventCode = new Int32(); int firstEventParam = new Int32(); if (BR.BaseStream.Position + sizeof(char) < BR.BaseStream.Length) { firstEventCode = Convert.ToInt32(BR.ReadChar()); } if (BR.BaseStream.Position + sizeof(char) < BR.BaseStream.Length) { firstEventParam = Convert.ToInt32(BR.ReadChar()); } bool firstEventEntered = false; //MOE.Common.Business.ControllerEvent firstEvent = new MOE.Common.Business.ControllerEvent(SignalID, StartTime, firstEventCode, firstEventParam); //After that, we can probably start reading while ((BR.BaseStream.Position + (sizeof(byte) * 4)) <= BR.BaseStream.Length) //we need ot make sure we are more that 4 characters from the end { DateTime EventTime = new DateTime(); int EventCode = new Int32(); int EventParam = new Int32(); //MOE.Common.Business.ControllerEvent controllerEvent = null; for (int EventPart = 1; EventPart < 4; EventPart++) { //getting the time offset if (EventPart == 1) { byte[] rawoffset = new byte[2]; //char[] offset = new char[2]; rawoffset = BR.ReadBytes(2); Array.Reverse(rawoffset); int offset = BitConverter.ToInt16(rawoffset, 0); double tenths = Convert.ToDouble(offset) / 10; EventTime = StartTime.AddSeconds(tenths); } //getting the EventCode if (EventPart == 2) { //Char EventCodeChar = BR.ReadChar(); //EventCode = Convert.ToInt32(BR.ReadChar()); EventCode = Convert.ToInt32(BR.ReadByte()); } if (EventPart == 3) { //Char EventParamChar = BR.ReadChar(); //EventParam = Convert.ToInt32(BR.ReadChar()); EventParam = Convert.ToInt32(BR.ReadByte()); } } //controllerEvent = new MOE.Common.Business.ControllerEvent(SignalID, EventTime, EventCode, EventParam); if ((EventTime) <= DateTime.Now && (EventTime > EarliestAcceptableDate)) { if (!firstEventEntered) { try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime.AddMilliseconds(-1); eventrow.SignalID = SignalId; eventrow.EventCode = firstEventCode; eventrow.EventParam = firstEventParam; if (!RowBag.Contains(eventrow)) { RowBag.Add(eventrow); } //elTable.AddController_Event_LogRow(SignalID, EventTime.AddMilliseconds(-1), firstEventCode, firstEventParam); } catch { } firstEventEntered = true; } else { try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; if (!RowBag.Contains(eventrow)) { RowBag.Add(eventrow); } //elTable.AddController_Event_LogRow(Convert.ToInt32(SignalID), EventTime, EventCode, EventParam); } catch { } } } } } //this is what we do when the datestring doesn't parse else { // return false; } } //this is what we do when the datestring doesn't parse else { //return false; } //return true; } }