private static void SaveToDB(XmlDocument xml, string SignalId) { System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings; string destTable = appSettings["DestinationTableName"]; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = new DateTime(); int EventCode = 0; int EventParam = 0; DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime); int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode); int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate) { elTable.AddController_Event_LogRow(eventrow); } } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable); }
private static void SaveToDB(XmlDocument xml, string SignalId) { MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event"); foreach (XmlNode node in list) { XmlAttributeCollection attrColl = node.Attributes; DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value); int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value); int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value); try { MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow(); eventrow.Timestamp = EventTime; eventrow.SignalID = SignalId; eventrow.EventCode = EventCode; eventrow.EventParam = EventParam; elTable.AddController_Event_LogRow(eventrow); } catch { } } MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.Signal.BulktoDB(elTable, Options); }
public static void DealWithMoab(MOE.Common.Business.BulkCopyOptions Options) { //int configuredRegion = 4; //int SNMPTimeout = Properties.Settings.Default.SNMPTimeout; //int SNMPRetry = Properties.Settings.Default.SNMPRetry; //int SNMPPort = Properties.Settings.Default.SNMPPort; //bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP; //bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP; //int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles; //int MoabTimeout = 100; //string Username = "******"; //string Password = "******"; //string RemoteDir = "\\Set1"; //bool ActiveMode = Properties.Settings.Default.FTPActiveMode; //string LocalDir = Properties.Settings.Default.HostDir + "8301" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.21", "8301", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); //LocalDir = Properties.Settings.Default.HostDir + "8302" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.27", "8302", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); //LocalDir = Properties.Settings.Default.HostDir + "8303" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.33", "8303", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); //LocalDir = Properties.Settings.Default.HostDir + "8304" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.45", "8304", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); //LocalDir = Properties.Settings.Default.HostDir + "8305" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.51", "8305", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); //LocalDir = Properties.Settings.Default.HostDir + "8306" + "\\"; //MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.57", "8306", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout); }
private static void BulkImportRecordsAndDeleteFiles(NameValueCollection appSettings, ConcurrentBag <string> toDelete, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable) { string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; string destTable = appSettings["DestinationTableNAme"]; MOE.Common.Business.BulkCopyOptions options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Convert.ToBoolean(appSettings["WriteToConsole"]), Convert.ToBoolean(appSettings["forceNonParallel"]), Convert.ToInt32(appSettings["MaxThreads"]), Convert.ToBoolean(appSettings["DeleteFile"]), Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), Convert.ToInt32(appSettings["BulkCopyBatchSize"]), Convert.ToInt32(appSettings["BulkCopyTimeOut"])); if (elTable.Count > 0) { if (MOE.Common.Business.SignalFtp.BulktoDb(elTable, options, destTable) && Convert.ToBoolean(appSettings["DeleteFile"])) { DeleteFiles(toDelete); } } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in toDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = FileByFileASC3Decoder.Properties.Settings.Default.ASC3LogsPath; foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } foreach (string dir in dirList) { if (FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an string //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var files = (Directory.GetFiles(dir, "*.dat")); foreach (string s in files) { try { FileInfo f = new FileInfo(s); if (f.Name.Contains("INT") || f.Name.Contains("_1970_") || f.Length < 367) { try { File.Delete(s); } catch { } continue; } } catch { } try { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable()) { mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges); mergedEventsTable.Dispose(); string connectionString = FileByFileASC3Decoder.Properties.Settings.Default.SPMConnectionString; string destTable = FileByFileASC3Decoder.Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole, true, 0, FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile, FileByFileASC3Decoder.Properties.Settings.Default.EarliestAcceptableDate, 5000, 30); if (EventsTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile) { try { File.Delete(s); } catch { } } } } } catch { } } } }
//subroutine to write the decoded log to the database. //this is where most of the work is done. //The only way we match signalid to the collected logs is by the directory name. //static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA) private void SaveEvents() { int insertErrorCount = 0; //int duplicateLineCount = 0; int insertedLinecount = 0; double errorRatio = 0; //bool fileHasBeenRead = false; DateTime startTime = new DateTime(); DateTime endTime = new DateTime(); TimeSpan elapsedTime = new TimeSpan(); string CWD = Properties.Settings.Default.PeekDatPath; List<string> dirList = new List<string>(); ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>(); foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth); if(Properties.Settings.Default.WriteToConsole) { Console.WriteLine(s); } } var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(dirList.AsEnumerable(), options, dir => //Parallel.ForEach(dirList.AsEnumerable(), dir => //foreach (string dir in dirList) { if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine(dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit[strsplit.Count() - 2]; string sigid = dirname; Console.WriteLine("Starting signal " + sigid); var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file => //Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file => //foreach (string file in Directory.GetFiles(dir, "*.csv")) { bool fileHasBeenRead = false; MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); startTime = DateTime.Now; foreach (string line in File.ReadAllLines(file)) { //Every other line is blank. We only care about the lines that have data, and //every data line has a comma if (line.Contains(',')) { //the first five lines or so are header information. They need to be filtered out. if (line.Contains(",ATC")) { //even if there is nothing but header in the file, we want it marked as being read //so we can delete is later. fileHasBeenRead = true; } else if (line.Contains(".dat")) { fileHasBeenRead = true; } else if (line.Contains(",IP Address:")) { fileHasBeenRead = true; } else if (line.Contains(",MAC Address")) { fileHasBeenRead = true; } else if (line.Contains(",Data Log Beginning")) { fileHasBeenRead = true; } else if (line.Contains(",Phases in use")) { fileHasBeenRead = true; } else if (line.Contains(",Binary")) { fileHasBeenRead = true; } else if (line.Contains(",Start Of")) { fileHasBeenRead = true; } else { //split the line on commas and assign each split to a var string[] lineSplit = line.Split(new char[] { ',' }); DateTime timeStamp = new DateTime(); int eventCode = 0; int eventParam = 0; bool lineError = false; //it might happen that the character on the line are not quite right. //the Try/catch stuff is an attempt to deal with that. try { timeStamp = Convert.ToDateTime(lineSplit[0]); } catch { Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]); lineError = true; } try { eventCode = Convert.ToInt32(lineSplit[1]); } catch { Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]); lineError = true; } try { eventParam = Convert.ToInt32(lineSplit[2]); } catch { Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]); lineError = true; } //If there were no errors on the line, then put the line into the bulk queue if (!lineError) { try { elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } //If it gets this far, the file has been opened fileHasBeenRead = true; } } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("NEXT LINE"); } } // Array.Clear(lines, 0, lines.Count()); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("$$$ Entire file has been read $$$"); } //Do the Math to find out if the error ratio is intolerably high before deleting the file if (insertErrorCount > 0) { errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount)); } else { errorRatio = 0; } if (file.Length == 0) { fileHasBeenRead = true; } endTime = DateTime.Now; //the Signal class has a static method to insert the tableinto the DB. We are using that. string connectionString = Properties.Settings.Default.SPM.ToString(); MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName); elapsedTime = endTime - startTime; if (Properties.Settings.Default.DeleteFiles) { try{ Directory.Delete(dir); } catch { } string d = Properties.Settings.Default.PeekDatPath; foreach (string f in Directory.GetFiles(d, "*.dat")) { File.Delete(f); } //try //{ // //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead)) // //{ // try // { // File.Delete(file); // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} Deleted", file); // } // } // catch (SystemException sysex) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file); // } // Thread.Sleep(100); // try // { // File.Delete(file); // } // catch (SystemException sysex2) // { // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file); // } // } // } // try // { // // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage); // } // catch (SqlException ex) // { // Console.WriteLine(ex); // } // // } // //else // //{ // // if (Properties.Settings.Default.WriteToConsole) // // { // // Console.WriteLine("Too many insertion errors to delete {0}", file); // // } // //} //} //catch (Exception ex) //{ // if (Properties.Settings.Default.WriteToConsole) // { // Console.WriteLine("{0} while deleting file {1}", ex, file); // } // FilesToDelete.Enqueue(file); //} } if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of file Loop%%%"); Thread.Sleep(100); } } ); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("%%%Start of DIRECTORY Loop%%%"); Thread.Sleep(100); } //CleanUpFiles(FilesToDelete); //}); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("###Start of Queue Build Hit###"); } } ); }
static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string cwd = appSettings["ASC3LogsPath"]; bool writeToConsole = Convert.ToBoolean(appSettings["WriteToConsole"]); bool deleteFile = Convert.ToBoolean(appSettings["DeleteFile"]); bool writeToCsv = Convert.ToBoolean(appSettings["WriteToCsv"]); foreach (string s in Directory.GetDirectories(cwd)) { dirList.Add(s); } foreach (string dir in dirList) { if (writeToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an string //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var files = (Directory.GetFiles(dir, "*.dat")); foreach (string s in files) { try { FileInfo f = new FileInfo(s); if (f.Name.Contains("INT") || f.Name.Contains("_1970_")) { try { File.Delete(s); } catch { } continue; } } catch { } try { var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(s, sigid, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"])); using (MOE.Common.Data.MOE.Controller_Event_LogDataTable eventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable()) { mergedEventsTable.CopyToDataTable(eventsTable, LoadOption.PreserveChanges); mergedEventsTable.Dispose(); string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; string destTable = appSettings["DestinationTableNAme"]; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, writeToConsole, true, 0, deleteFile, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), 5000, 30); if (eventsTable.Count > 0) { if (MOE.Common.Business.SignalFtp.BulktoDb(eventsTable, Options) && deleteFile) { try { File.Delete(s); } catch { } } } if (writeToCsv) { FileInfo fileInfo = new FileInfo(s); StringBuilder sb = new StringBuilder(); IEnumerable <string> columnNames = eventsTable.Columns.Cast <DataColumn>(). Select(column => column.ColumnName); sb.AppendLine(string.Join(",", columnNames)); foreach (MOE.Common.Data.MOE.Controller_Event_LogRow row in eventsTable.Rows) { List <string> fields = new List <string>(); fields.Add(row.SignalID); fields.Add(row.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff")); fields.Add(row.EventCode.ToString()); fields.Add(row.EventParam.ToString()); sb.AppendLine(string.Join(",", fields)); } var csvFilePath = fileInfo.FullName.TrimEnd(new char[] { '.', 'd', 'a', 't' }) + ".csv"; File.WriteAllText(csvFilePath, sb.ToString()); } } } catch { } } } }
//subroutine to write the decoded log to the database. //this is where most of the work is done. //The only way we match signalid to the collected logs is by the directory name. //static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA) private void SaveEvents() { int insertErrorCount = 0; int insertedLinecount = 0; double errorRatio = 0; DateTime startTime = new DateTime(); DateTime endTime = new DateTime(); TimeSpan elapsedTime = new TimeSpan(); string CWD = Properties.Settings.Default.LogPath; List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); ConcurrentQueue <string> FilesToDelete = new ConcurrentQueue <string>(); var lastrecords = new Dictionary <string, DateTime>(); var countrecords = new Dictionary <string, int>(); MOE.Common.Models.Repositories.IControllerEventLogRepository celRepository = MOE.Common.Models.Repositories.ControllerEventLogRepositoryFactory.Create(); foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); var signalID = s.Split(new char[] { '\\' }).Last(); lastrecords.Add(signalID, celRepository.GetMostRecentRecordTimestamp(signalID)); foreach (var file in Directory.GetFiles(s)) { countrecords.Add(file, ExistingRecords(signalID, file, celRepository)); } } var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; Parallel.ForEach(dirList.AsEnumerable(), options, dir => //foreach (var dir in dirList.AsEnumerable()) { //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var dstOffset = Math.Abs(DateTimeOffset.Now.Offset.Hours); WriteToConsole("Starting signal " + dirname); var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) }; //Parallel.ForEach(Directory.GetFiles(dir, "*.csv").OrderBy(f => f), options1, file => foreach (var file in Directory.GetFiles(dir, "*.csv").OrderBy(f => f)) { if (countrecords[file] >= File.ReadAllLines(file).Length - 1) { var delete = Properties.Settings.Default.DeleteFiles; WriteToConsole(String.Format("Skipping {0} {1}, we already imported this.", (delete ? "and deleting" : ""), file)); if (delete) { try { File.Delete(file); } catch (Exception e) { WriteToConsole(String.Format("Unable to delete {0}: {1}", file, e.Message)); } } continue; //return; } int skippedrecords = 0; DataTable elTable = new DataTable(); elTable.Columns.Add("sigid", System.Type.GetType("System.String")); elTable.Columns.Add("timeStamp", System.Type.GetType("System.DateTime")); elTable.Columns.Add("eventCode", System.Type.GetType("System.Int32")); elTable.Columns.Add("eventParam", System.Type.GetType("System.Int32")); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns[0], elTable.Columns[1], elTable.Columns[2], elTable.Columns[3] }); elTable.Constraints.Add(custUnique); startTime = DateTime.Now; //Siemens decoder makes the first line the IP address, so skip it. foreach (string line in File.ReadAllLines(file).Skip(1)) { //Every other line is blank. We only care about the lines that have data, and //every data line has a comma if (line.Contains(',')) { //split the line on commas and assign each split to a var string[] lineSplit = line.Split(new char[] { ',' }); DateTime timeStamp = new DateTime(); int eventCode = 0; int eventParam = 0; //it might happen that the character on the line are not quite right. //the Try/catch stuff is an attempt to deal with that. try { timeStamp = Convert.ToDateTime(lineSplit[0]); //Siemens decoder is converting to local time from UTC, so convert back to local time //Not perfect during DST transitions (at 2:00 AM twice per year) timeStamp = timeStamp + TimeSpan.FromHours(dstOffset); if (timeStamp < lastrecords[sigid]) { skippedrecords++; continue; } eventCode = Convert.ToInt32(lineSplit[1]); eventParam = Convert.ToInt32(lineSplit[2]); } catch (Exception ex) { WriteToConsole(String.Format("{0} while converting {1} to event. Skipping line", ex, lineSplit[0])); continue; } try { elTable.Rows.Add(sigid, timeStamp, eventCode, eventParam); } catch (Exception ex) { WriteToConsole(String.Format("{0} while adding event to data table", ex.ToString())); } } } WriteToConsole(String.Format("{0} has been parsed. Skipped {1} old records", file, skippedrecords)); //Do the Math to find out if the error ratio is intolerably high before deleting the file if (insertErrorCount > 0) { errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount)); } else { errorRatio = 0; } string connectionString = Properties.Settings.Default.SPM.ToString(); MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); endTime = DateTime.Now; //the Signal class has a static methods to insert the table into the DB. We are using that. MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions); elapsedTime = endTime - startTime; if (Properties.Settings.Default.DeleteFiles) { try { File.Delete(file); WriteToConsole(String.Format("{0} Deleted", file)); } catch (SystemException sysex) { WriteToConsole(String.Format("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file)); Thread.Sleep(100); try { File.Delete(file); } catch (SystemException sysex2) { WriteToConsole(String.Format("{0} while Deleting {1}, giving up", sysex2, file)); } } catch (Exception ex) { WriteToConsole(String.Format("{0} while deleting file {1}", ex, file)); FilesToDelete.Enqueue(file); } } } //); } ); }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = Properties.Settings.Default.ASC3LogsPath; string CSV = Properties.Settings.Default.CSVOutPAth; //var tableCollection = new BlockingCollection<DataTable>(); //DataTable mergedEventsTable = new DataTable(); ParallelOptions options; if (!Properties.Settings.Default.forceNonParallel) { options = new ParallelOptions { MaxDegreeOfParallelism = -1 }; } else { if (Properties.Settings.Default.MaxThreads < 2) { options = new ParallelOptions { MaxDegreeOfParallelism = 1 }; } else { options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; } } foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); //foreach (string dir in dirList) ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain }; Parallel.ForEach(sp, optionsMain, dir => { var ToDelete = new ConcurrentBag <string>(); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); //SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat")); //Parallel.ForEach(sp2, options, s => foreach (var s in Directory.GetFiles(dir, "*.dat")) { try { MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); ToDelete.Add(s); } catch { } } //); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } mergedEventsTable.Dispose(); string connectionString = Properties.Settings.Default.SPMConnectionString; string destTable = Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); if (elTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile) { DeleteFiles(ToDelete); } //string filename = sigid.ToString(); //filename += "_"; //filename += DateTime.Now.Month.ToString(); //filename += "_"; //filename += DateTime.Now.Day.ToString(); //filename += "_"; //filename += DateTime.Now.Year.ToString(); //filename += "_"; //filename += DateTime.Now.Hour.ToString(); //filename += "_"; //filename += DateTime.Now.Minute.ToString(); //filename += "_"; //filename += DateTime.Now.Second.ToString(); //filename += ".csv"; //SaveAsCSV(EventsTable, Path.Combine(CSV, filename)); //if (Properties.Settings.Default.DeleteFile) //{ // DeleteFiles(ToDelete); //} } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in ToDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } } ); }
public void UploadControllerData(string IPAddress, string SignalID, string UserName, string Password, string LocalDir, string RemoteDir, bool DeleteFiles, int SNMPRetry, int SNMPTimeout, int SNMPPort, bool ImportAfterFTP, bool ActiveMode, int WaitBetweenRecords, MOE.Common.Business.BulkCopyOptions Options) { MOE.Common.Business.Signal.GetCurrentRecords(IPAddress, SignalID, UserName, Password, LocalDir, RemoteDir, DeleteFiles, SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenRecords, Options); }
static void Main(string[] args) { MOE.Common.Models.Repositories.IApplicationEventRepository ErrorRepository = MOE.Common.Models.Repositories.ApplicationEventRepositoryFactory.Create(); int MaxThreads = Properties.Settings.Default.MaxThreads; int SNMPTimeout = Properties.Settings.Default.SNMPTimeout; int SNMPRetry = Properties.Settings.Default.SNMPRetry; int SNMPPort = Properties.Settings.Default.SNMPPort; bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP; bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP; int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles; var connection = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connection, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); //MOE.Common.Data.Signals.SignalFTPparamsDataTable SignalsDT = new MOE.Common.Data.Signals.SignalFTPparamsDataTable(); //MOE.Common.Data.SignalsTableAdapters.SignalFTPparamsTableAdapter SignalsTA = new MOE.Common.Data.SignalsTableAdapters.SignalFTPparamsTableAdapter(); //SignalsTA.Fill(SignalsDT); MOE.Common.Models.SPM db = new MOE.Common.Models.SPM(); var SignalsDT = from r in db.Signals join f in db.ControllerType on r.ControllerTypeID equals f.ControllerTypeID where r.ControllerTypeID != 4 select new { SignalId = r.SignalID, PrimaryName = r.PrimaryName, Secondary_Name = r.SecondaryName, Region = r.Region, IP_Address = r.IPAddress, UserName = f.UserName, Password = f.Password, FTPDirectory = f.FTPDirectory, ActiveFTP = f.ActiveFTP }; var options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; Parallel.ForEach(SignalsDT.AsEnumerable(), options, row => //foreach (var row in SignalsDT) { try { MOE.Common.Business.Signal signal = new MOE.Common.Business.Signal(); //Initialize the signal, because I didn't make a proper constructor signal.PrimaryName = row.PrimaryName.ToString(); signal.SecondaryName = row.Secondary_Name.ToString(); signal.Region = row.Region.ToString(); signal.IpAddress = row.IP_Address.ToString(); signal.SignalID = row.SignalId.ToString(); string Username = row.UserName; string Password = row.Password; string LocalDir = Properties.Settings.Default.HostDir + signal.SignalID + "\\"; string RemoteDir = row.FTPDirectory; bool ActiveMode = row.ActiveFTP; if (!Directory.Exists(LocalDir)) { Directory.CreateDirectory(LocalDir); } //Get the records over FTP if (CheckIfIPAddressIsValid(signal)) { try { //var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); //var token = tokenSource.Token; //Task task = Task.Factory.StartNew(() => MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, // SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout, token), token); //task.Wait(); //if (token.IsCancellationRequested) // token.ThrowIfCancellationRequested(); MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout); } catch (AggregateException ex) { Console.WriteLine("Error At Highest Level for signal " + ex.Message); ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalId); } } } catch (AggregateException ex) { Console.WriteLine("Error At Highest Level for signal " + ex.Message); ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalId); } } ); //if (Properties.Settings.Default.DealWithMoab) //{ // try // { // DealWithMoab(Options); // } // catch // { // } //} }
static void Main(string[] args) { MOE.Common.Models.Repositories.IApplicationEventRepository ErrorRepository = MOE.Common.Models.Repositories.ApplicationEventRepositoryFactory.Create(); int MaxThreads = Properties.Settings.Default.MaxThreads; int SNMPTimeout = Properties.Settings.Default.SNMPTimeout; int SNMPRetry = Properties.Settings.Default.SNMPRetry; int SNMPPort = Properties.Settings.Default.SNMPPort; bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP; bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP; int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles; var connection = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connection, Properties.Settings.Default.DestinationTableName, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); MOE.Common.Models.SPM _db = new MOE.Common.Models.SPM(); MOE.Common.Models.Repositories.ISignalsRepository _sr = SignalsRepositoryFactory.Create(_db); var _signals = _sr.GetSignalFTPInfoForAllFTPSignals(); var options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; Parallel.ForEach(_signals.AsEnumerable(), options, row => //foreach (var row in SignalsDT) { try { MOE.Common.Business.Signal signal = new MOE.Common.Business.Signal(); //Initialize the signal, because I didn't make a proper constructor signal.PrimaryName = row.PrimaryName.ToString(); signal.SecondaryName = row.Secondary_Name.ToString(); signal.IpAddress = row.IP_Address.ToString(); signal.SignalID = row.SignalID.ToString(); string Username = row.User_Name; string Password = row.Password; string LocalDir = Properties.Settings.Default.HostDir + signal.SignalID + "\\"; string RemoteDir = row.FTP_Directory; bool ActiveMode = row.ActiveFTP; if (!Directory.Exists(LocalDir)) { Directory.CreateDirectory(LocalDir); } //Get the records over FTP if (CheckIfIPAddressIsValid(signal)) { try { MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP, SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout); } catch (AggregateException ex) { Console.WriteLine("Error At Highest Level for signal " + ex.Message); ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalID); } } } catch (AggregateException ex) { Console.WriteLine("Error At Highest Level for signal " + ex.Message); ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalID); } } ); }