static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; List <string> dirList = new List <string>(); string cwd = appSettings["ASC3LogsPath"]; foreach (string s in Directory.GetDirectories(cwd)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(appSettings["MaxThreadsMain"]) }; Parallel.ForEach(sp, optionsMain, dir => { var toDelete = new ConcurrentBag <string>(); var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); if (Convert.ToBoolean(appSettings["WriteToConsole"])) { Console.WriteLine("-----------------------------Starting Signal " + dir); } string signalId; string[] fileNames; GetFileNamesAndSignalId(dir, out signalId, out fileNames); foreach (var fileName in fileNames) { try { MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(fileName, signalId, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"])); toDelete.Add(fileName); } catch (Exception ex) { Console.WriteLine(ex.Message); } } MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateDataTableForImport(); AddEventsToImportTable(mergedEventsTable, elTable); mergedEventsTable.Dispose(); BulkImportRecordsAndDeleteFiles(appSettings, toDelete, elTable); }); }
static void Main(string[] args) { List <string> dirList = new List <string>(); List <string> fileList = new List <string>(); string CWD = Properties.Settings.Default.ASC3LogsPath; string CSV = Properties.Settings.Default.CSVOutPAth; //var tableCollection = new BlockingCollection<DataTable>(); //DataTable mergedEventsTable = new DataTable(); ParallelOptions options; if (!Properties.Settings.Default.forceNonParallel) { options = new ParallelOptions { MaxDegreeOfParallelism = -1 }; } else { if (Properties.Settings.Default.MaxThreads < 2) { options = new ParallelOptions { MaxDegreeOfParallelism = 1 }; } else { options = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads }; } } foreach (string s in Directory.GetDirectories(CWD)) { dirList.Add(s); } SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList); //foreach (string dir in dirList) ParallelOptions optionsMain = new ParallelOptions { MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain }; Parallel.ForEach(sp, optionsMain, dir => { var ToDelete = new ConcurrentBag <string>(); if (Properties.Settings.Default.WriteToConsole) { Console.WriteLine("-----------------------------Starting Signal " + dir); } //get the name of the directory and casting it to an int //This is the only way the program knows the signal number of the controller. string[] strsplit = dir.Split(new char[] { '\\' }); string dirname = strsplit.Last(); string sigid = dirname; var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>(); //SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat")); //Parallel.ForEach(sp2, options, s => foreach (var s in Directory.GetFiles(dir, "*.dat")) { try { MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable); ToDelete.Add(s); } catch { } } //); MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable(); UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"], elTable.Columns["Timestamp"], elTable.Columns["EventCode"], elTable.Columns["EventParam"] }); elTable.Constraints.Add(custUnique); //mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges); foreach (var r in mergedEventsTable) { try { elTable.AddController_Event_LogRow(r); } catch { } } mergedEventsTable.Dispose(); string connectionString = Properties.Settings.Default.SPMConnectionString; string destTable = Properties.Settings.Default.DestinationTableNAme; MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable, Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile, Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut); if (elTable.Count > 0) { if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile) { DeleteFiles(ToDelete); } //string filename = sigid.ToString(); //filename += "_"; //filename += DateTime.Now.Month.ToString(); //filename += "_"; //filename += DateTime.Now.Day.ToString(); //filename += "_"; //filename += DateTime.Now.Year.ToString(); //filename += "_"; //filename += DateTime.Now.Hour.ToString(); //filename += "_"; //filename += DateTime.Now.Minute.ToString(); //filename += "_"; //filename += DateTime.Now.Second.ToString(); //filename += ".csv"; //SaveAsCSV(EventsTable, Path.Combine(CSV, filename)); //if (Properties.Settings.Default.DeleteFile) //{ // DeleteFiles(ToDelete); //} } else { ConcurrentBag <String> td = new ConcurrentBag <String>(); foreach (string s in ToDelete) { if (s.Contains("1970_01_01")) { td.Add(s); } } if (td.Count > 0) { DeleteFiles(td); } } } ); }