static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = FileByFileASC3Decoder.Properties.Settings.Default.ASC3LogsPath;
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_") || f.Length < 367)
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = FileByFileASC3Decoder.Properties.Settings.Default.SPMConnectionString;
string destTable = FileByFileASC3Decoder.Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole, true, 0, FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile,
FileByFileASC3Decoder.Properties.Settings.Default.EarliestAcceptableDate, 5000, 30);
if (EventsTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile)
{
try
{
File.Delete(s);
}
catch { }
}
}
}
}
catch { }
}
}
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string cwd = appSettings["ASC3LogsPath"];
bool writeToConsole = Convert.ToBoolean(appSettings["WriteToConsole"]);
bool deleteFile = Convert.ToBoolean(appSettings["DeleteFile"]);
bool writeToCsv = Convert.ToBoolean(appSettings["WriteToCsv"]);
foreach (string s in Directory.GetDirectories(cwd))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (writeToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_"))
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(s, sigid, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]));
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable eventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(eventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
string destTable = appSettings["DestinationTableNAme"];
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
writeToConsole, true, 0, deleteFile, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), 5000, 30);
if (eventsTable.Count > 0)
{
if (MOE.Common.Business.SignalFtp.BulktoDb(eventsTable, Options) && deleteFile)
{
try
{
File.Delete(s);
}
catch
{
}
}
}
if (writeToCsv)
{
FileInfo fileInfo = new FileInfo(s);
StringBuilder sb = new StringBuilder();
IEnumerable <string> columnNames = eventsTable.Columns.Cast <DataColumn>().
Select(column => column.ColumnName);
sb.AppendLine(string.Join(",", columnNames));
foreach (MOE.Common.Data.MOE.Controller_Event_LogRow row in eventsTable.Rows)
{
List <string> fields = new List <string>();
fields.Add(row.SignalID);
fields.Add(row.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff"));
fields.Add(row.EventCode.ToString());
fields.Add(row.EventParam.ToString());
sb.AppendLine(string.Join(",", fields));
}
var csvFilePath = fileInfo.FullName.TrimEnd(new char[] { '.', 'd', 'a', 't' }) + ".csv";
File.WriteAllText(csvFilePath, sb.ToString());
}
}
}
catch { }
}
}
}
public void DoesCopyTakeLongerThanForEach()
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = CreateTableObject(true);
for (int i = 0; i < 100000; i++)
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow1 = elTable.NewController_Event_LogRow();
eventrow1.Timestamp = DateTime.Now;
eventrow1.SignalID = "101";
eventrow1.EventCode = rnd.Next(1, 256);
eventrow1.EventParam = rnd.Next(1, 256);
mergedEventsTable.Add(eventrow1);
}
DateTime startCopy = DateTime.Now;
try
{
mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The copy method didn't work");
}
elTable.Clear();
DateTime endCopy = DateTime.Now;
DateTime startForEach = DateTime.Now;
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
if (elTable.Count != mergedEventsTable.Count)
{
Assert.Fail("The foreach method didn't work");
}
DateTime endForEach = DateTime.Now;
var copyDiff = (endCopy - startCopy);
var forEachDiff = (endForEach - startForEach);
Assert.IsTrue(forEachDiff < copyDiff);
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = Properties.Settings.Default.ASC3LogsPath;
string CSV = Properties.Settings.Default.CSVOutPAth;
//var tableCollection = new BlockingCollection<DataTable>();
//DataTable mergedEventsTable = new DataTable();
ParallelOptions options;
if (!Properties.Settings.Default.forceNonParallel)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = -1
};
}
else
{
if (Properties.Settings.Default.MaxThreads < 2)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = 1
};
}
else
{
options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
}
}
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList);
//foreach (string dir in dirList)
ParallelOptions optionsMain = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain
};
Parallel.ForEach(sp, optionsMain, dir =>
{
var ToDelete = new ConcurrentBag <string>();
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
SimplePartitioner <string> sp2 = new SimplePartitioner <string>(Directory.GetFiles(dir, "*.dat"));
Parallel.ForEach(sp2, options, s =>
{
try
{
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
ToDelete.Add(s);
}
catch { }
}
);
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = Properties.Settings.Default.SPMConnectionString;
string destTable = Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
if (EventsTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && Properties.Settings.Default.DeleteFile)
{
DeleteFiles(ToDelete);
}
//string filename = sigid.ToString();
//filename += "_";
//filename += DateTime.Now.Month.ToString();
//filename += "_";
//filename += DateTime.Now.Day.ToString();
//filename += "_";
//filename += DateTime.Now.Year.ToString();
//filename += "_";
//filename += DateTime.Now.Hour.ToString();
//filename += "_";
//filename += DateTime.Now.Minute.ToString();
//filename += "_";
//filename += DateTime.Now.Second.ToString();
//filename += ".csv";
//SaveAsCSV(EventsTable, Path.Combine(CSV, filename));
//if (Properties.Settings.Default.DeleteFile)
//{
// DeleteFiles(ToDelete);
//}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in ToDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
}
);
}