private static void SaveToDB(XmlDocument xml, string SignalId)
{
System.Collections.Specialized.NameValueCollection appSettings = ConfigurationManager.AppSettings;
string destTable = appSettings["DestinationTableName"];
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = new DateTime();
int EventCode = 0;
int EventParam = 0;
DateTime.TryParse(attrColl.GetNamedItem("TimeStamp").Value, out EventTime);
int.TryParse(attrColl.GetNamedItem("EventTypeID").Value, out EventCode);
int.TryParse(attrColl.GetNamedItem("Parameter").Value, out EventParam);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
if (eventrow.Timestamp > Properties.Settings.Default.EarliestAcceptableDate)
{
elTable.AddController_Event_LogRow(eventrow);
}
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(ConfigurationManager.ConnectionStrings["SPM"].ConnectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, Options, destTable);
}
private static void SaveToDB(XmlDocument xml, string SignalId)
{
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
XmlNodeList list = xml.SelectNodes("/EventResponses/EventResponse/Event");
foreach (XmlNode node in list)
{
XmlAttributeCollection attrColl = node.Attributes;
DateTime EventTime = Convert.ToDateTime(attrColl.GetNamedItem("TimeStamp").Value);
int EventCode = Convert.ToInt32(attrColl.GetNamedItem("EventTypeID").Value);
int EventParam = Convert.ToInt32(attrColl.GetNamedItem("Parameter").Value);
try
{
MOE.Common.Data.MOE.Controller_Event_LogRow eventrow = elTable.NewController_Event_LogRow();
eventrow.Timestamp = EventTime;
eventrow.SignalID = SignalId;
eventrow.EventCode = EventCode;
eventrow.EventParam = EventParam;
elTable.AddController_Event_LogRow(eventrow);
}
catch
{
}
}
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(Properties.Settings.Default.SPM, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, false,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.Signal.BulktoDB(elTable, Options);
}
public static void DealWithMoab(MOE.Common.Business.BulkCopyOptions Options)
{
//int configuredRegion = 4;
//int SNMPTimeout = Properties.Settings.Default.SNMPTimeout;
//int SNMPRetry = Properties.Settings.Default.SNMPRetry;
//int SNMPPort = Properties.Settings.Default.SNMPPort;
//bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP;
//bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP;
//int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles;
//int MoabTimeout = 100;
//string Username = "******";
//string Password = "******";
//string RemoteDir = "\\Set1";
//bool ActiveMode = Properties.Settings.Default.FTPActiveMode;
//string LocalDir = Properties.Settings.Default.HostDir + "8301" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.21", "8301", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
//LocalDir = Properties.Settings.Default.HostDir + "8302" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.27", "8302", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
//LocalDir = Properties.Settings.Default.HostDir + "8303" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.33", "8303", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
//LocalDir = Properties.Settings.Default.HostDir + "8304" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.45", "8304", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
//LocalDir = Properties.Settings.Default.HostDir + "8305" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.51", "8305", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
//LocalDir = Properties.Settings.Default.HostDir + "8306" + "\\";
//MOE.Common.Business.Signal.GetCurrentRecords("10.135.5.57", "8306", Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenFiles, Options, MoabTimeout);
}
private static void BulkImportRecordsAndDeleteFiles(NameValueCollection appSettings, ConcurrentBag <string> toDelete, MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable)
{
string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
string destTable = appSettings["DestinationTableNAme"];
MOE.Common.Business.BulkCopyOptions options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Convert.ToBoolean(appSettings["WriteToConsole"]),
Convert.ToBoolean(appSettings["forceNonParallel"]),
Convert.ToInt32(appSettings["MaxThreads"]),
Convert.ToBoolean(appSettings["DeleteFile"]),
Convert.ToDateTime(appSettings["EarliestAcceptableDate"]),
Convert.ToInt32(appSettings["BulkCopyBatchSize"]),
Convert.ToInt32(appSettings["BulkCopyTimeOut"]));
if (elTable.Count > 0)
{
if (MOE.Common.Business.SignalFtp.BulktoDb(elTable, options, destTable) && Convert.ToBoolean(appSettings["DeleteFile"]))
{
DeleteFiles(toDelete);
}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in toDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = FileByFileASC3Decoder.Properties.Settings.Default.ASC3LogsPath;
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_") || f.Length < 367)
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable EventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(EventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = FileByFileASC3Decoder.Properties.Settings.Default.SPMConnectionString;
string destTable = FileByFileASC3Decoder.Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
FileByFileASC3Decoder.Properties.Settings.Default.WriteToConsole, true, 0, FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile,
FileByFileASC3Decoder.Properties.Settings.Default.EarliestAcceptableDate, 5000, 30);
if (EventsTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(EventsTable, Options) && FileByFileASC3Decoder.Properties.Settings.Default.DeleteFile)
{
try
{
File.Delete(s);
}
catch { }
}
}
}
}
catch { }
}
}
}
//subroutine to write the decoded log to the database.
//this is where most of the work is done.
//The only way we match signalid to the collected logs is by the directory name.
//static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA)
private void SaveEvents()
{
int insertErrorCount = 0;
//int duplicateLineCount = 0;
int insertedLinecount = 0;
double errorRatio = 0;
//bool fileHasBeenRead = false;
DateTime startTime = new DateTime();
DateTime endTime = new DateTime();
TimeSpan elapsedTime = new TimeSpan();
string CWD = Properties.Settings.Default.PeekDatPath;
List<string> dirList = new List<string>();
ConcurrentQueue<string> FilesToDelete = new ConcurrentQueue<string>();
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s+"\\"+Properties.Settings.Default.PeekCSVPAth);
if(Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(s);
}
}
var options = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(dirList.AsEnumerable(), options, dir =>
//Parallel.ForEach(dirList.AsEnumerable(), dir =>
//foreach (string dir in dirList)
{
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine(dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit[strsplit.Count() - 2];
string sigid = dirname;
Console.WriteLine("Starting signal " + sigid);
var options1 = new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads) };
Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), options1, file =>
//Parallel.ForEach(Directory.GetFiles(dir, "*.csv"), file =>
//foreach (string file in Directory.GetFiles(dir, "*.csv"))
{
bool fileHasBeenRead = false;
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalId"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"]
});
elTable.Constraints.Add(custUnique);
startTime = DateTime.Now;
foreach (string line in File.ReadAllLines(file))
{
//Every other line is blank. We only care about the lines that have data, and
//every data line has a comma
if (line.Contains(','))
{
//the first five lines or so are header information. They need to be filtered out.
if (line.Contains(",ATC"))
{
//even if there is nothing but header in the file, we want it marked as being read
//so we can delete is later.
fileHasBeenRead = true;
}
else if (line.Contains(".dat"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",IP Address:"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",MAC Address"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Data Log Beginning"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Phases in use"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Binary"))
{
fileHasBeenRead = true;
}
else if (line.Contains(",Start Of"))
{
fileHasBeenRead = true;
}
else
{
//split the line on commas and assign each split to a var
string[] lineSplit = line.Split(new char[] { ',' });
DateTime timeStamp = new DateTime();
int eventCode = 0;
int eventParam = 0;
bool lineError = false;
//it might happen that the character on the line are not quite right.
//the Try/catch stuff is an attempt to deal with that.
try
{
timeStamp = Convert.ToDateTime(lineSplit[0]);
}
catch
{
Console.Write("Error converting {0} to Datetime. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventCode = Convert.ToInt32(lineSplit[1]);
}
catch
{
Console.Write("Error converting {0} to eventCode Interger. Skipping line", lineSplit[0]);
lineError = true;
}
try
{
eventParam = Convert.ToInt32(lineSplit[2]);
}
catch
{
Console.Write("Error converting {0} to eventParam Interger. Skipping line", lineSplit[0]);
lineError = true;
}
//If there were no errors on the line, then put the line into the bulk queue
if (!lineError)
{
try
{
elTable.AddController_Event_LogRow(sigid, timeStamp, eventCode, eventParam);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
//If it gets this far, the file has been opened
fileHasBeenRead = true;
}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("NEXT LINE");
}
}
// Array.Clear(lines, 0, lines.Count());
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("$$$ Entire file has been read $$$");
}
//Do the Math to find out if the error ratio is intolerably high before deleting the file
if (insertErrorCount > 0)
{
errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount));
}
else
{
errorRatio = 0;
}
if (file.Length == 0)
{
fileHasBeenRead = true;
}
endTime = DateTime.Now;
//the Signal class has a static method to insert the tableinto the DB. We are using that.
string connectionString = Properties.Settings.Default.SPM.ToString();
MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.ForceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions, Properties.Settings.Default.DestinationTableName);
elapsedTime = endTime - startTime;
if (Properties.Settings.Default.DeleteFiles)
{
try{
Directory.Delete(dir);
}
catch
{
}
string d = Properties.Settings.Default.PeekDatPath;
foreach (string f in Directory.GetFiles(d, "*.dat"))
{
File.Delete(f);
}
//try
//{
// //if ((errorRatio < Properties.Settings.Default.ErrorRatio) && (fileHasBeenRead))
// //{
// try
// {
// File.Delete(file);
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} Deleted", file);
// }
// }
// catch (SystemException sysex)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file);
// }
// Thread.Sleep(100);
// try
// {
// File.Delete(file);
// }
// catch (SystemException sysex2)
// {
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while Deleting {1}, waiting 100 ms before trying again", sysex2, file);
// }
// }
// }
// try
// {
// // MoeTA.sp_ProgramMessageInsert("Low", "ProcessASC3Logs", DBMessage);
// }
// catch (SqlException ex)
// {
// Console.WriteLine(ex);
// }
// // }
// //else
// //{
// // if (Properties.Settings.Default.WriteToConsole)
// // {
// // Console.WriteLine("Too many insertion errors to delete {0}", file);
// // }
// //}
//}
//catch (Exception ex)
//{
// if (Properties.Settings.Default.WriteToConsole)
// {
// Console.WriteLine("{0} while deleting file {1}", ex, file);
// }
// FilesToDelete.Enqueue(file);
//}
}
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of file Loop%%%");
Thread.Sleep(100);
}
}
);
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("%%%Start of DIRECTORY Loop%%%");
Thread.Sleep(100);
}
//CleanUpFiles(FilesToDelete);
//});
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("###Start of Queue Build Hit###");
}
}
);
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string cwd = appSettings["ASC3LogsPath"];
bool writeToConsole = Convert.ToBoolean(appSettings["WriteToConsole"]);
bool deleteFile = Convert.ToBoolean(appSettings["DeleteFile"]);
bool writeToCsv = Convert.ToBoolean(appSettings["WriteToCsv"]);
foreach (string s in Directory.GetDirectories(cwd))
{
dirList.Add(s);
}
foreach (string dir in dirList)
{
if (writeToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an string
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var files = (Directory.GetFiles(dir, "*.dat"));
foreach (string s in files)
{
try
{
FileInfo f = new FileInfo(s);
if (f.Name.Contains("INT") || f.Name.Contains("_1970_"))
{
try
{
File.Delete(s);
}
catch { }
continue;
}
}
catch { }
try
{
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
MOE.Common.Business.LogDecoder.Asc3Decoder.DecodeAsc3File(s, sigid, mergedEventsTable, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]));
using (MOE.Common.Data.MOE.Controller_Event_LogDataTable eventsTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable())
{
mergedEventsTable.CopyToDataTable(eventsTable, LoadOption.PreserveChanges);
mergedEventsTable.Dispose();
string connectionString = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
string destTable = appSettings["DestinationTableNAme"];
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
writeToConsole, true, 0, deleteFile, Convert.ToDateTime(appSettings["EarliestAcceptableDate"]), 5000, 30);
if (eventsTable.Count > 0)
{
if (MOE.Common.Business.SignalFtp.BulktoDb(eventsTable, Options) && deleteFile)
{
try
{
File.Delete(s);
}
catch
{
}
}
}
if (writeToCsv)
{
FileInfo fileInfo = new FileInfo(s);
StringBuilder sb = new StringBuilder();
IEnumerable <string> columnNames = eventsTable.Columns.Cast <DataColumn>().
Select(column => column.ColumnName);
sb.AppendLine(string.Join(",", columnNames));
foreach (MOE.Common.Data.MOE.Controller_Event_LogRow row in eventsTable.Rows)
{
List <string> fields = new List <string>();
fields.Add(row.SignalID);
fields.Add(row.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff"));
fields.Add(row.EventCode.ToString());
fields.Add(row.EventParam.ToString());
sb.AppendLine(string.Join(",", fields));
}
var csvFilePath = fileInfo.FullName.TrimEnd(new char[] { '.', 'd', 'a', 't' }) + ".csv";
File.WriteAllText(csvFilePath, sb.ToString());
}
}
}
catch { }
}
}
}
//subroutine to write the decoded log to the database.
//this is where most of the work is done.
//The only way we match signalid to the collected logs is by the directory name.
//static void WritetoDB(string dir, string file, MOEDataSetTableAdapters.QueriesTableAdapter MoeTA)
private void SaveEvents()
{
int insertErrorCount = 0;
int insertedLinecount = 0;
double errorRatio = 0;
DateTime startTime = new DateTime();
DateTime endTime = new DateTime();
TimeSpan elapsedTime = new TimeSpan();
string CWD = Properties.Settings.Default.LogPath;
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
ConcurrentQueue <string> FilesToDelete = new ConcurrentQueue <string>();
var lastrecords = new Dictionary <string, DateTime>();
var countrecords = new Dictionary <string, int>();
MOE.Common.Models.Repositories.IControllerEventLogRepository celRepository =
MOE.Common.Models.Repositories.ControllerEventLogRepositoryFactory.Create();
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
var signalID = s.Split(new char[] { '\\' }).Last();
lastrecords.Add(signalID, celRepository.GetMostRecentRecordTimestamp(signalID));
foreach (var file in Directory.GetFiles(s))
{
countrecords.Add(file, ExistingRecords(signalID, file, celRepository));
}
}
var options = new ParallelOptions {
MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads)
};
Parallel.ForEach(dirList.AsEnumerable(), options, dir =>
//foreach (var dir in dirList.AsEnumerable())
{
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var dstOffset = Math.Abs(DateTimeOffset.Now.Offset.Hours);
WriteToConsole("Starting signal " + dirname);
var options1 = new ParallelOptions {
MaxDegreeOfParallelism = Convert.ToInt32(Properties.Settings.Default.MaxThreads)
};
//Parallel.ForEach(Directory.GetFiles(dir, "*.csv").OrderBy(f => f), options1, file =>
foreach (var file in Directory.GetFiles(dir, "*.csv").OrderBy(f => f))
{
if (countrecords[file] >= File.ReadAllLines(file).Length - 1)
{
var delete = Properties.Settings.Default.DeleteFiles;
WriteToConsole(String.Format("Skipping {0} {1}, we already imported this.", (delete ? "and deleting" : ""), file));
if (delete)
{
try
{
File.Delete(file);
}
catch (Exception e)
{
WriteToConsole(String.Format("Unable to delete {0}: {1}", file, e.Message));
}
}
continue;
//return;
}
int skippedrecords = 0;
DataTable elTable = new DataTable();
elTable.Columns.Add("sigid", System.Type.GetType("System.String"));
elTable.Columns.Add("timeStamp", System.Type.GetType("System.DateTime"));
elTable.Columns.Add("eventCode", System.Type.GetType("System.Int32"));
elTable.Columns.Add("eventParam", System.Type.GetType("System.Int32"));
UniqueConstraint custUnique = new UniqueConstraint(new DataColumn[] {
elTable.Columns[0],
elTable.Columns[1],
elTable.Columns[2],
elTable.Columns[3]
});
elTable.Constraints.Add(custUnique);
startTime = DateTime.Now;
//Siemens decoder makes the first line the IP address, so skip it.
foreach (string line in File.ReadAllLines(file).Skip(1))
{
//Every other line is blank. We only care about the lines that have data, and
//every data line has a comma
if (line.Contains(','))
{
//split the line on commas and assign each split to a var
string[] lineSplit = line.Split(new char[] { ',' });
DateTime timeStamp = new DateTime();
int eventCode = 0;
int eventParam = 0;
//it might happen that the character on the line are not quite right.
//the Try/catch stuff is an attempt to deal with that.
try
{
timeStamp = Convert.ToDateTime(lineSplit[0]);
//Siemens decoder is converting to local time from UTC, so convert back to local time
//Not perfect during DST transitions (at 2:00 AM twice per year)
timeStamp = timeStamp + TimeSpan.FromHours(dstOffset);
if (timeStamp < lastrecords[sigid])
{
skippedrecords++;
continue;
}
eventCode = Convert.ToInt32(lineSplit[1]);
eventParam = Convert.ToInt32(lineSplit[2]);
}
catch (Exception ex)
{
WriteToConsole(String.Format("{0} while converting {1} to event. Skipping line", ex, lineSplit[0]));
continue;
}
try
{
elTable.Rows.Add(sigid, timeStamp, eventCode, eventParam);
}
catch (Exception ex)
{
WriteToConsole(String.Format("{0} while adding event to data table", ex.ToString()));
}
}
}
WriteToConsole(String.Format("{0} has been parsed. Skipped {1} old records", file, skippedrecords));
//Do the Math to find out if the error ratio is intolerably high before deleting the file
if (insertErrorCount > 0)
{
errorRatio = Convert.ToDouble(insertErrorCount) / Convert.ToDouble((insertedLinecount + insertErrorCount));
}
else
{
errorRatio = 0;
}
string connectionString = Properties.Settings.Default.SPM.ToString();
MOE.Common.Business.BulkCopyOptions bulkOptions = new MOE.Common.Business.BulkCopyOptions(connectionString, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
endTime = DateTime.Now;
//the Signal class has a static methods to insert the table into the DB. We are using that.
MOE.Common.Business.SignalFtp.BulktoDb(elTable, bulkOptions);
elapsedTime = endTime - startTime;
if (Properties.Settings.Default.DeleteFiles)
{
try
{
File.Delete(file);
WriteToConsole(String.Format("{0} Deleted", file));
}
catch (SystemException sysex)
{
WriteToConsole(String.Format("{0} while Deleting {1}, waiting 100 ms before trying again", sysex, file));
Thread.Sleep(100);
try
{
File.Delete(file);
}
catch (SystemException sysex2)
{
WriteToConsole(String.Format("{0} while Deleting {1}, giving up", sysex2, file));
}
}
catch (Exception ex)
{
WriteToConsole(String.Format("{0} while deleting file {1}", ex, file));
FilesToDelete.Enqueue(file);
}
}
}
//);
}
);
}
static void Main(string[] args)
{
List <string> dirList = new List <string>();
List <string> fileList = new List <string>();
string CWD = Properties.Settings.Default.ASC3LogsPath;
string CSV = Properties.Settings.Default.CSVOutPAth;
//var tableCollection = new BlockingCollection<DataTable>();
//DataTable mergedEventsTable = new DataTable();
ParallelOptions options;
if (!Properties.Settings.Default.forceNonParallel)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = -1
};
}
else
{
if (Properties.Settings.Default.MaxThreads < 2)
{
options = new ParallelOptions {
MaxDegreeOfParallelism = 1
};
}
else
{
options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
}
}
foreach (string s in Directory.GetDirectories(CWD))
{
dirList.Add(s);
}
SimplePartitioner <string> sp = new SimplePartitioner <string>(dirList);
//foreach (string dir in dirList)
ParallelOptions optionsMain = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreadsMain
};
Parallel.ForEach(sp, optionsMain, dir =>
{
var ToDelete = new ConcurrentBag <string>();
if (Properties.Settings.Default.WriteToConsole)
{
Console.WriteLine("-----------------------------Starting Signal " + dir);
}
//get the name of the directory and casting it to an int
//This is the only way the program knows the signal number of the controller.
string[] strsplit = dir.Split(new char[] { '\\' });
string dirname = strsplit.Last();
string sigid = dirname;
var mergedEventsTable = new BlockingCollection <MOE.Common.Data.MOE.Controller_Event_LogRow>();
//SimplePartitioner<string> sp2 = new SimplePartitioner<string>(Directory.GetFiles(dir, "*.dat"));
//Parallel.ForEach(sp2, options, s =>
foreach (var s in Directory.GetFiles(dir, "*.dat"))
{
try
{
MOE.Common.Business.LogDecoder.ASC3Decoder.DecodeASC3File(s, sigid, mergedEventsTable);
ToDelete.Add(s);
}
catch { }
}
//);
MOE.Common.Data.MOE.Controller_Event_LogDataTable elTable = new MOE.Common.Data.MOE.Controller_Event_LogDataTable();
UniqueConstraint custUnique =
new UniqueConstraint(new DataColumn[] { elTable.Columns["SignalID"],
elTable.Columns["Timestamp"],
elTable.Columns["EventCode"],
elTable.Columns["EventParam"] });
elTable.Constraints.Add(custUnique);
//mergedEventsTable.CopyToDataTable(elTable, LoadOption.PreserveChanges);
foreach (var r in mergedEventsTable)
{
try
{
elTable.AddController_Event_LogRow(r);
}
catch { }
}
mergedEventsTable.Dispose();
string connectionString = Properties.Settings.Default.SPMConnectionString;
string destTable = Properties.Settings.Default.DestinationTableNAme;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connectionString, destTable,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFile,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
if (elTable.Count > 0)
{
if (MOE.Common.Business.Signal.BulktoDB(elTable, Options) && Properties.Settings.Default.DeleteFile)
{
DeleteFiles(ToDelete);
}
//string filename = sigid.ToString();
//filename += "_";
//filename += DateTime.Now.Month.ToString();
//filename += "_";
//filename += DateTime.Now.Day.ToString();
//filename += "_";
//filename += DateTime.Now.Year.ToString();
//filename += "_";
//filename += DateTime.Now.Hour.ToString();
//filename += "_";
//filename += DateTime.Now.Minute.ToString();
//filename += "_";
//filename += DateTime.Now.Second.ToString();
//filename += ".csv";
//SaveAsCSV(EventsTable, Path.Combine(CSV, filename));
//if (Properties.Settings.Default.DeleteFile)
//{
// DeleteFiles(ToDelete);
//}
}
else
{
ConcurrentBag <String> td = new ConcurrentBag <String>();
foreach (string s in ToDelete)
{
if (s.Contains("1970_01_01"))
{
td.Add(s);
}
}
if (td.Count > 0)
{
DeleteFiles(td);
}
}
}
);
}
public void UploadControllerData(string IPAddress, string SignalID, string UserName, string Password, string LocalDir, string RemoteDir, bool DeleteFiles, int SNMPRetry, int SNMPTimeout, int SNMPPort, bool ImportAfterFTP, bool ActiveMode, int WaitBetweenRecords, MOE.Common.Business.BulkCopyOptions Options)
{
MOE.Common.Business.Signal.GetCurrentRecords(IPAddress, SignalID, UserName, Password, LocalDir, RemoteDir, DeleteFiles, SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, WaitBetweenRecords, Options);
}
static void Main(string[] args)
{
MOE.Common.Models.Repositories.IApplicationEventRepository ErrorRepository = MOE.Common.Models.Repositories.ApplicationEventRepositoryFactory.Create();
int MaxThreads = Properties.Settings.Default.MaxThreads;
int SNMPTimeout = Properties.Settings.Default.SNMPTimeout;
int SNMPRetry = Properties.Settings.Default.SNMPRetry;
int SNMPPort = Properties.Settings.Default.SNMPPort;
bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP;
bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP;
int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles;
var connection = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connection, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
//MOE.Common.Data.Signals.SignalFTPparamsDataTable SignalsDT = new MOE.Common.Data.Signals.SignalFTPparamsDataTable();
//MOE.Common.Data.SignalsTableAdapters.SignalFTPparamsTableAdapter SignalsTA = new MOE.Common.Data.SignalsTableAdapters.SignalFTPparamsTableAdapter();
//SignalsTA.Fill(SignalsDT);
MOE.Common.Models.SPM db = new MOE.Common.Models.SPM();
var SignalsDT = from r in db.Signals
join f in db.ControllerType on r.ControllerTypeID equals f.ControllerTypeID
where r.ControllerTypeID != 4
select new
{
SignalId = r.SignalID,
PrimaryName = r.PrimaryName,
Secondary_Name = r.SecondaryName,
Region = r.Region,
IP_Address = r.IPAddress,
UserName = f.UserName,
Password = f.Password,
FTPDirectory = f.FTPDirectory,
ActiveFTP = f.ActiveFTP
};
var options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
Parallel.ForEach(SignalsDT.AsEnumerable(), options, row =>
//foreach (var row in SignalsDT)
{
try
{
MOE.Common.Business.Signal signal = new MOE.Common.Business.Signal();
//Initialize the signal, because I didn't make a proper constructor
signal.PrimaryName = row.PrimaryName.ToString();
signal.SecondaryName = row.Secondary_Name.ToString();
signal.Region = row.Region.ToString();
signal.IpAddress = row.IP_Address.ToString();
signal.SignalID = row.SignalId.ToString();
string Username = row.UserName;
string Password = row.Password;
string LocalDir = Properties.Settings.Default.HostDir + signal.SignalID + "\\";
string RemoteDir = row.FTPDirectory;
bool ActiveMode = row.ActiveFTP;
if (!Directory.Exists(LocalDir))
{
Directory.CreateDirectory(LocalDir);
}
//Get the records over FTP
if (CheckIfIPAddressIsValid(signal))
{
try
{
//var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
//var token = tokenSource.Token;
//Task task = Task.Factory.StartNew(() => MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
// SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout, token), token);
//task.Wait();
//if (token.IsCancellationRequested)
// token.ThrowIfCancellationRequested();
MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout);
}
catch (AggregateException ex)
{
Console.WriteLine("Error At Highest Level for signal " + ex.Message);
ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalId);
}
}
}
catch (AggregateException ex)
{
Console.WriteLine("Error At Highest Level for signal " + ex.Message);
ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalId);
}
}
);
//if (Properties.Settings.Default.DealWithMoab)
//{
// try
// {
// DealWithMoab(Options);
// }
// catch
// {
// }
//}
}
static void Main(string[] args)
{
MOE.Common.Models.Repositories.IApplicationEventRepository ErrorRepository = MOE.Common.Models.Repositories.ApplicationEventRepositoryFactory.Create();
int MaxThreads = Properties.Settings.Default.MaxThreads;
int SNMPTimeout = Properties.Settings.Default.SNMPTimeout;
int SNMPRetry = Properties.Settings.Default.SNMPRetry;
int SNMPPort = Properties.Settings.Default.SNMPPort;
bool DeleteAfterFTP = Properties.Settings.Default.DeleteFilesAfterFTP;
bool ImportAfterFTP = Properties.Settings.Default.ImportAfterFTP;
int WaitBetweenFiles = Properties.Settings.Default.WaitBetweenFiles;
var connection = ConfigurationManager.ConnectionStrings["SPM"].ConnectionString;
MOE.Common.Business.BulkCopyOptions Options = new MOE.Common.Business.BulkCopyOptions(connection, Properties.Settings.Default.DestinationTableName,
Properties.Settings.Default.WriteToConsole, Properties.Settings.Default.forceNonParallel, Properties.Settings.Default.MaxThreads, Properties.Settings.Default.DeleteFiles,
Properties.Settings.Default.EarliestAcceptableDate, Properties.Settings.Default.BulkCopyBatchSize, Properties.Settings.Default.BulkCopyTimeOut);
MOE.Common.Models.SPM _db = new MOE.Common.Models.SPM();
MOE.Common.Models.Repositories.ISignalsRepository _sr = SignalsRepositoryFactory.Create(_db);
var _signals = _sr.GetSignalFTPInfoForAllFTPSignals();
var options = new ParallelOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.MaxThreads
};
Parallel.ForEach(_signals.AsEnumerable(), options, row =>
//foreach (var row in SignalsDT)
{
try
{
MOE.Common.Business.Signal signal = new MOE.Common.Business.Signal();
//Initialize the signal, because I didn't make a proper constructor
signal.PrimaryName = row.PrimaryName.ToString();
signal.SecondaryName = row.Secondary_Name.ToString();
signal.IpAddress = row.IP_Address.ToString();
signal.SignalID = row.SignalID.ToString();
string Username = row.User_Name;
string Password = row.Password;
string LocalDir = Properties.Settings.Default.HostDir + signal.SignalID + "\\";
string RemoteDir = row.FTP_Directory;
bool ActiveMode = row.ActiveFTP;
if (!Directory.Exists(LocalDir))
{
Directory.CreateDirectory(LocalDir);
}
//Get the records over FTP
if (CheckIfIPAddressIsValid(signal))
{
try
{
MOE.Common.Business.Signal.GetCurrentRecords(signal.IpAddress, signal.SignalID, Username, Password, LocalDir, RemoteDir, DeleteAfterFTP,
SNMPRetry, SNMPTimeout, SNMPPort, ImportAfterFTP, ActiveMode, 0, Options, Properties.Settings.Default.FTPTimeout);
}
catch (AggregateException ex)
{
Console.WriteLine("Error At Highest Level for signal " + ex.Message);
ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalID);
}
}
}
catch (AggregateException ex)
{
Console.WriteLine("Error At Highest Level for signal " + ex.Message);
ErrorRepository.QuickAdd("FTPFromAllControllers", "Main", "Main Loop", MOE.Common.Models.ApplicationEvent.SeverityLevels.Medium, "Error At Highest Level for signal " + row.SignalID);
}
}
);
}
