public Object DiffMatchPatch(string firstString, string secondString, float timeOut = 0)
{
//Garbage Collector Clearification
GC.Collect();
GC.WaitForPendingFinalizers();
DateTime timeStart = DateTime.Now;
DiffMatchPatch dmp = new DiffMatchPatch();
dmp.DiffTimeout = timeOut;
// Execute one reverse diff as a warmup.
List <Diff> diff = dmp.DiffMain(firstString, secondString);
return(new {
CalculationTime = (DateTime.Now - timeStart),
AllDifferences = diff,
PrettyHtml = dmp.DiffPrettyHtml(diff)
});
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.EmaxEventHis)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
List <EMAXEventHisRecord> records = new List <EMAXEventHisRecord>();
// retrieve last change for this file
EmaxDiagnosticFileChanges lastChanges = new TableOperations <EmaxDiagnosticFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new EmaxDiagnosticFileChanges();
}
// parse each line
foreach (string line in data)
{
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { ". " }, StringSplitOptions.RemoveEmptyEntries);
EMAXEventHisRecord newRecord = new EMAXEventHisRecord();
// date has 2 spaces if date is a single digit to keep specific column width
string format = "ddd MMM d HH:mm:ss yyyy";
if (section[0].Contains(" "))
{
format = "ddd MMM d HH:mm:ss yyyy";
}
newRecord.Line = line;
newRecord.Time = DateTime.ParseExact(section[0], format, CultureInfo.InvariantCulture);
if (newRecord.Time > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.Time = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Line += ". MiMD Parsing Alarm: DFR time set in the future.\n";
}
newRecord.Version = section[1];
records.Add(newRecord);
}
// if no records, or if the last record is same or before record in database, stop. There was probably an error.
if (!records.Any() || records.Last().Time <= lastChanges.LastWriteTime)
{
return(false);
}
List <EMAXEventHisRecord> newRecords = records.Where(x => x.Time > lastChanges.LastWriteTime).ToList();
// instantiate new changes object
EmaxDiagnosticFileChanges fileChanges = new EmaxDiagnosticFileChanges();
// create new record
fileChanges.MeterID = meterDataSet.Meter.ID;
fileChanges.FileName = fi.Name;
fileChanges.FileSize = (int)(fi.Length / 1000);
fileChanges.LastWriteTime = newRecords.Last().Time;
fileChanges.Span = (newRecords.Last().Time - newRecords.First().Time).Days;
fileChanges.NewRecords = newRecords.Count();
fileChanges.Alarms = 0;
for (int i = 0; i < newRecords.Count; ++i)
{
if (newRecords[i].Line.ToLower().Contains("operation alarm"))
{
}
else if (newRecords[i].Line.ToLower().Contains("offline. system offline"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("buffer full"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("error"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("alarm"))
{
if (newRecords[i].Line.ToLower().Contains("time sync failed"))
{
if (newRecords.Where(x => x.Line.ToLower().Contains("system started") && newRecords[i].Time.Subtract(x.Time).TotalSeconds <= 60).Any())
{
}
else
{
fileChanges.Alarms++;
}
}
else
{
fileChanges.Alarms++;
}
}
}
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain("", string.Join("\n", records.Where(x => x.Time > lastChanges.LastWriteTime).Select(x => x.Line)));
dmp.DiffCleanupSemantic(diff);
fileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = fileChanges.Alarms;
new TableOperations <EmaxDiagnosticFileChanges>(connection).AddNewRecord(fileChanges);
return(true);
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.AppStatus)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
DateTime lastWriteTime = TimeZoneInfo.ConvertTimeFromUtc(fi.LastWriteTime.ToUniversalTime(), TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
AppStatusFileChanges newRecord = new AppStatusFileChanges();
// retrieve last change for this file
AppStatusFileChanges lastChanges = new TableOperations <AppStatusFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new AppStatusFileChanges();
}
// if lastChanges LastWriteTime equals new LastWriteTime return
if (lastChanges.LastWriteTime.ToString("MM/dd/yyyy HH:mm:ss") == lastWriteTime.ToString("MM/dd/yyyy HH:mm:ss"))
{
return(false);
}
newRecord.MeterID = meterDataSet.Meter.ID;
newRecord.FileName = fi.Name;
newRecord.LastWriteTime = lastWriteTime;
newRecord.FileSize = (int)(fi.Length / 1000);
newRecord.Text = meterDataSet.Text;
newRecord.Alarms = 0;
if (lastChanges.LastWriteTime > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.LastWriteTime = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: DFR time set in the future.\n";
}
// parse each line
foreach (string line in data)
{
// if line is empty go to the next line
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { "=" }, StringSplitOptions.RemoveEmptyEntries);
if (section[0].ToLower() == "recorder")
{
newRecord.Version = section[1];
}
else if (section[0].ToLower() == "dfr")
{
newRecord.DFR = section[1];
if (newRecord.DFR.ToLower() != "online")
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: DFR not set to ONLINE.\n";
}
}
else if (section[0].ToLower() == "pc_time")
{
try
{
newRecord.PCTime = DateTime.ParseExact(section[1], "MM/dd/yyyy-HH:mm:ss", CultureInfo.InvariantCulture);
}
catch (Exception ex)
{
newRecord.PCTime = DateTime.MinValue;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect date format for PC_Time.\n";
}
}
else if (section[0].ToLower() == "time_mark_source")
{
newRecord.TimeMarkSource = section[1];
if (newRecord.TimeMarkSource.ToLower() == "irig-b")
{
}
else if (newRecord.TimeMarkSource.ToLower() == "pc")
{
}
else
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: TIME_MARK_SOURCE not set to IRIG-B.\n";
}
}
else if (section[0].ToLower() == "time_mark_time")
{
try
{
newRecord.TimeMarkTime = DateTime.ParseExact(section[1], "MM/dd/yyyy-HH:mm:ss.ffffff", CultureInfo.InvariantCulture);
if (newRecord.TimeMarkTime.Subtract(newRecord.PCTime).TotalSeconds > 2)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Time_Mark_Time and PC_Time difference greater than 2 seconds.";
}
}
catch (Exception ex)
{
newRecord.TimeMarkTime = DateTime.MinValue;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect date format for Time_Mark_Time.";
}
}
else if (section[0].ToLower() == "clock")
{
if (section[1].ToLower() == "sync(lock)")
{
}
else if (section[1].ToLower() == "unsync(unknown)" && newRecord.TimeMarkSource.ToLower() == "pc")
{
int count = connection.ExecuteScalar <int>(@"
with cte as
(SELECT TOP 1 * FROM AppStatusFileChanges
where meterid = {0}
order by LastWriteTime desc)
SELECT COUNT(*) FROM cte WHERE Text LIKE '%TIME_MARK_SOURCE=PC%' AND Text LIKE '%Clock=UNSYNC(unknown)%'
", meterDataSet.Meter.ID);
if (count > 0)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Time_Mark_Source Set to PC and Clock set to UNSYNC(unknown) in consecutive files.\n";
}
}
else
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Clock not set to SYNC(lock).\n";
}
}
else if (section[0].ToLower() == "data_drive")
{
try
{
string[] values = section[1].Replace("GB", "").Split('/');
newRecord.DataDriveUsage = double.Parse(values[0]) / double.Parse(values[1]);
}
catch (Exception ex)
{
newRecord.DataDriveUsage = 0;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect format for Data_Drive.\n";
}
}
else if (section[0].ToLower() == "chassis_not_comm")
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: CHASSIS_NOT_COMM field present.\n";
}
else if (section[0].ToLower() == "timemark")
{
// sometimes this lines ends in a comma, remove it
string timeString = section[1];
IEnumerable <string> times = timeString.Split(',').Where(x => !Regex.IsMatch(x, "\\s*") && x != string.Empty);
bool flag = times.Distinct().Count() > 1;
if (flag)
{
int count = connection.RetrieveData(@"
with cte as
(SELECT TOP 14 * FROM AppStatusFileChanges
where meterid = {0}
order by LastWriteTime desc)
SELECT * FROM cte WHERE Text LIKE '%TimeMark values not equal%'
", meterDataSet.Meter.ID).Rows.Count;
newRecord.Text += "\nMiMD Parsing Warning: TimeMark values not equal.\n";
if (count == 14)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: TimeMark values not equal in 15 consecutive files.\n";
}
}
}
else if (section[0].ToLower() == "dsp_board")
{
newRecord.DSPBoard = section[1];
}
else if (section[0].ToLower() == "dsp_revision")
{
newRecord.DSPRevision = section[1];
}
else if (section[0].ToLower().Contains("packet"))
{
newRecord.Packet = section[1];
}
else if (section[0].ToLower().Contains("recovery"))
{
newRecord.Recovery = section[1];
}
else if (section[0].ToLower() == "(>65c,c)")
{
newRecord.BoardTemp = section[1];
}
else if (section[0].ToLower() == "speedfan")
{
newRecord.SpeedFan = section[1].Replace("\"", "").Replace(" ", "");
}
else if (section[0].ToLower() == "alarmon" || section[0].ToLower() == "anfail")
{
newRecord.Alarms += 1;
}
}
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(lastChanges.Text ?? "", newRecord.Text);
dmp.DiffCleanupSemantic(diff);
newRecord.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = newRecord.Alarms;
new TableOperations <AppStatusFileChanges>(connection).AddNewRecord(newRecord);
return(true);
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.AppTrace)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
List <AppTraceRecord> records = new List <AppTraceRecord>();
// retrieve last change for this file
AppTraceFileChanges lastChanges = new TableOperations <AppTraceFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new AppTraceFileChanges();
}
// parse each line
foreach (string line in data)
{
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { ". " }, StringSplitOptions.RemoveEmptyEntries);
AppTraceRecord newRecord = new AppTraceRecord();
// date has 2 spaces if date is a single digit to keep specific column width
string regex = @"(\[\d+\/\d+\/\d+\s\d+:\d+:\d+\s[AP]M\])";
string[] results = Regex.Split(line, regex);
string format = "[M/d/yyyy h:mm:ss tt]";
if (results.Length <= 1)
{
regex = @"(\[\d+\/\d+\/\d+\])";
results = Regex.Split(line, regex);
format = "[M/d/yyyy]";
}
newRecord.Line = line;
newRecord.Time = DateTime.ParseExact(results[1], format, CultureInfo.InvariantCulture);
if (newRecord.Time > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.Time = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Line += ". MiMD Parsing Alarm: DFR time set in the future.\n";
}
newRecord.Description = results[2];
records.Add(newRecord);
}
// if no records, or if the last record is same or before record in database, stop. There was probably an error.
if (!records.Any() || records.Last().Time <= lastChanges.LastWriteTime)
{
return(false);
}
// instantiate new changes object
AppTraceFileChanges fileChanges = new AppTraceFileChanges();
List <string> alarmKeyWords = new List <string> {
"unsync<invalid(no signal)>", "[alarmon]", "sync loss", "chassis comm. error", "disk full", "master comm. error", "dsp board temperature", "analog fail", "pc health", "offline", "time in future"
};
IEnumerable <AppTraceRecord> newRecords = records.Where(x => x.Time > lastChanges.LastWriteTime);
// create new record
fileChanges.MeterID = meterDataSet.Meter.ID;
fileChanges.FileName = fi.Name;
fileChanges.FileSize = (int)(fi.Length / 1000);
fileChanges.LastWriteTime = records.Last().Time;
fileChanges.Span = (records.Last().Time - records.First().Time).Days;
fileChanges.NewRecords = newRecords.Count();
fileChanges.Alarms = newRecords.Where(x => alarmKeyWords.Contains(x.Line.ToLower())).Count();
if (records.Where(x => x.Line.ToLower().Contains("fault f")).Any())
{
fileChanges.LastFaultTime = records.Where(x => x.Line.ToLower().Contains("fault f")).Last().Time;
}
else
{
fileChanges.LastFaultTime = lastChanges.LastFaultTime;
}
fileChanges.FaultCount48hr = records.Where(x => x.Line.ToLower().Contains("fault f") && x.Time >= fileChanges.LastWriteTime.AddHours(-48)).Count();
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain("", string.Join("\n", records.Where(x => x.Time > lastChanges.LastWriteTime).Select(x => x.Line)));
dmp.DiffCleanupSemantic(diff);
fileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = fileChanges.Alarms;
new TableOperations <AppTraceFileChanges>(connection).AddNewRecord(fileChanges);
return(true);
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.BENConfig)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
FileInfo fi = new FileInfo(meterDataSet.FilePath);
DateTime lastWriteTime = DateTime.ParseExact(string.Join(",", fi.Name.Split(',').Take(2)), "yyMMdd,HHmmssfff", CultureInfo.InvariantCulture);
// see if there is already a record that matches this file
ConfigFileChanges configFileChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecordWhere("MeterID = {0} AND FileName = {1} AND LastWriteTime = {2}", meterDataSet.Meter.ID, $"{meterDataSet.Meter.AssetKey}.cfg", lastWriteTime);
// if a record already exists for this file skip it. There was probably an error.
if (configFileChanges != null)
{
return(false);
}
configFileChanges = new ConfigFileChanges();
// create new record
configFileChanges.MeterID = meterDataSet.Meter.ID;
configFileChanges.FileName = $"{meterDataSet.Meter.AssetKey}.cfg";
configFileChanges.LastWriteTime = lastWriteTime;
configFileChanges.Text = meterDataSet.Text.Replace("\r", "");
string[] data = File.ReadAllLines(meterDataSet.FilePath);
int[] channelCounts = data[1].Split(',').Select(x => int.Parse(x.Replace("A", "").Replace("D", ""))).ToArray();
int totalChannels = channelCounts[0];
// get portion of cfg file that contains channel mappings
string relevantPortion = string.Join("\n", data.Take(2 + totalChannels));
// get the previous record for this file
ConfigFileChanges lastChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1} AND LastWriteTime < {2}", meterDataSet.Meter.ID, $"{meterDataSet.Meter.AssetKey}.cfg", lastWriteTime));
// if there were no previous records for this file
if (lastChanges == null)
{
lastChanges = new ConfigFileChanges();
lastChanges.Text = configFileChanges.Text.Replace("\r", "");
// make diffs
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(relevantPortion, relevantPortion);
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = 0;
}
else
{
string[] data2 = lastChanges.Text.Split('\n');
int[] channelCounts2 = data2[1].Split(',').Select(x => int.Parse(x.Replace("A", "").Replace("D", ""))).ToArray();
int totalChannels2 = channelCounts2[0];
// get portion of cfg file that contains channel mappings
string relevantPortion2 = string.Join("\n", data2.Take(2 + totalChannels2));
// make diffs
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(relevantPortion2, relevantPortion);
List <Patch> patch = dmp.PatchMake(relevantPortion2, relevantPortion);
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = patch.Count;
if (patch.Count == 0)
{
return(false);
}
}
// write new record to db
meterDataSet.ConfigChanges = configFileChanges.Changes;
new TableOperations <ConfigFileChanges>(connection).AddNewRecord(configFileChanges);
return(true);
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.Config)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
FileInfo fi = new FileInfo(meterDataSet.FilePath);
// see if there is already a record that matches this file
ConfigFileChanges configFileChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecordWhere("MeterID = {0} AND FileName = {1} AND LastWriteTime = {2}", meterDataSet.Meter.ID, fi.Name, fi.LastWriteTime);
// if a record already exists for this file skip it. There was probably an error.
if (configFileChanges != null)
{
return(false);
}
configFileChanges = new ConfigFileChanges();
// create new record
configFileChanges.MeterID = meterDataSet.Meter.ID;
configFileChanges.FileName = fi.Name;
configFileChanges.LastWriteTime = fi.LastWriteTime;
configFileChanges.Text = meterDataSet.Text;
// get the previous record for this file
ConfigFileChanges lastChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1} AND LastWriteTime < {2}", meterDataSet.Meter.ID, fi.Name, fi.LastWriteTime));
// if there were no previous records for this file, just diff it against itself because we need an intial record.
if (lastChanges == null)
{
lastChanges = new ConfigFileChanges();
lastChanges.Text = meterDataSet.Text;
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(lastChanges.Text, configFileChanges.Text);
List <Patch> patch = dmp.PatchMake(lastChanges.Text, configFileChanges.Text);
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = patch.Count;
// write new record to db
meterDataSet.ConfigChanges = configFileChanges.Changes;
}
else
{
// make diffs
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(lastChanges.Text, configFileChanges.Text);
List <Patch> patch = dmp.PatchMake(lastChanges.Text, configFileChanges.Text);
if (patch.Count == 0)
{
return(false);
}
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = patch.Count;
// write new record to db
meterDataSet.ConfigChanges = configFileChanges.Changes;
}
new TableOperations <ConfigFileChanges>(connection).AddNewRecord(configFileChanges);
return(true);
}
}
