private WorkerItem ModifiedLogFile()
{
SetStatus("Parser.ModifiedLogFile:enter");
List <IFile <LogFileItem> > currentLogFiles = CurrentLogFiles();
LogFile currentLogFile = CurrentLogFile();
if (currentLogFiles == null) // || currentLogFile == null)
{
SetStatus("Parser.ModifiedLogFile:currentLog empty");
// SyncLogFiles();
return(new WorkerItem());
}
if (_previousLogFiles.Count == currentLogFiles.Count)
{
WorkerItem workerItem = new WorkerItem()
{
FilterFile = (FilterFile)_filterViewModel.CurrentFile(),
LogFile = currentLogFile,
WorkerModification = WorkerItem.Modification.Unknown,
FilterNeed = FilterNeed.Current
};
if (currentLogFile != _logFilePrevious)
{
workerItem.WorkerModification = WorkerItem.Modification.LogIndex;
workerItem.FilterNeed = FilterNeed.Filter;
_logFilePrevious = currentLogFile;
}
SetStatus("Parser.ModifiedLogFile:logfile count same");
return(workerItem);
}
else if (_previousLogFiles.Count < currentLogFiles.Count)
{
SetStatus("Parser.ModifiedLogFiles:logfile file added");
SyncLogFiles();
return(new WorkerItem()
{
LogFile = CurrentLogFile(),
FilterNeed = FilterNeed.Filter,
WorkerModification = WorkerItem.Modification.LogAdded
});
}
else
{
SetStatus("Parser.ModifiedLogFiles:logfile file removed");
SyncLogFiles();
return(new WorkerItem()
{
LogFile = CurrentLogFile(),
FilterNeed = FilterNeed.Filter,
WorkerModification = WorkerItem.Modification.LogRemoved
});
}
//return new WorkerItem()
//{
// FilterNeed = FilterNeed.Unknown,
// WorkerModification = WorkerItem.Modification.Unknown
//};
}
public ObservableCollection <LogFileItem> ApplyFilter(LogTabViewModel logTab, LogFile logFile, List <FilterFileItem> filterFileItems, FilterCommand filterCommand)
{
Mouse.OverrideCursor = Cursors.Wait;
DateTime timer = DateTime.Now;
SetStatus(string.Format("ApplyFilter:start time: {0} log file: {1} ", timer.ToString("hh:mm:ss.fffffff"), logFile.Tag));
List <FilterFileItem> filterItems = VerifyFilterPatterns(filterFileItems, logTab);
Debug.Print(string.Format("ApplyFilter: filterItems.Count={0}:{1}", Thread.CurrentThread.ManagedThreadId, filterItems.Count));
// set regex cache size to number of filter items for better performance
// https: //msdn.microsoft.com/en-us/library/gg578045(v=vs.110).aspx
Regex.CacheSize = filterItems.Count;
int inclusionFilterCount = filterItems.Count(x => x.Include == true);
ParallelOptions po = new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount // / 2
};
try
{
Parallel.ForEach(logFile.ContentItems, po, logItem =>
{
if (string.IsNullOrEmpty(logItem.Content))
{
Debug.Print(string.Format("ApplyFilter: logItem.Content empty={0}:{1}", Thread.CurrentThread.ManagedThreadId, logItem.Content));
// used for goto line as it needs all line items
logItem.FilterIndex = int.MinValue;
return;
}
int filterIndex = int.MaxValue; // int.MinValue;
int includeFilters = inclusionFilterCount;
if (Settings.CountMaskedMatches)
{
logItem.Masked = new int[filterItems.Count, 1];
}
// clear out groups
logItem.Group1 = string.Empty;
logItem.Group2 = string.Empty;
logItem.Group3 = string.Empty;
logItem.Group4 = string.Empty;
bool matchSet = false;
for (int fItem = 0; fItem < filterItems.Count; fItem++)
{
int filterItemIndex = filterFileItems[fItem].Index;
bool match = false;
FilterFileItem filterItem = filterItems[fItem];
Debug.Print(string.Format("ApplyFilter: loop:{0} filterItem.Pattern={1}:{2} logItem.Content:{3}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterItem.Filterpattern, logItem.Content));
// unnamed and named groups
//if (logTab.GroupCount > 0 && filterItem.Regex)
if (filterItem.GroupCount > 0 && filterItem.Regex)
{
MatchCollection mc = Regex.Matches(logItem.Content, filterItem.Filterpattern, RegexOptions.Singleline | (filterItem.CaseSensitive ? RegexOptions.None : RegexOptions.IgnoreCase));
if (mc.Count > 0)
{
match = true;
foreach (Match m in mc)
{
if (!string.IsNullOrEmpty(m.Groups[1].Value.ToString()))
{
logItem.Group1 += (string.IsNullOrEmpty(logItem.Group1) ? "" : ";\n") + m.Groups[1].Value.ToString();
}
if (!string.IsNullOrEmpty(m.Groups[2].Value.ToString()))
{
logItem.Group2 += (string.IsNullOrEmpty(logItem.Group2) ? "" : ";\n") + m.Groups[2].Value.ToString();
}
if (!string.IsNullOrEmpty(m.Groups[3].Value.ToString()))
{
logItem.Group3 += (string.IsNullOrEmpty(logItem.Group3) ? "" : ";\n") + m.Groups[3].Value.ToString();
}
if (!string.IsNullOrEmpty(m.Groups[4].Value.ToString()))
{
logItem.Group4 += (string.IsNullOrEmpty(logItem.Group4) ? "" : ";\n") + m.Groups[4].Value.ToString();
}
}
}
}
else if (filterItem.Regex && Regex.IsMatch(logItem.Content, filterItem.Filterpattern, RegexOptions.Singleline | (filterItem.CaseSensitive ? RegexOptions.None : RegexOptions.IgnoreCase)))
{
match = true;
}
else if (!filterItem.Regex)
{
bool andMatch = true;
if (filterItem.StringOperators)
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string with operators "
+ "thread id:{1} filter index:{2} filter string: {3}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
filterItem.Filterpattern));
// check for ' AND ' and ' OR ' operators
foreach (string andPattern in Regex.Split(filterItem.Filterpattern, " AND "))
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern "
+ "thread id:{1} filter index:{2} filter string: {3}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
andPattern));
match = false;
string[] ors = Regex.Split(andPattern, " OR ");
if (ors.Length > 1)
{
foreach (string orPattern in ors)
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string orPattern "
+ "thread id:{1} filter index:{2} filter string: {3}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
orPattern));
// only match one
if (filterItem.CaseSensitive && logItem.Content.Contains(orPattern))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string orPattern match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
orPattern,
logItem.Content));
break;
}
else if (logItem.Content.ToLower().Contains(orPattern.ToLower()))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string orPattern match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
orPattern,
logItem.Content));
break;
}
else
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string orPattern NO match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
orPattern,
logItem.Content));
}
}
}
else
{
// match all
if (filterItem.CaseSensitive && logItem.Content.Contains(andPattern))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern all match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
andPattern,
logItem.Content));
}
else if (logItem.Content.ToLower().Contains(andPattern.ToLower()))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern all match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
andPattern,
logItem.Content));
}
else
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern all NO match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
andPattern,
logItem.Content));
}
}
andMatch &= match;
}
match = andMatch;
}
else
{
// normal string match
if (filterItem.CaseSensitive && logItem.Content.Contains(filterItem.Filterpattern))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern one match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
filterItem.Filterpattern,
logItem.Content));
}
else if (logItem.Content.ToLower().Contains(filterItem.Filterpattern.ToLower()))
{
match = true;
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern one match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
filterItem.Filterpattern,
logItem.Content));
}
else
{
Debug.Print(string.Format("ApplyFilter: loop:{0} string andPattern one NO match "
+ "thread id:{1} filter index:{2} filter string: {3} logItem content: {4}",
filterItemIndex,
Thread.CurrentThread.ManagedThreadId,
filterIndex,
filterItem.Filterpattern,
logItem.Content));
}
}
}
Debug.Print(string.Format("ApplyFilter:** loop:{0} filterItem Match={1}:{2} **", filterItemIndex, Thread.CurrentThread.ManagedThreadId, match));
if (!matchSet)
{
if (match && filterItem.Include && !filterItem.Exclude)
{
filterIndex = filterItemIndex;
Debug.Print(string.Format("ApplyFilter: loop:{0} filterItem.Include not exclude setting filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterIndex));
matchSet = true;
includeFilters--;
// break;
}
else if (!match && filterItem.Include && filterItem.Exclude)
{
// dynamic filter with and quickfindand but no match so exit
filterIndex = int.MinValue;
Debug.Print(string.Format("ApplyFilter: loop:{0} no match filterItem.Include and exclude setting filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterIndex));
matchSet = true;
includeFilters = 0;
// break;
}
else if (match && filterItem.Include && filterItem.Exclude)
{
// dynamic filter with and quickfindand but with match
filterIndex = int.MinValue;
Debug.Print(string.Format("ApplyFilter: loop:{0} match filterItem.Include and exlude setting filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterIndex));
matchSet = false;
includeFilters = 0;
// break;
}
else if (match && filterItem.Exclude)
{
filterIndex = (fItem * -1) - 2;
Debug.Print(string.Format("ApplyFilter: loop:{0} filterItem.Exclude and match filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterIndex));
matchSet = true;
// break;
}
else if (!match && !filterItem.Exclude)
{
filterIndex = int.MinValue;
Debug.Print(string.Format("ApplyFilter: loop:{0} not filterItem.Exclude and not match filterIndex={1}:{2}",
filterItemIndex, Thread.CurrentThread.ManagedThreadId, filterIndex));
}
else if (match)
{
filterIndex = filterItemIndex;
Debug.Print(string.Format("ApplyFilter: loop:{0} setting filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterIndex));
matchSet = true;
// break;
}
else if (filterItem.Include)
{
includeFilters--;
}
}
else if (matchSet && match && Settings.CountMaskedMatches)
{
logItem.Masked[fItem, 0] = 1;
Debug.Print(string.Format("ApplyFilter: loop:{0} masked match filterIndex={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, filterItemIndex));
}
if (matchSet && !Settings.CountMaskedMatches)
{
Debug.Print(string.Format("ApplyFilter: loop:{0} not filterItem.Exclude CountMaskedMatches={1}:{2}", filterItemIndex,
Thread.CurrentThread.ManagedThreadId, Settings.CountMaskedMatches));
if (includeFilters == 0)
{
break;
}
}
}
Debug.Print(string.Format("ApplyFilter: loop finished set filterIndex={0}:{1}", Thread.CurrentThread.ManagedThreadId, filterIndex));
logItem.FilterIndex = filterIndex;
});
// write totals negative indexes arent displayed and are only used for counting
int filterCount = 0;
for (int i = 0; i < filterFileItems.Count; i++)
{
int filterItemIndex = filterFileItems[i].Index;
filterFileItems[i].Count = logFile.ContentItems.Count(x => x.FilterIndex == filterItemIndex | x.FilterIndex == (i * -1) - 2);
if (Settings.CountMaskedMatches)
{
filterFileItems[i].MaskedCount = logFile.ContentItems.Count(x => (x.FilterIndex != int.MaxValue) & (x.FilterIndex != int.MinValue) && x.Masked[i, 0] == 1);
SetStatus(string.Format("ApplyFilter:filterItem masked counttotal: {0}", filterFileItems[i].MaskedCount));
}
SetStatus(string.Format("ApplyFilter:filterItem counttotal: {0}", filterFileItems[i].Count));
filterCount += filterFileItems[i].Count;
}
double totalSeconds = DateTime.Now.Subtract(timer).TotalSeconds;
SetStatus(string.Format("ApplyFilter:total time in seconds: {0}\n\tlines per second: {1} "
+ "\n\tlogfile total lines count: {2}\n\tlogfile filter lines count: {3}\n\tlog file: {4}"
+ "\n\tnumber of filters: {5}\n\tcalculated lines per second for single query: {6}",
totalSeconds,
logFile.ContentItems.Count / totalSeconds,
logFile.ContentItems.Count,
filterCount,
logFile.Tag,
filterItems.Count,
logFile.ContentItems.Count / totalSeconds * filterItems.Count));
Mouse.OverrideCursor = null;
return(new ObservableCollection <LogFileItem>(logFile.ContentItems.Where(x => x.FilterIndex > -2)));
}
catch (Exception e)
{
SetStatus("ApplyFilter:exception" + e.ToString());
Mouse.OverrideCursor = null;
return(new ObservableCollection <LogFileItem>());
}
}
public LogFile MMFConcurrentRead(LogFile logFile, BackgroundWorker backgroundWorker)
{
Debug.Print("MMFConcurrentRead: enter");
GetEncoding(logFile);
// not sure why this here. messing up temp file
//logFile.IsNew = false;
//logFile.Modified = false;
if (!File.Exists(logFile.Tag))
{
Debug.Print("MMFConcurrentRead:error, file does not exist: " + logFile.Tag);
return(logFile);
}
byte[] bytes = new byte[new FileInfo(logFile.Tag).Length];
// 4,8,12
int threadCount = 1;
if (bytes.Length > 1000000)
{
threadCount = 8;
}
int bposition = logFile.HasBom ? logFile.Encoding.GetPreamble().Length : 0;
int blen = bytes.Length / threadCount;
MemoryMappedFile memoryMappedFile = MemoryMappedFile.CreateFromFile(logFile.Tag, FileMode.Open, "mmf", bytes.Length, MemoryMappedFileAccess.Read);
ManualResetEvent[] completedEvents = new ManualResetEvent[threadCount];
List <List <LogFileItem> > list = new List <List <LogFileItem> >();
for (int i = 0; i < threadCount; i++)
{
list.Add(new List <LogFileItem>());
completedEvents[i] = new ManualResetEvent(false);
}
for (int mmfCount = 0; mmfCount < threadCount; mmfCount++)
{
if (mmfCount != 0)
{
bposition += blen;
}
if (mmfCount == threadCount - 1)
{
blen = bytes.Length - bposition;
}
TaskMMFInfo taskInfo = new TaskMMFInfo()
{
mmf = memoryMappedFile,
stringList = list[mmfCount],
logFile = logFile,
position = bposition,
length = blen,
completedEvent = completedEvents[mmfCount],
bgWorker = backgroundWorker
};
ThreadPool.QueueUserWorkItem(new WaitCallback(ParallelMMFRead), taskInfo);
}
Debug.Print(string.Format("mmf thread length: {0}", blen));
WaitHandle.WaitAll(completedEvents);
if (memoryMappedFile != null)
{
memoryMappedFile.Dispose();
}
if (backgroundWorker.CancellationPending)
{
Debug.Print("MMFConcurrentRead:cancelled");
return(logFile);
}
bool patch = false;
List <LogFileItem> finalList = new List <LogFileItem>();
for (int listCount = 0; listCount < threadCount; listCount++)
{
if (patch)
{
// merge last line with first line in current list
finalList[finalList.Count - 1].Content = finalList[finalList.Count - 1].Content + list[listCount][0].Content;
list[listCount].RemoveAt(0);
patch = false;
}
finalList.AddRange(list[listCount]);
list[listCount].Clear();
// need to check for partial lines look in last index for needsPatch string
if (finalList.Count > 0)
{
if (finalList[finalList.Count - 1].Content == _needsPatch)
{
finalList.RemoveAt(finalList.Count - 1);
patch = true;
}
else
{
patch = false;
}
}
}
// set index
int counter = 1;
foreach (LogFileItem item in finalList)
{
item.Index = counter++;
}
Debug.Print("MMFConcurrentRead:exit");
logFile.ContentItems = new ObservableCollection <LogFileItem>(finalList);
return(logFile);
}
public override IFile <LogFileItem> ReadFile(string fileName)
{
// BOM UTF - 8 0xEF,0xBB,0xBF BOM UTF - 16 FE FF NO BOM assume ansi but utf-8 doesnt have
// to have one either
LogFile logFile = new LogFile();
try
{
SetStatus("ReadFile:enter: " + fileName);
Encoding encoding = Encoding.Default;
logFile.FileName = Path.GetFileName(fileName);
// find bom
using (System.IO.StreamReader sr = new System.IO.StreamReader(fileName, true))
{
encoding = sr.CurrentEncoding;
SetStatus("current encoding:" + encoding.EncodingName);
while (!sr.EndOfStream)
{
// if bom not supplied, try to determine utf-16 (unicode)
string line = sr.ReadLine();
byte[] bytes = Encoding.UTF8.GetBytes(line);
string newLine = Encoding.UTF8.GetString(bytes).Replace("\0", "");
SetStatus(string.Format("check encoding: bytes:{0} string: {1}", bytes.Length, newLine.Length));
if (bytes.Length > 0 && newLine.Length > 0 &&
((bytes.Length - newLine.Length) * 2 - 1 == bytes.Length
| (bytes.Length - newLine.Length) * 2 == bytes.Length))
{
SetStatus(string.Format("new encoding:Unicode bytes:{0} string: {1}", bytes.Length, newLine.Length));
encoding = Encoding.Unicode;
break;
}
else if (bytes.Length > 0 && newLine.Length > 0)
{
break;
}
}
}
// todo: use mapped file only for large files?
string[] lines = File.ReadAllLines(fileName, encoding);
ConcurrentBag <LogFileItem> cLogFileItems = new ConcurrentBag <LogFileItem>();
Parallel.For(0, lines.Length, x =>
{
LogFileItem logFileItem = new LogFileItem()
{
Content = lines[x],
Background = Settings.BackgroundColor,
Foreground = Settings.ForegroundColor,
Index = x + 1
};
cLogFileItems.Add(logFileItem);
});
logFile.ContentItems = new ObservableCollection <LogFileItem>(cLogFileItems.OrderBy(x => x.Index));
SetStatus("ReadFile:exit: " + fileName);
logFile.IsNew = false;
return(logFile);
}
catch (Exception e)
{
SetStatus("Fatal:ReadFile:exception: " + e.ToString());
return(logFile);
}
}
private bool GetEncoding(LogFile logFile)
{
logFile.HasBom = false;
logFile.Encoding = Encoding.GetEncoding("ISO-8859-1"); // extended ascii
try
{
// ascii will be default as it never has bom if all fail utf-7 rarely used and does
// not have preamble. wiki says 2b,2f,76,variable
SetStatus("GetEncoding:enter: " + logFile.Tag);
Encoding[] encodings = new Encoding[] { Encoding.UTF32, Encoding.UTF8, Encoding.BigEndianUnicode, Encoding.Unicode };
//logFile.FileName = Path.GetFileName(fileName);
// find bom
bool foundEncoding = false;
using (System.IO.StreamReader sr = new System.IO.StreamReader(logFile.Tag, true))
{
// encoding = sr.CurrentEncoding;
SetStatus("current encoding:" + logFile.Encoding.EncodingName);
// biggest preamble is 4 bytes
if (sr.BaseStream.Length < 4)
{
return(true);
}
int[] srBytes = new int[4];
for (int c = 0; c < srBytes.Length; c++)
{
srBytes[c] = sr.BaseStream.ReadByte();
}
sr.DiscardBufferedData();
foreach (Encoding enc in encodings)
{
byte[] bytes = enc.GetPreamble();
for (int i = 0; i < bytes.Length; i++)
{
if (bytes[i] != srBytes[i])
{
foundEncoding = false;
break;
}
else
{
foundEncoding = true;
}
}
if (foundEncoding)
{
logFile.HasBom = true;
logFile.Encoding = enc;
return(true);
}
}
// if bom not supplied, try to determine utf-16 (unicode)
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
byte[] bytes = Encoding.UTF8.GetBytes(line);
string newLine = Encoding.UTF8.GetString(bytes).Replace("\0", "");
SetStatus(string.Format("check encoding: bytes:{0} string: {1}", bytes.Length, newLine.Length));
if (bytes.Length > 0 && newLine.Length > 0 &&
((bytes.Length - newLine.Length) * 2 - 1 == bytes.Length
| (bytes.Length - newLine.Length) * 2 == bytes.Length))
{
SetStatus(string.Format("new encoding:Unicode bytes:{0} string: {1}", bytes.Length, newLine.Length));
logFile.Encoding = Encoding.Unicode;
SetStatus("new encoding:" + logFile.Encoding.EncodingName);
break;
}
else if (bytes.Length > 0 && newLine.Length > 0)
{
break;
}
}
}
return(true);
}
catch (Exception e)
{
SetStatus("Exception:GetEncoding:" + e.ToString());
return(false);
}
}
