private IEnumerable <NewSourceEntry> ParseFile(string file) { if (!File.Exists(file)) { Debug.Write("Can not access {0}", file); yield break; } var lines = File.ReadAllLines(file); var logTokenizer = new WhiteSpaceLogTokenizer(); for (var lineN = 0; lineN < lines.Length; lineN++) { var line = lines[lineN]; foreach (Match match in _scopeRegex.Matches(line)) { foreach (var wordHash in logTokenizer.TokenizeLine(RemoveQuotes(match.Value))) { yield return(new NewSourceEntry { Line = lineN, Position = match.Index, WordHash = wordHash, FileHash = file.GetHashCode() }); } } } }
private void InitBackgroundCorrelation() { if (!_scanTask.IsCompleted || _sourceFileHasher == null) { return; } _scanTask = Task.Run( () => { var tokenizer = new WhiteSpaceLogTokenizer(); for (var i = 0; i < Items.Count; i++) { var logEntry = Items[i]; var tokenizedLine = tokenizer.TokenizeLine(logEntry.Message); var sourceEntry = _sourceFileHasher.SearchNew( tokenizedLine.ToList(), storeDictionary) .FirstOrDefault(); logEntry.SourceEntry = sourceEntry; } Console.WriteLine("all done"); }); }
public void ProcessFile(string filePath) { if (!File.Exists(filePath)) { Debug.Write("Can not access {0}", filePath); } int fileHash = filePath.GetHashCode(); PersistencePart.FilesDictionary.Add(fileHash, filePath); var logTokenizer = new WhiteSpaceLogTokenizer(); var lines = File.ReadAllLines(filePath); //int previousHash = 0; SourceEntry prevSourceEntry = null; for (var lineN = 0; lineN < lines.Length; lineN++) { var line = lines[lineN]; foreach (Match match in _scopeRegex.Matches(line)) { foreach (var token in logTokenizer.TokenizeLine(RemoveQuotes(match.Value))) { var sourceEntry = new SourceEntry( fileHash: fileHash, lineNumber: lineN, lineOffset: match.Index, thisHash: token, previousHash: prevSourceEntry?.ThisHash ?? 0, // o-0~o nextHash: 0); // o-0~o if (prevSourceEntry != null) { prevSourceEntry.NextHash = sourceEntry.ThisHash; // o-0-o } AddSourceEntry(sourceEntry); prevSourceEntry = sourceEntry; // current entry is finished //match. } } } }