public Document AnalyzeFile(FileStats file) { Log.Logger.Debug(string.Format("Analyzing the file: {0}", file.FilePath)); string[] content; if (IOTools.ReadAllLines(file.FilePath, out content)) { // Build composite tree TokenizerWithStats tknz = new TokenizerWithStats(); Document root = new Document { Name = file.FileName }; foreach (string paragraph in content) { Item para = new Paragraph(); root.AddItem(para); foreach (Token token in tknz.Enumerate(paragraph)) { para.AddItem(token); } } return(lexer.AnalyzeText(root)); } else { return(null); } }
public IItem Compose(string[] content) { // Build composite tree TokenizerWithStats tknz = new TokenizerWithStats(); Document root = new Document(); foreach (string paragraph in content) { Item para = new Paragraph(); root.AddItem(para); foreach (Token token in tknz.Enumerate(paragraph)) { para.AddItem(token); } } return(root); }