public InvertedIndex CreateIndex(InvertedIndex result, params string[] documents) { int corpusCount = result._maxDocumentId; try { NumericComparer ns = new NumericComparer(); Array.Sort(documents, ns); } catch { } for (var i = 0; i < documents.Length; i++) { using (var reader = new StreamReader(documents[i])) { var tokenSource = new TokenSource(reader); while (tokenSource.Next()) { if (_analyzer.Process(tokenSource)) { result.Append( new TermSegment(tokenSource.Buffer, tokenSource.Size), corpusCount + i, tokenSource.Position); } } } } return(result); }
public string AnalyzeOnlyTheFirstToken(string source) { using (var reader = new StringReader(source)) { var tokenSource = new TokenSource(reader); tokenSource.Next(); Process(tokenSource); return(tokenSource.ToString()); } }