public InvertedIndex CreateIndex( params string[] documents ) { var result = new InvertedIndex(documents.Length); for (var i = 0; i < documents.Length; i++) { using (var reader = new StringReader(documents[i])) { var tokenSource = new TokenSource(reader); while (tokenSource.Next()) { if (_analyzer.Process(tokenSource)) { result.Append( new CharArraySegmentKey(tokenSource.Buffer, tokenSource.Size), i, tokenSource.Position); } } } } return(result); }
public InvertedIndex CreateIndex( params string[] documents ) { var result = new InvertedIndex(documents.Length); for (var i = 0; i < documents.Length; i++) { using (var reader = new StringReader(documents[i])) { ProcessDocument(result, i, reader); } } return(result); }
private void ProcessDocument( InvertedIndex result, int documentIndex, TextReader reader ) { var tokenSource = new TokenSource(reader); while (tokenSource.Next()) { if (_analyzer.Process(tokenSource)) { result.Append( new CharArraySegmentKey(tokenSource.Buffer, tokenSource.Size), documentIndex, tokenSource.Position); } } }