public InvertedIndex CreateIndex(
params string[] documents
)
{
var result = new InvertedIndex(documents.Length);
for (var i = 0; i < documents.Length; i++)
{
using (var reader = new StringReader(documents[i]))
{
var tokenSource = new TokenSource(reader);
while (tokenSource.Next())
{
if (_analyzer.Process(tokenSource))
{
result.Append(
new CharArraySegmentKey(tokenSource.Buffer, tokenSource.Size),
i, tokenSource.Position);
}
}
}
}
return(result);
}
public string AnalyzeOnlyTheFirstToken(string source)
{
using (var reader = new StringReader(source))
{
var tokenSource = new TokenSource(reader);
tokenSource.Next();
Process(tokenSource);
return(tokenSource.ToString());
}
}
private void ProcessDocument(
InvertedIndex result,
int documentIndex,
TextReader reader
)
{
var tokenSource = new TokenSource(reader);
while (tokenSource.Next())
{
if (_analyzer.Process(tokenSource))
{
result.Append(
new CharArraySegmentKey(tokenSource.Buffer, tokenSource.Size),
documentIndex, tokenSource.Position);
}
}
}