public InvertedIndex CreateIndex(InvertedIndex result, params string[] documents)
{
int corpusCount = result._maxDocumentId;
try
{
NumericComparer ns = new NumericComparer();
Array.Sort(documents, ns);
}
catch { }
for (var i = 0; i < documents.Length; i++)
{
using (var reader = new StreamReader(documents[i]))
{
var tokenSource = new TokenSource(reader);
while (tokenSource.Next())
{
if (_analyzer.Process(tokenSource))
{
result.Append(
new TermSegment(tokenSource.Buffer, tokenSource.Size),
corpusCount + i, tokenSource.Position);
}
}
}
}
return(result);
}
public IEnumerable <string> Analyze(string source)
{
using (var reader = new StringReader(source))
{
var tokenSource = new TokenSource(reader);
return(tokenSource.ReadAll(Process).ToArray());
}
}
public string AnalyzeOnlyTheFirstToken(string source)
{
using (var reader = new StringReader(source))
{
var tokenSource = new TokenSource(reader);
tokenSource.Next();
Process(tokenSource);
return(tokenSource.ToString());
}
}
public bool Process(TokenSource source)
{
for (var i = 0; i < _filters.Length; i++)
{
if (!_filters[i].Process(source))
{
return(false);
}
}
return(true);
}
