public Task <Stream> Run(RequestContext requestContext)
{
var tokens = requestContext.RequestBlob.Tokenise().ToList();
var doc = new TokenisedTextDocument(requestContext.OriginUrl.ToString(), tokens);
var indexInvoker = GetIndexInvoker(requestContext.OriginUrl.Host);
lock (indexInvoker)
{
indexInvoker.State.IndexDocument(doc);
indexInvoker.Trigger();
}
requestContext.RequestBlob.Position = 0;
return(Task.FromResult(requestContext.RequestBlob));
}
private void Append(TokenisedTextDocument document)
{
_index.IndexDocument(document);
var corpus = new Corpus(document.Tokens);
foreach (var block in corpus.Blocks)
{
_markovChain.AnalyseSequence(block.Select(b => b.Text));
}
}