public async Task <Stream> Run(RequestContext requestContext)
{
var data = await _logger.GetRequestsByPartialUrl(requestContext.OriginUrl.ToString());
var vector = new RequestVector()
{
Url = requestContext.OriginUrl,
ErrorCount = requestContext.OwinContext.Response.Header.StatusCode.GetValueOrDefault(0) >= 400 ? 1 : 0,
IsCacheable = IsCacheable(requestContext.OwinContext),
IsPublic = IsPublic(requestContext.OwinContext),
IsHtml = requestContext.OwinContext.Response.Header.ContentMimeType.Contains("text/html"),
IsJson = requestContext.OwinContext.Response.Header.ContentMimeType.Contains("json"),
PathCount = requestContext.OriginUrl.PathAndQuery.Split('/').Count(),
RequestCount = data.Items.Count,
ResponseSize = data.Items.Select(r => r.ResponseSize / 1024d).Mean(),
ResponseTime = data.Items.Select(r => r.ElapsedMilliseconds).Mean(),
UniqueRefererCount = data.Items.Where(r => r.RefererUrl != null).Select(r => r.RefererUrl).Distinct().Count()
};
var tokens = (await Tokenise(requestContext)).ToList();
vector.TokenCount = tokens.Count;
vector.SemanticTokenRatio = tokens.Where(t => t.Type == TokenType.Word && _dict.IsWord(t.Text)).Count() / (double)tokens.Count;
vector.NumericTokenRatio = tokens.Where(t => t.Type == TokenType.Number).Count() / (double)tokens.Count;
await Write(vector);
return(requestContext.RequestBlob);
}
private async Task Write(RequestVector vector)
{
var json = JsonConvert.SerializeObject(vector, Formatting.None);
await _featureIndexStream.WriteLineAsync(json);
await _featureIndexStream.FlushAsync();
}