public async Task <HttpResponseMessage> GetWorkspaceResultsByWorkspaceIdFormatedForSocial(Guid workspaceId, long offset = 0, int count = MaxResultsReturned, string order = "sequence:asc")
{
string[] acceptableOrderByTokens = { "sequence", "updated", "asc", "desc" };
try
{
count = count.ConstrainRange(1, MaxResultsReturned);
offset = offset.ConstrainMin(0);
ThrowIfOrderParameterIsInvalid(acceptableOrderByTokens, order);
using (var db = new Database())
{
IEnumerable <WorkspaceResult> results = await db.GetWorkspaceResults(workspaceId, offset, count, BuildSqlOrderByStringFromQueryStringOrderString(order));
var sresults = WorkspaceResultSocialFormat.FromWorkspaceResults(results);
var builder = new WorkspaceResultBuilder();
foreach (var sresult in sresults)
{
var data = await db.GetWebResourceCacheData(new MD5Hash(sresult.DataHash));
if (data == null)
{
continue;
}
using (var ms = new MemoryStream(data))
{
var t = builder.Build(ms, "", "content-type: text/html", out string s);
sresult.ImageUrl = t.PropertyCache.SafeGet("og_image")?.ToString();
sresult.Author = t.PropertyCache.SafeGet("og_author")?.ToString();
sresult.ResultType = t.PropertyCache.SafeGet("og_type")?.ToString();
sresult.SiteName = t.PropertyCache.SafeGet("og_site_name")?.ToString();
}
}
return(CreateOKResponse(sresults));
}
}
catch (Exception ex)
{
return(CreateExceptionResponse(ex));
}
}
public override async Task ProcessResponseStream(Stream dataStream)
{
try
{
if (dataStream == null)
{
return;
}
var postTo = new List <PostToWorkspaceInstruction>();
using (Stream stream = new MemoryStream())
{
dataStream.CopyTo(stream);
stream.Seek(0, SeekOrigin.Begin);
var builder = new WorkspaceResultBuilder();
var result = builder.Build(stream, requestString, responseHeaders, out string evaluationText);
// evaluate against the queries
foreach (var qry in Queries
.Where(x => ResourcesSeen == 0 || (x.Value.TotalCost / ResourcesSeen < FetchoConfiguration.Current.QueryBudgetForAverageQueryCost))
.OrderBy(x => x.Value.AvgCost))
{
try
{
stream.Seek(0, SeekOrigin.Begin);
var r = qry.Value.Evaluate(result, evaluationText.ToString(), stream);
if (r.Action == EvaluationResultAction.Include)
{
postTo.Add(new PostToWorkspaceInstruction()
{
WorkspaceId = qry.Key,
Tags = r.Tags.Distinct().Where(x => !result.Tags.Contains(x)).ToArray()
});
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
}
// if no matches, move onto the next resource
if (!postTo.Any())
{
return;
}
stream.Seek(0, SeekOrigin.Begin);
await fetchoClient.PostWebResourceDataCache(stream);
foreach (var workspace in postTo)
{
result.Tags.Clear();
result.Tags.AddRange(workspace.Tags);
await fetchoClient.PostWorkspaceResultsByWorkspaceIdAsync(workspace.WorkspaceId, new WorkspaceResult[] { result });
}
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
finally
{
if (++ResourcesSeen % 1000 == 0)
{
await SetupQueries(fetchoClient);
await PostQueryStatsToMaster(fetchoClient);
ReportStatus();
}
}
}
public async void OutputResponse(WebResponse response, byte[] buffer, int bytesRead)
{
try
{
Writer.OutputResponse(response, buffer, bytesRead);
// bail if we dont get anything
if (bytesRead == 0)
{
return;
}
// if we need to push it to this workspace
if (queueItem is ImmediateWorkspaceQueueItem wqi)
{
try
{
var sb = new StringBuilder();
if (response is HttpWebResponse httpWebResponse)
{
sb.AppendFormat("status: {0} {1}\n", httpWebResponse.StatusCode, httpWebResponse.StatusDescription);
}
foreach (string key in response.Headers)
{
sb.AppendFormat("{0}: {1}\n", key, response.Headers[key]);
}
responseHeaders = sb.ToString();
using (var ms = new MemoryStream(buffer))
{
var builder = new WorkspaceResultBuilder();
var result = builder.Build(ms, requestString, responseHeaders, out string evalText);
result.Tags.AddRange(wqi.Tags);
var hash = MD5Hash.Compute(buffer);
var db = await DatabasePool.GetDatabaseAsync();
try
{
await db.AddWorkspaceResults(wqi.DestinationWorkspaceId, new[] { result });
// OPTIMISE: Remove ToArray and just pass IEnumerable<>
await db.AddWebResourceDataCache(hash, buffer.Take(bytesRead).ToArray());
}
catch (Exception ex)
{
Utility.LogException(ex);
}
finally
{
await DatabasePool.GiveBackToPool(db);
}
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
}
