public async Task <HttpResponseMessage> GetWorkspaceResultsByWorkspaceIdFormatedForSocial(Guid workspaceId, long offset = 0, int count = MaxResultsReturned, string order = "sequence:asc") { string[] acceptableOrderByTokens = { "sequence", "updated", "asc", "desc" }; try { count = count.ConstrainRange(1, MaxResultsReturned); offset = offset.ConstrainMin(0); ThrowIfOrderParameterIsInvalid(acceptableOrderByTokens, order); using (var db = new Database()) { IEnumerable <WorkspaceResult> results = await db.GetWorkspaceResults(workspaceId, offset, count, BuildSqlOrderByStringFromQueryStringOrderString(order)); var sresults = WorkspaceResultSocialFormat.FromWorkspaceResults(results); var builder = new WorkspaceResultBuilder(); foreach (var sresult in sresults) { var data = await db.GetWebResourceCacheData(new MD5Hash(sresult.DataHash)); if (data == null) { continue; } using (var ms = new MemoryStream(data)) { var t = builder.Build(ms, "", "content-type: text/html", out string s); sresult.ImageUrl = t.PropertyCache.SafeGet("og_image")?.ToString(); sresult.Author = t.PropertyCache.SafeGet("og_author")?.ToString(); sresult.ResultType = t.PropertyCache.SafeGet("og_type")?.ToString(); sresult.SiteName = t.PropertyCache.SafeGet("og_site_name")?.ToString(); } } return(CreateOKResponse(sresults)); } } catch (Exception ex) { return(CreateExceptionResponse(ex)); } }
public override async Task ProcessResponseStream(Stream dataStream) { try { if (dataStream == null) { return; } var postTo = new List <PostToWorkspaceInstruction>(); using (Stream stream = new MemoryStream()) { dataStream.CopyTo(stream); stream.Seek(0, SeekOrigin.Begin); var builder = new WorkspaceResultBuilder(); var result = builder.Build(stream, requestString, responseHeaders, out string evaluationText); // evaluate against the queries foreach (var qry in Queries .Where(x => ResourcesSeen == 0 || (x.Value.TotalCost / ResourcesSeen < FetchoConfiguration.Current.QueryBudgetForAverageQueryCost)) .OrderBy(x => x.Value.AvgCost)) { try { stream.Seek(0, SeekOrigin.Begin); var r = qry.Value.Evaluate(result, evaluationText.ToString(), stream); if (r.Action == EvaluationResultAction.Include) { postTo.Add(new PostToWorkspaceInstruction() { WorkspaceId = qry.Key, Tags = r.Tags.Distinct().Where(x => !result.Tags.Contains(x)).ToArray() }); } } catch (Exception ex) { Utility.LogException(ex); } } // if no matches, move onto the next resource if (!postTo.Any()) { return; } stream.Seek(0, SeekOrigin.Begin); await fetchoClient.PostWebResourceDataCache(stream); foreach (var workspace in postTo) { result.Tags.Clear(); result.Tags.AddRange(workspace.Tags); await fetchoClient.PostWorkspaceResultsByWorkspaceIdAsync(workspace.WorkspaceId, new WorkspaceResult[] { result }); } } } catch (Exception ex) { Utility.LogException(ex); } finally { if (++ResourcesSeen % 1000 == 0) { await SetupQueries(fetchoClient); await PostQueryStatsToMaster(fetchoClient); ReportStatus(); } } }
public async void OutputResponse(WebResponse response, byte[] buffer, int bytesRead) { try { Writer.OutputResponse(response, buffer, bytesRead); // bail if we dont get anything if (bytesRead == 0) { return; } // if we need to push it to this workspace if (queueItem is ImmediateWorkspaceQueueItem wqi) { try { var sb = new StringBuilder(); if (response is HttpWebResponse httpWebResponse) { sb.AppendFormat("status: {0} {1}\n", httpWebResponse.StatusCode, httpWebResponse.StatusDescription); } foreach (string key in response.Headers) { sb.AppendFormat("{0}: {1}\n", key, response.Headers[key]); } responseHeaders = sb.ToString(); using (var ms = new MemoryStream(buffer)) { var builder = new WorkspaceResultBuilder(); var result = builder.Build(ms, requestString, responseHeaders, out string evalText); result.Tags.AddRange(wqi.Tags); var hash = MD5Hash.Compute(buffer); var db = await DatabasePool.GetDatabaseAsync(); try { await db.AddWorkspaceResults(wqi.DestinationWorkspaceId, new[] { result }); // OPTIMISE: Remove ToArray and just pass IEnumerable<> await db.AddWebResourceDataCache(hash, buffer.Take(bytesRead).ToArray()); } catch (Exception ex) { Utility.LogException(ex); } finally { await DatabasePool.GiveBackToPool(db); } } } catch (Exception ex) { Utility.LogException(ex); } } } catch (Exception ex) { Utility.LogException(ex); } }