public async Task <List <CommentAggregateView> > SearchAggregateAsync(CommentSearch search, Requester requester) { //Repeat code, be careful await FixWatchLimits(watchSource, requester, search.ContentLimit); var ids = converter.SearchIds(search, q => services.permissions.PermissionWhere(q, requester, Keys.ReadAction)); var groups = await converter.GroupAsync <EntityRelation, TempGroup>(ids, x => new TempGroup() { userId = -x.entityId2, contentId = x.entityId1 }); return(groups.ToLookup(x => x.Key.contentId).Select(x => new CommentAggregateView() { id = x.Key, count = x.Sum(y => y.Value.count), lastDate = x.Max(y => y.Value.lastDate), firstDate = x.Min(y => y.Value.firstDate), lastId = x.Max(y => y.Value.lastId), userIds = x.Select(y => y.Key.userId).Distinct().ToList() }).ToList()); }
protected async Task OptimizedCommentSearch(CommentSearch search, Requester requester, Func <Func <IQueryable <EntityGroup>, IQueryable <EntityGroup> >, Task> perform) { //If fixing the watch limits automatically safe limited us, we don't have to perform the (sometimes lengthy) permission calculation again. var safeLimited = await FixWatchLimits(watchSource, requester, search.ContentLimit); converter.JoinPermissions = false; try { var limitIds = search.ContentLimit.Limit.Select(x => x.id).ToList(); //An optimization: regardless of what the USER asked for, a content limit query will ONLY return //rows (comments) where the comment parent is within the limiter. This enables the later optimization if (limitIds.Count > 0) { search.ParentIds = limitIds; } //We can do some special stuff to increase performance if we STILL have no parentids if (search.ParentIds.Count == 0) { var tempSearch = services.mapper.Map <CommentSearch>(search); tempSearch.Limit = -1; //Remove limits, we're grouping. if (search.Ids.Count > 0) { //This is a more optimized group because it does not require a join search.ParentIds = (await converter.GroupAsync <EntityRelation, long>( await converter.GetBaseQuery(tempSearch), g => g.relation, converter.PermIdSelector)).Select(x => x.Key).ToList(); } else if (search.CreateEnd.Ticks > 0 || search.CreateStart.Ticks > 0 || search.MinId > 0 || search.MaxId > 0) { search.ParentIds = (await converter.GroupAsync(await converter.SearchIds(tempSearch), converter.PermIdSelector)).Select(x => x.Key).ToList(); } } if (search.ParentIds.Count > 0) { //If we're already safe limited, don't reperform the parent id limit if (!safeLimited) { //Limit parentids by the ones this requester is allowed to have. var ids = await contentSource.SearchIds(new ContentSearch() { Ids = search.ParentIds }, q => services.permissions.PermissionWhere(q, requester, Keys.ReadAction)); search.ParentIds = await services.provider.GetListAsync(ids); search.ParentIds.Add(long.MaxValue); //Ensures it stays at "empty" parentids } await perform(null); } else { converter.JoinPermissions = true; await perform(q => services.permissions.PermissionWhere(q, requester, Keys.ReadAction)); } } finally { converter.JoinPermissions = true; } }
public override async Task <List <ContentView> > PreparedSearchAsync(ContentSearch search, Requester requester) { var baseResult = await base.PreparedSearchAsync(search, requester); var baseIds = baseResult.Select(x => x.id).ToList(); if (baseIds.Count > 0 && search.IncludeAbout) { //This requires intimate knowledge of how watches work. it's increasing the complexity/dependency, //but at least... I don't know, it's way more performant. Come up with some system perhaps after //you see what you need in other instances. var watches = await watchSource.GroupAsync <EntityRelation, long>( watchSource.SearchIds(new WatchSearch() { ContentIds = baseIds }), watchSource.PermIdSelector); //x => x.entityId2); var comments = await commentSource.GroupAsync <EntityRelation, long>( commentSource.SearchIds(new CommentSearch() { ParentIds = baseIds }), commentSource.PermIdSelector); //x => x.entityId1); var votes = new Dictionary <string, Dictionary <long, SimpleAggregateData> >(); var watchSearch = new WatchSearch(); watchSearch.UserIds.Add(requester.userId); watchSearch.ContentIds.AddRange(baseIds); var userWatching = await watchSource.SimpleSearchAsync(watchSearch); //THIS could be an intensive query! Make sure you check the CPU usage! var voteSearch = new VoteSearch(); voteSearch.UserIds.Add(requester.userId); voteSearch.ContentIds.AddRange(baseIds); var userVotes = await voteSource.SimpleSearchAsync(voteSearch); foreach (var voteWeight in Votes.VoteWeights) { votes.Add(voteWeight.Key, await voteSource.GroupAsync <EntityRelation, long>( voteSource.SearchIds(new VoteSearch() { ContentIds = baseIds, Vote = voteWeight.Key }), voteSource.PermIdSelector)); //x => x.entityId2)); } baseResult.ForEach(x => { if (watches.ContainsKey(x.id)) //-x.id)) { x.about.watches = watches[x.id]; //-x.id]; } if (comments.ContainsKey(x.id)) { x.about.comments = comments[x.id]; } x.about.watching = userWatching.Any(y => y.contentId == x.id); x.about.myVote = userVotes.FirstOrDefault(y => y.contentId == x.id)?.vote; foreach (var voteWeight in Votes.VoteWeights) { x.about.votes.Add(voteWeight.Key, new SimpleAggregateData()); if (votes[voteWeight.Key].ContainsKey(x.id)) //-x.id)) { x.about.votes[voteWeight.Key] = votes[voteWeight.Key][x.id]; //-x.id]; } } }); } else if (!search.IncludeAbout) { //Eventually, about is going away anyway baseResult.ForEach(x => { x.about = null; }); } return(baseResult); }
public override async Task <List <ContentView> > PreparedSearchAsync(ContentSearch search, Requester requester) { string key = JsonSerializer.Serialize(search) + JsonSerializer.Serialize(requester); List <ContentView> baseResult = null; bool includeAbout = search.IncludeAbout != null && search.IncludeAbout.Count > 0; bool useCache = !includeAbout; if (useCache && cache.GetValue(key, ref baseResult)) { return(baseResult); } List <long> baseIds = null; Interlocked.Increment(ref rid); var t = services.timer.StartTimer(""); try { baseResult = await base.PreparedSearchAsync(search, requester); baseIds = baseResult.Select(x => x.id).ToList(); t.Name = $"[{rid}] content PreparedSearchAsync {baseResult.Count} ({string.Join(",", baseIds)})"; } finally { services.timer.EndTimer(t); } if (includeAbout && baseResult.Count > 0) { var desired = search.IncludeAbout.Select(x => x.ToLower()); //TODO: STOP THIS MADNESS!! commentSource.JoinPermissions = false; watchSource.JoinPermissions = false; voteSource.JoinPermissions = false; try { Dictionary <long, SimpleAggregateData> comments = null; Dictionary <long, SimpleAggregateData> watches = null; Dictionary <string, Dictionary <long, SimpleAggregateData> > votes = null; List <WatchView> userWatching = null; List <VoteView> userVotes = null; if (desired.Any(x => x.StartsWith("comment"))) { t = services.timer.StartTimer($"[{rid}] comment pull"); comments = await commentSource.GroupAsync <EntityRelation, long>( await commentSource.GetBaseQuery(new CommentSearch() { ParentIds = baseIds }), g => g.relation, commentSource.PermIdSelector); services.timer.EndTimer(t); } if (desired.Any(x => x.StartsWith("watch"))) { t = services.timer.StartTimer($"[{rid}] watch pull (both)"); //This requires intimate knowledge of how watches work. it's increasing the complexity/dependency, //but at least... I don't know, it's way more performant. Come up with some system perhaps after //you see what you need in other instances. watches = await watchSource.GroupAsync <EntityRelation, long>( await watchSource.SearchIds(new WatchSearch() { ContentIds = baseIds }), watchSource.PermIdSelector); var watchSearch = new WatchSearch(); watchSearch.UserIds.Add(requester.userId); watchSearch.ContentIds.AddRange(baseIds); userWatching = await watchSource.SimpleSearchAsync(watchSearch); services.timer.EndTimer(t); } if (desired.Any(x => x.StartsWith("watch"))) { //THIS could be an intensive query! Make sure you check the CPU usage! t = services.timer.StartTimer($"[{rid}] vote pull"); var voteSearch = new VoteSearch(); voteSearch.UserIds.Add(requester.userId); voteSearch.ContentIds.AddRange(baseIds); userVotes = await voteSource.SimpleSearchAsync(voteSearch); services.timer.EndTimer(t); t = services.timer.StartTimer($"[{rid}] vote aggregate"); votes = new Dictionary <string, Dictionary <long, SimpleAggregateData> >(); foreach (var voteWeight in Votes.VoteWeights) { votes.Add(voteWeight.Key, await voteSource.GroupAsync <EntityRelation, long>( await voteSource.SearchIds(new VoteSearch() { ContentIds = baseIds, Vote = voteWeight.Key }), voteSource.PermIdSelector)); //x => x.entityId2)); } services.timer.EndTimer(t); } baseResult.ForEach(x => { if (watches != null) { if (watches.ContainsKey(x.id)) { x.about.watches = watches[x.id]; } x.about.watching = userWatching.Any(y => y.contentId == x.id); } if (comments != null) { if (comments.ContainsKey(x.id)) { x.about.comments = comments[x.id]; } } if (votes != null) { x.about.myVote = userVotes.FirstOrDefault(y => y.contentId == x.id)?.vote; foreach (var voteWeight in Votes.VoteWeights) { x.about.votes.Add(voteWeight.Key, new SimpleAggregateData()); if (votes[voteWeight.Key].ContainsKey(x.id)) { x.about.votes[voteWeight.Key] = votes[voteWeight.Key][x.id]; } } } }); } finally { commentSource.JoinPermissions = true; watchSource.JoinPermissions = true; voteSource.JoinPermissions = true; } } else { //Eventually, about is going away anyway baseResult.ForEach(x => { x.about = null; }); } if (useCache) { cache.StoreItem(key, baseResult); } return(baseResult); }