public async Task <StatisticsDTO> Get() { using (var db = new PikabuContext()) { var commentsCount = await db.Comments.CountAsync(); var usersCount = await db.Users.CountAsync(); var storiesCount = await db.Stories.CountAsync(); var fetcherStats = await db.FetcherStats.ToArrayAsync(); return(new StatisticsDTO() { CommentsCount = commentsCount, UsersCount = usersCount, StoriesCount = storiesCount, Fetchers = fetcherStats.Select(f => new StatisticsDTO.StatisticsFetchersDTO() { FetcherName = f.FetcherName, StoriesPerSecondForLastHour = f.StoriesPerSecondForLastHour, StoriesPerSecondForLastMinute = f.StoriesPerSecondForLastMinute }).ToArray() }); } }
public async Task <IEnumerable <TopDTO> > GetTop(int users, int days) { var key = $"{CacheKeys.TopUsers}/{users}/{days}"; if (!_cache.TryGetValue(key, out var topUsers)) { var cacheEntryOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(DateTimeOffset.UtcNow + TimeSpan.FromSeconds(60)); var cacheEntry = await GetTopNonCached(); _cache.Set(key, cacheEntry, cacheEntryOptions); return(cacheEntry); } return((IEnumerable <TopDTO>)topUsers); async Task <TopDTO[]> GetTopNonCached() { using (var db = new PikabuContext()) { return(await db.Comments .Where(c => c.DateTimeUtc >= DateTime.UtcNow.AddDays(-days)) .Select(c => new { c.UserName, c.DateTimeUtc }) .GroupBy(c => c.UserName) .OrderByDescending(grouping => grouping.Count()) .Take(users) .Select(comments => new TopDTO { User = comments.Key, Comments = comments.Count() }) .ToArrayAsync()); } } }
public async Task <IEnumerable <string> > Get() { using (var db = new PikabuContext()) { return(await db.Users.Select(c => c.UserName).OrderBy(c => c).ToArrayAsync()); } }
public async Task <CommentDTO[]> Get(string userName, long skipTill = long.MaxValue) { System.Console.WriteLine(userName); using (var db = new PikabuContext()) { return(await db.Comments .Where(c => c.UserName == userName) .Where(c => c.CommentId < skipTill) .OrderByDescending(c => c.DateTimeUtc) .Take(50) .Select(c => new CommentDTO { StoryId = c.StoryId, StoryTitle = c.Story.Title, AvatarUrl = db.Users.Single(u => u.UserName == c.UserName).AvatarUrl, CommentId = c.CommentId, ParentId = c.ParentId, DateTimeUtc = c.DateTimeUtc, CommentBody = c.CommentContent.BodyHtml, IsAuthor = c.UserName == c.Story.Author, Rating = c.Rating }) .ToArrayAsync()); } }
static async Task Main(string[] args) { using (var db = new PikabuContext()) { var top1000Commenters = await db.Comments .Where(c => c.DateTimeUtc >= DateTime.UtcNow.AddDays(-30)) .GroupBy(c => c.UserName) .OrderByDescending(grouping => grouping.Count()) .Take(1000) .Select(comments => new { User = comments.Key, Comments = comments.Count() }) .ToArrayAsync(); } }
public async Task ProcessStory(CancellationToken cancellationToken) { var utcNow = DateTimeOffset.UtcNow; _latestHourStats.Enqueue(utcNow); _latestMinuteStats.Enqueue(utcNow); while (_latestHourStats.Peek() < utcNow.AddHours(-1)) { _latestHourStats.Dequeue(); } while (_latestMinuteStats.Peek() < utcNow.AddMinutes(-1)) { _latestMinuteStats.Dequeue(); } using (var db = new PikabuContext()) { var stat = await db.FetcherStats.SingleOrDefaultAsync(s => s.FetcherName == _name, cancellationToken); if (stat == null) { stat = new FetcherStat(); stat.FetcherName = _name; db.FetcherStats.Add(stat); } if (_latestHourStats.Count >= 2) { stat.StoriesPerSecondForLastHour = _latestHourStats.Count / (utcNow - _latestHourStats.Peek()).TotalSeconds; } if (_latestMinuteStats.Count >= 2) { stat.StoriesPerSecondForLastMinute = _latestMinuteStats.Count / (utcNow - _latestMinuteStats.Peek()).TotalSeconds; } await db.SaveChangesAsync(cancellationToken); } }
public async Task <CommentDTO> Get(long id) { using (var db = new PikabuContext()) { return(await db.Comments .Where(comment => comment.CommentId == id) .Select(c => new CommentDTO { StoryId = c.StoryId, UserName = c.UserName, AvatarUrl = db.Users.Single(u => u.UserName == c.UserName).AvatarUrl, StoryTitle = c.Story.Title, CommentId = c.CommentId, ParentId = c.ParentId, DateTimeUtc = c.DateTimeUtc, CommentBody = c.CommentContent.BodyHtml, IsAuthor = c.UserName == c.Story.Author, Rating = c.Rating }) .SingleOrDefaultAsync()); } }
public override async Task FetchLoop() { var performanceCounter = new PerformanceCounter("Top" + _top); var savingTask = Task.CompletedTask; while (true) { try { int[] topStoryIds; using (var db = new PikabuContext()) { topStoryIds = await db.Stories .Where(story => story.DateTimeUtc >= DateTime.UtcNow - _duration) .OrderByDescending(story => story.Rating) .Select(story => story.StoryId) .Take(_top) .ToArrayAsync(); } if (topStoryIds.Length < _top) { Console.WriteLine($"!!!{DateTime.UtcNow} topStoryIds.Length < top ({topStoryIds.Length})"); await Task.Delay(TimeSpan.FromSeconds(1)); continue; } foreach (var storyId in topStoryIds) { try { if (savingTask.IsCanceled || savingTask.IsFaulted) { savingTask = Task.Delay(TimeSpan.FromSeconds(1)); } await savingTask; using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1))) { savingTask = await ProcessStory(storyId, '!', cancellationTokenSource.Token); await performanceCounter.ProcessStory(cancellationTokenSource.Token); } } catch (Exception e) { await Task.Delay(1000); Console.WriteLine($"!!!{DateTime.UtcNow} ERROR: {e}"); } } Console.WriteLine($"!!!{DateTime.UtcNow} RESTART"); } catch (TaskCanceledException) { Console.WriteLine($"!!!{DateTime.UtcNow} TERMINATED"); throw; } catch (Exception e) { await Task.Delay(1000); Console.WriteLine($"!!!{DateTime.UtcNow} ERROR: {e}"); } } }
public IndexModel(PikabuContext db) { _db = db; }
private async Task SaveStory(StoryComments storyComments, char fetcher, CancellationToken cancellationToken) { using (var db = new PikabuContext()) using (var transaction = await db.Database.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken)) { var scanTime = DateTime.UtcNow; var story = await db.Stories.SingleOrDefaultAsync(s => s.StoryId == storyComments.StoryId, cancellationToken); if (story == null) { story = new Story { StoryId = storyComments.StoryId, Rating = storyComments.Rating, Title = storyComments.StoryTitle, Author = storyComments.Author, DateTimeUtc = storyComments.Timestamp.UtcDateTime, Comments = new List <Comment>() }; await db.Stories.AddAsync(story, cancellationToken); } story.Rating = storyComments.Rating; story.Title = storyComments.StoryTitle; story.Author = storyComments.Author; story.LastScanUtc = scanTime; var storyCommentIds = storyComments.Comments.Select(c => c.CommentId).ToArray(); var existingComments = await db.Comments.Where(c => storyCommentIds.Contains(c.CommentId)).ToDictionaryAsync(c => c.CommentId, cancellationToken); var storyUserNames = new HashSet <string>(storyComments.Comments.Select(c => c.User)); var existingUsers = (await db.Users.Where(c => storyUserNames.Contains(c.UserName)).ToDictionaryAsync(u => u.UserName, cancellationToken)); var newComments = 0; foreach (var comment in storyComments.Comments) { if (!existingUsers.TryGetValue(comment.User, out var user)) { user = new User { UserName = comment.User, AvatarUrl = comment.UserAvatarUrl, Comments = new List <Comment>() }; await db.Users.AddAsync(user, cancellationToken); existingUsers[user.UserName] = user; } else { user.AvatarUrl = comment.UserAvatarUrl; } if (!existingComments.TryGetValue(comment.CommentId, out var c)) { var item = new Comment { CommentId = comment.CommentId, ParentId = comment.ParentId, DateTimeUtc = comment.Timestamp.UtcDateTime, Rating = comment.Rating, Story = story, UserName = comment.User, CommentContent = new CommentContent { BodyHtml = comment.Body } }; await db.Comments.AddAsync(item, cancellationToken); existingComments[item.CommentId] = item; } else { newComments++; c.Rating = comment.Rating; } } /*Console.Write($"{fetcher}{DateTime.UtcNow} ({storyComments.StoryId}) {storyComments.Rating?.ToString("+0;-#") ?? "?"} {storyComments.StoryTitle}"); * if (newComments > 0) { * var color = Console.ForegroundColor; * Console.ForegroundColor = ConsoleColor.Yellow; * Console.WriteLine($" +{newComments}"); * Console.ForegroundColor = color; * } * else * { * Console.WriteLine(); * }*/ await db.SaveChangesAsync(cancellationToken); transaction.Commit(); } }