public async Task <StatisticsDTO> Get()
{
using (var db = new PikabuContext())
{
var commentsCount = await db.Comments.CountAsync();
var usersCount = await db.Users.CountAsync();
var storiesCount = await db.Stories.CountAsync();
var fetcherStats = await db.FetcherStats.ToArrayAsync();
return(new StatisticsDTO()
{
CommentsCount = commentsCount,
UsersCount = usersCount,
StoriesCount = storiesCount,
Fetchers = fetcherStats.Select(f => new StatisticsDTO.StatisticsFetchersDTO()
{
FetcherName = f.FetcherName,
StoriesPerSecondForLastHour = f.StoriesPerSecondForLastHour,
StoriesPerSecondForLastMinute = f.StoriesPerSecondForLastMinute
}).ToArray()
});
}
}
public async Task <IEnumerable <TopDTO> > GetTop(int users, int days)
{
var key = $"{CacheKeys.TopUsers}/{users}/{days}";
if (!_cache.TryGetValue(key, out var topUsers))
{
var cacheEntryOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(DateTimeOffset.UtcNow + TimeSpan.FromSeconds(60));
var cacheEntry = await GetTopNonCached();
_cache.Set(key, cacheEntry, cacheEntryOptions);
return(cacheEntry);
}
return((IEnumerable <TopDTO>)topUsers);
async Task <TopDTO[]> GetTopNonCached()
{
using (var db = new PikabuContext())
{
return(await db.Comments
.Where(c => c.DateTimeUtc >= DateTime.UtcNow.AddDays(-days))
.Select(c => new { c.UserName, c.DateTimeUtc })
.GroupBy(c => c.UserName)
.OrderByDescending(grouping => grouping.Count())
.Take(users)
.Select(comments => new TopDTO {
User = comments.Key, Comments = comments.Count()
})
.ToArrayAsync());
}
}
}
public async Task <IEnumerable <string> > Get()
{
using (var db = new PikabuContext())
{
return(await db.Users.Select(c => c.UserName).OrderBy(c => c).ToArrayAsync());
}
}
public async Task <CommentDTO[]> Get(string userName, long skipTill = long.MaxValue)
{
System.Console.WriteLine(userName);
using (var db = new PikabuContext())
{
return(await db.Comments
.Where(c => c.UserName == userName)
.Where(c => c.CommentId < skipTill)
.OrderByDescending(c => c.DateTimeUtc)
.Take(50)
.Select(c => new CommentDTO
{
StoryId = c.StoryId,
StoryTitle = c.Story.Title,
AvatarUrl = db.Users.Single(u => u.UserName == c.UserName).AvatarUrl,
CommentId = c.CommentId,
ParentId = c.ParentId,
DateTimeUtc = c.DateTimeUtc,
CommentBody = c.CommentContent.BodyHtml,
IsAuthor = c.UserName == c.Story.Author,
Rating = c.Rating
})
.ToArrayAsync());
}
}
static async Task Main(string[] args)
{
using (var db = new PikabuContext())
{
var top1000Commenters = await db.Comments
.Where(c => c.DateTimeUtc >= DateTime.UtcNow.AddDays(-30))
.GroupBy(c => c.UserName)
.OrderByDescending(grouping => grouping.Count())
.Take(1000)
.Select(comments => new { User = comments.Key, Comments = comments.Count() })
.ToArrayAsync();
}
}
public async Task ProcessStory(CancellationToken cancellationToken)
{
var utcNow = DateTimeOffset.UtcNow;
_latestHourStats.Enqueue(utcNow);
_latestMinuteStats.Enqueue(utcNow);
while (_latestHourStats.Peek() < utcNow.AddHours(-1))
{
_latestHourStats.Dequeue();
}
while (_latestMinuteStats.Peek() < utcNow.AddMinutes(-1))
{
_latestMinuteStats.Dequeue();
}
using (var db = new PikabuContext())
{
var stat = await db.FetcherStats.SingleOrDefaultAsync(s => s.FetcherName == _name, cancellationToken);
if (stat == null)
{
stat = new FetcherStat();
stat.FetcherName = _name;
db.FetcherStats.Add(stat);
}
if (_latestHourStats.Count >= 2)
{
stat.StoriesPerSecondForLastHour =
_latestHourStats.Count / (utcNow - _latestHourStats.Peek()).TotalSeconds;
}
if (_latestMinuteStats.Count >= 2)
{
stat.StoriesPerSecondForLastMinute =
_latestMinuteStats.Count / (utcNow - _latestMinuteStats.Peek()).TotalSeconds;
}
await db.SaveChangesAsync(cancellationToken);
}
}
public async Task <CommentDTO> Get(long id)
{
using (var db = new PikabuContext())
{
return(await db.Comments
.Where(comment => comment.CommentId == id)
.Select(c => new CommentDTO
{
StoryId = c.StoryId,
UserName = c.UserName,
AvatarUrl = db.Users.Single(u => u.UserName == c.UserName).AvatarUrl,
StoryTitle = c.Story.Title,
CommentId = c.CommentId,
ParentId = c.ParentId,
DateTimeUtc = c.DateTimeUtc,
CommentBody = c.CommentContent.BodyHtml,
IsAuthor = c.UserName == c.Story.Author,
Rating = c.Rating
})
.SingleOrDefaultAsync());
}
}
public override async Task FetchLoop()
{
var performanceCounter = new PerformanceCounter("Top" + _top);
var savingTask = Task.CompletedTask;
while (true)
{
try
{
int[] topStoryIds;
using (var db = new PikabuContext())
{
topStoryIds = await db.Stories
.Where(story => story.DateTimeUtc >= DateTime.UtcNow - _duration)
.OrderByDescending(story => story.Rating)
.Select(story => story.StoryId)
.Take(_top)
.ToArrayAsync();
}
if (topStoryIds.Length < _top)
{
Console.WriteLine($"!!!{DateTime.UtcNow} topStoryIds.Length < top ({topStoryIds.Length})");
await Task.Delay(TimeSpan.FromSeconds(1));
continue;
}
foreach (var storyId in topStoryIds)
{
try
{
if (savingTask.IsCanceled || savingTask.IsFaulted)
{
savingTask = Task.Delay(TimeSpan.FromSeconds(1));
}
await savingTask;
using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)))
{
savingTask = await ProcessStory(storyId, '!', cancellationTokenSource.Token);
await performanceCounter.ProcessStory(cancellationTokenSource.Token);
}
}
catch (Exception e)
{
await Task.Delay(1000);
Console.WriteLine($"!!!{DateTime.UtcNow} ERROR: {e}");
}
}
Console.WriteLine($"!!!{DateTime.UtcNow} RESTART");
}
catch (TaskCanceledException)
{
Console.WriteLine($"!!!{DateTime.UtcNow} TERMINATED");
throw;
}
catch (Exception e)
{
await Task.Delay(1000);
Console.WriteLine($"!!!{DateTime.UtcNow} ERROR: {e}");
}
}
}
public IndexModel(PikabuContext db)
{
_db = db;
}
private async Task SaveStory(StoryComments storyComments, char fetcher, CancellationToken cancellationToken)
{
using (var db = new PikabuContext())
using (var transaction = await db.Database.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken))
{
var scanTime = DateTime.UtcNow;
var story = await db.Stories.SingleOrDefaultAsync(s => s.StoryId == storyComments.StoryId, cancellationToken);
if (story == null)
{
story = new Story
{
StoryId = storyComments.StoryId,
Rating = storyComments.Rating,
Title = storyComments.StoryTitle,
Author = storyComments.Author,
DateTimeUtc = storyComments.Timestamp.UtcDateTime,
Comments = new List <Comment>()
};
await db.Stories.AddAsync(story, cancellationToken);
}
story.Rating = storyComments.Rating;
story.Title = storyComments.StoryTitle;
story.Author = storyComments.Author;
story.LastScanUtc = scanTime;
var storyCommentIds = storyComments.Comments.Select(c => c.CommentId).ToArray();
var existingComments = await db.Comments.Where(c => storyCommentIds.Contains(c.CommentId)).ToDictionaryAsync(c => c.CommentId, cancellationToken);
var storyUserNames = new HashSet <string>(storyComments.Comments.Select(c => c.User));
var existingUsers = (await db.Users.Where(c => storyUserNames.Contains(c.UserName)).ToDictionaryAsync(u => u.UserName, cancellationToken));
var newComments = 0;
foreach (var comment in storyComments.Comments)
{
if (!existingUsers.TryGetValue(comment.User, out var user))
{
user = new User {
UserName = comment.User, AvatarUrl = comment.UserAvatarUrl, Comments = new List <Comment>()
};
await db.Users.AddAsync(user, cancellationToken);
existingUsers[user.UserName] = user;
}
else
{
user.AvatarUrl = comment.UserAvatarUrl;
}
if (!existingComments.TryGetValue(comment.CommentId, out var c))
{
var item = new Comment
{
CommentId = comment.CommentId,
ParentId = comment.ParentId,
DateTimeUtc = comment.Timestamp.UtcDateTime,
Rating = comment.Rating,
Story = story,
UserName = comment.User,
CommentContent = new CommentContent {
BodyHtml = comment.Body
}
};
await db.Comments.AddAsync(item, cancellationToken);
existingComments[item.CommentId] = item;
}
else
{
newComments++;
c.Rating = comment.Rating;
}
}
/*Console.Write($"{fetcher}{DateTime.UtcNow} ({storyComments.StoryId}) {storyComments.Rating?.ToString("+0;-#") ?? "?"} {storyComments.StoryTitle}");
* if (newComments > 0) {
* var color = Console.ForegroundColor;
* Console.ForegroundColor = ConsoleColor.Yellow;
* Console.WriteLine($" +{newComments}");
* Console.ForegroundColor = color;
* }
* else
* {
* Console.WriteLine();
* }*/
await db.SaveChangesAsync(cancellationToken);
transaction.Commit();
}
}