public PostCountsByDay Get()
{
lock (Lock)
{
if (memoryCache.TryGetValue(nameof(PostCountsByDay), out PostCountsByDay cachedResult))
{
return(cachedResult);
}
if (!DateRangeTable.TryRead(connection, out var range))
{
throw new InvalidOperationException("Empty date range table in SQLite database.");
}
var min = range.from;
var max = range.to;
var totalsByDay = new Dictionary <DateTime, ushort>();
foreach (var entry in StoryTable.GetEntries(connection))
{
var day = entry.Date.Date;
if (!totalsByDay.ContainsKey(day))
{
totalsByDay[day] = 1;
}
else
{
totalsByDay[day]++;
}
}
var days = new List <DateTime>();
var counts = new List <ushort>();
var current = min;
while (current.Date < max.Date)
{
days.Add(current.Date);
if (totalsByDay.TryGetValue(current.Date, out var count))
{
counts.Add(count);
}
else
{
counts.Add(0);
}
current = current.AddDays(1);
}
cachedResult = new PostCountsByDay(min, max, counts, days);
memoryCache.Set(nameof(PostCountsByDay), cachedResult);
return(cachedResult);
}
}
public async Task Run(CancellationToken cancellationToken = default(CancellationToken))
{
using var connection = connectionFactory.Open();
LastWriteTable.TryRead(connection, out var lastId);
DateTime max;
DateTime min;
if (!DateRangeTable.TryRead(connection, out var range))
{
max = DateTime.MinValue;
min = DateTime.MaxValue;
}
else
{
max = range.to;
min = range.from;
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
var maxItemResponse = await httpClient
.GetStringAsync("https://hacker-news.firebaseio.com/v0/maxitem.json")
.ConfigureAwait(false);
var maxItem = int.Parse(maxItemResponse);
if (lastId >= maxItem)
{
return;
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
var count = maxItem - Math.Max(0, lastId);
Trace.WriteLine($"Running from {lastId} to {maxItem} ({count} items).");
var entries = new ConcurrentBag <Entry>();
var interval = (ThreadBucketSize * maxThreads);
for (var i = lastId + 1; i <= maxItem; i += interval)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var tasks = new Task[maxThreads];
for (var threadIndex = 0; threadIndex < maxThreads; threadIndex++)
{
var offset = threadIndex * ThreadBucketSize;
var threadStartId = i + offset;
var end = Math.Min(maxItem + 1, threadStartId + ThreadBucketSize);
var task = RunBucket(threadStartId, end, entries, cancellationToken);
tasks[threadIndex] = task;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
if (entries.Count > 10)
{
Trace.WriteLine("Flushing to database.");
WriteEntries(entries.OrderBy(x => x.Id), connection, i + interval - 1, ref min, ref max);
entries.Clear();
}
}
if (entries.Count > 0)
{
Trace.WriteLine("Flushing to database.");
WriteEntries(entries.OrderBy(x => x.Id), connection, maxItem, ref min, ref max);
}
}