private static void StartQueue()
{
Task.Factory.StartNew(() =>
{
while (_running)
{
try
{
var firstCmd = IncrementQueue.Take();
// Try and batch up a lot of the commands
var commands = new List <IncrementCommand> {
firstCmd
};
while (commands.Count < 1000 && IncrementQueue.Count > 0)
{
commands.Add(IncrementQueue.Take());
}
// Convert all the individual commands into aggregated updates
var aggregated = commands
.GroupBy(c => new { c.Emoji, c.SourceType })
.Select(g => new IncrementCommand()
{
SourceType = g.Key.SourceType, Emoji = g.Key.Emoji, Amount = g.Sum(c => c.Amount)
});
using (var db = new WorkerDb())
{
foreach (var cmd in aggregated)
{
var record = db.EmojiCounts.Find(cmd.Emoji, cmd.SourceType);
if (record == null)
{
log.Warning("No record for {0}/{1}", cmd.SourceType, cmd.Emoji);
return;
}
record.Count += cmd.Amount;
}
db.SaveChanges();
}
log.Information("Updated stored emoji");
}
catch (Exception e)
{
log.Error(e, "Failed to handle incoming command queue");
}
}
});
}
private static void SyncRedisToDb()
{
log.Information("Synchronizing emoji with redis/database...");
using (var db = new WorkerDb())
{
var allEmoji = db.EmojiCounts.ToArray();
var mapped = allEmoji.GroupBy(e => e.Emoji).ToDictionary(e => e.Key, e => e.Sum(p => p.Count));
var changed = false;
foreach (var emoji in EmojiData.KnownSymbols)
{
var record = allEmoji.First(e => e.Emoji == emoji && e.Source == EmojiCount.SourceUnknown);
var efCount = mapped.ContainsKey(emoji) ? mapped[emoji] : 0;
var reCountStr = _redisDb.StringGet(emoji);
if (string.IsNullOrWhiteSpace(reCountStr))
{
reCountStr = "0";
}
var reCount = long.Parse(reCountStr);
if (reCount == efCount)
{
continue;
}
if (reCount < efCount)
{
// Redis is wrong - fix it
_redisDb.StringSet(emoji, efCount, flags: CommandFlags.FireAndForget);
return;
}
record.Count += (reCount - efCount);
changed = true;
}
if (changed)
{
db.SaveChanges();
}
}
log.Information("Done synchronizing emoji with redis/database");
}
private static void StartDatabase()
{
WorkerDb.ConnectionString = _config.GetConnectionString("emoji");
using (var context = new WorkerDb()) context.Database.Migrate();
using (var context = new WorkerDb())
{
var sources = new string[]
{
EmojiCount.SourceMessage,
EmojiCount.SourceReaction,
EmojiCount.SourceUnknown,
};
var records = context.EmojiCounts.ToArray();
if (records.Count() == (sources.Length * EmojiData.KnownSymbols.Count()))
{
log.Warning(
"Skipping database record update - suspecting the emoji are already present. This may cause problems");
return;
}
var changed = false;
foreach (var emoji in EmojiData.KnownSymbols)
{
foreach (var source in sources)
{
log.Information("Checking for {0}/{1} emoji...", source, emoji);
var record = records.FirstOrDefault(e => e.Emoji == emoji && e.Source == source);
if (record != null)
{
continue;
}
context.EmojiCounts.Add(new EmojiCount(emoji, source));
changed = true;
}
}
if (changed)
{
context.SaveChanges();
}
}
}