private void HandleFollowStream(RedisChannel ch, RedisValue val)
=> Task.Run(() =>
{
var info = JsonConvert.DeserializeAnonymousType(
val.ToString(),
new { Key = default(StreamDataKey), GuildId = 0ul });
_streamTracker.CacheAddData(info.Key, null, replace: false);
lock (_shardLock)
{
var key = info.Key;
if (_trackCounter.ContainsKey(key))
{
_trackCounter[key].Add(info.GuildId);
}
else
{
_trackCounter[key] = new HashSet <ulong>()
{
info.GuildId
};
}
}
});
public StreamNotificationService(DbService db, DiscordSocketClient client,
NadekoStrings strings, IDataCache cache, IBotCredentials creds, IHttpClientFactory httpFactory,
NadekoBot bot)
{
_db = db;
_client = client;
_strings = strings;
_multi = cache.Redis;
_creds = creds;
_streamTracker = new NotifChecker(httpFactory, cache.Redis, creds.RedisKey(), client.ShardId == 0);
using (var uow = db.GetDbContext())
{
var ids = client.GetGuildIds();
var guildConfigs = uow._context.Set <GuildConfig>()
.AsQueryable()
.Include(x => x.FollowedStreams)
.Where(x => ids.Contains(x.GuildId))
.ToList();
_offlineNotificationServers = new ConcurrentHashSet <ulong>(guildConfigs
.Where(gc => gc.NotifyStreamOffline)
.Select(x => x.GuildId)
.ToList());
var followedStreams = guildConfigs
.SelectMany(x => x.FollowedStreams)
.ToList();
_shardTrackedStreams = followedStreams
.GroupBy(x => new { Type = x.Type, Name = x.Username.ToLower() })
.ToList()
.ToDictionary(
x => new StreamDataKey(x.Key.Type, x.Key.Name.ToLower()),
x => x.GroupBy(y => y.GuildId)
.ToDictionary(y => y.Key, y => y.AsEnumerable().ToHashSet()));
// shard 0 will keep track of when there are no more guilds which track a stream
if (client.ShardId == 0)
{
var allFollowedStreams = uow._context.Set <FollowedStream>()
.AsQueryable()
.ToList();
foreach (var fs in allFollowedStreams)
{
_streamTracker.CacheAddData(fs.CreateKey(), null, replace: false);
}
_trackCounter = allFollowedStreams
.GroupBy(x => new { Type = x.Type, Name = x.Username.ToLower() })
.ToDictionary(
x => new StreamDataKey(x.Key.Type, x.Key.Name),
x => x.Select(fs => fs.GuildId).ToHashSet());
}
}
var sub = _multi.GetSubscriber();
sub.Subscribe($"{_creds.RedisKey()}_streams_offline", HandleStreamsOffline);
sub.Subscribe($"{_creds.RedisKey()}_streams_online", HandleStreamsOnline);
if (client.ShardId == 0)
{
// only shard 0 will run the tracker,
// and then publish updates with redis to other shards
_streamTracker.OnStreamsOffline += OnStreamsOffline;
_streamTracker.OnStreamsOnline += OnStreamsOnline;
_ = _streamTracker.RunAsync();
sub.Subscribe($"{_creds.RedisKey()}_follow_stream", HandleFollowStream);
sub.Subscribe($"{_creds.RedisKey()}_unfollow_stream", HandleUnfollowStream);
}
bot.JoinedGuild += ClientOnJoinedGuild;
client.LeftGuild += ClientOnLeftGuild;
}
public StreamNotificationService(DbService db, DiscordSocketClient client,
IBotStrings strings, IDataCache cache, IBotCredentials creds, IHttpClientFactory httpFactory,
NadekoBot bot)
{
_db = db;
_client = client;
_strings = strings;
_multi = cache.Redis;
_creds = creds;
_streamTracker = new NotifChecker(httpFactory, cache.Redis, creds.RedisKey(), client.ShardId == 0);
using (var uow = db.GetDbContext())
{
var ids = client.GetGuildIds();
var guildConfigs = uow._context.Set <GuildConfig>()
.AsQueryable()
.Include(x => x.FollowedStreams)
.Where(x => ids.Contains(x.GuildId))
.ToList();
_offlineNotificationServers = new ConcurrentHashSet <ulong>(guildConfigs
.Where(gc => gc.NotifyStreamOffline)
.Select(x => x.GuildId)
.ToList());
var followedStreams = guildConfigs
.SelectMany(x => x.FollowedStreams)
.ToList();
_shardTrackedStreams = followedStreams
.GroupBy(x => new { Type = x.Type, Name = x.Username.ToLower() })
.ToList()
.ToDictionary(
x => new StreamDataKey(x.Key.Type, x.Key.Name.ToLower()),
x => x.GroupBy(y => y.GuildId)
.ToDictionary(y => y.Key, y => y.AsEnumerable().ToHashSet()));
// shard 0 will keep track of when there are no more guilds which track a stream
if (client.ShardId == 0)
{
var allFollowedStreams = uow._context.Set <FollowedStream>()
.AsQueryable()
.ToList();
foreach (var fs in allFollowedStreams)
{
_streamTracker.CacheAddData(fs.CreateKey(), null, replace: false);
}
_trackCounter = allFollowedStreams
.GroupBy(x => new { Type = x.Type, Name = x.Username.ToLower() })
.ToDictionary(
x => new StreamDataKey(x.Key.Type, x.Key.Name),
x => x.Select(fs => fs.GuildId).ToHashSet());
}
}
var sub = _multi.GetSubscriber();
sub.Subscribe($"{_creds.RedisKey()}_streams_offline", HandleStreamsOffline);
sub.Subscribe($"{_creds.RedisKey()}_streams_online", HandleStreamsOnline);
if (client.ShardId == 0)
{
// only shard 0 will run the tracker,
// and then publish updates with redis to other shards
_streamTracker.OnStreamsOffline += OnStreamsOffline;
_streamTracker.OnStreamsOnline += OnStreamsOnline;
_ = _streamTracker.RunAsync();
_notifCleanupTimer = new Timer(_ =>
{
try
{
var errorLimit = TimeSpan.FromHours(12);
var failingStreams = _streamTracker.GetFailingStreams(errorLimit, true)
.ToList();
if (!failingStreams.Any())
{
return;
}
var deleteGroups = failingStreams.GroupBy(x => x.Type)
.ToDictionary(x => x.Key, x => x.Select(x => x.Name).ToList());
using (var uow = _db.GetDbContext())
{
foreach (var kvp in deleteGroups)
{
Log.Information($"Deleting {kvp.Value.Count} {kvp.Key} streams because " +
$"they've been erroring for more than {errorLimit}: {string.Join(", ", kvp.Value)}");
var toDelete = uow._context.Set <FollowedStream>()
.AsQueryable()
.Where(x => x.Type == kvp.Key && kvp.Value.Contains(x.Username))
.ToList();
uow._context.RemoveRange(toDelete);
uow.SaveChanges();
foreach (var loginToDelete in kvp.Value)
{
_streamTracker.UntrackStreamByKey(new StreamDataKey(kvp.Key, loginToDelete));
}
}
}
}
catch (Exception ex)
{
Log.Error("Error cleaning up FollowedStreams");
Log.Error(ex.ToString());
}
}, null, TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(30));
sub.Subscribe($"{_creds.RedisKey()}_follow_stream", HandleFollowStream);
sub.Subscribe($"{_creds.RedisKey()}_unfollow_stream", HandleUnfollowStream);
}
bot.JoinedGuild += ClientOnJoinedGuild;
client.LeftGuild += ClientOnLeftGuild;
}