public ShardsCoordinator()
{
//load main stuff
LogSetup.SetupLogger();
_log = LogManager.GetCurrentClassLogger();
_creds = new BotCredentials();
_log.Info("Starting NadekoBot v" + StatsService.BotVersion);
_key = _creds.RedisKey();
_redis = ConnectionMultiplexer.Connect("127.0.0.1");
//setup initial shard statuses
_defaultShardState = new ShardComMessage()
{
ConnectionState = Discord.ConnectionState.Disconnected,
Guilds = 0,
Time = DateTime.Now - TimeSpan.FromMinutes(1)
};
var db = _redis.GetDatabase();
_shardProcesses = new Process[_creds.TotalShards];
for (int i = 0; i < _creds.TotalShards; i++)
{
_defaultShardState.ShardId = i;
db.ListRightPush(_key + "_shardstats",
JsonConvert.SerializeObject(_defaultShardState),
flags: CommandFlags.FireAndForget);
}
_curProcessId = Process.GetCurrentProcess().Id;
_redis = ConnectionMultiplexer.Connect("127.0.0.1");
var sub = _redis.GetSubscriber();
sub.Subscribe(_key + "_shardcoord_send",
OnDataReceived,
CommandFlags.FireAndForget);
sub.Subscribe(_key + "_shardcoord_restart",
OnRestart,
CommandFlags.FireAndForget);
sub.Subscribe(_key + "_shardcoord_stop",
OnStop,
CommandFlags.FireAndForget);
}
public ShardsCoordinator()
{
//load main stuff
LogSetup.SetupLogger(-1);
_log = LogManager.GetCurrentClassLogger();
_creds = new BotCredentials();
_log.Info("Starting NadekoBot v" + StatsService.BotVersion);
_key = _creds.RedisKey();
_redis = ConnectionMultiplexer.Connect("127.0.0.1");
new RedisImagesCache(_redis, _creds).Reload(); //reload images into redis
//setup initial shard statuses
_defaultShardState = new ShardComMessage()
{
ConnectionState = Discord.ConnectionState.Disconnected,
Guilds = 0,
Time = DateTime.UtcNow
};
var db = _redis.GetDatabase();
//clear previous statuses
db.KeyDelete(_key + "_shardstats");
_shardProcesses = new Process[_creds.TotalShards];
for (int i = 0; i < _creds.TotalShards; i++)
{
//add it to the list of shards which should be started
#if DEBUG
if (i > 0)
{
_shardStartQueue.Enqueue(i);
}
else
{
_shardProcesses[i] = Process.GetCurrentProcess();
}
#else
_shardStartQueue.Enqueue(i);
#endif
//set the shard's initial state in redis cache
var msg = _defaultShardState.Clone();
msg.ShardId = i;
//this is to avoid the shard coordinator thinking that
//the shard is unresponsive while starting up
var delay = 45;
#if GLOBAL_NADEKO
delay = 180;
#endif
msg.Time = DateTime.UtcNow + TimeSpan.FromSeconds(delay * (i + 1));
db.ListRightPush(_key + "_shardstats",
JsonConvert.SerializeObject(msg),
flags: CommandFlags.FireAndForget);
}
_curProcessId = Process.GetCurrentProcess().Id;
//subscribe to shardcoord events
var sub = _redis.GetSubscriber();
//send is called when shard status is updated. Every 7.5 seconds atm
sub.Subscribe(_key + "_shardcoord_send",
OnDataReceived,
CommandFlags.FireAndForget);
//called to stop the shard, although the shard will start again when it finds out it's dead
sub.Subscribe(_key + "_shardcoord_stop",
OnStop,
CommandFlags.FireAndForget);
//called kill the bot
sub.Subscribe(_key + "_die",
(ch, x) => Environment.Exit(0),
CommandFlags.FireAndForget);
}
public async Task RunAsync()
{
//this task will complete when the initial start of the shards
//is complete, but will keep running in order to restart shards
//which are disconnected for too long
TaskCompletionSource <bool> tsc = new TaskCompletionSource <bool>();
var _ = Task.Run(async() =>
{
do
{
//start a shard which is scheduled for start every 6 seconds
while (_shardStartQueue.TryPeek(out var id))
{
// if the shard is on the waiting list again
// remove it since it's starting up now
_shardRestartWaitingList.TryRemove(id);
//if the task is already completed,
//it means the initial shard starting is done,
//and this is an auto-restart
if (tsc.Task.IsCompleted)
{
_log.Warn("Auto-restarting shard {0}", id);
}
var rem = _shardProcesses[id];
if (rem != null)
{
try
{
rem.KillTree();
rem.Dispose();
}
catch { }
}
_shardProcesses[id] = StartShard(id);
_shardStartQueue.TryDequeue(out var __);
await Task.Delay(6000).ConfigureAwait(false);
}
tsc.TrySetResult(true);
await Task.Delay(6000).ConfigureAwait(false);
}while (true);
// ^ keep checking for shards which need to be restarted
});
//restart unresponsive shards
_ = Task.Run(async() =>
{
//after all shards have started initially
await tsc.Task.ConfigureAwait(false);
while (true)
{
await Task.Delay(15000).ConfigureAwait(false);
try
{
var db = _redis.GetDatabase();
//get all shards which didn't communicate their status in the last 30 seconds
var all = db.ListRange(_creds.RedisKey() + "_shardstats")
.Select(x => JsonConvert.DeserializeObject <ShardComMessage>(x));
var statuses = all
.Where(x => x.Time < DateTime.UtcNow - TimeSpan.FromSeconds(30));
if (!statuses.Any())
{
#if DEBUG
for (var i = 0; i < _shardProcesses.Length; i++)
{
var p = _shardProcesses[i];
if (p == null || p.HasExited)
{
_log.Warn("Scheduling shard {0} for restart because it's process is stopped.", i);
_shardStartQueue.Enqueue(i);
}
}
#endif
}
else
{
foreach (var s in statuses)
{
OnStop(s.ShardId);
_shardStartQueue.Enqueue(s.ShardId);
//to prevent shards which are already scheduled for restart to be scheduled again
s.Time = DateTime.UtcNow + TimeSpan.FromSeconds(30 * _shardStartQueue.Count);
db.ListSetByIndex(_key + "_shardstats", s.ShardId,
JsonConvert.SerializeObject(s), CommandFlags.FireAndForget);
_log.Warn("Shard {0} is scheduled for a restart because it's unresponsive.", s.ShardId);
}
}
}
catch (Exception ex) { _log.Error(ex); throw; }
}
});
await tsc.Task.ConfigureAwait(false);
return;
}
public ShardsCoordinator()
{
//load main stuff
LogSetup.SetupLogger(-1);
_log = LogManager.GetCurrentClassLogger();
_creds = new BotCredentials();
_log.Info("Starting NadekoBot v" + StatsService.BotVersion);
_key = _creds.RedisKey();
_log.Info("Redis options: " + _creds.RedisOptions);
var conf = ConfigurationOptions.Parse(_creds.RedisOptions);
_redis = ConnectionMultiplexer.Connect(conf);
var imgCache = new RedisImagesCache(_redis, _creds); //reload images into redis
if (!imgCache.AllKeysExist().GetAwaiter().GetResult()) // but only if the keys don't exist. If images exist, you have to reload them manually
{
imgCache.Reload().GetAwaiter().GetResult();
}
else
{
_log.Info("Images are already present in redis. Use .imagesreload to force update if needed.");
}
//setup initial shard statuses
_defaultShardState = new ShardComMessage()
{
ConnectionState = Discord.ConnectionState.Disconnected,
Guilds = 0,
Time = DateTime.UtcNow
};
var db = _redis.GetDatabase();
//clear previous statuses
db.KeyDelete(_key + "_shardstats");
_shardProcesses = new Process[_creds.TotalShards];
var shardIds = Enumerable.Range(1, _creds.TotalShards - 1)
.Shuffle()
.Prepend(0)
.ToArray();
for (var i = 0; i < shardIds.Length; i++)
{
var id = shardIds[i];
//add it to the list of shards which should be started
#if DEBUG
if (id > 0)
{
_shardStartQueue.Enqueue(id);
}
else
{
_shardProcesses[id] = Process.GetCurrentProcess();
}
#else
_shardStartQueue.Enqueue(id);
#endif
//set the shard's initial state in redis cache
var msg = _defaultShardState.Clone();
msg.ShardId = id;
//this is to avoid the shard coordinator thinking that
//the shard is unresponsive while starting up
var delay = 45;
#if GLOBAL_NADEKO
delay = 180;
#endif
msg.Time = DateTime.UtcNow + TimeSpan.FromSeconds(delay * (id + 1));
db.ListRightPush(_key + "_shardstats",
JsonConvert.SerializeObject(msg),
flags: CommandFlags.FireAndForget);
}
_curProcessId = Process.GetCurrentProcess().Id;
//subscribe to shardcoord events
var sub = _redis.GetSubscriber();
//send is called when shard status is updated. Every 7.5 seconds atm
sub.Subscribe(_key + "_shardcoord_send",
OnDataReceived,
CommandFlags.FireAndForget);
//called to stop the shard, although the shard will start again when it finds out it's dead
sub.Subscribe(_key + "_shardcoord_stop",
OnStop,
CommandFlags.FireAndForget);
//called kill the bot
sub.Subscribe(_key + "_die",
(ch, x) => Environment.Exit(0),
CommandFlags.FireAndForget);
}