public async Task CollectStats()
{
var stopwatch = new Stopwatch();
stopwatch.Start();
// Aggregate guild/channel stats
_metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, _client.Guilds.Count);
_metrics.Measure.Gauge.SetValue(BotMetrics.Channels, _client.Guilds.Sum(g => g.TextChannels.Count));
_metrics.Measure.Gauge.SetValue(BotMetrics.ShardsConnected, _client.Shards.Count(shard => shard.ConnectionState == ConnectionState.Connected));
// Aggregate member stats
var usersKnown = new HashSet <ulong>();
var usersOnline = new HashSet <ulong>();
foreach (var guild in _client.Guilds)
{
foreach (var user in guild.Users)
{
usersKnown.Add(user.Id);
if (user.Status == UserStatus.Online)
{
usersOnline.Add(user.Id);
}
}
}
_metrics.Measure.Gauge.SetValue(BotMetrics.MembersTotal, usersKnown.Count);
_metrics.Measure.Gauge.SetValue(BotMetrics.MembersOnline, usersOnline.Count);
// Aggregate DB stats
_metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, await _data.GetTotalSystems());
_metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, await _data.GetTotalMembers());
_metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, await _data.GetTotalSwitches());
_metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, await _data.GetTotalMessages());
// Process info
var process = Process.GetCurrentProcess();
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await EstimateCpuUsage());
// Database info
_metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount);
// Other shiz
_metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize);
stopwatch.Stop();
_logger.Information("Updated metrics in {Time}", stopwatch.ElapsedDuration());
}
public async Task CollectStats()
{
var stopwatch = new Stopwatch();
stopwatch.Start();
// Aggregate guild/channel stats
var guildCount = 0;
var channelCount = 0;
// No LINQ today, sorry
await foreach (var guild in _cache.GetAllGuilds())
{
guildCount++;
foreach (var channel in _cache.GetGuildChannels(guild.Id))
{
if (channel.Type == Channel.ChannelType.GuildText)
{
channelCount++;
}
}
}
_metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, guildCount);
_metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount);
// Aggregate DB stats
var counts = await _db.Execute(c => c.QueryFirstAsync <Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages, (select count(*) from groups) as groups"));
_metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members);
_metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages);
_metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.Groups);
// Process info
var process = Process.GetCurrentProcess();
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await _cpu.EstimateCpuUsage());
// Database info
_metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount);
// Other shiz
_metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize);
stopwatch.Stop();
_logger.Debug("Updated metrics in {Time}", stopwatch.ElapsedDuration());
}
private async Task UpdatePeriodic()
{
_logger.Information("Running per-minute scheduled tasks.");
var stopwatch = new Stopwatch();
stopwatch.Start();
_logger.Information("Updating database stats...");
await _repo.UpdateStats();
// Collect bot cluster statistics from Redis (if it's enabled)
if (_useRedisMetrics)
{
await CollectBotStats();
}
// Clean up message cache in postgres
await CleanupOldMessages();
stopwatch.Stop();
_logger.Information("Ran scheduled tasks in {Time}", stopwatch.ElapsedDuration());
}
public async Task CollectStats()
{
var stopwatch = new Stopwatch();
stopwatch.Start();
// Aggregate guild/channel stats
var guildCount = 0;
var channelCount = 0;
// No LINQ today, sorry
foreach (var shard in _client.ShardClients.Values)
{
guildCount += shard.Guilds.Count;
foreach (var guild in shard.Guilds.Values)
{
foreach (var channel in guild.Channels.Values)
{
if (channel.Type == ChannelType.Text)
{
channelCount++;
}
}
}
}
_metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, guildCount);
_metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount);
// Aggregate member stats
var usersKnown = new HashSet <ulong>();
var usersOnline = new HashSet <ulong>();
foreach (var shard in _client.ShardClients.Values)
{
foreach (var guild in shard.Guilds.Values)
{
foreach (var user in guild.Members.Values)
{
usersKnown.Add(user.Id);
if (user.Presence?.Status == UserStatus.Online)
{
usersOnline.Add(user.Id);
}
}
}
}
_metrics.Measure.Gauge.SetValue(BotMetrics.MembersTotal, usersKnown.Count);
_metrics.Measure.Gauge.SetValue(BotMetrics.MembersOnline, usersOnline.Count);
// Aggregate DB stats
var counts = await _db.Execute(c => c.QueryFirstAsync <Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages"));
_metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members);
_metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages);
// Process info
var process = Process.GetCurrentProcess();
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count);
_metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await _cpu.EstimateCpuUsage());
// Database info
_metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount);
// Other shiz
_metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize);
stopwatch.Stop();
_logger.Information("Updated metrics in {Time}", stopwatch.ElapsedDuration());
}
public void Execute(
Run run,
IReadOnlyList <SalesArea> salesAreas,
Action <string> raiseInfo,
Action <string> raiseWarning,
Action <string, Exception> raiseException
)
{
if (raiseInfo is null)
{
throw new ArgumentNullException(nameof(raiseInfo));
}
if (raiseWarning is null)
{
throw new ArgumentNullException(nameof(raiseWarning));
}
if (raiseException is null)
{
throw new ArgumentNullException(nameof(raiseException));
}
if (run is null)
{
raiseWarning("Run passed to Smooth as null");
return;
}
if (salesAreas is null)
{
raiseWarning($"Sales area list passed to Smooth as null for (RunID={Log(run.Id)})");
return;
}
if (salesAreas.Count == 0)
{
raiseWarning($"No sales areas to execute Smooth for (RunID={Log(run.Id)})");
return;
}
IReadOnlyCollection <SmoothStatistics> smoothStatistics = PrepareSmoothStatisticsForSalesAreas(salesAreas);
var smoothOutputs = new List <SmoothOutput>();
int smoothInstancesFailed = 0;
var smoothStopWatch = new Stopwatch();
smoothStopWatch.Start();
try
{
DateTimeRange runPeriod = (
DateHelper.CreateStartDateTime(run.StartDate, run.StartTime),
DateHelper.CreateEndDateTime(run.EndDate, run.EndTime)
);
DateTime processorDateTime = DateTime.UtcNow;
DateTimeRange smoothPeriod = (
DateHelper.CreateStartDateTime(run.SmoothDateRange.Start, run.StartTime),
DateHelper.CreateEndDateTime(run.SmoothDateRange.End, run.EndTime)
);
IReadOnlyCollection <string> salesAreaNames = new List <string>(
salesAreas.Select(s => s.Name)
);
ImmutableSmoothData threadSafeCollections = LoadSmoothData(
_modelLoaders,
salesAreaNames,
smoothPeriod,
raiseInfo);
raiseInfo($"Smoothing {Log(salesAreas.Count)} sales areas (RunID={Log(run.Id)})");
LogSmoothConfiguration(
run.Id,
threadSafeCollections.SmoothConfiguration,
_rootFolder,
processorDateTime,
runPeriod,
smoothPeriod,
raiseInfo,
raiseWarning
);
using (var processSmoothOutputMutex = new Mutex())
{
_smoothEngine.OnSmoothBatchComplete += (
sender,
salesArea,
currentFromDateTime,
currentToDateTime,
dayRecommendations,
smoothFailures) =>
{
_ = processSmoothOutputMutex.WaitOne();
var timestamp = $"{Log(currentFromDateTime)} - {Log(currentToDateTime)} (RunID={Log(run.Id)})";
try
{
raiseInfo($"Processed Smooth output for sales area {salesArea.Name} for {timestamp}");
}
catch (Exception exception)
{
raiseException(
$"Error processing Smooth output for sales area {salesArea.Name} for {timestamp}",
exception
);
}
finally
{
processSmoothOutputMutex.ReleaseMutex();
}
};
// Define handler for Smooth output.
_smoothEngine.OnSmoothComplete += (
sender,
salesArea,
exception,
smoothOutput) =>
{
smoothOutputs.Add(smoothOutput);
var salesAreaName = salesArea.Name;
if (exception is null)
{
SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == salesAreaName);
smoothStatistic.TimeEnded = DateTime.UtcNow;
TimeSpan elapsedTime = smoothStatistic.TimeEnded - smoothStatistic.TimeStarted;
var info = new StringBuilder(320);
_ = info
.Append("Completed Smooth processing")
.Append(": RunID=")
.Append(Log(run.Id))
.Append(": Sales Area=")
.Append(salesAreaName)
.Append(": Started=")
.Append(Log(smoothStatistic.TimeStarted))
.Append(": Ended=")
.Append(Log(smoothStatistic.TimeEnded))
.Append(": Elapsed=")
.Append(Log((long)elapsedTime.TotalSeconds) + "s")
.Append(": Breaks=")
.Append(smoothOutput.Breaks.ToString())
.Append(": Spots set=")
.Append(smoothOutput.SpotsSet.ToString())
.Append(": Spots not set=")
.Append(smoothOutput.SpotsNotSet.ToString())
.Append(": Recommendations=")
.Append(smoothOutput.Recommendations.ToString())
;
raiseInfo(info.ToString());
return;
}
try
{
_ = processSmoothOutputMutex.WaitOne();
smoothInstancesFailed++;
raiseException(
$"Error executing Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})",
exception);
if (exception is AggregateException aggEx)
{
AggregateException flatEx = aggEx.Flatten();
raiseException(
$"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})",
flatEx
);
}
else
{
raiseException(
$"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})",
exception
);
if (exception.InnerException != null)
{
raiseException(
$"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})",
exception
);
}
}
}
catch (Exception exception2)
{
raiseException(
$"Guru meditation: Exception during outputting Smooth exception for sales area {salesArea.Name} (RunID={Log(run.Id)})",
exception2
);
}
finally
{
processSmoothOutputMutex.ReleaseMutex();
}
};
_ = Parallel.ForEach(salesAreas, salesArea =>
{
string salesAreaName = salesArea.Name;
raiseInfo($"Executing Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})");
SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == salesAreaName);
smoothStatistic.TimeStarted = DateTime.UtcNow;
_smoothEngine.SmoothSalesAreaForDateTimePeriod(
run.Id,
run.Scenarios[0].Id,
salesArea,
processorDateTime,
smoothPeriod,
threadSafeCollections,
raiseInfo,
raiseWarning,
raiseException
);
});
}
}
finally
{
smoothStopWatch.Stop();
var allSmoothOutput = new SmoothOutput()
{
SpotsByFailureMessage = new Dictionary <int, int>()
};
foreach (var smoothOutput in smoothOutputs.Where(o => o != null))
{
allSmoothOutput.Append(smoothOutput);
SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == smoothOutput.SalesAreaName);
TimeSpan elapsedTimeSalesArea = smoothStatistic.TimeEnded - smoothStatistic.TimeStarted;
var passDetails = new StringBuilder();
foreach (var passSequence in smoothOutput.OutputByPass.Keys)
{
_ = passDetails
.AppendFormat("(Pass={0}, ", smoothOutput.OutputByPass[passSequence].PassSequence.ToString())
.AppendFormat("Spots set={0}); ", smoothOutput.OutputByPass[passSequence].CountSpotsSet.ToString());
}
// Remove the trailing space from the string.
passDetails.Length--;
// Log final statistics
var finalStats = new StringBuilder(800);
_ = finalStats
.Append("Smooth results")
.Append(": RunID=" + run.Id.ToString())
.Append(": Sales area=" + smoothOutput.SalesAreaName)
.Append(": Started=" + Log(smoothStatistic.TimeStarted))
.Append(": Ended=" + Log(smoothStatistic.TimeEnded))
.Append(": Elapsed=" + Log((long)elapsedTimeSalesArea.TotalSeconds) + "s")
.Append(": Breaks=" + smoothOutput.Breaks.ToString())
.Append(": Breaks with reduced Optimizer availability for unplaced spots=" + smoothOutput.BreaksWithReducedOptimizerAvailForUnplacedSpots.ToString())
.Append(": Spots set=" + smoothOutput.SpotsSet.ToString())
.Append(": Spots set after moving other spots=" + smoothOutput.SpotsSetAfterMovingOtherSpots.ToString())
.Append(": Spots not set=" + smoothOutput.SpotsNotSet.ToString())
.Append(": Spots not set due to excluded campaign=" + smoothOutput.SpotsNotSetDueToExternalCampaignRef.ToString())
.Append(": Booked spots unplaced due to restrictions=" + smoothOutput.BookedSpotsUnplacedDueToRestrictions.ToString())
.Append(": Recommendations=" + smoothOutput.Recommendations.ToString())
.Append(": Failures=" + smoothOutput.Failures.ToString())
.Append(": Passes Results=" + passDetails.ToString())
;
raiseInfo(finalStats.ToString());
}
LogSmoothFailureMessages(allSmoothOutput.SpotsByFailureMessage, raiseInfo);
var completedSmoothStats = new StringBuilder(512);
_ = completedSmoothStats
.Append("Completed Smooth")
.Append(": RunID=" + Log(run.Id))
.Append(": Elapsed=" + Log(smoothStopWatch.ElapsedDuration()) + "s")
.Append(": Breaks=" + allSmoothOutput.Breaks.ToString())
.Append(": Breaks with reduced Optimizer availability for unplaced spots=" + allSmoothOutput.BreaksWithReducedOptimizerAvailForUnplacedSpots.ToString())
.Append(": Spots set=" + allSmoothOutput.SpotsSet.ToString())
.Append(": Spots set after moving other spots=" + allSmoothOutput.SpotsSetAfterMovingOtherSpots.ToString())
.Append(": Spots not set=" + allSmoothOutput.SpotsNotSet.ToString())
.Append(": Spots not set due to excluded campaign=" + allSmoothOutput.SpotsNotSetDueToExternalCampaignRef.ToString())
.Append(": Booked spots unplaced due to restrictions=" + allSmoothOutput.BookedSpotsUnplacedDueToRestrictions.ToString())
.Append(": Recommendations=" + allSmoothOutput.Recommendations.ToString())
.Append(": Failures=" + allSmoothOutput.Failures.ToString())
.Append(": Fail Smooth instance=" + smoothInstancesFailed.ToString())
;
raiseInfo(completedSmoothStats.ToString());
}
}
