public async Task CollectStats() { var stopwatch = new Stopwatch(); stopwatch.Start(); // Aggregate guild/channel stats _metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, _client.Guilds.Count); _metrics.Measure.Gauge.SetValue(BotMetrics.Channels, _client.Guilds.Sum(g => g.TextChannels.Count)); _metrics.Measure.Gauge.SetValue(BotMetrics.ShardsConnected, _client.Shards.Count(shard => shard.ConnectionState == ConnectionState.Connected)); // Aggregate member stats var usersKnown = new HashSet <ulong>(); var usersOnline = new HashSet <ulong>(); foreach (var guild in _client.Guilds) { foreach (var user in guild.Users) { usersKnown.Add(user.Id); if (user.Status == UserStatus.Online) { usersOnline.Add(user.Id); } } } _metrics.Measure.Gauge.SetValue(BotMetrics.MembersTotal, usersKnown.Count); _metrics.Measure.Gauge.SetValue(BotMetrics.MembersOnline, usersOnline.Count); // Aggregate DB stats _metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, await _data.GetTotalSystems()); _metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, await _data.GetTotalMembers()); _metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, await _data.GetTotalSwitches()); _metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, await _data.GetTotalMessages()); // Process info var process = Process.GetCurrentProcess(); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount); _metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await EstimateCpuUsage()); // Database info _metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount); // Other shiz _metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize); stopwatch.Stop(); _logger.Information("Updated metrics in {Time}", stopwatch.ElapsedDuration()); }
public async Task CollectStats() { var stopwatch = new Stopwatch(); stopwatch.Start(); // Aggregate guild/channel stats var guildCount = 0; var channelCount = 0; // No LINQ today, sorry await foreach (var guild in _cache.GetAllGuilds()) { guildCount++; foreach (var channel in _cache.GetGuildChannels(guild.Id)) { if (channel.Type == Channel.ChannelType.GuildText) { channelCount++; } } } _metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, guildCount); _metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount); // Aggregate DB stats var counts = await _db.Execute(c => c.QueryFirstAsync <Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages, (select count(*) from groups) as groups")); _metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems); _metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members); _metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches); _metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages); _metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.Groups); // Process info var process = Process.GetCurrentProcess(); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount); _metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await _cpu.EstimateCpuUsage()); // Database info _metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount); // Other shiz _metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize); stopwatch.Stop(); _logger.Debug("Updated metrics in {Time}", stopwatch.ElapsedDuration()); }
private async Task UpdatePeriodic() { _logger.Information("Running per-minute scheduled tasks."); var stopwatch = new Stopwatch(); stopwatch.Start(); _logger.Information("Updating database stats..."); await _repo.UpdateStats(); // Collect bot cluster statistics from Redis (if it's enabled) if (_useRedisMetrics) { await CollectBotStats(); } // Clean up message cache in postgres await CleanupOldMessages(); stopwatch.Stop(); _logger.Information("Ran scheduled tasks in {Time}", stopwatch.ElapsedDuration()); }
public async Task CollectStats() { var stopwatch = new Stopwatch(); stopwatch.Start(); // Aggregate guild/channel stats var guildCount = 0; var channelCount = 0; // No LINQ today, sorry foreach (var shard in _client.ShardClients.Values) { guildCount += shard.Guilds.Count; foreach (var guild in shard.Guilds.Values) { foreach (var channel in guild.Channels.Values) { if (channel.Type == ChannelType.Text) { channelCount++; } } } } _metrics.Measure.Gauge.SetValue(BotMetrics.Guilds, guildCount); _metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount); // Aggregate member stats var usersKnown = new HashSet <ulong>(); var usersOnline = new HashSet <ulong>(); foreach (var shard in _client.ShardClients.Values) { foreach (var guild in shard.Guilds.Values) { foreach (var user in guild.Members.Values) { usersKnown.Add(user.Id); if (user.Presence?.Status == UserStatus.Online) { usersOnline.Add(user.Id); } } } } _metrics.Measure.Gauge.SetValue(BotMetrics.MembersTotal, usersKnown.Count); _metrics.Measure.Gauge.SetValue(BotMetrics.MembersOnline, usersOnline.Count); // Aggregate DB stats var counts = await _db.Execute(c => c.QueryFirstAsync <Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages")); _metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems); _metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members); _metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches); _metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages); // Process info var process = Process.GetCurrentProcess(); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPhysicalMemory, process.WorkingSet64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessVirtualMemory, process.VirtualMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count); _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount); _metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await _cpu.EstimateCpuUsage()); // Database info _metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount); // Other shiz _metrics.Measure.Gauge.SetValue(BotMetrics.WebhookCacheSize, _webhookCache.CacheSize); stopwatch.Stop(); _logger.Information("Updated metrics in {Time}", stopwatch.ElapsedDuration()); }
public void Execute( Run run, IReadOnlyList <SalesArea> salesAreas, Action <string> raiseInfo, Action <string> raiseWarning, Action <string, Exception> raiseException ) { if (raiseInfo is null) { throw new ArgumentNullException(nameof(raiseInfo)); } if (raiseWarning is null) { throw new ArgumentNullException(nameof(raiseWarning)); } if (raiseException is null) { throw new ArgumentNullException(nameof(raiseException)); } if (run is null) { raiseWarning("Run passed to Smooth as null"); return; } if (salesAreas is null) { raiseWarning($"Sales area list passed to Smooth as null for (RunID={Log(run.Id)})"); return; } if (salesAreas.Count == 0) { raiseWarning($"No sales areas to execute Smooth for (RunID={Log(run.Id)})"); return; } IReadOnlyCollection <SmoothStatistics> smoothStatistics = PrepareSmoothStatisticsForSalesAreas(salesAreas); var smoothOutputs = new List <SmoothOutput>(); int smoothInstancesFailed = 0; var smoothStopWatch = new Stopwatch(); smoothStopWatch.Start(); try { DateTimeRange runPeriod = ( DateHelper.CreateStartDateTime(run.StartDate, run.StartTime), DateHelper.CreateEndDateTime(run.EndDate, run.EndTime) ); DateTime processorDateTime = DateTime.UtcNow; DateTimeRange smoothPeriod = ( DateHelper.CreateStartDateTime(run.SmoothDateRange.Start, run.StartTime), DateHelper.CreateEndDateTime(run.SmoothDateRange.End, run.EndTime) ); IReadOnlyCollection <string> salesAreaNames = new List <string>( salesAreas.Select(s => s.Name) ); ImmutableSmoothData threadSafeCollections = LoadSmoothData( _modelLoaders, salesAreaNames, smoothPeriod, raiseInfo); raiseInfo($"Smoothing {Log(salesAreas.Count)} sales areas (RunID={Log(run.Id)})"); LogSmoothConfiguration( run.Id, threadSafeCollections.SmoothConfiguration, _rootFolder, processorDateTime, runPeriod, smoothPeriod, raiseInfo, raiseWarning ); using (var processSmoothOutputMutex = new Mutex()) { _smoothEngine.OnSmoothBatchComplete += ( sender, salesArea, currentFromDateTime, currentToDateTime, dayRecommendations, smoothFailures) => { _ = processSmoothOutputMutex.WaitOne(); var timestamp = $"{Log(currentFromDateTime)} - {Log(currentToDateTime)} (RunID={Log(run.Id)})"; try { raiseInfo($"Processed Smooth output for sales area {salesArea.Name} for {timestamp}"); } catch (Exception exception) { raiseException( $"Error processing Smooth output for sales area {salesArea.Name} for {timestamp}", exception ); } finally { processSmoothOutputMutex.ReleaseMutex(); } }; // Define handler for Smooth output. _smoothEngine.OnSmoothComplete += ( sender, salesArea, exception, smoothOutput) => { smoothOutputs.Add(smoothOutput); var salesAreaName = salesArea.Name; if (exception is null) { SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == salesAreaName); smoothStatistic.TimeEnded = DateTime.UtcNow; TimeSpan elapsedTime = smoothStatistic.TimeEnded - smoothStatistic.TimeStarted; var info = new StringBuilder(320); _ = info .Append("Completed Smooth processing") .Append(": RunID=") .Append(Log(run.Id)) .Append(": Sales Area=") .Append(salesAreaName) .Append(": Started=") .Append(Log(smoothStatistic.TimeStarted)) .Append(": Ended=") .Append(Log(smoothStatistic.TimeEnded)) .Append(": Elapsed=") .Append(Log((long)elapsedTime.TotalSeconds) + "s") .Append(": Breaks=") .Append(smoothOutput.Breaks.ToString()) .Append(": Spots set=") .Append(smoothOutput.SpotsSet.ToString()) .Append(": Spots not set=") .Append(smoothOutput.SpotsNotSet.ToString()) .Append(": Recommendations=") .Append(smoothOutput.Recommendations.ToString()) ; raiseInfo(info.ToString()); return; } try { _ = processSmoothOutputMutex.WaitOne(); smoothInstancesFailed++; raiseException( $"Error executing Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})", exception); if (exception is AggregateException aggEx) { AggregateException flatEx = aggEx.Flatten(); raiseException( $"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})", flatEx ); } else { raiseException( $"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})", exception ); if (exception.InnerException != null) { raiseException( $"Guru meditation: Exception during Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})", exception ); } } } catch (Exception exception2) { raiseException( $"Guru meditation: Exception during outputting Smooth exception for sales area {salesArea.Name} (RunID={Log(run.Id)})", exception2 ); } finally { processSmoothOutputMutex.ReleaseMutex(); } }; _ = Parallel.ForEach(salesAreas, salesArea => { string salesAreaName = salesArea.Name; raiseInfo($"Executing Smooth for sales area {salesAreaName} (RunID={Log(run.Id)})"); SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == salesAreaName); smoothStatistic.TimeStarted = DateTime.UtcNow; _smoothEngine.SmoothSalesAreaForDateTimePeriod( run.Id, run.Scenarios[0].Id, salesArea, processorDateTime, smoothPeriod, threadSafeCollections, raiseInfo, raiseWarning, raiseException ); }); } } finally { smoothStopWatch.Stop(); var allSmoothOutput = new SmoothOutput() { SpotsByFailureMessage = new Dictionary <int, int>() }; foreach (var smoothOutput in smoothOutputs.Where(o => o != null)) { allSmoothOutput.Append(smoothOutput); SmoothStatistics smoothStatistic = smoothStatistics.First(ss => ss.SalesAreaName == smoothOutput.SalesAreaName); TimeSpan elapsedTimeSalesArea = smoothStatistic.TimeEnded - smoothStatistic.TimeStarted; var passDetails = new StringBuilder(); foreach (var passSequence in smoothOutput.OutputByPass.Keys) { _ = passDetails .AppendFormat("(Pass={0}, ", smoothOutput.OutputByPass[passSequence].PassSequence.ToString()) .AppendFormat("Spots set={0}); ", smoothOutput.OutputByPass[passSequence].CountSpotsSet.ToString()); } // Remove the trailing space from the string. passDetails.Length--; // Log final statistics var finalStats = new StringBuilder(800); _ = finalStats .Append("Smooth results") .Append(": RunID=" + run.Id.ToString()) .Append(": Sales area=" + smoothOutput.SalesAreaName) .Append(": Started=" + Log(smoothStatistic.TimeStarted)) .Append(": Ended=" + Log(smoothStatistic.TimeEnded)) .Append(": Elapsed=" + Log((long)elapsedTimeSalesArea.TotalSeconds) + "s") .Append(": Breaks=" + smoothOutput.Breaks.ToString()) .Append(": Breaks with reduced Optimizer availability for unplaced spots=" + smoothOutput.BreaksWithReducedOptimizerAvailForUnplacedSpots.ToString()) .Append(": Spots set=" + smoothOutput.SpotsSet.ToString()) .Append(": Spots set after moving other spots=" + smoothOutput.SpotsSetAfterMovingOtherSpots.ToString()) .Append(": Spots not set=" + smoothOutput.SpotsNotSet.ToString()) .Append(": Spots not set due to excluded campaign=" + smoothOutput.SpotsNotSetDueToExternalCampaignRef.ToString()) .Append(": Booked spots unplaced due to restrictions=" + smoothOutput.BookedSpotsUnplacedDueToRestrictions.ToString()) .Append(": Recommendations=" + smoothOutput.Recommendations.ToString()) .Append(": Failures=" + smoothOutput.Failures.ToString()) .Append(": Passes Results=" + passDetails.ToString()) ; raiseInfo(finalStats.ToString()); } LogSmoothFailureMessages(allSmoothOutput.SpotsByFailureMessage, raiseInfo); var completedSmoothStats = new StringBuilder(512); _ = completedSmoothStats .Append("Completed Smooth") .Append(": RunID=" + Log(run.Id)) .Append(": Elapsed=" + Log(smoothStopWatch.ElapsedDuration()) + "s") .Append(": Breaks=" + allSmoothOutput.Breaks.ToString()) .Append(": Breaks with reduced Optimizer availability for unplaced spots=" + allSmoothOutput.BreaksWithReducedOptimizerAvailForUnplacedSpots.ToString()) .Append(": Spots set=" + allSmoothOutput.SpotsSet.ToString()) .Append(": Spots set after moving other spots=" + allSmoothOutput.SpotsSetAfterMovingOtherSpots.ToString()) .Append(": Spots not set=" + allSmoothOutput.SpotsNotSet.ToString()) .Append(": Spots not set due to excluded campaign=" + allSmoothOutput.SpotsNotSetDueToExternalCampaignRef.ToString()) .Append(": Booked spots unplaced due to restrictions=" + allSmoothOutput.BookedSpotsUnplacedDueToRestrictions.ToString()) .Append(": Recommendations=" + allSmoothOutput.Recommendations.ToString()) .Append(": Failures=" + allSmoothOutput.Failures.ToString()) .Append(": Fail Smooth instance=" + smoothInstancesFailed.ToString()) ; raiseInfo(completedSmoothStats.ToString()); } }