public void RegisterServerStats() { // lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); PerformanceCounter tempPC; Stat tempStat; string tempName; try { tempName = "CPUPercent"; tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); processorPercentPerfCounter = new PerfCounterControl(tempPC); // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor, StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); }, StatVerbosity.Info); StatsManager.RegisterStat(tempStat); RegisteredStats.Add(tempName, tempStat); MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; }); MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; }); MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; }); MakeStat("Threads", null, "threads", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); } catch (Exception e) { m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); } MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = workerThreads; }); MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = iocpThreads; }); if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null) { MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads); MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads); MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems); MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads); MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads); MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks); } MakeStat( "HTTPRequestsMade", "Number of outbound HTTP requests made", "requests", ContainerNetwork, s => s.Value = WebUtil.RequestNumber, MeasuresOfInterest.AverageChangeOverTime); try { List <string> okInterfaceTypes = new List <string>(NetworkInterfaceTypes.Split(',')); IEnumerable <NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces(); foreach (NetworkInterface nic in nics) { if (nic.OperationalStatus != OperationalStatus.Up) { continue; } string nicInterfaceType = nic.NetworkInterfaceType.ToString(); if (!okInterfaceTypes.Contains(nicInterfaceType)) { m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.", LogHeader, nic.Name, nicInterfaceType); m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}", LogHeader, NetworkInterfaceTypes); continue; } if (nic.Supports(NetworkInterfaceComponent.IPv4)) { IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); if (nicStats != null) { MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return(ns.BytesReceived); }, 1024.0); }); MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return(ns.BytesSent); }, 1024.0); }); MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return(ns.BytesSent + ns.BytesReceived); }, 1024.0); }); } } // TODO: add IPv6 (it may actually happen someday) } } catch (Exception e) { m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); } MakeStat("ProcessMemory", null, "MB", ContainerMemory, (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); }); MakeStat("HeapMemory", null, "MB", ContainerMemory, (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); }); MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); }
/// <summary> /// Check watched threads. Fire alarm if appropriate. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { int now = Environment.TickCount & Int32.MaxValue; int msElapsed = now - LastWatchdogThreadTick; if (msElapsed > WATCHDOG_INTERVAL_MS * 2) { m_log.WarnFormat( "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms", msElapsed, WATCHDOG_INTERVAL_MS); } LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout; if (callback != null) { List <ThreadWatchdogInfo> callbackInfos = null; lock (m_threads) { foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) { if (threadInfo.Thread.ThreadState == ThreadState.Stopped) { RemoveThread(threadInfo.Thread.ManagedThreadId); if (callbackInfos == null) { callbackInfos = new List <ThreadWatchdogInfo>(); } callbackInfos.Add(threadInfo); } else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { threadInfo.IsTimedOut = true; if (threadInfo.AlarmIfTimeout) { if (callbackInfos == null) { callbackInfos = new List <ThreadWatchdogInfo>(); } // Send a copy of the watchdog info to prevent race conditions where the watchdog // thread updates the monitoring info after an alarm has been sent out. callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); } } } } if (callbackInfos != null) { foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) { callback(callbackInfo); } } } if (MemoryWatchdog.Enabled) { MemoryWatchdog.Update(); } StatsManager.RecordStats(); m_watchdogTimer.Start(); }
/// <summary> /// Report back collected statistical information. /// </summary> /// <returns></returns> public override string Report() { StringBuilder sb = new StringBuilder(Environment.NewLine); // sb.Append("ASSET STATISTICS"); // sb.Append(Environment.NewLine); /* * sb.Append( * string.Format( * @"Asset cache contains {0,6} non-texture assets using {1,10} K * Texture cache contains {2,6} texture assets using {3,10} K * Latest asset request time after cache miss: {4}s * Blocked client requests for missing textures: {5} * Asset service request failures: {6}"+ Environment.NewLine, * AssetsInCache, Math.Round(AssetCacheMemoryUsage / 1024.0), * TexturesInCache, Math.Round(TextureCacheMemoryUsage / 1024.0), * assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0, * BlockedMissingTextureRequests, * AssetServiceRequestFailures)); */ /* * sb.Append( * string.Format( * @"Asset cache contains {0,6} assets * Latest asset request time after cache miss: {1}s * Blocked client requests for missing textures: {2} * Asset service request failures: {3}" + Environment.NewLine, * AssetsInCache, * assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0, * BlockedMissingTextureRequests, * AssetServiceRequestFailures)); */ sb.Append(Environment.NewLine); sb.Append("CONNECTION STATISTICS"); sb.Append(Environment.NewLine); List <Stat> stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives"); sb.AppendFormat( "Client logouts due to no data receive timeout: {0}\n\n", stats != null ? stats.Sum(s => s.Value).ToString() : "unknown"); // sb.Append(Environment.NewLine); // sb.Append("INVENTORY STATISTICS"); // sb.Append(Environment.NewLine); // sb.Append( // string.Format( // "Initial inventory caching failures: {0}" + Environment.NewLine, // InventoryServiceRetrievalFailures)); sb.Append(Environment.NewLine); sb.Append("SAMPLE FRAME STATISTICS"); sb.Append(Environment.NewLine); sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS"); sb.Append(Environment.NewLine); sb.Append( string.Format( "{0,6:0.00} {1,6:0} {2,6:0.0} {3,6:0.0} {4,6:0} {5,6:0} {6,6:0} {7,6:0} {8,6:0} {9,6:0}", timeDilation, simFps, physicsFps, agentUpdates, rootAgents, childAgents, totalPrims, activePrims, activeScripts, scriptLinesPerSecond)); sb.Append(Environment.NewLine); sb.Append(Environment.NewLine); // There is no script frame time currently because we don't yet collect it sb.Append("PktsIn PktOut PendDl PendUl UnackB TotlFt NetFt PhysFt OthrFt AgntFt ImgsFt"); sb.Append(Environment.NewLine); sb.Append( string.Format( "{0,6:0} {1,6:0} {2,6:0} {3,6:0} {4,6:0} {5,6:0.0} {6,6:0.0} {7,6:0.0} {8,6:0.0} {9,6:0.0} {10,6:0.0}\n\n", inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime, netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime)); /* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by * the two formatted printouts above. * SortedDictionary<string, SortedDictionary<string, Stat>> sceneStats; * if (StatsManager.TryGetStats("scene", out sceneStats)) * { * foreach (KeyValuePair<string, SortedDictionary<string, Stat>> kvp in sceneStats) * { * foreach (Stat stat in kvp.Value.Values) * { * if (stat.Verbosity == StatVerbosity.Info) * { * sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName); * } * } * } * } */ /* * sb.Append(Environment.NewLine); * sb.Append("PACKET QUEUE STATISTICS"); * sb.Append(Environment.NewLine); * sb.Append("Agent UUID "); * sb.Append( * string.Format( * " {0,7} {1,7} {2,7} {3,7} {4,7} {5,7} {6,7} {7,7} {8,7} {9,7}", * "Send", "In", "Out", "Resend", "Land", "Wind", "Cloud", "Task", "Texture", "Asset")); * sb.Append(Environment.NewLine); * * foreach (UUID key in packetQueueStatsCollectors.Keys) * { * sb.Append(string.Format("{0}: ", key)); * sb.Append(packetQueueStatsCollectors[key].Report()); * sb.Append(Environment.NewLine); * } */ sb.Append(base.Report()); return(sb.ToString()); }
public void Cleanup() { StatsManager.DeregisterStat(Stat); }
/// <summary> /// Check watched threads. Fire alarm if appropriate. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private static void WatchdogTimerElapsed(object sender) { if (!m_enabled) { return; } int now = Environment.TickCount & Int32.MaxValue; int msElapsed = now - LastWatchdogThreadTick; if (msElapsed > WATCHDOG_INTERVAL_MS * 2) { m_log.WarnFormat( "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms", msElapsed, WATCHDOG_INTERVAL_MS); } LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout; if (callback != null) { List <ThreadWatchdogInfo> callbackInfos = null; List <ThreadWatchdogInfo> threadsToRemove = null; const ThreadState thgone = ThreadState.Stopped; lock (m_threads) { foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) { if (!m_enabled) { return; } if ((threadInfo.Thread.ThreadState & thgone) != 0) { if (threadsToRemove == null) { threadsToRemove = new List <ThreadWatchdogInfo>(); } threadsToRemove.Add(threadInfo); /* * if(callbackInfos == null) * callbackInfos = new List<ThreadWatchdogInfo>(); * * callbackInfos.Add(threadInfo); */ } else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { threadInfo.IsTimedOut = true; if (threadInfo.AlarmIfTimeout) { if (callbackInfos == null) { callbackInfos = new List <ThreadWatchdogInfo>(); } // Send a copy of the watchdog info to prevent race conditions where the watchdog // thread updates the monitoring info after an alarm has been sent out. callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); } } } if (threadsToRemove != null) { foreach (ThreadWatchdogInfo twi in threadsToRemove) { RemoveThread(twi.Thread.ManagedThreadId); } } } if (callbackInfos != null) { foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) { callback(callbackInfo); } } } if (MemoryWatchdog.Enabled) { MemoryWatchdog.Update(); } ChecksManager.CheckChecks(); StatsManager.RecordStats(); m_watchdogTimer.Change(WATCHDOG_INTERVAL_MS, Timeout.Infinite); }