public void RegisterServerStats()
{
// lastperformanceCounterSampleTime = Util.EnvironmentTickCount();
PerformanceCounter tempPC;
Stat tempStat;
string tempName;
try
{
tempName = "CPUPercent";
tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total");
processorPercentPerfCounter = new PerfCounterControl(tempPC);
// A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy.
tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor,
StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); },
StatVerbosity.Info);
StatsManager.RegisterStat(tempStat);
RegisteredStats.Add(tempName, tempStat);
MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor,
(s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; });
MakeStat("UserProcessorTime", null, "sec", ContainerProcessor,
(s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; });
MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor,
(s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; });
MakeStat("Threads", null, "threads", ContainerProcessor,
(s) => { s.Value = Process.GetCurrentProcess().Threads.Count; });
}
catch (Exception e)
{
m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e);
}
MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool,
s =>
{
int workerThreads, iocpThreads;
ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
s.Value = workerThreads;
});
MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool,
s =>
{
int workerThreads, iocpThreads;
ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
s.Value = iocpThreads;
});
if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null)
{
MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads);
MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads);
MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems);
MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads);
MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads);
MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks);
}
MakeStat(
"HTTPRequestsMade",
"Number of outbound HTTP requests made",
"requests",
ContainerNetwork,
s => s.Value = WebUtil.RequestNumber,
MeasuresOfInterest.AverageChangeOverTime);
try
{
List <string> okInterfaceTypes = new List <string>(NetworkInterfaceTypes.Split(','));
IEnumerable <NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces();
foreach (NetworkInterface nic in nics)
{
if (nic.OperationalStatus != OperationalStatus.Up)
{
continue;
}
string nicInterfaceType = nic.NetworkInterfaceType.ToString();
if (!okInterfaceTypes.Contains(nicInterfaceType))
{
m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.",
LogHeader, nic.Name, nicInterfaceType);
m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}",
LogHeader, NetworkInterfaceTypes);
continue;
}
if (nic.Supports(NetworkInterfaceComponent.IPv4))
{
IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics();
if (nicStats != null)
{
MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork,
(s) => { LookupNic(s, (ns) => { return(ns.BytesReceived); }, 1024.0); });
MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork,
(s) => { LookupNic(s, (ns) => { return(ns.BytesSent); }, 1024.0); });
MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork,
(s) => { LookupNic(s, (ns) => { return(ns.BytesSent + ns.BytesReceived); }, 1024.0); });
}
}
// TODO: add IPv6 (it may actually happen someday)
}
}
catch (Exception e)
{
m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e);
}
MakeStat("ProcessMemory", null, "MB", ContainerMemory,
(s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); });
MakeStat("HeapMemory", null, "MB", ContainerMemory,
(s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); });
MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory,
(s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory,
(s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
}
/// <summary>
/// Check watched threads. Fire alarm if appropriate.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
int now = Environment.TickCount & Int32.MaxValue;
int msElapsed = now - LastWatchdogThreadTick;
if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
{
m_log.WarnFormat(
"[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms",
msElapsed, WATCHDOG_INTERVAL_MS);
}
LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout;
if (callback != null)
{
List <ThreadWatchdogInfo> callbackInfos = null;
lock (m_threads)
{
foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
{
if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
{
RemoveThread(threadInfo.Thread.ManagedThreadId);
if (callbackInfos == null)
{
callbackInfos = new List <ThreadWatchdogInfo>();
}
callbackInfos.Add(threadInfo);
}
else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
{
threadInfo.IsTimedOut = true;
if (threadInfo.AlarmIfTimeout)
{
if (callbackInfos == null)
{
callbackInfos = new List <ThreadWatchdogInfo>();
}
// Send a copy of the watchdog info to prevent race conditions where the watchdog
// thread updates the monitoring info after an alarm has been sent out.
callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
}
}
}
}
if (callbackInfos != null)
{
foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
{
callback(callbackInfo);
}
}
}
if (MemoryWatchdog.Enabled)
{
MemoryWatchdog.Update();
}
StatsManager.RecordStats();
m_watchdogTimer.Start();
}
/// <summary>
/// Report back collected statistical information.
/// </summary>
/// <returns></returns>
public override string Report()
{
StringBuilder sb = new StringBuilder(Environment.NewLine);
// sb.Append("ASSET STATISTICS");
// sb.Append(Environment.NewLine);
/*
* sb.Append(
* string.Format(
* @"Asset cache contains {0,6} non-texture assets using {1,10} K
* Texture cache contains {2,6} texture assets using {3,10} K
* Latest asset request time after cache miss: {4}s
* Blocked client requests for missing textures: {5}
* Asset service request failures: {6}"+ Environment.NewLine,
* AssetsInCache, Math.Round(AssetCacheMemoryUsage / 1024.0),
* TexturesInCache, Math.Round(TextureCacheMemoryUsage / 1024.0),
* assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0,
* BlockedMissingTextureRequests,
* AssetServiceRequestFailures));
*/
/*
* sb.Append(
* string.Format(
* @"Asset cache contains {0,6} assets
* Latest asset request time after cache miss: {1}s
* Blocked client requests for missing textures: {2}
* Asset service request failures: {3}" + Environment.NewLine,
* AssetsInCache,
* assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0,
* BlockedMissingTextureRequests,
* AssetServiceRequestFailures));
*/
sb.Append(Environment.NewLine);
sb.Append("CONNECTION STATISTICS");
sb.Append(Environment.NewLine);
List <Stat> stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives");
sb.AppendFormat(
"Client logouts due to no data receive timeout: {0}\n\n",
stats != null ? stats.Sum(s => s.Value).ToString() : "unknown");
// sb.Append(Environment.NewLine);
// sb.Append("INVENTORY STATISTICS");
// sb.Append(Environment.NewLine);
// sb.Append(
// string.Format(
// "Initial inventory caching failures: {0}" + Environment.NewLine,
// InventoryServiceRetrievalFailures));
sb.Append(Environment.NewLine);
sb.Append("SAMPLE FRAME STATISTICS");
sb.Append(Environment.NewLine);
sb.Append("Dilatn SimFPS PhyFPS AgntUp RootAg ChldAg Prims AtvPrm AtvScr ScrLPS");
sb.Append(Environment.NewLine);
sb.Append(
string.Format(
"{0,6:0.00} {1,6:0} {2,6:0.0} {3,6:0.0} {4,6:0} {5,6:0} {6,6:0} {7,6:0} {8,6:0} {9,6:0}",
timeDilation, simFps, physicsFps, agentUpdates, rootAgents,
childAgents, totalPrims, activePrims, activeScripts, scriptLinesPerSecond));
sb.Append(Environment.NewLine);
sb.Append(Environment.NewLine);
// There is no script frame time currently because we don't yet collect it
sb.Append("PktsIn PktOut PendDl PendUl UnackB TotlFt NetFt PhysFt OthrFt AgntFt ImgsFt");
sb.Append(Environment.NewLine);
sb.Append(
string.Format(
"{0,6:0} {1,6:0} {2,6:0} {3,6:0} {4,6:0} {5,6:0.0} {6,6:0.0} {7,6:0.0} {8,6:0.0} {9,6:0.0} {10,6:0.0}\n\n",
inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime,
netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime));
/* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by
* the two formatted printouts above.
* SortedDictionary<string, SortedDictionary<string, Stat>> sceneStats;
* if (StatsManager.TryGetStats("scene", out sceneStats))
* {
* foreach (KeyValuePair<string, SortedDictionary<string, Stat>> kvp in sceneStats)
* {
* foreach (Stat stat in kvp.Value.Values)
* {
* if (stat.Verbosity == StatVerbosity.Info)
* {
* sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName);
* }
* }
* }
* }
*/
/*
* sb.Append(Environment.NewLine);
* sb.Append("PACKET QUEUE STATISTICS");
* sb.Append(Environment.NewLine);
* sb.Append("Agent UUID ");
* sb.Append(
* string.Format(
* " {0,7} {1,7} {2,7} {3,7} {4,7} {5,7} {6,7} {7,7} {8,7} {9,7}",
* "Send", "In", "Out", "Resend", "Land", "Wind", "Cloud", "Task", "Texture", "Asset"));
* sb.Append(Environment.NewLine);
*
* foreach (UUID key in packetQueueStatsCollectors.Keys)
* {
* sb.Append(string.Format("{0}: ", key));
* sb.Append(packetQueueStatsCollectors[key].Report());
* sb.Append(Environment.NewLine);
* }
*/
sb.Append(base.Report());
return(sb.ToString());
}
public void Cleanup()
{
StatsManager.DeregisterStat(Stat);
}
/// <summary>
/// Check watched threads. Fire alarm if appropriate.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private static void WatchdogTimerElapsed(object sender)
{
if (!m_enabled)
{
return;
}
int now = Environment.TickCount & Int32.MaxValue;
int msElapsed = now - LastWatchdogThreadTick;
if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
{
m_log.WarnFormat(
"[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms",
msElapsed, WATCHDOG_INTERVAL_MS);
}
LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout;
if (callback != null)
{
List <ThreadWatchdogInfo> callbackInfos = null;
List <ThreadWatchdogInfo> threadsToRemove = null;
const ThreadState thgone = ThreadState.Stopped;
lock (m_threads)
{
foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
{
if (!m_enabled)
{
return;
}
if ((threadInfo.Thread.ThreadState & thgone) != 0)
{
if (threadsToRemove == null)
{
threadsToRemove = new List <ThreadWatchdogInfo>();
}
threadsToRemove.Add(threadInfo);
/*
* if(callbackInfos == null)
* callbackInfos = new List<ThreadWatchdogInfo>();
*
* callbackInfos.Add(threadInfo);
*/
}
else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
{
threadInfo.IsTimedOut = true;
if (threadInfo.AlarmIfTimeout)
{
if (callbackInfos == null)
{
callbackInfos = new List <ThreadWatchdogInfo>();
}
// Send a copy of the watchdog info to prevent race conditions where the watchdog
// thread updates the monitoring info after an alarm has been sent out.
callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
}
}
}
if (threadsToRemove != null)
{
foreach (ThreadWatchdogInfo twi in threadsToRemove)
{
RemoveThread(twi.Thread.ManagedThreadId);
}
}
}
if (callbackInfos != null)
{
foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
{
callback(callbackInfo);
}
}
}
if (MemoryWatchdog.Enabled)
{
MemoryWatchdog.Update();
}
ChecksManager.CheckChecks();
StatsManager.RecordStats();
m_watchdogTimer.Change(WATCHDOG_INTERVAL_MS, Timeout.Infinite);
}