public void RegisterServerStats()
        {
//            lastperformanceCounterSampleTime = Util.EnvironmentTickCount();
            PerformanceCounter tempPC;
            Stat   tempStat;
            string tempName;

            try
            {
                tempName = "CPUPercent";
                tempPC   = new PerformanceCounter("Processor", "% Processor Time", "_Total");
                processorPercentPerfCounter = new PerfCounterControl(tempPC);
                // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy.
                tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor,
                                    StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); },
                                    StatVerbosity.Info);
                StatsManager.RegisterStat(tempStat);
                RegisteredStats.Add(tempName, tempStat);

                MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor,
                         (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; });

                MakeStat("UserProcessorTime", null, "sec", ContainerProcessor,
                         (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; });

                MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor,
                         (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; });

                MakeStat("Threads", null, "threads", ContainerProcessor,
                         (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; });
            }
            catch (Exception e)
            {
                m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e);
            }

            MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool,
                     s =>
            {
                int workerThreads, iocpThreads;
                ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
                s.Value = workerThreads;
            });

            MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool,
                     s =>
            {
                int workerThreads, iocpThreads;
                ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
                s.Value = iocpThreads;
            });

            if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null)
            {
                MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value       = Util.GetSmartThreadPoolInfo().MaxThreads);
                MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value       = Util.GetSmartThreadPoolInfo().MinThreads);
                MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value      = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems);
                MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value    = Util.GetSmartThreadPoolInfo().ActiveThreads);
                MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value     = Util.GetSmartThreadPoolInfo().InUseThreads);
                MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks);
            }

            MakeStat(
                "HTTPRequestsMade",
                "Number of outbound HTTP requests made",
                "requests",
                ContainerNetwork,
                s => s.Value = WebUtil.RequestNumber,
                MeasuresOfInterest.AverageChangeOverTime);

            try
            {
                List <string> okInterfaceTypes = new List <string>(NetworkInterfaceTypes.Split(','));

                IEnumerable <NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces();
                foreach (NetworkInterface nic in nics)
                {
                    if (nic.OperationalStatus != OperationalStatus.Up)
                    {
                        continue;
                    }

                    string nicInterfaceType = nic.NetworkInterfaceType.ToString();
                    if (!okInterfaceTypes.Contains(nicInterfaceType))
                    {
                        m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.",
                                          LogHeader, nic.Name, nicInterfaceType);
                        m_log.DebugFormat("{0}     To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}",
                                          LogHeader, NetworkInterfaceTypes);
                        continue;
                    }

                    if (nic.Supports(NetworkInterfaceComponent.IPv4))
                    {
                        IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics();
                        if (nicStats != null)
                        {
                            MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork,
                                     (s) => { LookupNic(s, (ns) => { return(ns.BytesReceived); }, 1024.0); });
                            MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork,
                                     (s) => { LookupNic(s, (ns) => { return(ns.BytesSent); }, 1024.0); });
                            MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork,
                                     (s) => { LookupNic(s, (ns) => { return(ns.BytesSent + ns.BytesReceived); }, 1024.0); });
                        }
                    }
                    // TODO: add IPv6 (it may actually happen someday)
                }
            }
            catch (Exception e)
            {
                m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e);
            }

            MakeStat("ProcessMemory", null, "MB", ContainerMemory,
                     (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); });
            MakeStat("HeapMemory", null, "MB", ContainerMemory,
                     (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); });
            MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory,
                     (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
            MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory,
                     (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
        }
Beispiel #2
0
        /// <summary>
        /// Check watched threads.  Fire alarm if appropriate.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            int now       = Environment.TickCount & Int32.MaxValue;
            int msElapsed = now - LastWatchdogThreadTick;

            if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
            {
                m_log.WarnFormat(
                    "[WATCHDOG]: {0} ms since Watchdog last ran.  Interval should be approximately {1} ms",
                    msElapsed, WATCHDOG_INTERVAL_MS);
            }

            LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;

            Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout;

            if (callback != null)
            {
                List <ThreadWatchdogInfo> callbackInfos = null;

                lock (m_threads)
                {
                    foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
                    {
                        if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
                        {
                            RemoveThread(threadInfo.Thread.ManagedThreadId);

                            if (callbackInfos == null)
                            {
                                callbackInfos = new List <ThreadWatchdogInfo>();
                            }

                            callbackInfos.Add(threadInfo);
                        }
                        else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
                        {
                            threadInfo.IsTimedOut = true;

                            if (threadInfo.AlarmIfTimeout)
                            {
                                if (callbackInfos == null)
                                {
                                    callbackInfos = new List <ThreadWatchdogInfo>();
                                }

                                // Send a copy of the watchdog info to prevent race conditions where the watchdog
                                // thread updates the monitoring info after an alarm has been sent out.
                                callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
                            }
                        }
                    }
                }

                if (callbackInfos != null)
                {
                    foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
                    {
                        callback(callbackInfo);
                    }
                }
            }

            if (MemoryWatchdog.Enabled)
            {
                MemoryWatchdog.Update();
            }

            StatsManager.RecordStats();

            m_watchdogTimer.Start();
        }
Beispiel #3
0
        /// <summary>
        /// Report back collected statistical information.
        /// </summary>
        /// <returns></returns>
        public override string Report()
        {
            StringBuilder sb = new StringBuilder(Environment.NewLine);

//            sb.Append("ASSET STATISTICS");
//            sb.Append(Environment.NewLine);

            /*
             * sb.Append(
             *  string.Format(
             * @"Asset cache contains   {0,6} non-texture assets using {1,10} K
             * Texture cache contains {2,6} texture     assets using {3,10} K
             * Latest asset request time after cache miss: {4}s
             * Blocked client requests for missing textures: {5}
             * Asset service request failures: {6}"+ Environment.NewLine,
             *      AssetsInCache, Math.Round(AssetCacheMemoryUsage / 1024.0),
             *      TexturesInCache, Math.Round(TextureCacheMemoryUsage / 1024.0),
             *      assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0,
             *      BlockedMissingTextureRequests,
             *      AssetServiceRequestFailures));
             */

            /*
             * sb.Append(
             *  string.Format(
             * @"Asset cache contains   {0,6} assets
             * Latest asset request time after cache miss: {1}s
             * Blocked client requests for missing textures: {2}
             * Asset service request failures: {3}" + Environment.NewLine,
             *      AssetsInCache,
             *      assetRequestTimeAfterCacheMiss.Milliseconds / 1000.0,
             *      BlockedMissingTextureRequests,
             *      AssetServiceRequestFailures));
             */

            sb.Append(Environment.NewLine);
            sb.Append("CONNECTION STATISTICS");
            sb.Append(Environment.NewLine);

            List <Stat> stats = StatsManager.GetStatsFromEachContainer("clientstack", "ClientLogoutsDueToNoReceives");

            sb.AppendFormat(
                "Client logouts due to no data receive timeout: {0}\n\n",
                stats != null ? stats.Sum(s => s.Value).ToString() : "unknown");

//            sb.Append(Environment.NewLine);
//            sb.Append("INVENTORY STATISTICS");
//            sb.Append(Environment.NewLine);
//            sb.Append(
//                string.Format(
//                    "Initial inventory caching failures: {0}" + Environment.NewLine,
//                    InventoryServiceRetrievalFailures));

            sb.Append(Environment.NewLine);
            sb.Append("SAMPLE FRAME STATISTICS");
            sb.Append(Environment.NewLine);
            sb.Append("Dilatn  SimFPS  PhyFPS  AgntUp  RootAg  ChldAg  Prims   AtvPrm  AtvScr  ScrLPS");
            sb.Append(Environment.NewLine);
            sb.Append(
                string.Format(
                    "{0,6:0.00}  {1,6:0}  {2,6:0.0}  {3,6:0.0}  {4,6:0}  {5,6:0}  {6,6:0}  {7,6:0}  {8,6:0}  {9,6:0}",
                    timeDilation, simFps, physicsFps, agentUpdates, rootAgents,
                    childAgents, totalPrims, activePrims, activeScripts, scriptLinesPerSecond));

            sb.Append(Environment.NewLine);
            sb.Append(Environment.NewLine);
            // There is no script frame time currently because we don't yet collect it
            sb.Append("PktsIn  PktOut  PendDl  PendUl  UnackB  TotlFt  NetFt   PhysFt  OthrFt  AgntFt  ImgsFt");
            sb.Append(Environment.NewLine);
            sb.Append(
                string.Format(
                    "{0,6:0}  {1,6:0}  {2,6:0}  {3,6:0}  {4,6:0}  {5,6:0.0}  {6,6:0.0}  {7,6:0.0}  {8,6:0.0}  {9,6:0.0}  {10,6:0.0}\n\n",
                    inPacketsPerSecond, outPacketsPerSecond, pendingDownloads, pendingUploads, unackedBytes, totalFrameTime,
                    netFrameTime, physicsFrameTime, otherFrameTime, agentFrameTime, imageFrameTime));

            /* 20130319 RA: For the moment, disable the dump of 'scene' catagory as they are mostly output by
             * the two formatted printouts above.
             * SortedDictionary<string, SortedDictionary<string, Stat>> sceneStats;
             * if (StatsManager.TryGetStats("scene", out sceneStats))
             * {
             *  foreach (KeyValuePair<string, SortedDictionary<string, Stat>> kvp in sceneStats)
             *  {
             *      foreach (Stat stat in kvp.Value.Values)
             *      {
             *          if (stat.Verbosity == StatVerbosity.Info)
             *          {
             *              sb.AppendFormat("{0} ({1}): {2}{3}\n", stat.Name, stat.Container, stat.Value, stat.UnitName);
             *          }
             *      }
             *  }
             * }
             */

            /*
             * sb.Append(Environment.NewLine);
             * sb.Append("PACKET QUEUE STATISTICS");
             * sb.Append(Environment.NewLine);
             * sb.Append("Agent UUID                          ");
             * sb.Append(
             *  string.Format(
             *      "  {0,7}  {1,7}  {2,7}  {3,7}  {4,7}  {5,7}  {6,7}  {7,7}  {8,7}  {9,7}",
             *      "Send", "In", "Out", "Resend", "Land", "Wind", "Cloud", "Task", "Texture", "Asset"));
             * sb.Append(Environment.NewLine);
             *
             * foreach (UUID key in packetQueueStatsCollectors.Keys)
             * {
             *  sb.Append(string.Format("{0}: ", key));
             *  sb.Append(packetQueueStatsCollectors[key].Report());
             *  sb.Append(Environment.NewLine);
             * }
             */

            sb.Append(base.Report());

            return(sb.ToString());
        }
Beispiel #4
0
 public void Cleanup()
 {
     StatsManager.DeregisterStat(Stat);
 }
Beispiel #5
0
        /// <summary>
        /// Check watched threads.  Fire alarm if appropriate.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private static void WatchdogTimerElapsed(object sender)
        {
            if (!m_enabled)
            {
                return;
            }
            int now       = Environment.TickCount & Int32.MaxValue;
            int msElapsed = now - LastWatchdogThreadTick;

            if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
            {
                m_log.WarnFormat(
                    "[WATCHDOG]: {0} ms since Watchdog last ran.  Interval should be approximately {1} ms",
                    msElapsed, WATCHDOG_INTERVAL_MS);
            }

            LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;

            Action <ThreadWatchdogInfo> callback = OnWatchdogTimeout;

            if (callback != null)
            {
                List <ThreadWatchdogInfo> callbackInfos   = null;
                List <ThreadWatchdogInfo> threadsToRemove = null;

                const ThreadState thgone = ThreadState.Stopped;

                lock (m_threads)
                {
                    foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
                    {
                        if (!m_enabled)
                        {
                            return;
                        }
                        if ((threadInfo.Thread.ThreadState & thgone) != 0)
                        {
                            if (threadsToRemove == null)
                            {
                                threadsToRemove = new List <ThreadWatchdogInfo>();
                            }

                            threadsToRemove.Add(threadInfo);

/*
 *                          if(callbackInfos == null)
 *                              callbackInfos = new List<ThreadWatchdogInfo>();
 *
 *                          callbackInfos.Add(threadInfo);
 */
                        }
                        else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
                        {
                            threadInfo.IsTimedOut = true;

                            if (threadInfo.AlarmIfTimeout)
                            {
                                if (callbackInfos == null)
                                {
                                    callbackInfos = new List <ThreadWatchdogInfo>();
                                }

                                // Send a copy of the watchdog info to prevent race conditions where the watchdog
                                // thread updates the monitoring info after an alarm has been sent out.
                                callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
                            }
                        }
                    }

                    if (threadsToRemove != null)
                    {
                        foreach (ThreadWatchdogInfo twi in threadsToRemove)
                        {
                            RemoveThread(twi.Thread.ManagedThreadId);
                        }
                    }
                }

                if (callbackInfos != null)
                {
                    foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
                    {
                        callback(callbackInfo);
                    }
                }
            }

            if (MemoryWatchdog.Enabled)
            {
                MemoryWatchdog.Update();
            }

            ChecksManager.CheckChecks();
            StatsManager.RecordStats();

            m_watchdogTimer.Change(WATCHDOG_INTERVAL_MS, Timeout.Infinite);
        }