/// <summary> /// Event handler that gets fired when a worker process is identified as being healthy after start up. /// </summary> private void WorkerIsHealthy(SIPAppServerWorker worker) { if (worker.IsHealthy()) { m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_healthyPriority); } else { logger.Warn("An app server worker was unhealthy after the process initialisation period."); } }
/// <summary> /// Method that gets spawned on a dedicated thread to manage an app server process. /// </summary> private void ManageWorker(XmlNode appServerWorkerNode) { try { SIPAppServerWorker appServerWorker = RecycleAppServer(null, 0, appServerWorkerNode); string appServerSocket = appServerWorker.AppServerEndpoint.ToString(); Thread.CurrentThread.Name = "manage-" + appServerWorker.AppServerEndpoint.Port; DateTime lastProbeTime = DateTime.MinValue; while (!m_exit) { // Process checks. if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart) { if (appServerWorker.WorkerProcess != null && appServerWorker.WorkerProcess.HasExited) { // This is the only case where action is taken to immediately disable the use of a worker process outside of the access controlled // RecycleAppServer method. The reason being there's no point attempting to use a worker process that has completely gone. logger.Debug("Worker has disappeared for " + appServerSocket + ". Deactivating and marking for immediate restart."); appServerWorker.NeedsImmediateRestart = true; appServerWorker.IsDeactivated = true; lock (m_appServerWorkers) { m_appServerWorkers.Remove(appServerWorker); } m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_unhealthyPriority); } else { appServerWorker.WorkerProcess.Refresh(); if (appServerWorker.WorkerProcess.PrivateMemorySize64 >= MAX_PHYSICAL_MEMORY) { logger.Debug("Memory limit reached on worker " + appServerSocket + " pid " + appServerWorker.ProcessID + ". Marking for graceful restart."); appServerWorker.NeedsToRestart = true; } } } // Periodically send a SIP call probe to the app server socket. if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart && DateTime.Now.Subtract(lastProbeTime).TotalSeconds > m_probePeriodSeconds) { appServerWorker.SendProbe(); lastProbeTime = DateTime.Now; } // Restarts. if (appServerWorker.NeedsImmediateRestart || appServerWorker.NeedsToRestart) { double secondsSinceLastStart = DateTime.Now.Subtract(appServerWorker.WorkerProcessStartTime).TotalSeconds; if (secondsSinceLastStart < PROCESS_RESTART_DELAY_SECONDS) { int secondsDelay = PROCESS_RESTART_DELAY_SECONDS - Convert.ToInt32(secondsSinceLastStart % Int32.MaxValue); logger.Debug("Waiting " + secondsDelay + " seconds before attempting to recycle worker on " + appServerSocket + "."); Thread.Sleep(secondsDelay * 1000); } string recycleType = (appServerWorker.NeedsImmediateRestart) ? "immediate" : "graceful"; logger.Debug("Attempting to get lock for " + recycleType + " recycle for " + appServerSocket + " pid " + appServerWorker.ProcessID + "."); if (Monitor.TryEnter(m_recycleLock, WAIT_FOR_RECYCLE_LOCK_SECONDS * 1000)) { int recycleDelay = (appServerWorker.NeedsToRestart) ? PROCESS_RESTART_DELAY_SECONDS : 0; appServerWorker = RecycleAppServer(appServerWorker, recycleDelay, appServerWorkerNode); lastProbeTime = DateTime.Now; Monitor.Exit(m_recycleLock); } else { logger.Debug("Failed to acquire recycle lock for " + appServerSocket + " pid " + appServerWorker.ProcessID + "."); } } Thread.Sleep(CHECK_WORKER_PERIOD_SECONDS * 1000); } } catch (Exception excp) { logger.Error("Exception ManageWorker. " + excp.Message); } finally { try { Monitor.Exit(m_recycleLock); } catch { } } }