/// <summary> /// Event handler for a failed probe response. /// </summary> private void AppServerCallFailed(ISIPClientUserAgent uac, string errorMessage, int workerProcessID, DateTime probeSentAt, bool isInitialProbe) { try { string workerSocket = SIPURI.ParseSIPURI(uac.CallDescriptor.Uri).Host; logger.Warn("SIPAppServerManager call to " + workerSocket + " for PID " + workerProcessID + " failed, initial probe " + isInitialProbe + " , sent at " + probeSentAt.ToString("dd MMM yyyy HH:mm:ss") + ", " + errorMessage); // Find the worker for the failed end point. SIPAppServerWorker failedWorker = GetWorkerForEndPoint(workerSocket); // Make sure the worker process hasn't changed in the meantime and don't restart for initial probes. if (failedWorker != null && failedWorker.WorkerProcess != null && failedWorker.WorkerProcess.Id == workerProcessID) { if (!isInitialProbe) { failedWorker.InitialProbeResponseReceived = true; logger.Debug("Scheduling immediate restart on app server worker process pid=" + failedWorker.WorkerProcess.Id + ", " + workerSocket + " due to failed probe."); failedWorker.ScheduleRestart(DateTime.Now); } else if (failedWorker.InitialProbeCount >= INITIAL_PROBE_RETRANSMIT_LIMIT) { failedWorker.InitialProbeResponseReceived = true; logger.Debug("Initial probe retransmit limit reached, scheduling immediate restart on app server worker process pid=" + failedWorker.WorkerProcess.Id + ", " + workerSocket + " due to failed probe."); failedWorker.ScheduleRestart(DateTime.Now); } } } catch (Exception excp) { logger.Error("Exception AppServerCallFailed. " + excp.Message); } }
private SIPAppServerWorker RecycleAppServer(SIPAppServerWorker badAppServerWorker, int delaySeconds, XmlNode appServerWorkerNode) { try { lock (m_recycleLock) { // Shutdown existing worker process. if (badAppServerWorker != null) { if (!badAppServerWorker.IsDeactivated) { logger.Debug("Deactivating worker on " + badAppServerWorker.AppServerEndpoint.ToString() + "."); m_sipCallDispatcherFile.UpdateAppServerPriority(badAppServerWorker.AppServerEndpoint, m_unhealthyPriority); lock (m_appServerWorkers) { m_appServerWorkers.Remove(badAppServerWorker); } } if (delaySeconds > 0) { logger.Debug("Delaying process restart for " + badAppServerWorker.AppServerEndpoint.ToString() + " by " + delaySeconds + "s."); Thread.Sleep(delaySeconds * 1000); } badAppServerWorker.Kill(); } // Start new worker process and wait for a successful probe response before returning. SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode, m_sipTransport); logger.Debug("Starting new worker on " + appServerWorker.AppServerEndpoint.ToString() + "."); DateTime startTime = DateTime.Now; if (appServerWorker.StartProcess()) { logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " ready after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds."); m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_healthyPriority); lock (m_appServerWorkers) { m_appServerWorkers.Add(appServerWorker); } } else { logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " failed to reach a ready state after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds."); appServerWorker.NeedsImmediateRestart = true; } return(appServerWorker); } } catch (Exception excp) { logger.Error("Exception RecycleAppServer. " + excp.Message); throw; } }
/// <summary> /// Event handler that gets fired when a worker process is identified as being healthy after start up. /// </summary> private void WorkerIsHealthy(SIPAppServerWorker worker) { if (worker.IsHealthy()) { m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_healthyPriority); } else { logger.Warn("An app server worker was unhealthy after the process initialisation period."); } }
/// <summary> /// Starts a new SIP application server worker process. /// </summary> private void StartWorkerProcess(SIPAppServerWorker worker) { string errorMessage = worker.StartProcess(); if (errorMessage == null) { ProbeWorker(worker, true); } else { logger.Warn("Error starting worker process for " + worker.AppServerEndpoint.ToString() + ". " + errorMessage); } }
private List <string> m_workerSIPEndPoints = new List <string>(); // Allow quick lookups to determine whether a remote end point is that of a worker process. public SIPAppServerManager( SIPMonitorLogDelegate logDelegate, SIPTransport sipTransport, XmlNode appServerWorkersNode, string appServerEndPointsPath) { if (appServerWorkersNode == null || appServerWorkersNode.ChildNodes.Count == 0) { throw new ArgumentNullException("A SIPAppServerManager cannot be created with an empty workers node."); } SIPMonitorLogEvent_External = logDelegate; m_sipTransport = sipTransport; m_appServerWorkersNode = appServerWorkersNode; m_appServerEndPointsPath = appServerEndPointsPath; if (!appServerEndPointsPath.IsNullOrBlank() && File.Exists(appServerEndPointsPath)) { m_sipCallDispatcherFile = new SIPCallDispatcherFile(logDelegate, appServerEndPointsPath); } try { CallManagerPassThruServiceInstanceProvider callManagerPassThruSvcInstanceProvider = new CallManagerPassThruServiceInstanceProvider(this); m_callManagerPassThruSvcHost = new ServiceHost(typeof(CallManagerPassThruService)); m_callManagerPassThruSvcHost.Description.Behaviors.Add(callManagerPassThruSvcInstanceProvider); m_callManagerPassThruSvcHost.Open(); logger.Debug("SIPAppServerManager CallManagerPassThru hosted service successfully started on " + m_callManagerPassThruSvcHost.BaseAddresses[0].AbsoluteUri + "."); } catch (Exception excp) { logger.Warn("Exception starting SIPAppServerManager CallManagerPassThru hosted service. " + excp.Message); } foreach (XmlNode appServerWorkerNode in m_appServerWorkersNode.ChildNodes) { SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode); if (m_sipCallDispatcherFile != null) { appServerWorker.Healthy += WorkerIsHealthy; appServerWorker.Unhealthy += WorkerIsUnhealthy; } m_appServerWorkers.Add(appServerWorker); m_workerSIPEndPoints.Add(appServerWorker.AppServerEndpoint.ToString()); logger.Debug("SIPAppServerManager worker added for " + appServerWorker.AppServerEndpoint.ToString() + " and " + appServerWorker.CallManagerAddress.ToString() + "."); } ThreadPool.QueueUserWorkItem(delegate { SpawnWorkers(); }); ThreadPool.QueueUserWorkItem(delegate { ProbeWorkers(); }); }
/// <summary> /// Event handler for a successful probe response. /// </summary> private void AppServerCallSucceeded(ISIPClientUserAgent uac) { try { string workerSocket = SIPURI.ParseSIPURI(uac.CallDescriptor.Uri).Host; SIPAppServerWorker worker = GetWorkerForEndPoint(workerSocket); if (!worker.InitialProbeResponseReceived) { logger.Debug("Initial probe received for " + workerSocket + "."); worker.InitialCallSuccessful(); } } catch (Exception excp) { logger.Error("Exception AppServerCallSucceeded. " + excp.Message); } }
/// <summary> /// Sends the SIP INVITE probe request. /// </summary> private void ProbeWorker(SIPAppServerWorker worker, bool isInitialProbe) { try { if (isInitialProbe) { worker.InitialProbeCount++; } int workerProcessID = worker.WorkerProcess.Id; SIPEndPoint workerEndPoint = worker.AppServerEndpoint; DateTime probeSentAt = DateTime.Now; SIPCallDescriptor callDescriptor = new SIPCallDescriptor(m_dispatcherUsername, null, "sip:" + m_dispatcherUsername + "@" + workerEndPoint.GetIPEndPoint().ToString(), "sip:" + m_dispatcherUsername + "@sipcalldispatcher", "sip:" + workerEndPoint.GetIPEndPoint().ToString(), null, null, null, SIPCallDirection.Out, null, null, null); SIPClientUserAgent uac = new SIPClientUserAgent(m_sipTransport, null, null, null, null); uac.CallFailed += (failedUAC, errorMessage) => { AppServerCallFailed(failedUAC, errorMessage, workerProcessID, probeSentAt, isInitialProbe); }; uac.CallAnswered += (call, sipResponse) => { if (sipResponse.Status != SIPResponseStatusCodesEnum.BadExtension) { //logger.Warn("Probe call answered with unexpected response code of " + sipResponse.StatusCode + "."); AppServerCallFailed(call, "Unexpected response of " + ((int)sipResponse.StatusCode) + " on probe call.", workerProcessID, probeSentAt, isInitialProbe); } else { AppServerCallSucceeded(call); } }; uac.Call(callDescriptor); } catch (Exception excp) { logger.Error("Exception SIPAppServerManager ProberWorker. " + excp.Message); } }
/// <summary> /// Method that gets spawned on a dedicated thread to manage an app server process. /// </summary> private void ManageWorker(XmlNode appServerWorkerNode) { try { SIPAppServerWorker appServerWorker = RecycleAppServer(null, 0, appServerWorkerNode); string appServerSocket = appServerWorker.AppServerEndpoint.ToString(); Thread.CurrentThread.Name = "manage-" + appServerWorker.AppServerEndpoint.Port; DateTime lastProbeTime = DateTime.MinValue; while (!m_exit) { // Process checks. if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart) { if (appServerWorker.WorkerProcess != null && appServerWorker.WorkerProcess.HasExited) { // This is the only case where action is taken to immediately disable the use of a worker process outside of the access controlled // RecycleAppServer method. The reason being there's no point attempting to use a worker process that has completely gone. logger.Debug("Worker has disappeared for " + appServerSocket + ". Deactivating and marking for immediate restart."); appServerWorker.NeedsImmediateRestart = true; appServerWorker.IsDeactivated = true; lock (m_appServerWorkers) { m_appServerWorkers.Remove(appServerWorker); } m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_unhealthyPriority); } else { appServerWorker.WorkerProcess.Refresh(); if (appServerWorker.WorkerProcess.PrivateMemorySize64 >= MAX_PHYSICAL_MEMORY) { logger.Debug("Memory limit reached on worker " + appServerSocket + " pid " + appServerWorker.ProcessID + ". Marking for graceful restart."); appServerWorker.NeedsToRestart = true; } } } // Periodically send a SIP call probe to the app server socket. if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart && DateTime.Now.Subtract(lastProbeTime).TotalSeconds > m_probePeriodSeconds) { appServerWorker.SendProbe(); lastProbeTime = DateTime.Now; } // Restarts. if (appServerWorker.NeedsImmediateRestart || appServerWorker.NeedsToRestart) { double secondsSinceLastStart = DateTime.Now.Subtract(appServerWorker.WorkerProcessStartTime).TotalSeconds; if (secondsSinceLastStart < PROCESS_RESTART_DELAY_SECONDS) { int secondsDelay = PROCESS_RESTART_DELAY_SECONDS - Convert.ToInt32(secondsSinceLastStart % Int32.MaxValue); logger.Debug("Waiting " + secondsDelay + " seconds before attempting to recycle worker on " + appServerSocket + "."); Thread.Sleep(secondsDelay * 1000); } string recycleType = (appServerWorker.NeedsImmediateRestart) ? "immediate" : "graceful"; logger.Debug("Attempting to get lock for " + recycleType + " recycle for " + appServerSocket + " pid " + appServerWorker.ProcessID + "."); if (Monitor.TryEnter(m_recycleLock, WAIT_FOR_RECYCLE_LOCK_SECONDS * 1000)) { int recycleDelay = (appServerWorker.NeedsToRestart) ? PROCESS_RESTART_DELAY_SECONDS : 0; appServerWorker = RecycleAppServer(appServerWorker, recycleDelay, appServerWorkerNode); lastProbeTime = DateTime.Now; Monitor.Exit(m_recycleLock); } else { logger.Debug("Failed to acquire recycle lock for " + appServerSocket + " pid " + appServerWorker.ProcessID + "."); } } Thread.Sleep(CHECK_WORKER_PERIOD_SECONDS * 1000); } } catch (Exception excp) { logger.Error("Exception ManageWorker. " + excp.Message); } finally { try { Monitor.Exit(m_recycleLock); } catch { } } }
private SIPAppServerWorker RecycleAppServer(SIPAppServerWorker badAppServerWorker, int delaySeconds, XmlNode appServerWorkerNode) { try { lock (m_recycleLock) { // Shutdown existing worker process. if (badAppServerWorker != null) { if (!badAppServerWorker.IsDeactivated) { logger.Debug("Deactivating worker on " + badAppServerWorker.AppServerEndpoint.ToString() + "."); m_sipCallDispatcherFile.UpdateAppServerPriority(badAppServerWorker.AppServerEndpoint, m_unhealthyPriority); lock (m_appServerWorkers) { m_appServerWorkers.Remove(badAppServerWorker); } } if (delaySeconds > 0) { logger.Debug("Delaying process restart for " + badAppServerWorker.AppServerEndpoint.ToString() + " by " + delaySeconds + "s."); Thread.Sleep(delaySeconds * 1000); } badAppServerWorker.Kill(); } // Start new worker process and wait for a successful probe response before returning. SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode, m_sipTransport); logger.Debug("Starting new worker on " + appServerWorker.AppServerEndpoint.ToString() + "."); DateTime startTime = DateTime.Now; if (appServerWorker.StartProcess()) { logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " ready after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds."); m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_healthyPriority); lock (m_appServerWorkers) { m_appServerWorkers.Add(appServerWorker); } } else { logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " failed to reach a ready state after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds."); appServerWorker.NeedsImmediateRestart = true; } return appServerWorker; } } catch (Exception excp) { logger.Error("Exception RecycleAppServer. " + excp.Message); throw; } }
/// <summary> /// Event handler that gets fired when a worker process is identified as being unhealthy at any point in its lifetime. /// </summary> /// <param name="worker"></param> private void WorkerIsUnhealthy(SIPAppServerWorker worker) { m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_unhealthyPriority); }
public SIPAppServerManager( SIPMonitorLogDelegate logDelegate, SIPTransport sipTransport, XmlNode appServerWorkersNode, string appServerEndPointsPath) { if (appServerWorkersNode == null || appServerWorkersNode.ChildNodes.Count == 0) { throw new ArgumentNullException("A SIPAppServerManager cannot be created with an empty workers node."); } SIPMonitorLogEvent_External = logDelegate; m_sipTransport = sipTransport; m_appServerWorkersNode = appServerWorkersNode; m_appServerEndPointsPath = appServerEndPointsPath; if (!appServerEndPointsPath.IsNullOrBlank() && File.Exists(appServerEndPointsPath)) { m_sipCallDispatcherFile = new SIPCallDispatcherFile(logDelegate, appServerEndPointsPath); } try { CallManagerPassThruServiceInstanceProvider callManagerPassThruSvcInstanceProvider = new CallManagerPassThruServiceInstanceProvider(this); m_callManagerPassThruSvcHost = new ServiceHost(typeof(CallManagerPassThruService)); m_callManagerPassThruSvcHost.Description.Behaviors.Add(callManagerPassThruSvcInstanceProvider); m_callManagerPassThruSvcHost.Open(); logger.Debug("SIPAppServerManager CallManagerPassThru hosted service successfully started on " + m_callManagerPassThruSvcHost.BaseAddresses[0].AbsoluteUri + "."); } catch (Exception excp) { logger.Warn("Exception starting SIPAppServerManager CallManagerPassThru hosted service. " + excp.Message); } foreach (XmlNode appServerWorkerNode in m_appServerWorkersNode.ChildNodes) { SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode); if (m_sipCallDispatcherFile != null) { appServerWorker.Healthy += WorkerIsHealthy; appServerWorker.Unhealthy += WorkerIsUnhealthy; } m_appServerWorkers.Add(appServerWorker); m_workerSIPEndPoints.Add(appServerWorker.AppServerEndpoint.ToString()); logger.Debug("SIPAppServerManager worker added for " + appServerWorker.AppServerEndpoint.ToString() + " and " + appServerWorker.CallManagerAddress.ToString() + "."); } ThreadPool.QueueUserWorkItem(delegate { SpawnWorkers(); }); ThreadPool.QueueUserWorkItem(delegate { ProbeWorkers(); }); }