/// <summary>
/// Event handler for a failed probe response.
/// </summary>
private void AppServerCallFailed(ISIPClientUserAgent uac, string errorMessage, int workerProcessID, DateTime probeSentAt, bool isInitialProbe)
{
try
{
string workerSocket = SIPURI.ParseSIPURI(uac.CallDescriptor.Uri).Host;
logger.Warn("SIPAppServerManager call to " + workerSocket + " for PID " + workerProcessID + " failed, initial probe " + isInitialProbe + " , sent at " + probeSentAt.ToString("dd MMM yyyy HH:mm:ss") + ", " + errorMessage);
// Find the worker for the failed end point.
SIPAppServerWorker failedWorker = GetWorkerForEndPoint(workerSocket);
// Make sure the worker process hasn't changed in the meantime and don't restart for initial probes.
if (failedWorker != null && failedWorker.WorkerProcess != null && failedWorker.WorkerProcess.Id == workerProcessID)
{
if (!isInitialProbe)
{
failedWorker.InitialProbeResponseReceived = true;
logger.Debug("Scheduling immediate restart on app server worker process pid=" + failedWorker.WorkerProcess.Id + ", " + workerSocket + " due to failed probe.");
failedWorker.ScheduleRestart(DateTime.Now);
}
else if (failedWorker.InitialProbeCount >= INITIAL_PROBE_RETRANSMIT_LIMIT)
{
failedWorker.InitialProbeResponseReceived = true;
logger.Debug("Initial probe retransmit limit reached, scheduling immediate restart on app server worker process pid=" + failedWorker.WorkerProcess.Id + ", " + workerSocket + " due to failed probe.");
failedWorker.ScheduleRestart(DateTime.Now);
}
}
}
catch (Exception excp)
{
logger.Error("Exception AppServerCallFailed. " + excp.Message);
}
}
private SIPAppServerWorker RecycleAppServer(SIPAppServerWorker badAppServerWorker, int delaySeconds, XmlNode appServerWorkerNode)
{
try
{
lock (m_recycleLock)
{
// Shutdown existing worker process.
if (badAppServerWorker != null)
{
if (!badAppServerWorker.IsDeactivated)
{
logger.Debug("Deactivating worker on " + badAppServerWorker.AppServerEndpoint.ToString() + ".");
m_sipCallDispatcherFile.UpdateAppServerPriority(badAppServerWorker.AppServerEndpoint, m_unhealthyPriority);
lock (m_appServerWorkers)
{
m_appServerWorkers.Remove(badAppServerWorker);
}
}
if (delaySeconds > 0)
{
logger.Debug("Delaying process restart for " + badAppServerWorker.AppServerEndpoint.ToString() + " by " + delaySeconds + "s.");
Thread.Sleep(delaySeconds * 1000);
}
badAppServerWorker.Kill();
}
// Start new worker process and wait for a successful probe response before returning.
SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode, m_sipTransport);
logger.Debug("Starting new worker on " + appServerWorker.AppServerEndpoint.ToString() + ".");
DateTime startTime = DateTime.Now;
if (appServerWorker.StartProcess())
{
logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " ready after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds.");
m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_healthyPriority);
lock (m_appServerWorkers)
{
m_appServerWorkers.Add(appServerWorker);
}
}
else
{
logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " failed to reach a ready state after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds.");
appServerWorker.NeedsImmediateRestart = true;
}
return(appServerWorker);
}
}
catch (Exception excp)
{
logger.Error("Exception RecycleAppServer. " + excp.Message);
throw;
}
}
/// <summary>
/// Event handler that gets fired when a worker process is identified as being healthy after start up.
/// </summary>
private void WorkerIsHealthy(SIPAppServerWorker worker)
{
if (worker.IsHealthy())
{
m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_healthyPriority);
}
else
{
logger.Warn("An app server worker was unhealthy after the process initialisation period.");
}
}
/// <summary>
/// Starts a new SIP application server worker process.
/// </summary>
private void StartWorkerProcess(SIPAppServerWorker worker)
{
string errorMessage = worker.StartProcess();
if (errorMessage == null)
{
ProbeWorker(worker, true);
}
else
{
logger.Warn("Error starting worker process for " + worker.AppServerEndpoint.ToString() + ". " + errorMessage);
}
}
private List <string> m_workerSIPEndPoints = new List <string>(); // Allow quick lookups to determine whether a remote end point is that of a worker process.
public SIPAppServerManager(
SIPMonitorLogDelegate logDelegate,
SIPTransport sipTransport,
XmlNode appServerWorkersNode,
string appServerEndPointsPath)
{
if (appServerWorkersNode == null || appServerWorkersNode.ChildNodes.Count == 0)
{
throw new ArgumentNullException("A SIPAppServerManager cannot be created with an empty workers node.");
}
SIPMonitorLogEvent_External = logDelegate;
m_sipTransport = sipTransport;
m_appServerWorkersNode = appServerWorkersNode;
m_appServerEndPointsPath = appServerEndPointsPath;
if (!appServerEndPointsPath.IsNullOrBlank() && File.Exists(appServerEndPointsPath))
{
m_sipCallDispatcherFile = new SIPCallDispatcherFile(logDelegate, appServerEndPointsPath);
}
try
{
CallManagerPassThruServiceInstanceProvider callManagerPassThruSvcInstanceProvider = new CallManagerPassThruServiceInstanceProvider(this);
m_callManagerPassThruSvcHost = new ServiceHost(typeof(CallManagerPassThruService));
m_callManagerPassThruSvcHost.Description.Behaviors.Add(callManagerPassThruSvcInstanceProvider);
m_callManagerPassThruSvcHost.Open();
logger.Debug("SIPAppServerManager CallManagerPassThru hosted service successfully started on " + m_callManagerPassThruSvcHost.BaseAddresses[0].AbsoluteUri + ".");
}
catch (Exception excp)
{
logger.Warn("Exception starting SIPAppServerManager CallManagerPassThru hosted service. " + excp.Message);
}
foreach (XmlNode appServerWorkerNode in m_appServerWorkersNode.ChildNodes)
{
SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode);
if (m_sipCallDispatcherFile != null)
{
appServerWorker.Healthy += WorkerIsHealthy;
appServerWorker.Unhealthy += WorkerIsUnhealthy;
}
m_appServerWorkers.Add(appServerWorker);
m_workerSIPEndPoints.Add(appServerWorker.AppServerEndpoint.ToString());
logger.Debug("SIPAppServerManager worker added for " + appServerWorker.AppServerEndpoint.ToString() + " and " + appServerWorker.CallManagerAddress.ToString() + ".");
}
ThreadPool.QueueUserWorkItem(delegate { SpawnWorkers(); });
ThreadPool.QueueUserWorkItem(delegate { ProbeWorkers(); });
}
/// <summary>
/// Event handler for a successful probe response.
/// </summary>
private void AppServerCallSucceeded(ISIPClientUserAgent uac)
{
try
{
string workerSocket = SIPURI.ParseSIPURI(uac.CallDescriptor.Uri).Host;
SIPAppServerWorker worker = GetWorkerForEndPoint(workerSocket);
if (!worker.InitialProbeResponseReceived)
{
logger.Debug("Initial probe received for " + workerSocket + ".");
worker.InitialCallSuccessful();
}
}
catch (Exception excp)
{
logger.Error("Exception AppServerCallSucceeded. " + excp.Message);
}
}
/// <summary>
/// Sends the SIP INVITE probe request.
/// </summary>
private void ProbeWorker(SIPAppServerWorker worker, bool isInitialProbe)
{
try
{
if (isInitialProbe)
{
worker.InitialProbeCount++;
}
int workerProcessID = worker.WorkerProcess.Id;
SIPEndPoint workerEndPoint = worker.AppServerEndpoint;
DateTime probeSentAt = DateTime.Now;
SIPCallDescriptor callDescriptor = new SIPCallDescriptor(m_dispatcherUsername, null, "sip:" + m_dispatcherUsername + "@" + workerEndPoint.GetIPEndPoint().ToString(),
"sip:" + m_dispatcherUsername + "@sipcalldispatcher", "sip:" + workerEndPoint.GetIPEndPoint().ToString(), null, null, null, SIPCallDirection.Out, null, null, null);
SIPClientUserAgent uac = new SIPClientUserAgent(m_sipTransport, null, null, null, null);
uac.CallFailed += (failedUAC, errorMessage) =>
{
AppServerCallFailed(failedUAC, errorMessage, workerProcessID, probeSentAt, isInitialProbe);
};
uac.CallAnswered += (call, sipResponse) =>
{
if (sipResponse.Status != SIPResponseStatusCodesEnum.BadExtension)
{
//logger.Warn("Probe call answered with unexpected response code of " + sipResponse.StatusCode + ".");
AppServerCallFailed(call, "Unexpected response of " + ((int)sipResponse.StatusCode) + " on probe call.", workerProcessID, probeSentAt, isInitialProbe);
}
else
{
AppServerCallSucceeded(call);
}
};
uac.Call(callDescriptor);
}
catch (Exception excp)
{
logger.Error("Exception SIPAppServerManager ProberWorker. " + excp.Message);
}
}
/// <summary>
/// Method that gets spawned on a dedicated thread to manage an app server process.
/// </summary>
private void ManageWorker(XmlNode appServerWorkerNode)
{
try
{
SIPAppServerWorker appServerWorker = RecycleAppServer(null, 0, appServerWorkerNode);
string appServerSocket = appServerWorker.AppServerEndpoint.ToString();
Thread.CurrentThread.Name = "manage-" + appServerWorker.AppServerEndpoint.Port;
DateTime lastProbeTime = DateTime.MinValue;
while (!m_exit)
{
// Process checks.
if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart)
{
if (appServerWorker.WorkerProcess != null && appServerWorker.WorkerProcess.HasExited)
{
// This is the only case where action is taken to immediately disable the use of a worker process outside of the access controlled
// RecycleAppServer method. The reason being there's no point attempting to use a worker process that has completely gone.
logger.Debug("Worker has disappeared for " + appServerSocket + ". Deactivating and marking for immediate restart.");
appServerWorker.NeedsImmediateRestart = true;
appServerWorker.IsDeactivated = true;
lock (m_appServerWorkers)
{
m_appServerWorkers.Remove(appServerWorker);
}
m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_unhealthyPriority);
}
else
{
appServerWorker.WorkerProcess.Refresh();
if (appServerWorker.WorkerProcess.PrivateMemorySize64 >= MAX_PHYSICAL_MEMORY)
{
logger.Debug("Memory limit reached on worker " + appServerSocket + " pid " + appServerWorker.ProcessID + ". Marking for graceful restart.");
appServerWorker.NeedsToRestart = true;
}
}
}
// Periodically send a SIP call probe to the app server socket.
if (!appServerWorker.NeedsImmediateRestart && !appServerWorker.NeedsToRestart &&
DateTime.Now.Subtract(lastProbeTime).TotalSeconds > m_probePeriodSeconds)
{
appServerWorker.SendProbe();
lastProbeTime = DateTime.Now;
}
// Restarts.
if (appServerWorker.NeedsImmediateRestart || appServerWorker.NeedsToRestart)
{
double secondsSinceLastStart = DateTime.Now.Subtract(appServerWorker.WorkerProcessStartTime).TotalSeconds;
if (secondsSinceLastStart < PROCESS_RESTART_DELAY_SECONDS)
{
int secondsDelay = PROCESS_RESTART_DELAY_SECONDS - Convert.ToInt32(secondsSinceLastStart % Int32.MaxValue);
logger.Debug("Waiting " + secondsDelay + " seconds before attempting to recycle worker on " + appServerSocket + ".");
Thread.Sleep(secondsDelay * 1000);
}
string recycleType = (appServerWorker.NeedsImmediateRestart) ? "immediate" : "graceful";
logger.Debug("Attempting to get lock for " + recycleType + " recycle for " + appServerSocket + " pid " + appServerWorker.ProcessID + ".");
if (Monitor.TryEnter(m_recycleLock, WAIT_FOR_RECYCLE_LOCK_SECONDS * 1000))
{
int recycleDelay = (appServerWorker.NeedsToRestart) ? PROCESS_RESTART_DELAY_SECONDS : 0;
appServerWorker = RecycleAppServer(appServerWorker, recycleDelay, appServerWorkerNode);
lastProbeTime = DateTime.Now;
Monitor.Exit(m_recycleLock);
}
else
{
logger.Debug("Failed to acquire recycle lock for " + appServerSocket + " pid " + appServerWorker.ProcessID + ".");
}
}
Thread.Sleep(CHECK_WORKER_PERIOD_SECONDS * 1000);
}
}
catch (Exception excp)
{
logger.Error("Exception ManageWorker. " + excp.Message);
}
finally
{
try
{
Monitor.Exit(m_recycleLock);
}
catch { }
}
}
private SIPAppServerWorker RecycleAppServer(SIPAppServerWorker badAppServerWorker, int delaySeconds, XmlNode appServerWorkerNode)
{
try
{
lock (m_recycleLock)
{
// Shutdown existing worker process.
if (badAppServerWorker != null)
{
if (!badAppServerWorker.IsDeactivated)
{
logger.Debug("Deactivating worker on " + badAppServerWorker.AppServerEndpoint.ToString() + ".");
m_sipCallDispatcherFile.UpdateAppServerPriority(badAppServerWorker.AppServerEndpoint, m_unhealthyPriority);
lock (m_appServerWorkers)
{
m_appServerWorkers.Remove(badAppServerWorker);
}
}
if (delaySeconds > 0)
{
logger.Debug("Delaying process restart for " + badAppServerWorker.AppServerEndpoint.ToString() + " by " + delaySeconds + "s.");
Thread.Sleep(delaySeconds * 1000);
}
badAppServerWorker.Kill();
}
// Start new worker process and wait for a successful probe response before returning.
SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode, m_sipTransport);
logger.Debug("Starting new worker on " + appServerWorker.AppServerEndpoint.ToString() + ".");
DateTime startTime = DateTime.Now;
if (appServerWorker.StartProcess())
{
logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " ready after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds.");
m_sipCallDispatcherFile.UpdateAppServerPriority(appServerWorker.AppServerEndpoint, m_healthyPriority);
lock (m_appServerWorkers)
{
m_appServerWorkers.Add(appServerWorker);
}
}
else
{
logger.Debug("Worker on " + appServerWorker.AppServerEndpoint.ToString() + " failed to reach a ready state after " + DateTime.Now.Subtract(startTime).TotalSeconds.ToString("0.##") + " seconds.");
appServerWorker.NeedsImmediateRestart = true;
}
return appServerWorker;
}
}
catch (Exception excp)
{
logger.Error("Exception RecycleAppServer. " + excp.Message);
throw;
}
}
/// <summary>
/// Event handler that gets fired when a worker process is identified as being unhealthy at any point in its lifetime.
/// </summary>
/// <param name="worker"></param>
private void WorkerIsUnhealthy(SIPAppServerWorker worker)
{
m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_unhealthyPriority);
}
/// <summary>
/// Event handler that gets fired when a worker process is identified as being healthy after start up.
/// </summary>
private void WorkerIsHealthy(SIPAppServerWorker worker)
{
if (worker.IsHealthy())
{
m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_healthyPriority);
}
else
{
logger.Warn("An app server worker was unhealthy after the process initialisation period.");
}
}
/// <summary>
/// Starts a new SIP application server worker process.
/// </summary>
private void StartWorkerProcess(SIPAppServerWorker worker)
{
string errorMessage = worker.StartProcess();
if (errorMessage == null)
{
ProbeWorker(worker, true);
}
else
{
logger.Warn("Error starting worker process for " + worker.AppServerEndpoint.ToString() + ". " + errorMessage);
}
}
/// <summary>
/// Sends the SIP INVITE probe request.
/// </summary>
private void ProbeWorker(SIPAppServerWorker worker, bool isInitialProbe)
{
try
{
if (isInitialProbe)
{
worker.InitialProbeCount++;
}
int workerProcessID = worker.WorkerProcess.Id;
SIPEndPoint workerEndPoint = worker.AppServerEndpoint;
DateTime probeSentAt = DateTime.Now;
SIPCallDescriptor callDescriptor = new SIPCallDescriptor(m_dispatcherUsername, null, "sip:" + m_dispatcherUsername + "@" + workerEndPoint.GetIPEndPoint().ToString(),
"sip:" + m_dispatcherUsername + "@sipcalldispatcher", "sip:" + workerEndPoint.GetIPEndPoint().ToString(), null, null, null, SIPCallDirection.Out, null, null, null);
SIPClientUserAgent uac = new SIPClientUserAgent(m_sipTransport, null, null, null, null);
uac.CallFailed += (failedUAC, errorMessage) =>
{
AppServerCallFailed(failedUAC, errorMessage, workerProcessID, probeSentAt, isInitialProbe);
};
uac.CallAnswered += (call, sipResponse) =>
{
if (sipResponse.Status != SIPResponseStatusCodesEnum.BadExtension)
{
//logger.Warn("Probe call answered with unexpected response code of " + sipResponse.StatusCode + ".");
AppServerCallFailed(call, "Unexpected response of " + ((int)sipResponse.StatusCode) + " on probe call.", workerProcessID, probeSentAt, isInitialProbe);
}
else
{
AppServerCallSucceeded(call);
}
};
uac.Call(callDescriptor);
}
catch (Exception excp)
{
logger.Error("Exception SIPAppServerManager ProberWorker. " + excp.Message);
}
}
public SIPAppServerManager(
SIPMonitorLogDelegate logDelegate,
SIPTransport sipTransport,
XmlNode appServerWorkersNode,
string appServerEndPointsPath)
{
if (appServerWorkersNode == null || appServerWorkersNode.ChildNodes.Count == 0)
{
throw new ArgumentNullException("A SIPAppServerManager cannot be created with an empty workers node.");
}
SIPMonitorLogEvent_External = logDelegate;
m_sipTransport = sipTransport;
m_appServerWorkersNode = appServerWorkersNode;
m_appServerEndPointsPath = appServerEndPointsPath;
if (!appServerEndPointsPath.IsNullOrBlank() && File.Exists(appServerEndPointsPath))
{
m_sipCallDispatcherFile = new SIPCallDispatcherFile(logDelegate, appServerEndPointsPath);
}
try
{
CallManagerPassThruServiceInstanceProvider callManagerPassThruSvcInstanceProvider = new CallManagerPassThruServiceInstanceProvider(this);
m_callManagerPassThruSvcHost = new ServiceHost(typeof(CallManagerPassThruService));
m_callManagerPassThruSvcHost.Description.Behaviors.Add(callManagerPassThruSvcInstanceProvider);
m_callManagerPassThruSvcHost.Open();
logger.Debug("SIPAppServerManager CallManagerPassThru hosted service successfully started on " + m_callManagerPassThruSvcHost.BaseAddresses[0].AbsoluteUri + ".");
}
catch (Exception excp)
{
logger.Warn("Exception starting SIPAppServerManager CallManagerPassThru hosted service. " + excp.Message);
}
foreach (XmlNode appServerWorkerNode in m_appServerWorkersNode.ChildNodes)
{
SIPAppServerWorker appServerWorker = new SIPAppServerWorker(appServerWorkerNode);
if (m_sipCallDispatcherFile != null)
{
appServerWorker.Healthy += WorkerIsHealthy;
appServerWorker.Unhealthy += WorkerIsUnhealthy;
}
m_appServerWorkers.Add(appServerWorker);
m_workerSIPEndPoints.Add(appServerWorker.AppServerEndpoint.ToString());
logger.Debug("SIPAppServerManager worker added for " + appServerWorker.AppServerEndpoint.ToString() + " and " + appServerWorker.CallManagerAddress.ToString() + ".");
}
ThreadPool.QueueUserWorkItem(delegate { SpawnWorkers(); });
ThreadPool.QueueUserWorkItem(delegate { ProbeWorkers(); });
}
/// <summary>
/// Event handler that gets fired when a worker process is identified as being unhealthy at any point in its lifetime.
/// </summary>
/// <param name="worker"></param>
private void WorkerIsUnhealthy(SIPAppServerWorker worker)
{
m_sipCallDispatcherFile.UpdateAppServerPriority(worker.AppServerEndpoint, m_unhealthyPriority);
}
