/// <summary>
/// Initialise the properties of this executor collection.
/// This involves verfiying the connection to all the dedicated executors in the database.
/// </summary>
public void Init()
{
logger.Debug("Init-ing executor collection from db");
ExecutorStorageView[] executors = ManagerStorageFactory.ManagerStorage().GetExecutors(TriStateBoolean.True);
logger.Debug("# of dedicated executors = " + executors.Length);
foreach (ExecutorStorageView executor in executors)
{
string executorId = executor.ExecutorId;
EndPoint ep = new EndPoint(executor.HostName, executor.Port, RemotingMechanism.TcpBinary);
MExecutor me = new MExecutor(executorId);
try
{
logger.Debug("Creating a MExecutor and connecting-dedicated to it");
me.ConnectDedicated(ep);
}
catch (Exception)
{
logger.Debug("Exception while init-ing exec.collection. Continuing with other executors...");
}
}
logger.Debug("Executor collection init done");
}
private void StartDispatch()
{
logger.Info("Dispatcher thread started.");
try
{
// TODO: allow scheduling of multiple threads in one go
while (!_stop)
{
try
{
//logger.Debug("WaitOne for 1000 millis on DedicatedSchedulerActive");
InternalShared.Instance.DedicatedSchedulerActive.WaitOne(1000, false);
//logger.Debug("Getting a dedicated schedule");
DedicatedSchedule ds = InternalShared.Instance.Scheduler.ScheduleDedicated();
if (ds == null)
{
//to avoid blocking again if stop has been called.
if (!_stop)
{
InternalShared.Instance.DedicatedSchedulerActive.Reset();
//logger.Debug("Dedicated schedule is null. Reset the DedicatedSchedulerActive waithandle");
}
}
else
{
logger.Debug(String.Format("Scheduler mapped thread {0} to executor: {1}", ds.TI.ThreadId, ds.ExecutorId));
MExecutor me = new MExecutor(ds.ExecutorId);
try
{
// dispatch thread
me.ExecuteThread(ds);
/// [email protected] - Feb 28, 2006:
/// updating the thread's status is done inside ExecuteThread after it was decided
/// whether this executor can take it or not
/// this prevents race conditions if the Executor finishes very quickly
/// and we overwrite here the Executor's status back to Scheduled after it was finished
// // update thread state 'after' it is dispatched. (kna changed this: aug19,05). to prevent the scheduler from hanging here.
// mt.CurrentExecutorId = ds.ExecutorId;
// mt.State = ThreadState.Scheduled;
// logger.Debug("Scheduled thread " + ds.TI.ThreadId + " to executor:"+ds.ExecutorId);
}
catch (Exception e)
{
logger.Error("Some error occured trying to schedule. Reset-ing the thread to be scheduled. Continuing...", e);
MThread mt = new MThread(ds.TI);
mt.Reset(); // this should happen as part of the disconnection
}
}
}
catch (ThreadAbortException)
{
logger.Debug("Dispatcher Thread resetting abort (1)...");
Thread.ResetAbort();
}
catch (Exception e)
{
if (e is DbException)
{
//some problem contacting database
//wait for a bit and try again
logger.Debug("Error contacting database:", e);
Thread.Sleep(30000);
//TODO: need to provide fault tolerance here: what if the database/storage cannot be contacted.?
//TODO: in that case, just raise an event, and let the Service/exec deal with it.
}
else
{
logger.Error("Dispatch thread error. Continuing...", e);
}
}
} //while
}
catch (ThreadAbortException)
{
logger.Debug("Scheduler Thread resetting abort (2)...");
Thread.ResetAbort();
}
catch (Exception e)
{
logger.Error("StartDispatch thread error. Scheduler thread stopped.", e);
}
logger.Info("Scheduler thread exited.");
}
private void StartDispatch()
{
logger.Info("Dispatcher thread started.");
try
{
// TODO: allow scheduling of multiple threads in one go
while (!_stop)
{
try
{
//logger.Debug("WaitOne for 1000 millis on DedicatedSchedulerActive");
InternalShared.Instance.DedicatedSchedulerActive.WaitOne(1000, false);
//logger.Debug("Getting a dedicated schedule");
DedicatedSchedule ds = InternalShared.Instance.Scheduler.ScheduleDedicated();
if (ds == null)
{
//to avoid blocking again if stop has been called.
if (!_stop)
{
InternalShared.Instance.DedicatedSchedulerActive.Reset();
//logger.Debug("Dedicated schedule is null. Reset the DedicatedSchedulerActive waithandle");
}
}
else
{
logger.Debug(String.Format("Scheduler mapped thread {0} to executor: {1}", ds.TI.ThreadId, ds.ExecutorId));
MExecutor me = new MExecutor(ds.ExecutorId);
try
{
// dispatch thread
me.ExecuteThread(ds);
/// [email protected] - Feb 28, 2006:
/// updating the thread's status is done inside ExecuteThread after it was decided
/// whether this executor can take it or not
/// this prevents race conditions if the Executor finishes very quickly
/// and we overwrite here the Executor's status back to Scheduled after it was finished
// // update thread state 'after' it is dispatched. (kna changed this: aug19,05). to prevent the scheduler from hanging here.
// mt.CurrentExecutorId = ds.ExecutorId;
// mt.State = ThreadState.Scheduled;
// logger.Debug("Scheduled thread " + ds.TI.ThreadId + " to executor:"+ds.ExecutorId);
}
catch (Exception e)
{
logger.Error("Some error occured trying to schedule. Reset-ing the thread to be scheduled. Continuing...", e);
MThread mt = new MThread(ds.TI);
mt.Reset(); // this should happen as part of the disconnection
}
}
}
catch (ThreadAbortException)
{
logger.Debug("Dispatcher Thread resetting abort (1)...");
Thread.ResetAbort();
}
catch (Exception e)
{
if (e is DbException)
{
//some problem contacting database
//wait for a bit and try again
logger.Debug("Error contacting database:", e);
Thread.Sleep(30000);
//TODO: need to provide fault tolerance here: what if the database/storage cannot be contacted.?
//TODO: in that case, just raise an event, and let the Service/exec deal with it.
}
else
{
logger.Error("Dispatch thread error. Continuing...", e);
}
}
} //while
}
catch (ThreadAbortException)
{
logger.Debug("Scheduler Thread resetting abort (2)...");
Thread.ResetAbort();
}
catch (Exception e)
{
logger.Error("StartDispatch thread error. Scheduler thread stopped.", e);
}
logger.Info("Scheduler thread exited.");
}
//-----------------------------------------------------------------------------------------------
/// <summary>
/// Clean up all application related files on the executors.
/// </summary>
/// <param name="sc">security credentials to verify if the owner has permission to perform this operation
/// (i.e stop application, which is associated with the ManageOwnApp / ManageAllApps permission)</param>
/// <param name="appid"></param>
public void Owner_CleanupApplication(SecurityCredentials sc, string appid)
{
try
{
AuthenticateUser(sc);
ApplicationAuthorizationCheck(sc, appid);
ExecutorStorageView[] executorsStorage = _Executors.AvailableDedicatedExecutors;
foreach (ExecutorStorageView executorStorage in executorsStorage)
{
try
{
string executorId = executorStorage.ExecutorId;
MExecutor me = new MExecutor(executorId);
//do it for dedicated ones only.
if (me.RemoteRef!=null)
me.RemoteRef.Manager_CleanupApplication(appid);
}
catch{}
}
logger.Debug("Cleaning up files on the manager for app: "+appid);
string appDir = string.Format("{0}\\dat\\application_{1}", AppDomain.CurrentDomain.BaseDirectory, appid);
logger.Debug("Deleting: " + appDir);
Directory.Delete(appDir,true);
logger.Debug("Clean up finished for app: "+appid);
}
catch (Exception e)
{
logger.Debug("Clean up app: " + appid + " error: " + e.Message );
}
}
//-----------------------------------------------------------------------------------------------
/// <summary>
/// Aborts the thread with the given id, on the given executor.
/// </summary>
/// <param name="ti">ThreadIdentifier</param>
/// <param name="executorId">Executor id</param>
internal static void AbortThread(ThreadIdentifier ti, string executorId)
{
if (executorId == null)
{
// not being executed on any executor
logger.Debug(string.Format("Null executorID passed in to AbortThread: {0}:{1}. Not aborting...",ti.ApplicationId,ti.ThreadId));
return;
}
MExecutor me = new MExecutor(executorId);
if (me.RemoteRef == null)
{
// not being executed on a dedicated executor .. so can't abort
logger.Debug("AbortThread: not doing anything since the executor is not dedicated.");
return;
}
try
{
logger.Debug("Aborting thread "+ ti.ThreadId +" on executor:"+executorId);
me.RemoteRef.Manager_AbortThread(ti);
logger.Debug("Aborting thread "+ ti.ThreadId +" on executor:"+executorId + ":: Complete.");
}
catch (ExecutorCommException ece)
{
logger.Error("ExecutorCommException...while aborting thread. Disconnecting executor...",ece);
me.Disconnect();
}
catch (Exception e)
{
logger.Debug("Exception aborting thread on executor : " + e.Message,e);
}
}
/// <summary>
/// Clean up all application related files on the executors.
/// </summary>
/// <param name="sc">security credentials to verify if the owner has permission to perform this operation
/// (i.e stop application, which is associated with the ManageOwnApp / ManageAllApps permission)</param>
/// <param name="appid"></param>
public void Owner_CleanupApplication(SecurityCredentials sc, string appid)
{
try
{
AuthenticateUser(sc);
ApplicationAuthorizationCheck(sc, appid);
ExecutorStorageView[] executorsStorage = _Executors.AvailableDedicatedExecutors;
foreach (ExecutorStorageView executorStorage in executorsStorage)
{
try
{
string executorId = executorStorage.ExecutorId;
MExecutor me = new MExecutor(executorId);
//do it for dedicated ones only.
if (me.RemoteRef != null)
me.RemoteRef.Manager_CleanupApplication(appid);
}
catch(Exception e)
{
logger.Warn("Exception calling Manager_CleanupApplication: " + appid + " error: " + e.Message);
}
}
Cleanup(appid);
}
catch (Exception e)
{
logger.Debug("Clean up app: " + appid + " error: " + e.Message);
}
}
private void StartWatch()
{
logger.Info("WatchDog thread started.");
try
{
while (!_stop)
{
try
{
Thread.Sleep(7000);
// ping dedicated executors running threads and reset executor and thread if can't ping
ThreadStorageView[] dedicatedRunningThreadsStorage = ManagerStorageFactory.ManagerStorage().GetExecutorThreads(true,
Alchemi.Core.Owner.ThreadState.Scheduled, Alchemi.Core.Owner.ThreadState.Started);
foreach (ThreadStorageView threadStorage in dedicatedRunningThreadsStorage)
{
MExecutor me = _Executors[threadStorage.ExecutorId];
try
{
me.RemoteRef.PingExecutor();
}
catch
{
me.Disconnect();
new MThread(threadStorage.ApplicationId, threadStorage.ThreadId).Reset();
}
}
// disconnect nde if not recd alive notification in the last 90 seconds
// TODO: make time interval configurable
int secondsToTimeout = 90;
DisconnectTimedOutExecutors(new TimeSpan(0, 0, secondsToTimeout));
// reset threads whose executors have been disconnected
ThreadStorageView[] nonDedicatedLostThreadsStorage = ManagerStorageFactory.ManagerStorage().GetExecutorThreads(
false,
false,
Alchemi.Core.Owner.ThreadState.Scheduled,
Alchemi.Core.Owner.ThreadState.Started,
Alchemi.Core.Owner.ThreadState.Finished,
Alchemi.Core.Owner.ThreadState.Dead);
foreach (ThreadStorageView threadStorage in nonDedicatedLostThreadsStorage)
{
new MThread(threadStorage.ApplicationId, threadStorage.ThreadId).Reset();
new MExecutor(threadStorage.ExecutorId).Disconnect();
}
}
catch (ThreadAbortException)
{
logger.Debug("StartWatch thread aborting...");
Thread.ResetAbort();
}
catch (Exception ex)
{
logger.Debug("Error in WatchDog thread. Continuing after error...", ex);
}
} //while
}
catch (ThreadAbortException)
{
logger.Debug("StartWatch thread aborting...");
Thread.ResetAbort();
}
catch (Exception e)
{
logger.Error("WatchDog thread error. WatchDog thread stopped.", e);
}
logger.Info("WatchDog thread exited.");
}
/// <summary>
/// Initialise the properties of this executor collection.
/// This involves verfiying the connection to all the dedicated executors in the database.
/// </summary>
public void Init()
{
logger.Debug("Init-ing executor collection from db");
ExecutorStorageView[] executors = ManagerStorageFactory.ManagerStorage().GetExecutors(TriStateBoolean.True);
logger.Debug("# of dedicated executors = " + executors.Length);
foreach (ExecutorStorageView executor in executors)
{
string executorId = executor.ExecutorId;
EndPoint ep = new EndPoint(executor.HostName, executor.Port, RemotingMechanism.TcpBinary);
MExecutor me = new MExecutor(executorId);
try
{
logger.Debug("Creating a MExecutor and connecting-dedicated to it");
me.ConnectDedicated(ep);
}
catch (Exception)
{
logger.Debug("Exception while init-ing exec.collection. Continuing with other executors...");
}
}
logger.Debug("Executor collection init done");
}