/// <summary>
/// Verify if the executor exists in the database and the remote endpoint host matches the database setting
/// </summary>
/// <param name="ep"></param>
/// <returns></returns>
private bool VerifyExists(EndPoint ep)
{
bool exists = false;
//TODO: review the use of this method!
try
{
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
if (executorStorage != null)
{
bool remoteEndPointNullOrHostIsSameAsExecutor;
remoteEndPointNullOrHostIsSameAsExecutor =
ep == null || (ep != null && executorStorage.HostName == ep.Host);
exists = remoteEndPointNullOrHostIsSameAsExecutor;
}
}
catch (Exception ex)
{
logger.Error("Executor :" + _Id + " invalid id? ", ex);
throw new InvalidExecutorException("The supplied Executor ID is invalid.", ex);
}
return(exists);
}
/// <summary>
/// Return the thread with the highest priority from the pool of Ready threads.
///
/// Note: pathetic way of selecting the highest priority thread.
/// This should be moved into the storage for a more efficient implementation.
/// </summary>
/// <returns></returns>
protected ThreadStorageView GetNextAvailableThread()
{
// achieve thread safety by locking on a static variable
// this lock is not enough, we should lock until the thread status changes
lock (threadChooserLock)
{
ThreadStorageView[] threads = ManagerStorageFactory.ManagerStorage().GetThreads(
ApplicationState.Ready,
ThreadState.Ready);
if (threads == null || threads.Length == 0)
{
return(null);
}
ThreadStorageView highestPriorityThread = threads[0];
foreach (ThreadStorageView thread in threads)
{
if (thread.Priority > highestPriorityThread.Priority)
{
highestPriorityThread = thread;
}
}
return(highestPriorityThread);
}
}
/// <summary>
/// Stops an application.
/// </summary>
public void Stop()
{
//first check if the application is all finished
IManagerStorage store = ManagerStorageFactory.ManagerStorage();
ApplicationStorageView app = store.GetApplication(_Id);
if (app.State != ApplicationState.Stopped)
{
ThreadStorageView[] threads = store.GetThreads(_Id);
foreach (ThreadStorageView thread in threads)
{
if (thread.State != ThreadState.Dead && thread.State != ThreadState.Finished)
{
GManager.AbortThread(new ThreadIdentifier(_Id, thread.ThreadId), thread.ExecutorId);
// clean up the thread status
thread.State = ThreadState.Dead;
store.UpdateThread(thread);
}
}
//update the application
app.State = ApplicationState.Stopped;
app.TimeCompleted = DateTime.Now;
store.UpdateApplication(app);
logger.Debug("Stopped the current application." + _Id);
}
else
{
logger.Debug(string.Format("Application {0} already stopped.", _Id));
}
}
/// <summary>
/// Initialise the properties of this executor collection.
/// This involves verfiying the connection to all the dedicated executors in the database.
/// </summary>
public void Init()
{
logger.Debug("Init-ing executor collection from db");
ExecutorStorageView[] executors = ManagerStorageFactory.ManagerStorage().GetExecutors(TriStateBoolean.True);
logger.Debug("# of dedicated executors = " + executors.Length);
foreach (ExecutorStorageView executor in executors)
{
string executorId = executor.ExecutorId;
EndPoint ep = new EndPoint(executor.HostName, executor.Port, RemotingMechanism.TcpBinary);
MExecutor me = new MExecutor(executorId);
try
{
logger.Debug("Creating a MExecutor and connecting-dedicated to it");
me.ConnectDedicated(ep);
}
catch (Exception)
{
logger.Debug("Exception while init-ing exec.collection. Continuing with other executors...");
}
}
logger.Debug("Executor collection init done");
}
/// <summary>
/// Verify if the executor exists in the database and the remote endpoint host matches the database setting
/// </summary>
/// <param name="ep"></param>
/// <returns></returns>
private bool VerifyExists(RemoteEndPoint ep)
{
bool exists = false;
try
{
logger.Debug("Checking if executor :" + _Id + " exists in db");
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
bool remoteEndPointNullOrHostIsSameAsExecutor;
remoteEndPointNullOrHostIsSameAsExecutor =
ep == null || (ep != null && executorStorage.HostName == ep.Host);
if (executorStorage != null && remoteEndPointNullOrHostIsSameAsExecutor)
{
exists = true;
}
logger.Debug("Executor :" + _Id + " exists in db=" + exists);
}
catch (Exception ex)
{
logger.Error("Executor :" + _Id + " invalid id? ", ex);
throw new InvalidExecutorException("The supplied Executor ID is invalid.", ex);
}
return(exists);
}
public void SetUp()
{
_managerStorage = new InMemoryManagerStorage();
ManagerStorageFactory.SetManagerStorage(_managerStorage);
_executorCollection = new MExecutorCollection();
}
/// <summary>
/// Initializes this thread.
/// </summary>
/// <param name="primary">specifies if this thread is a primary thread. A thread is primary if it is created and scheduled by the same manager</param>
public void Init(bool primary)
{
if (primary)
{
ThreadStorageView threadStorage = new ThreadStorageView(_AppId, _Id, ThreadState.Ready);
ManagerStorageFactory.ManagerStorage().AddThread(threadStorage);
}
else
{
ApplicationStorageView applicationStorage = ManagerStorageFactory.ManagerStorage().GetApplication(_AppId);
if (applicationStorage == null)
{
applicationStorage = new ApplicationStorageView(
_AppId,
ApplicationState.AwaitingManifest,
DateTime.Now,
false,
"" /* TODO: What's the username here?*/);
ManagerStorageFactory.ManagerStorage().AddApplication(applicationStorage);
}
ThreadStorageView threadStorage = new ThreadStorageView(_AppId, _Id, ThreadState.Ready);
ManagerStorageFactory.ManagerStorage().AddThread(threadStorage);
}
}
/// <summary>
/// Connects to the executor in dedicated mode.
/// </summary>
/// <param name="ep">end point of the executor</param>
public void ConnectDedicated(EndPoint ep)
{
logger.Debug("Trying to connect Dedicated to executor: " + _Id);
if (!VerifyExists(ep))
{
logger.Debug("The supplied Executor ID does not exist.");
throw new InvalidExecutorException("The supplied Executor ID does not exist.", null);
}
bool success = false;
IExecutor executor;
EndPointReference epr = null;
try
{
epr = GNode.GetRemoteRef(ep, typeof(IExecutor));
executor = (IExecutor)epr.Instance;
executor.PingExecutor(); //connect back to executor.
success = true;
logger.Debug("Connected dedicated. Executor_id=" + _Id);
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.Connected = success;
executorStorage.Dedicated = true;
executorStorage.HostName = ep.Host;
executorStorage.Port = ep.Port;
// update state in db (always happens even if cannnot connect back to executor
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
logger.Debug("Updated db after ping back to executor. dedicated executor_id=" + _Id + ", dedicated = true, connected = " + success);
// update hashtable
//for thread-safety
lock (_DedicatedExecutors)
{
//TODO: change this collection to a collection of EndPointReferences of the connections will not be properly disposed.
if (!_DedicatedExecutors.ContainsKey(_Id))
{
_DedicatedExecutors.Add(_Id, executor);
logger.Debug("Added to list of dedicated executors: executor_id=" + _Id);
}
else
{
//WCF ( doesnt remoting do that to ) closes the connection if executor connects and disconects.
//So we must remove the old and add the new record
//Jure Subara
_DedicatedExecutors.Remove(_Id);
_DedicatedExecutors.Add(_Id, executor);
logger.Debug("Refreshed the record in list od dedicated executors: executor_id=" + _Id);
}
}
}
catch (Exception e)
{
logger.Error("Error connecting to exec: " + _Id, e);
throw new ExecutorCommException(_Id, e);
}
}
/// <summary>
/// Disconnect from the executor.
/// (Updates the database to reflect the executor disconnection.)
/// </summary>
public void Disconnect()
{
// maybe should reset threads here as part of the disconnection rather than explicitly ...
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.Connected = false;
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
}
/// <summary>
/// Gets all the threads from the given application ids.
/// </summary>
/// <param name="cApplicationIds">application ids</param>
/// <returns>all threads from application ids</returns>
private IList GetThreads(IList cApplicationIds)
{
ArrayList cThreads = new ArrayList();
foreach (string strApplicationId in cApplicationIds)
{
cThreads.AddRange(ManagerStorageFactory.ManagerStorage().GetThreads(strApplicationId, ThreadState.Ready));
}
return(cThreads);
}
/// <summary>
/// Creates a new application
/// </summary>
/// <param name="username">the user associated with the application</param>
/// <returns>Id of the newly created application</returns>
public string CreateNew(string username)
{
//Krishna: changed to use the constructor with created-time
ApplicationStorageView application =
new ApplicationStorageView(ApplicationState.Ready, DateTime.Now, true, username);
//new ApplicationStorageView(username);
string appId = ManagerStorageFactory.ManagerStorage().AddApplication(application);
this[appId].CreateDataDirectory();
return(appId);
}
/// <summary>
/// Updates the database with the heartbeat info of this executor
/// </summary>
/// <param name="info"></param>
public void HeartbeatUpdate(HeartbeatInfo info)
{
// update ping time and other heartbeatinfo
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.PingTime = DateTime.Now;
executorStorage.Connected = true;
executorStorage.CpuUsage = info.PercentUsedCpuPower;
executorStorage.AvailableCpu = info.PercentAvailCpuPower;
executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100;
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
}
/// <summary>
/// Execute a thread on a dedicated Executor.
///
/// Right before the execution the thread's status is set the Scheduled
/// and the thread's executor is set to the executor's ID.
/// Spawn a new thread that remotes the execution to the Executor.
///
/// </summary>
/// <param name="ds">Containes the Thread ID and the Executor ID.</param>
/// <returns>True if the thread was successfully started on the Executor.</returns>
public bool ExecuteThread(DedicatedSchedule ds)
{
bool success = false;
//kna added this to allow controlling MAX_CONCURRENT_THREADS per executor. Aug19. 05
//find # of executing threads from the db.
int numConcurrentThreads = 0;
MThread mt = new MThread(ds.TI);
try
{
numConcurrentThreads = ManagerStorageFactory.ManagerStorage().GetExecutorThreadCount(
_Id,
ThreadState.Ready,
ThreadState.Scheduled,
ThreadState.Started);
if (numConcurrentThreads >= MAX_CONCURRENT_THREADS)
{
success = false;
}
else
{
/// [email protected] - Feb 28, 2006:
/// moved the thread status updating here from ManagerContainer.ScheduleDedicated
/// to make sure that the thread state is written in the right order
mt.CurrentExecutorId = ds.ExecutorId;
mt.State = ThreadState.Scheduled;
logger.Debug(String.Format("Scheduling thread {0} to executor: {1}", ds.TI.ThreadId, ds.ExecutorId));
this.currentTI = ds.TI;
Thread dispatchThread = new Thread(new ThreadStart(this.ExecuteCurrentThread));
dispatchThread.Name = "ScheduleDispatchThread";
dispatchThread.Start();
success = true;
}
}
catch (Exception ex)
{
// restore the thread status so another executor can pick it up
mt.CurrentExecutorId = null;
mt.State = ThreadState.Ready;
logger.Debug("Error scheduling thread on executor " + _Id, ex);
}
return(success);
}
/// <summary>
/// Gets the active executor ids.
/// </summary>
/// <returns></returns>
private IList GetActiveExecutorIds()
{
IList cActiveExecutorIds = new ArrayList();
ExecutorStorageView[] cExecutors = ManagerStorageFactory.ManagerStorage().GetExecutors();
foreach (ExecutorStorageView oExecutor in cExecutors)
{
if (oExecutor.Connected)
{
cActiveExecutorIds.Add(oExecutor.ExecutorId);
}
}
return(cActiveExecutorIds);
}
/// <summary>
/// Disconnect executors if the alive notification was not received in the alloted time.
/// </summary>
/// <param name="time"></param>
private void DisconnectTimedOutExecutors(TimeSpan time)
{
DateTime currentTime = DateTime.Now; // storing the current time so we don't unfairly disconnect executors in case this process takes some time to complete
ExecutorStorageView[] connectedExecutorsStorage = ManagerStorageFactory.ManagerStorage().GetExecutors(TriStateBoolean.Undefined, TriStateBoolean.True);
foreach (ExecutorStorageView executorStorage in connectedExecutorsStorage)
{
if (executorStorage.PingTime.Add(time) < currentTime)
{
executorStorage.Connected = false;
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
}
}
}
/// <summary>
/// Gets the active application ids.
/// </summary>
/// <returns></returns>
private IList GetActiveApplicationIds()
{
IList cActiveApplicationIds = new ArrayList();
ApplicationStorageView[] cApplications = ManagerStorageFactory.ManagerStorage().GetApplications();
foreach (ApplicationStorageView oApplication in cApplications)
{
if (oApplication.State == ApplicationState.Ready)
{
cActiveApplicationIds.Add(oApplication.ApplicationId);
}
}
return(cActiveApplicationIds);
}
/// <summary>
/// Gets the next executor for the given application id.
/// </summary>
/// <param name="strApplicationId">application id</param>
/// <returns>next executor for application</returns>
private string GetNextExecutor(string strApplicationId)
{
IList cExecutorIds = _Mapping.GetExecutorIds(strApplicationId);
foreach (string strExecutorId in cExecutorIds)
{
int nThreadCount = ManagerStorageFactory.ManagerStorage().GetExecutorThreadCount(strExecutorId, ThreadState.Ready, ThreadState.Scheduled, ThreadState.Started);
if (nThreadCount == 0)
{
return(strExecutorId);
}
}
return(null);
}
/// <summary>
/// Resets this MThread so it can be rescheduled.
/// </summary>
public void Reset()
{
ThreadStorageView threadStorage = ManagerStorageFactory.ManagerStorage().GetThread(_AppId, _Id);
//take care that only threads that are not already aborted are reset.
if (threadStorage.State != ThreadState.Dead)
{
threadStorage.State = ThreadState.Ready;
threadStorage.ExecutorId = null;
threadStorage.ResetTimeStarted();
threadStorage.ResetTimeFinished();
ManagerStorageFactory.ManagerStorage().UpdateThread(threadStorage);
}
}
/// <summary>
/// Verify if the database is properly set up.
///
/// </summary>
/// <param name="id">application id</param>
/// <returns>true if the application is set up in the database</returns>
public bool VerifyApp(string id)
{
bool appSetup = true;
//create directory can be repeated, and wont throw an error even if the dir already exists.
this[id].CreateDataDirectory();
ApplicationStorageView application = ManagerStorageFactory.ManagerStorage().GetApplication(id);
if (application == null)
{
appSetup = false;
}
return(appSetup);
}
/// <summary>
/// Connects to the executor in dedicated mode.
/// </summary>
/// <param name="ep">end point of the executor</param>
public void ConnectDedicated(RemoteEndPoint ep)
{
logger.Debug("Trying to connect Dedicated to executor: " + _Id);
if (!VerifyExists(ep))
{
logger.Debug("The supplied Executor ID does not exist.");
throw new InvalidExecutorException("The supplied Executor ID does not exist.", null);
}
bool success = false;
IExecutor executor;
try
{
executor = (IExecutor)GNode.GetRemoteRef(ep, _Id);
executor.PingExecutor(); //connect back to executor.
success = true;
logger.Debug("Connected dedicated. Executor_id=" + _Id);
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.Connected = success;
executorStorage.Dedicated = true;
executorStorage.HostName = ep.Host;
executorStorage.Port = ep.Port;
// update state in db (always happens even if cannnot connect back to executor
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
logger.Debug("Updated db after ping back to executor. dedicated executor_id=" + _Id + ", dedicated = true, connected = " + success);
// update hashtable
if (!_DedicatedExecutors.ContainsKey(_Id))
{
_DedicatedExecutors.Add(_Id, executor);
logger.Debug("Added to list of dedicated executors: executor_id=" + _Id);
}
}
catch (Exception e)
{
logger.Error("Error connecting to exec: " + _Id, e);
throw new ExecutorCommException(_Id, e);
}
}
/// <summary>
/// Connects to the executor in non-dedicated mode.
/// </summary>
public void ConnectNonDedicated(RemoteEndPoint ep)
{
logger.Debug("Trying to connect NON-Dedicated to executor: " + _Id);
if (!VerifyExists(ep))
{
logger.Debug("The supplied Executor ID does not exist.");
throw new InvalidExecutorException("The supplied Executor ID does not exist.", null);
}
//here we don't ping back, since we know non-dedicated nodes can't be pinged back.
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.Connected = true;
executorStorage.Dedicated = false;
// update state in db
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
logger.Debug("Connected non-dedicated. Updated db. executor_id=" + _Id);
}
/// <summary>
/// Return the next available ExecutorId.
/// For an executor to be available the following considions have to be met:
/// - executor is dedicated
/// - executor is conected
/// - executor has no threads running or scheduled
/// </summary>
/// <returns></returns>
protected ExecutorStorageView GetNextAvailableExecutor()
{
// [email protected]; gets next available executor without bias based upon position in executors list.
ExecutorStorageView[] executors = ManagerStorageFactory.ManagerStorage().GetExecutors(TriStateBoolean.True, TriStateBoolean.True);
for (int i = 0; i < executors.Length; i++)
{
nextExecutorIndex++;
if (nextExecutorIndex >= executors.Length)
{
nextExecutorIndex = 0;
}
ExecutorStorageView executor = executors[nextExecutorIndex];
if (ManagerStorageFactory.ManagerStorage().GetExecutorThreadCount(executor.ExecutorId, ThreadState.Ready, ThreadState.Scheduled, ThreadState.Started) == 0)
{
return(executor);
}
}
return(null);
}
/// <summary>
/// Registers a new executor with the manager
/// </summary>
/// <param name="sc">security credentials used to perform this operation.</param>
/// <param name="executorId">The preferred executor ID. Set it to null to generate a new ID.</param>
/// <param name="info">information about the executor (see ExecutorInfo)</param>
/// <returns></returns>
public string RegisterNew(SecurityCredentials sc, string executorId, ExecutorInfo info)
{
//NOTE: when registering, all executors are initially set to non-dedicated.non-connected.
//once they connect and can be pinged back, then these values are set accordingly.
ExecutorStorageView executorStorage = new ExecutorStorageView(
executorId,
false,
false,
info.Hostname,
sc.Username,
info.MaxCpuPower,
info.MaxMemory,
info.MaxDiskSpace,
info.NumberOfCpus,
info.OS,
info.Architecture
);
if (this[executorId].VerifyExists())
{
// executor already exists, just update the details
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
logger.Debug("Updated executor details = " + executorId);
return(executorId);
}
else
{
string newExecutorId = ManagerStorageFactory.ManagerStorage().AddExecutor(executorStorage);
logger.Debug("Registered new executor id=" + newExecutorId);
return(newExecutorId);
}
}
/// <summary>
/// Verify if the database is properly set up.
///
/// </summary>
/// <param name="id">application id</param>
/// <returns>true if the application is set up in the database</returns>
public bool VerifyApp(string id)
{
IManagerStorage store = ManagerStorageFactory.ManagerStorage();
bool appSetup = true;
//create directory can be repeated, and wont throw an error even if the dir already exists.
this[id].CreateDataDirectory();
ApplicationStorageView application = store.GetApplication(id);
if (application == null)
{
appSetup = false;
}
else
{
//just make sure the status is set to ready, or else it may remain Stopped
//in case of re-started multi-use apps.
application.State = ApplicationState.Ready;
store.UpdateApplication(application);
}
return(appSetup);
}
/// <summary>
/// Gets the list of applications for the given user.
/// </summary>
/// <param name="user_name"></param>
/// <returns>ApplicationStorageView array with the requested information.</returns>
public ApplicationStorageView[] GetApplicationList(string user_name)
{
return(ManagerStorageFactory.ManagerStorage().GetApplications(user_name, true));
}
public void TearDown()
{
_managerStorage = null;
ManagerStorageFactory.SetManagerStorage(null);
}
public void SetUp()
{
_managerStorage = new InMemoryManagerStorage();
ManagerStorageFactory.SetManagerStorage(_managerStorage);
}
/// <summary>
/// Gets the list of threads with the given thread-state
/// </summary>
/// <param name="status"></param>
/// <returns>Dataset with thread info.</returns>
public ThreadStorageView[] GetThreadList(ThreadState status)
{
return(ManagerStorageFactory.ManagerStorage().GetThreads(_Id, status));
}
/// <summary>
/// Gets the count threads with the given thread-state.
/// </summary>
/// <param name="ts"></param>
/// <returns>Thread count</returns>
public int ThreadCount(ThreadState ts)
{
return(ManagerStorageFactory.ManagerStorage().GetApplicationThreadCount(_Id, ts));
}
/// <summary>
/// Gets the threads.
/// </summary>
/// <returns></returns>
private IList GetThreads()
{
return(new ArrayList(ManagerStorageFactory.ManagerStorage().GetThreads(ApplicationState.Ready, ThreadState.Ready)));
}
