/// <summary>
/// Shuts down this asset
/// </summary>
public override void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
// If the shutdown option is set to not shutdown the simulator, then
// set the state to offline and return.
if (!options.ShutdownDeviceSimulators)
{
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
return;
}
_inShutdown = true;
TraceFactory.Logger.Debug("Shutting down {0}".FormatWith(_machine.Name));
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
// Cancel the deployment if it's starting up.
CancelBootup();
if (_machine.IsPoweredOn())
{
try
{
JediSimulatorManager.ShutdownSimulator(((JediSimulatorDetail)Asset).MachineName);
}
catch (Exception ex)
{
TraceFactory.Logger.Debug("Simulator shutdown failed. Continuing with machine shutdown. {0}".FormatWith(ex.Message));
}
_machine.Shutdown();
}
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
private void ShutdownScenarioHandler(string sessionId, ShutdownOptions options)
{
try
{
// If a session exists, proceed with the shutdown, otherwise, just return.
SessionProxyController controller = null;
if (_proxyControllers.TryGetValue(sessionId, out controller))
{
_proxyControllers.Remove(controller);
TraceFactory.Logger.Debug("Found & removed controller: Cnt: {0} : {1}".FormatWith(_proxyControllers.Count, controller.Endpoint.AbsoluteUri));
if (!controller.SessionClosing) //Make sure the session isn't already closing
{
lock (controller)
{
controller.SessionClosing = true;
ThreadPool.QueueUserWorkItem(t => ShutdownScenario(controller, sessionId, options));
}
}
}
}
catch (Exception ex)
{
TraceFactory.Logger.Error(ex);
}
}
/// <summary>
/// Shuts down this print queue
/// </summary>
public virtual void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
if (options.PurgeRemotePrintQueues)
{
PurgeRemotePrintQueue(PrintServerName + '.' + GlobalSettings.Items[Setting.DnsDomain], PrintQueueName);
}
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
public override void Shutdown(ShutdownOptions options)
{
var credential = GlobalSettings.Items.DomainAdminCredential;
List <string> commands = new List <string>();
commands.Add(Properties.Resources.TaskKill.FormatWith("PrintMonitorService"));
commands.Add(Properties.Resources.TaskKill.FormatWith("ClientFactoryConsole"));
string command = string.Join(" && ", commands);
TraceFactory.Logger.Debug("Killing services on {0}".FormatWith(_machine.Name));
RunCommand(command, credential);
}
public override void Shutdown(ShutdownOptions options)
{
if (!options.PowerOff)
{
TraceFactory.Logger.Debug("{0} Power Off option not set, returning...".FormatWith(_machine.Name));
return;
}
if (!_machine.IsPoweredOn())
{
TraceFactory.Logger.Debug("{0} not powered on, returning...".FormatWith(_machine.Name));
return;
}
Action shutdownAction = new Action(() =>
{
switch (options.PowerOffOption)
{
case VMPowerOffOption.PowerOff:
try
{
_machine.Shutdown(wait: true);
}
catch (SoapException ex)
{
TraceFactory.Logger.Warn("{0} Failed graceful shutdown. Attempting power off: {1}".FormatWith(_machine.Name, ex.Message));
_machine.PowerOff();
}
break;
case VMPowerOffOption.RevertToSnapshot:
_machine.Revert(wait: true);
break;
default:
throw new NotSupportedException("Unsupported shutdown type: " + options.PowerOffOption);
}
});
try
{
MachineStop.Run(_machine.Name, shutdownAction);
TraceFactory.Logger.Debug("Shutdown completed for {0}".FormatWith(_machine.Name));
}
catch (Exception ex)
{
TraceFactory.Logger.Error("Error shutting down machine, continuing...", ex);
}
}
/// <summary>
/// Shuts down all resource hosts in this map using the specified options
/// </summary>
/// <param name="options">The options.</param>
/// <param name="loopState">State of the loop.</param>
public override void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
try
{
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
Parallel.ForEach <ResourceHost>(Hosts, (h, l) => h.Shutdown(options, l));
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
catch (AggregateException ex)
{
// Log the exception at this element level, then throw again to catch it higher
MapElement.UpdateStatus(RuntimeState.Error, "Shutdown error", ex);
throw;
}
}
/// <summary>
/// Shuts down this instance.
/// </summary>
public void Shutdown(ShutdownOptions options)
{
if (Endpoint == null)
{
TraceFactory.Logger.Info("{0} never registered, so marking as already shut down.".FormatWith(Id));
ChangeState(RuntimeState.Offline);
return;
}
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
try
{
TraceFactory.Logger.Debug("{0} - Shutdown Signaled".FormatWith(Endpoint));
ChangeState(RuntimeState.ShuttingDown);
var action = new Action(() =>
{
using (var client = VirtualResourceManagementConnection.Create(Endpoint))
{
client.Channel.Shutdown(options.CopyLogs);
}
});
var listofExceptions = new List <Type>()
{
typeof(EndpointNotFoundException), typeof(TimeoutException)
};
int retries = Retry.WhileThrowing(action, 2, TimeSpan.FromSeconds(20), listofExceptions);
TraceFactory.Logger.Debug("{0} -> Shutdown Complete: RETRIES: {1}".FormatWith(Endpoint, retries));
}
catch (Exception ex)
{
TraceFactory.Logger.Error("{0} -> Shutdown Failed, continuing: {1}".FormatWith(Endpoint, ex.ToString()));
//ChangeState(RuntimeState.Offline);
//MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
finally
{
//Only used to speed up shutdown in ResourceHost
ChangeState(RuntimeState.Offline);
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
}
/// <summary>
/// Shuts down all maps using the specified options
/// </summary>
/// <param name="options">The shutdown options (unused).</param>
/// <param name="loopState">State of the loop.</param>
public override void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
try
{
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
Parallel.ForEach <RemotePrintQueueElement>(RemotePrintQueueElements, (h, l) => h.Shutdown(options, l));
if (!SessionMapElement.AllElementsSetTo <RemotePrintQueueElement>(RemotePrintQueueElements, RuntimeState.Offline))
{
loopState.Break();
}
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
catch (AggregateException ex)
{
MapElement.UpdateStatus(RuntimeState.Error, "Shutdown error", ex);
throw;
}
}
/// <summary>
/// Shuts down all maps
/// </summary>
public override void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
// The shutdown options are not used by the asset map so just ignore them
try
{
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
Parallel.ForEach <AssetHost>(Hosts, (h, l) => h.Shutdown(options, l));
if (!SessionMapElement.AllElementsSetTo <AssetHost>(Hosts, RuntimeState.Offline))
{
loopState.Break();
}
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
catch (AggregateException ex)
{
// Log the exception at this element level, the throw again to catch it higher
MapElement.UpdateStatus(RuntimeState.Error, "Shutdown error", ex);
throw;
}
}
/// <summary>
/// Shuts down this asset host
/// </summary>
/// <param name="options"></param>
/// <param name="loopState"></param>
public override void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
//_useRecording = _cameraDetail.UseAutoStart;
if (_cameraDetail.UseAutoStart)
{
using (CameraSessionManager manager = new CameraSessionManager(_cameraDetail.Address, _cameraDetail.ServerUser, _cameraDetail.ServerPassword))
{
try
{
// Stop the camera
manager.StopRecordingCommand(_cameraDetail.CameraId);
}
catch (Exception ex)
{
TraceFactory.Logger.Error($"Cannot Stop the camera: {_cameraDetail.CameraId}.", ex);
MapElement.UpdateStatus($"Camera {_cameraDetail.CameraId} is not stopping properly.", RuntimeState.Warning);
}
}
}
base.Shutdown(options, loopState);
}
/// <summary>
/// Closes the specified session with given Shutdown Options
/// </summary>
/// <param name="sessionId"></param>
/// <param name="options"></param>
public void Close(string sessionId, ShutdownOptions options)
{
TraceFactory.Logger.Debug("SessionId: {0}".FormatWith(sessionId));
CallDispatcher((c) => c.Close(sessionId, options));
}
/// <summary>
/// Shuts down this host instance.
/// </summary>
/// <param name="options">The options.</param>
public virtual void Shutdown(ShutdownOptions options)
{
}
private void ShutdownScenario(SessionProxyController controller, string sessionId, ShutdownOptions options)
{
try
{
SetTraceSessionContext(sessionId);
TraceFactory.Logger.Debug("{0}: {1}".FormatWith(sessionId, controller.Endpoint.AbsoluteUri));
if (controller.ProxyProcessStarted)
{
controller.Channel.Shutdown(options);
}
else
{
TraceFactory.Logger.Debug("Proxy process not started");
}
}
catch (Exception ex)
{
TraceFactory.Logger.Warn("Error sending shutdown call, ignoring", ex);
}
finally
{
controller.Dispose();
}
}
/// <summary> /// Shuts down all maps using the specified options /// </summary> /// <param name="options">The options.</param> /// <param name="loopState">State of the loop.</param> public abstract void Shutdown(ShutdownOptions options, ParallelLoopState loopState);
/// <summary>
/// Shuts down this asset host
/// </summary>
public virtual void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
public override void Shutdown(ShutdownOptions options)
{
_controller.Stop();
}
private void CloseSessionHandler(string sessionId, ShutdownOptions options)
{
SetTraceSessionContext(sessionId);
SessionProxyController controller = null;
if (_proxyControllers.TryGetValue(sessionId, out controller)) //If a session exists, proceed with normal shutdown. Otherwise reset manually.
{
_proxyControllers.Remove(controller);
TraceFactory.Logger.Debug("Found & removed controller: Count: {0} : {1}".FormatWith(_proxyControllers.Count, controller.Endpoint.AbsoluteUri));
if (!controller.SessionClosing) //Make sure the session isn't already closing
{
lock (controller)
{
controller.SessionClosing = true;
try
{
ThreadPool.QueueUserWorkItem(t => ShutdownScenario(controller, sessionId, options));
}
catch (Exception ex)
{
TraceFactory.Logger.Error(ex);
}
}
}
}
else
{
TraceFactory.Logger.Debug(string.Format("Closing session {0}", sessionId));
// Manually reset DomainAccountReservation for all inactive SessionIds
using (AssetInventoryContext context = DbConnect.AssetInventoryContext())
{
List <string> activeSessionIds = context.FrameworkClients.Select(n => n.SessionId).Distinct().Where(n => n != null).ToList();
foreach (var reservation in context.DomainAccountReservations)
{
if (!activeSessionIds.Contains(reservation.SessionId))
{
context.DomainAccountReservations.Remove(reservation);
}
}
context.SaveChanges();
}
try
{
// Parallel process the two actions required to clean up this session, first define
// the collection of actions, then spawn them each off in the background and then
// wait for them to return.
var actions = new Collection <Action>()
{
() => CleanupAssetHosts(sessionId),
() => CleanupResourceHosts(sessionId)
};
Parallel.ForEach <Action>(actions, a => a());
using (DataLogContext context = DbConnect.DataLogContext())
{
SessionSummary summary = context.DbSessions.First(n => n.SessionId == sessionId);
summary.Status = SessionStatus.Aborted.ToString();
summary.EndDateTime = DateTime.UtcNow;
context.SaveChanges();
}
UpdateSessionShutdownState(MachineShutdownState.ManualReset, sessionId, ignoreMissing: true);
TraceFactory.Logger.Info("Done closing {0}".FormatWith(sessionId));
}
catch (Exception ex)
{
TraceFactory.Logger.Error(ex);
}
// Tell all clients to clean up any session info
EventPublisher.ReleaseSession(sessionId);
//Checks for any Virtual worker process which may have not been terminated correctly from the previous runs
if (GlobalSettings.IsDistributedSystem == false)
{
using (DataLogContext context = DbConnect.DataLogContext())
{
SessionSummary summary = context.DbSessions.First(n => n.SessionId == sessionId);
if (summary.Dispatcher == Environment.MachineName)
{
KillOrphanedWorkerProcesses(sessionId);
}
}
}
}
}
private void CloseSessionHandler(string sessionId)
{
ShutdownOptions options = new ShutdownOptions()
{
AllowWorkerToComplete = false,
CopyLogs = false,
PowerOff = true,
PowerOffOption = VMPowerOffOption.RevertToSnapshot
};
CloseSessionHandler(sessionId, options);
//SetTraceSessionContext(sessionId);
//SessionProxyController controller = null;
//if (_proxyControllers.TryGetValue(sessionId, out controller)) //If a session exists, proceed with normal shutdown. Otherwise reset manually.
//{
// _proxyControllers.Remove(controller);
// TraceFactory.Logger.Debug("Found & removed controller: Cnt: {0} : {1}".FormatWith(_proxyControllers.Count, controller.Endpoint.AbsoluteUri));
// if (!controller.SessionClosing) //Make sure the session isn't already closing
// {
// lock (controller)
// {
// controller.SessionClosing = true;
// ShutdownOptions options = new ShutdownOptions()
// {
// AllowWorkerToComplete = false,
// CopyLogs = false,
// PowerOff = true,
// PowerOffOption = VMPowerOffOption.RevertToSnapshot
// };
// try
// {
// ThreadPool.QueueUserWorkItem(t => ShutdownScenario(controller, sessionId, options));
// }
// catch (Exception ex)
// {
// TraceFactory.Logger.Error(ex);
// }
// }
// }
//}
//else
//{
// TraceFactory.Logger.Debug(string.Format("Closing session {0}", sessionId));
// // Manually reset DomainAccountReservation for all inactive SessionIds
// using (AssetInventoryContext context = new AssetInventoryContext())
// {
// List<string> activeSessionIds = VirtualMachineReservation.SelectSessionIds(context).ToList();
// foreach (var reservation in context.DomainAccountReservations)
// {
// if (!activeSessionIds.Contains(reservation.SessionId))
// {
// context.DeleteObject(reservation);
// }
// }
// context.SaveChanges();
// }
// try
// {
// // Parallel process the two actions required to clean up this session, first define
// // the collection of actions, then spawn them each off in the background and then
// // wait for them to return.
// var actions = new Collection<Action>()
// {
// () => CleanupAssetHosts(sessionId),
// () => CleanupResourceHosts(sessionId)
// };
// Parallel.ForEach<Action>(actions, a => a());
// UpdateSessionShutdownState(MachineShutdownState.ManualReset, sessionId, ignoreMissing: true);
// TraceFactory.Logger.Info("Done closing {0}".FormatWith(sessionId));
// }
// catch (Exception ex)
// {
// TraceFactory.Logger.Error(ex);
// }
// // Tell all clients to clean up any session info
// EventPublisher.ReleaseSession(sessionId);
//}
}
/// <summary>
/// Shuts down this resource host
/// </summary>
/// <param name="options">The options used to define how the machine is shutdown</param>
/// <param name="loopState">State of the loop.</param>
public void Shutdown(ShutdownOptions options, ParallelLoopState loopState)
{
_inShutdown = true;
MapElement.UpdateStatus("Shutdown", RuntimeState.ShuttingDown);
// If the Machine has not been configured yet, then just return
if (!Machine.Configured)
{
TraceFactory.Logger.Debug("This host is not configured, returning");
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
return;
}
TraceFactory.Logger.Debug("Shutting down {0}".FormatWith(Machine.Name));
if (CancelBootup())
{
TraceFactory.Logger.Debug("{0} was booting, so just shutdown machine".FormatWith(Machine.Name));
Machine.Shutdown(options);
}
else
{
TraceFactory.Logger.Debug("{0} already booted, shutdown resources, then machine".FormatWith(Machine.Name));
// Create the wait handle first, so that it is decrementing even before the
// wait occurs. That way if the resources send their offline state changes
// this count latch will receive them and appropriately decrement.
InitializeShutdownWaitHandle();
try
{
Parallel.ForEach <ResourceInstance>(Resources, r => r.Shutdown(options));
WaitForResourcesToShutdown();
TraceFactory.Logger.Debug("{0}: All virtual resources now exited".FormatWith(Machine.Name));
}
catch (AggregateException ex)
{
TraceFactory.Logger.Error("Error shutting down resources: {0}".FormatWith(ex.ToString()));
}
try
{
// If we are in a validated or offline state, then the client VM hasn't fully booted and won't
// have a service running to call Cleanup on. This is just here to speed up the
// shutdown process when in this condition.
if (options.PerformCleanup && MapElement.State != RuntimeState.Validated && MapElement.State != RuntimeState.Offline)
{
// Tell the host to cleanup as well. This will provide an opportunity for a host
// to perform any custom clean up items.
using (var resourceHostClient = ClientControllerServiceConnection.Create(Machine.Name))
{
resourceHostClient.Channel.Cleanup();
TraceFactory.Logger.Debug("Sent Cleanup signal to {0}".FormatWith(Machine.Name));
}
}
}
catch (Exception ex)
{
TraceFactory.Logger.Error("Unable to call Cleanup: {0}".FormatWith(ex.ToString()));
}
try
{
if (options.CopyLogs)
{
CopyClientControllerLogs();
}
}
catch (Exception ex)
{
TraceFactory.Logger.Error("Unable to copy Logs: {0}".FormatWith(ex.ToString()));
}
try
{
MapElement.UpdateStatus("Powering off");
TraceFactory.Logger.Debug("Shutdown machine {0}".FormatWith(Machine.Name));
Machine.Shutdown(options);
TraceFactory.Logger.Debug("Shutdown machine {0} COMPLETE".FormatWith(Machine.Name));
MapElement.UpdateStatus("Shut down");
}
catch (Exception ex)
{
TraceFactory.Logger.Error("Unable to shutdown {0}: {1}".FormatWith(Machine.Name, ex.ToString()));
}
}
if (options.ReleaseDeviceReservation)
{
try
{
MapElement.UpdateStatus("Release");
TraceFactory.Logger.Debug("Machine {0} releasing".FormatWith(Machine.Name));
Machine.Release();
TraceFactory.Logger.Debug("Machine {0} released".FormatWith(Machine.Name));
}
catch (Exception ex)
{
TraceFactory.Logger.Error("Error releasing {0}: {1}".FormatWith(Machine.Name, ex.ToString()));
}
}
MapElement.UpdateStatus("Offline", RuntimeState.Offline);
}
/// <summary>
/// Closes the specified session with specified shutdown options
/// </summary>
/// <param name="sessionId"></param>
/// <param name="options"></param>
public void Close(string sessionId, ShutdownOptions options)
{
TraceFactory.Logger.Debug("{0}{1}{2}".FormatWith(sessionId, Environment.NewLine, options));
CloseSessionHandler(sessionId, options);
}