/// <summary>
/// Runs the commands on the PlanetLab node at the specified index.
/// </summary>
/// <param name="state">The state.</param>
/// <param name="index">The node state index.</param>
private void OnRunNode(PlManagerState state, int index)
{
lock (state.Sync)
{
// If the operation has been canceled, return.
if (state.IsStopped) return;
// Get the node state.
PlManagerNodeState nodeState = state.GetNode(index);
// Execute the secure shell connection on the thread pool.
ThreadPool.QueueUserWorkItem((object threadState) =>
{
// Raise a node started event.
if (null != this.NodeStarted) this.NodeStarted(this, new PlManagerNodeEventArgs(state, nodeState.Node));
try
{
// Create the private key connection information.
ConnectionInfo connectionInfo;
// Create a memory stream with the key data.
using (MemoryStream memoryStream = new MemoryStream(state.Slice.Key))
{
// Create the private key file.
using (PrivateKeyFile keyFile = new PrivateKeyFile(memoryStream))
{
// Create a key connection info.
connectionInfo = new PrivateKeyConnectionInfo(nodeState.Node.Hostname, state.Slice.Name, keyFile);
}
}
// Open an SSH client to the PlanetLab node.
using (SshClient sshClient = new SshClient(connectionInfo))
{
// Connect the client.
sshClient.Connect();
// For all the slice commands.
foreach (PlCommand command in state.Slice.Commands)
{
// If the command has parameters.
if (command.HasParameters)
{
// Format the command with all parameter sets.
for (int indexSet = 0; indexSet < command.SetsCount; indexSet++)
{
// If the operation has been paused, wait for resume.
if (state.IsPaused)
{
state.WaitPause();
}
// If the operation has been canceled, return.
if (state.IsStopped)
{
// Remove the node from the running list.
state.UpdateNodeRunningToSkipped(index);
// Cancel all nodes in the pending list.
state.CancelPendingNodes();
// Raise the canceled event.
if (null != this.NodeCanceled) this.NodeCanceled(this, new PlManagerNodeEventArgs(state, nodeState.Node));
// Return.
return;
}
// Run the command with the current parameters set.
this.OnRunCommand(state, nodeState, command, indexSet, sshClient);
}
}
else
{
// Run the command without parameters.
this.OnRunCommand(state, nodeState, command, -1, sshClient);
}
}
// Disconnect the client.
sshClient.Disconnect();
}
// Remove the node from the running list.
state.UpdateNodeRunningToCompleted(index);
// Raise the success event.
if (null != this.NodeFinishedSuccess) this.NodeFinishedSuccess(this, new PlManagerNodeEventArgs(state, nodeState.Node));
}
catch (Exception exception)
{
// Remove the node from the running list.
state.UpdateNodeRunningToSkipped(index);
// Raise the fail event.
if (null != this.NodeFinishedFail) this.NodeFinishedFail(this, new PlManagerNodeEventArgs(state, nodeState.Node, exception));
}
finally
{
lock (state.Sync)
{
// If the operation has not been canceled.
if (!state.IsStopped)
{
// If the list of pending nodes and running nodes is empty, stop.
if ((0 == state.PendingCount) && (0 == state.RunningCount))
{
this.Stop(state);
}
// Get a cached copy of all the pending PlanetLab nodes.
int[] pendingCache = state.PendingNodes.ToArray();
// Find the next pending node to execute commands.
foreach (int newIndex in pendingCache)
{
// Get the node state corresponding to the pending node.
PlManagerNodeState newNode = state.GetNode(newIndex);
// If the slice configuration only allows one node per site.
if (state.Slice.OnlyRunOneNodePerSite)
{
// If the node site is completed.
if (state.IsSiteCompleted(newNode.Node.SiteId.Value))
{
// Change the node from the pending to the skipped state.
state.UpdateNodePendingToSkipped(newIndex);
// Raise an event indicating that the node has been skipped.
if (null != this.NodeSkipped) this.NodeSkipped(this, new PlManagerNodeEventArgs(state, newNode.Node));
// Skip the loop to the next node.
continue;
}
// If the node site is running.
if (state.IsSiteRunning(newNode.Node.SiteId.Value))
{
// Postpone the node for later, and skip the loop to the next node.
continue;
}
}
// Change the node from the pending to the running state.
state.UpdateNodePendingToRunning(newIndex);
// Execute the commands on specified node.
this.OnRunNode(state, newIndex);
// Exit the loop.
break;
}
}
}
}
});
}
}
/// <summary>
/// Runs a command on the specified PlanetLab node.
/// </summary>
/// <param name="state">The manager state.</param>
/// <param name="node">The PlanetLab node state.</param>
/// <param name="command">The PlanetLab command.</param>
/// <param name="set">The command parameter set.</param>
/// <param name="sshClient">The SSH client.</param>
private void OnRunCommand(PlManagerState state, PlManagerNodeState node, PlCommand command, int set, SshClient sshClient)
{
// Raise the command started event.
if (null != this.CommandStarted) this.CommandStarted(this, new PlManagerCommandEventArgs(state, node.Node, command, set));
try
{
// Compute the command text.
string commandText = set >= 0 ? command.GetCommand(set) : command.Command;
// The number of subcommands.
int success = 0;
int fail = 0;
// Divide the command into subcommands.
string[] subcommands = commandText.Split(PlManager.subcommandSeparators, StringSplitOptions.RemoveEmptyEntries);
// For all subcommands.
foreach (string subcommand in subcommands)
{
// If the operation has been paused, wait for resume.
if (state.IsPaused)
{
state.WaitPause();
}
// If the operation has been canceled, return.
if (state.IsStopped)
{
// Raise the canceled event.
if (null != this.CommandCanceled) this.CommandCanceled(this, new PlManagerCommandEventArgs(state, node.Node, command, set));
// Return.
return;
}
// If the subcommand is empty, continue to the next subcommand.
if (string.IsNullOrWhiteSpace(subcommand)) continue;
// Create a new SSH command.
using (SshCommand sshCommand = sshClient.CreateCommand(subcommand))
{
try
{
// The command duration.
TimeSpan duration;
// The retry count.
int retry = 0;
do
{
// The start time.
DateTime startTime = DateTime.Now;
// Execute the command.
sshCommand.Execute();
// Compute the command duration.
duration = DateTime.Now - startTime;
}
while ((retry++ < state.Slice.CommandRetries) && (sshCommand.ExitStatus != 0));
// Create a new command state.
PlManagerSubcommandState subcommandState = new PlManagerSubcommandState(node, sshCommand, duration, retry - 1);
// Increment the number of successful subcommands.
success++;
// Add the subcommand.
node.AddSubcommand(subcommandState);
// Raise a subcommand success event.
if (null != this.SubcommandSuccess) this.SubcommandSuccess(this, new PlManagerSubcommandEventArgs(state, node.Node, command, set, subcommandState));
}
catch (Exception exception)
{
// Create a new subcommand state.
PlManagerSubcommandState subcommandState = new PlManagerSubcommandState(node, sshCommand, exception);
// Increment the number of failed subcommands.
fail++;
// Add the subcommand.
node.AddSubcommand(subcommandState);
// Raise a subcommand fail event.
if (null != this.SubcommandFail) this.SubcommandFail(this, new PlManagerSubcommandEventArgs(state, node.Node, command, set, exception));
}
}
}
// Raise a command completed event.
if (null != this.CommandFinishedSuccess) this.CommandFinishedSuccess(this, new PlManagerCommandEventArgs(state, node.Node, command, set, success, fail));
}
catch (Exception exception)
{
// Raise a command completed event.
if (null != this.CommandFinishedFail) this.CommandFinishedFail(this, new PlManagerCommandEventArgs(state, node.Node, command, set, exception));
}
}