/// <summary>
/// Runs the commands on the PlanetLab nodes.
/// </summary>
/// <param name="state">The manager state.</param>
private void OnRunNodes(PlManagerState state)
{
lock (state.Sync)
{
// If the command has been stopped, return.
if (state.IsStopped) return;
// Compute the number of commands to execute.
int commandCount = 0;
foreach (PlCommand command in state.Slice.Commands)
{
commandCount += command.HasParameters ? command.SetsCount : 1;
}
// Create a list of the pending nodes.
foreach (PlNode node in state.Nodes)
{
// If using only the nodes in the boot state, and the node is not in the boot state.
if (state.Slice.OnlyRunOnBootNodes && (node.GetBootState() != PlBootState.Boot))
{
// Raise a node disabled event.
if (null != this.NodeDisabled) this.NodeDisabled(this, new PlManagerNodeEventArgs(state, node));
// Skip processing this node.
continue;
}
// Raise a node enabled event.
if (null != this.NodeEnabled) this.NodeEnabled(this, new PlManagerNodeEventArgs(state, node, commandCount));
// Add the node to the list of pending nodes.
state.AddNode(node);
}
}
// Compute the availability of worker threads for parallel processing.
int availableWorkerThreads;
int availableCompletionPortThreads;
int maxWorkerThreads;
int maxCompletionPortThreads;
int maxParallel = Math.Max(state.Slice.RunParallelNodes, state.PendingCount);
// Get the number of threads available on the thread pool.
ThreadPool.GetAvailableThreads(out availableWorkerThreads, out availableCompletionPortThreads);
// Get the maximum number of threads on the thread pool.
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxCompletionPortThreads);
// If the number of available worker threads is smaller than the number of parallel nodes.
if (availableWorkerThreads < maxParallel)
{
// Increment the number of maximum threads.
ThreadPool.SetMaxThreads(
maxWorkerThreads + maxParallel - availableWorkerThreads,
maxCompletionPortThreads
);
}
lock (state.Sync)
{
// If the list of pending nodes is empty, stop.
if (0 == state.PendingCount)
{
this.Stop(state);
}
// Get a cached copy of all the pending PlanetLab nodes.
int[] pendingCache = state.PendingNodes.ToArray();
// For all the pending nodes.
for (int index = 0; (index < pendingCache.Length) && (state.RunningCount < state.Slice.RunParallelNodes); index++)
{
// Get the node state corresponding to the pending node.
PlManagerNodeState nodeState = state.GetNode(pendingCache[index]);
// If the slice configuration only allows one node per site.
if (state.Slice.OnlyRunOneNodePerSite)
{
// If the node site is completed.
if (state.IsSiteCompleted(nodeState.Node.SiteId.Value))
{
// Change the node from the pending to the skipped state.
state.UpdateNodePendingToSkipped(index);
// Raise an event indicating that the node has been skipped.
if (null != this.NodeSkipped) this.NodeSkipped(this, new PlManagerNodeEventArgs(state, nodeState.Node));
// Skip the loop to the next node.
continue;
}
// If the node site is running.
if (state.IsSiteRunning(nodeState.Node.SiteId.Value))
{
// Postpone the node for later, and skip the loop to the next node.
continue;
}
}
// Change the node from the pending to the running state.
state.UpdateNodePendingToRunning(index);
// Execute the commands on specified node.
this.OnRunNode(state, index);
}
}
}
/// <summary>
/// Runs the commands on the PlanetLab node at the specified index.
/// </summary>
/// <param name="state">The state.</param>
/// <param name="index">The node state index.</param>
private void OnRunNode(PlManagerState state, int index)
{
lock (state.Sync)
{
// If the operation has been canceled, return.
if (state.IsStopped) return;
// Get the node state.
PlManagerNodeState nodeState = state.GetNode(index);
// Execute the secure shell connection on the thread pool.
ThreadPool.QueueUserWorkItem((object threadState) =>
{
// Raise a node started event.
if (null != this.NodeStarted) this.NodeStarted(this, new PlManagerNodeEventArgs(state, nodeState.Node));
try
{
// Create the private key connection information.
ConnectionInfo connectionInfo;
// Create a memory stream with the key data.
using (MemoryStream memoryStream = new MemoryStream(state.Slice.Key))
{
// Create the private key file.
using (PrivateKeyFile keyFile = new PrivateKeyFile(memoryStream))
{
// Create a key connection info.
connectionInfo = new PrivateKeyConnectionInfo(nodeState.Node.Hostname, state.Slice.Name, keyFile);
}
}
// Open an SSH client to the PlanetLab node.
using (SshClient sshClient = new SshClient(connectionInfo))
{
// Connect the client.
sshClient.Connect();
// For all the slice commands.
foreach (PlCommand command in state.Slice.Commands)
{
// If the command has parameters.
if (command.HasParameters)
{
// Format the command with all parameter sets.
for (int indexSet = 0; indexSet < command.SetsCount; indexSet++)
{
// If the operation has been paused, wait for resume.
if (state.IsPaused)
{
state.WaitPause();
}
// If the operation has been canceled, return.
if (state.IsStopped)
{
// Remove the node from the running list.
state.UpdateNodeRunningToSkipped(index);
// Cancel all nodes in the pending list.
state.CancelPendingNodes();
// Raise the canceled event.
if (null != this.NodeCanceled) this.NodeCanceled(this, new PlManagerNodeEventArgs(state, nodeState.Node));
// Return.
return;
}
// Run the command with the current parameters set.
this.OnRunCommand(state, nodeState, command, indexSet, sshClient);
}
}
else
{
// Run the command without parameters.
this.OnRunCommand(state, nodeState, command, -1, sshClient);
}
}
// Disconnect the client.
sshClient.Disconnect();
}
// Remove the node from the running list.
state.UpdateNodeRunningToCompleted(index);
// Raise the success event.
if (null != this.NodeFinishedSuccess) this.NodeFinishedSuccess(this, new PlManagerNodeEventArgs(state, nodeState.Node));
}
catch (Exception exception)
{
// Remove the node from the running list.
state.UpdateNodeRunningToSkipped(index);
// Raise the fail event.
if (null != this.NodeFinishedFail) this.NodeFinishedFail(this, new PlManagerNodeEventArgs(state, nodeState.Node, exception));
}
finally
{
lock (state.Sync)
{
// If the operation has not been canceled.
if (!state.IsStopped)
{
// If the list of pending nodes and running nodes is empty, stop.
if ((0 == state.PendingCount) && (0 == state.RunningCount))
{
this.Stop(state);
}
// Get a cached copy of all the pending PlanetLab nodes.
int[] pendingCache = state.PendingNodes.ToArray();
// Find the next pending node to execute commands.
foreach (int newIndex in pendingCache)
{
// Get the node state corresponding to the pending node.
PlManagerNodeState newNode = state.GetNode(newIndex);
// If the slice configuration only allows one node per site.
if (state.Slice.OnlyRunOneNodePerSite)
{
// If the node site is completed.
if (state.IsSiteCompleted(newNode.Node.SiteId.Value))
{
// Change the node from the pending to the skipped state.
state.UpdateNodePendingToSkipped(newIndex);
// Raise an event indicating that the node has been skipped.
if (null != this.NodeSkipped) this.NodeSkipped(this, new PlManagerNodeEventArgs(state, newNode.Node));
// Skip the loop to the next node.
continue;
}
// If the node site is running.
if (state.IsSiteRunning(newNode.Node.SiteId.Value))
{
// Postpone the node for later, and skip the loop to the next node.
continue;
}
}
// Change the node from the pending to the running state.
state.UpdateNodePendingToRunning(newIndex);
// Execute the commands on specified node.
this.OnRunNode(state, newIndex);
// Exit the loop.
break;
}
}
}
}
});
}
}
