/// <summary>
/// Asynchronous method that delays execution until the nodes within the specified pool reach the specified state.
/// </summary>
/// <param name="client">A fully intitialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the pool containing the nodes to monitor.</param>
/// <param name="targetNodeState">The node state to monitor.</param>
/// <param name="timeout">The maximum time to wait for the nodes to reach the specified state.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
public static async Task WaitForNodesToReachStateAsync(BatchClient client, string poolId, ComputeNodeState targetNodeState, TimeSpan timeout)
{
Console.WriteLine("Waiting for nodes to reach state {0}", targetNodeState);
DateTime startTime = DateTime.UtcNow;
DateTime timeoutAfterThisTimeUtc = startTime.Add(timeout);
CloudPool pool = await client.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false);
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,state");
IEnumerable <ComputeNode> computeNodes = pool.ListComputeNodes(detail);
while (computeNodes.Any(computeNode => computeNode.State != targetNodeState))
{
Console.Write(".");
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(continueOnCapturedContext: false);
computeNodes = pool.ListComputeNodes(detail).ToList();
if (DateTime.UtcNow > timeoutAfterThisTimeUtc)
{
throw new TimeoutException(string.Format("Timed out waiting for compute nodes in pool {0} to reach state {1}", poolId, targetNodeState.ToString()));
}
}
Console.WriteLine();
}
public static CloudPool WaitForPoolAllocation(BatchClient client, string poolId)
{
CloudPool thePool = client.PoolOperations.GetPool(poolId);
//Wait for pool to be in a usable state
//TODO: Use a Utilities waiter
TimeSpan computeNodeAllocationTimeout = TimeSpan.FromMinutes(10);
TestUtilities.WaitForPoolToReachStateAsync(client, poolId, AllocationState.Steady, computeNodeAllocationTimeout).Wait();
//Wait for the compute nodes in the pool to be in a usable state
//TODO: Use a Utilities waiter
TimeSpan computeNodeSteadyTimeout = TimeSpan.FromMinutes(25);
DateTime allocationWaitStartTime = DateTime.UtcNow;
DateTime timeoutAfterThisTimeUtc = allocationWaitStartTime.Add(computeNodeSteadyTimeout);
IEnumerable <ComputeNode> computeNodes = thePool.ListComputeNodes();
while (computeNodes.Any(computeNode => computeNode.State != ComputeNodeState.Idle))
{
Thread.Sleep(TimeSpan.FromSeconds(10));
computeNodes = thePool.ListComputeNodes().ToList();
if (DateTime.UtcNow > timeoutAfterThisTimeUtc)
{
throw new Exception("CreatePool: Timed out waiting for compute nodes in pool to reach idle state. Timeout: " + computeNodeSteadyTimeout.ToString());
}
}
return(thePool);
}
public void TestComputeNodeUserIaas()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
CloudPool sharedPool = this.poolFixture.Pool;
List <string> cnuNamesToDelete = new List <string>();
// pick a compute node to victimize with user accounts
var nodes = sharedPool.ListComputeNodes().ToList();
ComputeNode cn = nodes[0];
try
{
ComputeNodeUser bob = batchCli.PoolOperations.CreateComputeNodeUser(sharedPool.Id, cn.Id);
bob.Name = "bob";
bob.ExpiryTime = DateTime.UtcNow + TimeSpan.FromHours(25);
bob.Password = "******";
bob.SshPublicKey = "base64==";
cnuNamesToDelete.Add(bob.Name); // remember to clean this up
bob.Commit(ComputeNodeUserCommitSemantics.AddUser);
bob.SshPublicKey = "base65==";
bob.Commit(ComputeNodeUserCommitSemantics.UpdateUser);
// TODO: need to close the loop on this somehow... move to unit/interceptor-based?
// currently the server is timing out.
}
finally
{
// clear any old accounts
try
{
foreach (string curCNUName in cnuNamesToDelete)
{
this.testOutputHelper.WriteLine("TestComputeNodeUserIAAS attempting to delete the following <nodeid,user>: <{0},{1}>", cn.Id, curCNUName);
cn.DeleteComputeNodeUser(curCNUName);
}
}
catch (Exception ex)
{
this.testOutputHelper.WriteLine("TestComputeNodeUserIAAS: exception deleting user account. ex: " + ex.ToString());
}
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void TestGetRemoteLoginSettings()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
CloudPool sharedPool = this.poolFixture.Pool;
try
{
// get a compute node.
// get its RLS via PoolOps and via CN
// Assert there are values and that the values are equal
var nodes = sharedPool.ListComputeNodes().ToList();
Assert.Equal <int>(expected: 1, actual: nodes.Count);
ComputeNode cn = nodes[0];
// get the RLS via each object
RemoteLoginSettings rlsViaPoolOps = batchCli.PoolOperations.GetRemoteLoginSettings(sharedPool.Id, cn.Id);
RemoteLoginSettings rlsViaNode = cn.GetRemoteLoginSettings();
// they must have RLS
Assert.NotNull(rlsViaNode);
Assert.NotNull(rlsViaPoolOps);
// there must be an IP in each RLS
Assert.False(string.IsNullOrWhiteSpace(rlsViaPoolOps.IPAddress));
Assert.False(string.IsNullOrWhiteSpace(rlsViaNode.IPAddress));
// the ports must match
Assert.Equal(expected: rlsViaNode.Port, actual: rlsViaPoolOps.Port);
}
finally
{
// cleanup goes here
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
private void PrintStartTaskInfo(CloudPool pool)
{
IPagedEnumerable <ComputeNode> nodes = pool.ListComputeNodes();
PagedEnumerableExtensions.ForEachAsync <ComputeNode>(nodes, delegate(ComputeNode node)
{
Console.WriteLine("***** Start Task Details *****");
Console.WriteLine($"Start time: [{node.StartTaskInformation.StartTime}]");
Console.WriteLine($"State: [{node.StartTaskInformation.State}]");
if (node.StartTaskInformation.Result.Equals(TaskExecutionResult.Failure))
{
Console.WriteLine($"Error Category: [{node.StartTaskInformation.FailureInformation.Category}]");
Console.WriteLine($"Error Details: [{node.StartTaskInformation.FailureInformation.Details}]");
Console.WriteLine($"Error Message: [{node.StartTaskInformation.FailureInformation.Message}]");
Console.WriteLine($"Failure Code: [{node.StartTaskInformation.FailureInformation.Code}]");
}
Console.WriteLine($"End time: [{node.StartTaskInformation.EndTime}]");
}).Wait();
}
public void BugComputeNodeMissingStartTaskInfo_RunAfterPoolIsUsed()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
CloudPool pool = batchCli.PoolOperations.GetPool(poolFixture.PoolId);
// confirm start task info exists and has rational values
foreach (ComputeNode curComputeNode in pool.ListComputeNodes())
{
StartTaskInformation sti = curComputeNode.StartTaskInformation;
// set when pool was created
Assert.NotNull(sti);
Assert.True(StartTaskState.Running == sti.State || StartTaskState.Completed == sti.State);
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
private static async Task WaitForPoolToBeReady(CloudPool pool)
{
while (true)
{
await pool.RefreshAsync();
Log.Logger.Information("Pool {PoolId} is {AllocationState}.", pool.Id, pool.AllocationState);
if (pool.AllocationState == AllocationState.Steady)
{
var nodes = await pool.ListComputeNodes().ToListAsync();
foreach (var node in nodes)
{
Log.Logger.Information("Node {NodeId} is {NodeState}.", node.Id, node.State);
}
if (nodes.All(x => x.State == ComputeNodeState.Idle))
{
break;
}
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
public void CanAddTaskWithFilesToStage()
{
StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "TestTaskWithFilesToStage-" + TestUtilities.GetMyName();
try
{
CloudJob job = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = this.poolFixture.PoolId
});
job.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");
myTask.FilesToStage = new List <IFileStagingProvider>
{
new FileToStage(Resources.LocalWordsDotText, storageCreds)
};
// add the task to the job
var artifacts = boundJob.AddTask(myTask);
var specificArtifact = artifacts[typeof(FileToStage)];
SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;
Assert.NotNull(sfsa);
// Open the new Job as bound.
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
// wait for the task to complete
TaskStateMonitor taskStateMonitor = batchCli.Utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(10),
controlParams: null,
additionalBehaviors: new[]
{
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
try
{
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach(
(icn) =>
{
this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
}
catch (Exception ex)
{
// there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
}
})
});
List <CloudTask> tasks = boundJob.ListTasks().ToList();
CloudTask myCompletedTask = tasks.Single();
foreach (CloudTask curTask in tasks)
{
this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
}
boundPool.Refresh();
this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
this.testOutputHelper.WriteLine("StdOut: ");
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine("StdErr: ");
this.testOutputHelper.WriteLine(stdErr);
this.testOutputHelper.WriteLine("Task Files:");
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
{
this.testOutputHelper.WriteLine(" File path: " + curFile.Path);
}
var files = myCompletedTask.ListNodeFiles(recursive: true).ToList();
// confirm the files are there
Assert.True(files.Any(file => file.Path.Contains("localWords.txt")), "missing file: localWords.txt");
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
}
public async Task OnlineOfflineTest()
{
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
TimeSpan refreshPollingTimeout = TimeSpan.FromMinutes(3);
CloudPool pool = this.poolFixture.Pool;
List <ComputeNode> nodes = pool.ListComputeNodes().ToList();
Assert.True(nodes.Count > 0);
// pick a victim compute node. cleanup code needs this set
ComputeNode victim = nodes[0];
try
{
Assert.True(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState));
Assert.True(victim.State.HasValue && (ComputeNodeState.Idle == victim.State));
// PoolOperations methods
{
// disable task scheduling
batchCli.PoolOperations.DisableComputeNodeScheduling(pool.Id, victim.Id, DisableComputeNodeSchedulingOption.Terminate);
// Li says state change is not atomic so we will sleep
// asserted above this node is idle so no need to wait for task fussery
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Disabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Disabled, victim.SchedulingState);
// enable task scheduling
batchCli.PoolOperations.EnableComputeNodeScheduling(pool.Id, victim.Id);
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState)),
timeout: refreshPollingTimeout);
Assert.Equal <SchedulingState?>(SchedulingState.Enabled, victim.SchedulingState);
}
// ComputeNode methods
{
// disable task scheduling
victim.DisableScheduling(DisableComputeNodeSchedulingOption.TaskCompletion);
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Disabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Disabled, victim.SchedulingState);
// enable task scheduling
victim.EnableScheduling();
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Enabled, victim.SchedulingState);
// now test azureerror code for: NodeAlreadyInTargetSchedulingState
bool gotCorrectException = false;
try
{
victim.EnableScheduling(); // it is already enabled so this should trigger exception
}
catch (Exception ex)
{
TestUtilities.AssertIsBatchExceptionAndHasCorrectAzureErrorCode(ex, Microsoft.Azure.Batch.Common.BatchErrorCodeStrings.NodeAlreadyInTargetSchedulingState, this.testOutputHelper);
gotCorrectException = true;
}
if (!gotCorrectException)
{
throw new Exception("OnlineOfflineTest: failed to see an exception for NodeAlreadyInTargetSchedulingState test");
}
}
}
finally // restore state of victim compute node
{
try
{
// do not pollute the shared pool with disabled scheduling
if (null != victim)
{
victim.EnableScheduling();
}
}
catch (Exception ex)
{
this.testOutputHelper.WriteLine(string.Format("OnlineOfflineTest: exception during exit trying to restore scheduling state: {0}", ex.ToString()));
}
}
}
},
TestTimeout);
}
public void Bug2342986_StartTaskMissingOnComputeNode()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
CloudPool pool = batchCli.PoolOperations.GetPool(this.poolFixture.PoolId);
this.testOutputHelper.WriteLine("Getting pool");
StartTask poolStartTask = pool.StartTask;
Assert.NotNull(poolStartTask);
Assert.NotNull(poolStartTask.EnvironmentSettings);
IEnumerable <ComputeNode> computeNodes = pool.ListComputeNodes();
Assert.True(computeNodes.Any());
this.testOutputHelper.WriteLine("Checking every compute nodes start task in the pool matches the pools start task");
foreach (ComputeNode computeNode in computeNodes)
{
this.testOutputHelper.WriteLine("Checking start task of compute node: {0}", computeNode.Id);
//Check that the property is correctly set on each compute node
Assert.NotNull(computeNode.StartTask);
Assert.Equal(poolStartTask.CommandLine, computeNode.StartTask.CommandLine);
Assert.Equal(poolStartTask.MaxTaskRetryCount, computeNode.StartTask.MaxTaskRetryCount);
Assert.Equal(AutoUserScope.Task, poolStartTask.UserIdentity.AutoUser.Scope);
Assert.Equal(AutoUserScope.Task, computeNode.StartTask.UserIdentity.AutoUser.Scope);
Assert.Equal(poolStartTask.WaitForSuccess, computeNode.StartTask.WaitForSuccess);
if (poolStartTask.EnvironmentSettings != null)
{
Assert.Equal(poolStartTask.EnvironmentSettings.Count, computeNode.StartTask.EnvironmentSettings.Count);
foreach (EnvironmentSetting environmentSetting in poolStartTask.EnvironmentSettings)
{
EnvironmentSetting matchingEnvSetting = computeNode.StartTask.EnvironmentSettings.FirstOrDefault(envSetting => envSetting.Name == environmentSetting.Name);
Assert.NotNull(matchingEnvSetting);
Assert.Equal(environmentSetting.Name, matchingEnvSetting.Name);
Assert.Equal(environmentSetting.Value, matchingEnvSetting.Value);
}
}
if (poolStartTask.ResourceFiles != null)
{
Assert.Equal(poolStartTask.ResourceFiles.Count, computeNode.StartTask.ResourceFiles.Count);
foreach (ResourceFile resourceFile in poolStartTask.ResourceFiles)
{
ResourceFile matchingResourceFile = computeNode.StartTask.ResourceFiles.FirstOrDefault(item => item.HttpUrl == resourceFile.HttpUrl);
Assert.NotNull(matchingResourceFile);
Assert.Equal(resourceFile.HttpUrl, matchingResourceFile.HttpUrl);
Assert.Equal(resourceFile.FilePath, matchingResourceFile.FilePath);
}
}
//Try to set some properties of the compute node's start task and ensure it fails
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.CommandLine = "Test"; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.MaxTaskRetryCount = 5; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.UserIdentity = new UserIdentity("foo"); });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.WaitForSuccess = true; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.EnvironmentSettings = new List <EnvironmentSetting>(); });
if (computeNode.StartTask.EnvironmentSettings != null)
{
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.EnvironmentSettings.Add(new EnvironmentSetting("test", "test")); });
}
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.ResourceFiles = new List <ResourceFile>(); });
if (computeNode.StartTask.ResourceFiles != null)
{
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.ResourceFiles.Add(ResourceFile.FromUrl("test", "test")); });
}
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void TestSampleWithFilesAndPool()
{
Action test = () =>
{
StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
string jobId = "SampleWithFilesJob-" + TestUtilities.GetMyName();
try
{
CloudJob quickJob = batchCli.JobOperations.CreateJob();
quickJob.Id = jobId;
quickJob.PoolInformation = new PoolInformation()
{
PoolId = this.poolFixture.PoolId
};
quickJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");
// first we have local files that we want pushed to the compute node before the commandline is invoked
FileToStage wordsDotText = new FileToStage(Resources.LocalWordsDotText, storageCreds); // use "default" mapping to base name of local file
myTask.FilesToStage = new List <IFileStagingProvider>();
myTask.FilesToStage.Add(wordsDotText);
// add the task to the job
var artifacts = boundJob.AddTask(myTask);
var specificArtifact = artifacts[typeof(FileToStage)];
SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;
Assert.NotNull(sfsa);
// add a million more tasks...
// test to ensure the task is read only
TestUtilities.AssertThrows <InvalidOperationException>(() => myTask.FilesToStage = new List <IFileStagingProvider>());
// Open the new Job as bound.
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(10),
controlParams: null,
additionalBehaviors:
new[]
{
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
try
{
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach(
(icn) =>
{
this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
}
catch (Exception ex)
{
// there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
}
})
});
List <CloudTask> tasks = boundJob.ListTasks(null).ToList();
CloudTask myCompletedTask = tasks[0];
foreach (CloudTask curTask in tasks)
{
this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
}
boundPool.Refresh();
this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
this.testOutputHelper.WriteLine("StdOut: ");
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine("StdErr: ");
this.testOutputHelper.WriteLine(stdErr);
this.testOutputHelper.WriteLine("Task Files:");
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
{
this.testOutputHelper.WriteLine(" Filename: " + curFile.Name);
}
// confirm the files are there
Assert.True(FoundFile("localwords.txt", myCompletedTask.ListNodeFiles(recursive: true)), "mising file: localwords.txt");
// test validation of StagingStorageAccount
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: " ", storageAccountKey: "key", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: " ", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: "key", blobEndpoint: ""); });
if (null != sfsa)
{
// TODO: delete the container!
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void LongRunning_Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPool()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "Bug1965363Job-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
AutoPoolSpecification = new AutoPoolSpecification()
{
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
TargetDedicatedComputeNodes = 1
}
}
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
testOutputHelper.WriteLine("Commiting quickjob");
unboundJob.Commit();
CloudTask task = new CloudTask("Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPoolTask", "cmd /c echo Bug1965363");
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
boundJob.AddTask(task);
testOutputHelper.WriteLine("Getting pool name: {0}", boundJob.ExecutionInformation.PoolId);
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
TaskStateMonitor tsm = batchCli.Utilities.CreateTaskStateMonitor();
ODATAMonitorControl odControl = new ODATAMonitorControl();
// we know that the autopool compute nodes will take a long time to become scheduleable so we slow down polling/spam
odControl.DelayBetweenDataFetch = TimeSpan.FromSeconds(5);
testOutputHelper.WriteLine("Invoking TaskStateMonitor");
tsm.WaitAll(
boundJob.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(15),
odControl,
new[] {
// spam/logging interceptor
new Protocol.RequestInterceptor((x) =>
{
testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
testOutputHelper.WriteLine(" #comnpute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach((icn) => { testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State); });
testOutputHelper.WriteLine("");
})
});
// confirm the task ran by inspecting the stdOut
string stdOut = boundJob.ListTasks().ToList()[0].GetNodeFile(Constants.StandardOutFileName).ReadAsString();
Assert.Contains("Bug1965363", stdOut);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
public void Bug1665834TaskStateMonitor()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "Bug1665834Job-" + TestUtilities.GetMyName();
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// add some noise tasks
for (int j = 0; j < 5; j++)
{
CloudTask unboundTaskQuick = new CloudTask((10 + j).ToString(), "cmd /c hostname");
boundJob.AddTask(unboundTaskQuick);
}
Thread.Sleep(5000);
// wait for fast tasks to complete
{
bool repeat = true;
while (repeat)
{
CloudPool boundPool = batchCli.PoolOperations.GetPool(poolFixture.PoolId);
repeat = false;
foreach (CloudTask curTask in boundJob.ListTasks())
{
if (curTask.State != TaskState.Completed)
{
repeat = true;
testOutputHelper.WriteLine("Manual Wait Task Id: " + curTask.Id + ", state = " + curTask.State);
testOutputHelper.WriteLine(" poolstate: " + boundPool.State + ", currentdedicated: " + boundPool.CurrentDedicatedComputeNodes);
testOutputHelper.WriteLine(" compute nodes:");
foreach (ComputeNode curComputeNode in boundPool.ListComputeNodes())
{
testOutputHelper.WriteLine(" computeNode.Id: " + curComputeNode.Id + ", state: " + curComputeNode.State);
}
}
}
}
}
// add some longer running tasks
testOutputHelper.WriteLine("Adding longer running tasks");
for (int i = 0; i < 15; i++)
{
CloudTask unboundTask = new CloudTask(i.ToString() + "_a234567890a234567890a234567890a234567890a234567890a234567890", "cmd /c ping 127.0.0.1 -n 4");
boundJob.AddTask(unboundTask);
}
Utilities utilities = batchCli.Utilities;
TaskStateMonitor tsm = utilities.CreateTaskStateMonitor();
IPagedEnumerable <CloudTask> taskList = boundJob.ListTasks();
// try to set really low delay
ODATAMonitorControl odmc = new ODATAMonitorControl {
DelayBetweenDataFetch = new TimeSpan(0)
};
// confirm the floor is enforced
Assert.Equal(500, odmc.DelayBetweenDataFetch.Milliseconds);
testOutputHelper.WriteLine("Calling TaskStateMonitor.WaitAll(). This will take a while.");
TimeSpan timeToWait = TimeSpan.FromMinutes(5);
Task whenAll = tsm.WhenAll(taskList, TaskState.Completed, timeToWait, controlParams: odmc);
//This could throw, if it does the test will fail, which is what we want
whenAll.Wait();
foreach (CloudTask curTask in boundJob.ListTasks())
{
Assert.Equal(TaskState.Completed, curTask.State);
}
}
finally
{
// cleanup
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
private static void WriteVRayConfig(CloudPool pool)
{
var vrayConfig = Path.Combine(
Environment.GetEnvironmentVariable("LOCALAPPDATA"),
ConfigurationManager.AppSettings["VRayDRConfigPath"],
"vray_dr.cfg");
var vrayRtConfig = Path.Combine(
Environment.GetEnvironmentVariable("LOCALAPPDATA"),
ConfigurationManager.AppSettings["VRayDRConfigPath"],
"vrayrt_dr.cfg");
var vrayConfigContent = ConfigurationManager.AppSettings["VRayDRConfig"];
var vrayRtConfigContent = ConfigurationManager.AppSettings["VRayRTDRConfig"];
// If the config files already exist, preserve their content
if (File.Exists(vrayConfig))
{
var content = File.ReadAllText(vrayConfig);
if (!string.IsNullOrWhiteSpace(content))
{
vrayConfigContent = content;
}
}
if (File.Exists(vrayRtConfig))
{
var content = File.ReadAllText(vrayRtConfig);
if (!string.IsNullOrWhiteSpace(content))
{
vrayRtConfigContent = content;
}
}
File.WriteAllText(vrayConfig, "");
File.WriteAllText(vrayRtConfig, "");
foreach (var computeNode in pool.ListComputeNodes())
{
var computeNodeEntry = GetComputeNodeEntry(computeNode);
Console.WriteLine(" {0} {1}", computeNode.Id, computeNodeEntry);
if (vrayConfigContent.Contains(computeNodeEntry))
{
Console.WriteLine("Compute node {0} already exists in config file {1}", computeNodeEntry, vrayConfig);
}
else
{
File.AppendAllText(vrayConfig, computeNodeEntry);
}
if (vrayRtConfigContent.Contains(computeNodeEntry))
{
Console.WriteLine("Compute node {0} already exists in config file {1}", computeNodeEntry, vrayRtConfigContent);
}
else
{
File.AppendAllText(vrayRtConfig, computeNodeEntry);
}
}
File.AppendAllText(vrayConfig, vrayConfigContent);
File.AppendAllText(vrayRtConfig, vrayRtConfigContent);
Console.WriteLine("Updated VRay DR config file: " + vrayConfig);
Console.WriteLine("Updated VRayRT DR config file: " + vrayConfig);
}