public void TestSampleWithFilesAndPool()
{
Action test = () =>
{
StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
string jobId = "SampleWithFilesJob-" + TestUtilities.GetMyName();
try
{
CloudJob quickJob = batchCli.JobOperations.CreateJob();
quickJob.Id = jobId;
quickJob.PoolInformation = new PoolInformation()
{
PoolId = this.poolFixture.PoolId
};
quickJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");
// first we have local files that we want pushed to the compute node before the commandline is invoked
FileToStage wordsDotText = new FileToStage(Resources.LocalWordsDotText, storageCreds); // use "default" mapping to base name of local file
myTask.FilesToStage = new List <IFileStagingProvider>();
myTask.FilesToStage.Add(wordsDotText);
// add the task to the job
var artifacts = boundJob.AddTask(myTask);
var specificArtifact = artifacts[typeof(FileToStage)];
SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;
Assert.NotNull(sfsa);
// add a million more tasks...
// test to ensure the task is read only
TestUtilities.AssertThrows <InvalidOperationException>(() => myTask.FilesToStage = new List <IFileStagingProvider>());
// Open the new Job as bound.
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(10),
controlParams: null,
additionalBehaviors:
new[]
{
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
try
{
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach(
(icn) =>
{
this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
}
catch (Exception ex)
{
// there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
}
})
});
List <CloudTask> tasks = boundJob.ListTasks(null).ToList();
CloudTask myCompletedTask = tasks[0];
foreach (CloudTask curTask in tasks)
{
this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
}
boundPool.Refresh();
this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
this.testOutputHelper.WriteLine("StdOut: ");
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine("StdErr: ");
this.testOutputHelper.WriteLine(stdErr);
this.testOutputHelper.WriteLine("Task Files:");
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
{
this.testOutputHelper.WriteLine(" Filename: " + curFile.Name);
}
// confirm the files are there
Assert.True(FoundFile("localwords.txt", myCompletedTask.ListNodeFiles(recursive: true)), "mising file: localwords.txt");
// test validation of StagingStorageAccount
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: " ", storageAccountKey: "key", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: " ", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: "key", blobEndpoint: ""); });
if (null != sfsa)
{
// TODO: delete the container!
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void CanAddTaskWithFilesToStage()
{
StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "TestTaskWithFilesToStage-" + TestUtilities.GetMyName();
try
{
CloudJob job = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = this.poolFixture.PoolId
});
job.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");
myTask.FilesToStage = new List <IFileStagingProvider>
{
new FileToStage(Resources.LocalWordsDotText, storageCreds)
};
// add the task to the job
var artifacts = boundJob.AddTask(myTask);
var specificArtifact = artifacts[typeof(FileToStage)];
SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;
Assert.NotNull(sfsa);
// Open the new Job as bound.
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
// wait for the task to complete
TaskStateMonitor taskStateMonitor = batchCli.Utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(10),
controlParams: null,
additionalBehaviors: new[]
{
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
try
{
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach(
(icn) =>
{
this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
}
catch (Exception ex)
{
// there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
}
})
});
List <CloudTask> tasks = boundJob.ListTasks().ToList();
CloudTask myCompletedTask = tasks.Single();
foreach (CloudTask curTask in tasks)
{
this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
}
boundPool.Refresh();
this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
this.testOutputHelper.WriteLine("StdOut: ");
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine("StdErr: ");
this.testOutputHelper.WriteLine(stdErr);
this.testOutputHelper.WriteLine("Task Files:");
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
{
this.testOutputHelper.WriteLine(" File path: " + curFile.Path);
}
var files = myCompletedTask.ListNodeFiles(recursive: true).ToList();
// confirm the files are there
Assert.True(files.Any(file => file.Path.Contains("localWords.txt")), "missing file: localWords.txt");
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
}
public TaskModel(JobModel parentJob, CloudTask task)
{
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
this.SubtasksInfo = null;
this.OutputFiles = task.ListNodeFiles();
}