static void AddTasks(BatchClient client, CloudJob cloudJob, string jobId, IEnumerable<IComputeTask> computeTasks)
{
foreach (var computeTask in computeTasks)
{
var definition = computeTask.Definition;
var executable = new ResourceFile($"{definition.StorageUri}/{definition.ExecutableName}", definition.ExecutableName);
var resources = definition.Resources.Select(resource => new ResourceFile($"{definition.StorageUri}/{resource}", resource));
var inputs = computeTask.Inputs.Select(input => new ResourceFile($"{definition.StorageUri}/{input}", input));
var resourceFiles = new List<ResourceFile> { executable };
resourceFiles.AddRange(resources);
resourceFiles.AddRange(inputs);
var task = client.JobOperations.ListTasks(jobId).SingleOrDefault(t => t.Id == computeTask.Id);
if (task == null)
{
task = new CloudTask(computeTask.Id, computeTask.CommandLine)
{
ResourceFiles = resourceFiles
};
cloudJob.AddTask(task);
cloudJob.Commit();
cloudJob.Refresh();
}
}
client.Utilities.CreateTaskStateMonitor().WaitAll(cloudJob.ListTasks(), TaskState.Completed, new TimeSpan(0, 30, 0));
}
public void TestNode_GetListDeleteFiles()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "TestNodeGetListDeleteFiles-" + TestUtilities.GetMyName();
try
{
const string taskId = "hiWorld";
const string directoryCreationTaskId1 = "dirTask1";
const string directoryCreationTaskId2 = "dirTask2";
const string directoryNameOne = "Foo";
const string directoryNameTwo = "Bar";
//
// Create the job
//
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(taskId, "cmd /c echo hello world");
CloudTask directoryCreationTask1 = new CloudTask(directoryCreationTaskId1, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameOne));
CloudTask directoryCreationTask2 = new CloudTask(directoryCreationTaskId2, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameTwo));
boundJob.AddTask(myTask);
boundJob.AddTask(directoryCreationTask1);
boundJob.AddTask(directoryCreationTask2);
testOutputHelper.WriteLine("Initial job commit()");
//
// Wait for task to go to completion
//
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));
CloudTask boundTask = boundJob.GetTask(taskId);
//Since the compute node name comes back as "Node:<computeNodeId>" we need to split on : to get the actual compute node name
string computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];
ComputeNode computeNode = batchCli.PoolOperations.GetComputeNode(poolFixture.PoolId, computeNodeId);
testOutputHelper.WriteLine("Task ran on compute node: {0}", computeNodeId);
//Ensure that ListFiles done without a recursive option, or with recursive false return the same values
{
List <NodeFile> filesByComputeNodeRecursiveOmitted = batchCli.PoolOperations.ListNodeFiles(
poolFixture.PoolId,
computeNodeId).ToList();
List <NodeFile> filesByComputeNodeRecursiveFalse = batchCli.PoolOperations.ListNodeFiles(
poolFixture.PoolId,
computeNodeId,
recursive: false).ToList();
AssertFileListsMatch(filesByComputeNodeRecursiveOmitted, filesByComputeNodeRecursiveFalse);
}
{
List <NodeFile> filesByTaskRecursiveOmitted = batchCli.JobOperations.ListNodeFiles(
jobId,
taskId).ToList();
List <NodeFile> filesByTaskRecursiveFalse = batchCli.JobOperations.ListNodeFiles(
jobId,
taskId,
recursive: false).ToList();
AssertFileListsMatch(filesByTaskRecursiveOmitted, filesByTaskRecursiveFalse);
}
//
// List all node files from operations -- recursive true
//
//TODO: Detail level?
List <NodeFile> fileListFromComputeNodeOperations = batchCli.PoolOperations.ListNodeFiles(poolFixture.PoolId, computeNodeId, recursive: true).ToList();
foreach (NodeFile f in fileListFromComputeNodeOperations)
{
testOutputHelper.WriteLine("Found file: {0}", f.Path);
}
//Check to make sure the expected folder named "Shared" exists
Assert.Contains("shared", fileListFromComputeNodeOperations.Select(f => f.Path));
//
// List all node files from the compute node -- recursive true
//
List <NodeFile> fileListFromComputeNode = computeNode.ListNodeFiles(recursive: true).ToList();
foreach (NodeFile f in fileListFromComputeNodeOperations)
{
testOutputHelper.WriteLine("Found file: {0}", f.Path);
}
//Check to make sure the expected folder named "Shared" exists
Assert.Contains("shared", fileListFromComputeNode.Select(f => f.Path));
//
// Get file from operations
//
string filePathToGet = fileListFromComputeNode.First(f => !f.IsDirectory.Value && f.Properties.ContentLength > 0).Path;
testOutputHelper.WriteLine("Getting file: {0}", filePathToGet);
NodeFile computeNodeFileFromManager = batchCli.PoolOperations.GetNodeFile(poolFixture.PoolId, computeNodeId, filePathToGet);
testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
testOutputHelper.WriteLine("---- File data: ----");
var computeNodeFileContentFromManager = computeNodeFileFromManager.ReadAsString();
testOutputHelper.WriteLine(computeNodeFileContentFromManager);
Assert.NotEmpty(computeNodeFileContentFromManager);
//
// Get file directly from operations (bypassing the properties call)
//
var computeNodeFileContentDirect = batchCli.PoolOperations.CopyNodeFileContentToString(poolFixture.PoolId, computeNodeId, filePathToGet);
testOutputHelper.WriteLine("---- File data: ----");
testOutputHelper.WriteLine(computeNodeFileContentDirect);
Assert.NotEmpty(computeNodeFileContentDirect);
//
// Get file from compute node
//
testOutputHelper.WriteLine("Getting file: {0}", filePathToGet);
NodeFile fileFromComputeNode = computeNode.GetNodeFile(filePathToGet);
testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
testOutputHelper.WriteLine("---- File data: ----");
var computeNodeFileContentFromNode = fileFromComputeNode.ReadAsString();
testOutputHelper.WriteLine(computeNodeFileContentFromNode);
Assert.NotEmpty(computeNodeFileContentFromNode);
//
// Get file from compute node (bypassing the properties call)
//
computeNodeFileContentDirect = computeNode.CopyNodeFileContentToString(filePathToGet);
testOutputHelper.WriteLine("---- File data: ----");
testOutputHelper.WriteLine(computeNodeFileContentDirect);
Assert.NotEmpty(computeNodeFileContentDirect);
//
// NodeFile delete
//
string filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardOutFileName);
NodeFile nodeFile = batchCli.PoolOperations.GetNodeFile(poolFixture.PoolId, computeNodeId, filePath);
nodeFile.Delete();
//Ensure delete succeeded
TestUtilities.AssertThrows <BatchException>(() => nodeFile.Refresh());
//Delete directory
NodeFile directory = batchCli.PoolOperations.ListNodeFiles(poolFixture.PoolId, computeNodeId, recursive: true).First(item => item.Path.Contains(directoryNameOne));
Assert.True(directory.IsDirectory);
TestUtilities.AssertThrows <BatchException>(() => directory.Delete(recursive: false));
directory.Delete(recursive: true);
Assert.Null(batchCli.PoolOperations.ListNodeFiles(poolFixture.PoolId, computeNodeId, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameOne)));
//
// PoolManager delete node file
//
filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardErrorFileName);
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
batchCli.PoolOperations.DeleteNodeFile(poolFixture.PoolId, computeNodeId, filePath);
//Ensure delete succeeded
TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));
//Delete directory
directory = batchCli.PoolOperations.ListNodeFiles(poolFixture.PoolId, computeNodeId, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
Assert.True(directory.IsDirectory);
TestUtilities.AssertThrows <BatchException>(() => batchCli.PoolOperations.DeleteNodeFile(poolFixture.PoolId, computeNodeId, directory.Path, recursive: false));
batchCli.PoolOperations.DeleteNodeFile(poolFixture.PoolId, computeNodeId, directory.Path, recursive: true);
Assert.Null(batchCli.PoolOperations.ListNodeFiles(poolFixture.PoolId, computeNodeId, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameTwo)));
}
finally
{
batchCli.JobOperations.DeleteJob(jobId);
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void Bug230385SupportDeleteNodeFileByTask()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "Bug230285Job-" + TestUtilities.GetMyName();
try
{
const string taskId = "hiWorld";
const string directoryCreationTaskId1 = "dirTask1";
const string directoryCreationTaskId2 = "dirTask2";
const string directoryNameOne = "Foo";
const string directoryNameTwo = "Bar";
const string directory2PathOnNode = "wd/" + directoryNameTwo;
//
// Create the job
//
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(taskId, "cmd /c echo hello world");
CloudTask directoryCreationTask1 = new CloudTask(directoryCreationTaskId1, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameOne));
CloudTask directoryCreationTask2 = new CloudTask(directoryCreationTaskId2, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameTwo));
boundJob.AddTask(myTask);
boundJob.AddTask(directoryCreationTask1);
boundJob.AddTask(directoryCreationTask2);
testOutputHelper.WriteLine("Initial job commit()");
//
// Wait for task to go to completion
//
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));
//
// NodeFile delete
//
//Delete single file
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
file.Delete();
//Ensure delete succeeded
TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName));
//Delete directory
NodeFile directory = batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId1, recursive: true).First(item => item.Path.Contains(directoryNameOne));
Assert.True(directory.IsDirectory);
TestUtilities.AssertThrows <BatchException>(() => directory.Delete(recursive: false));
directory.Delete(recursive: true);
Assert.Null(batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId1, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameOne)));
//
// JobScheduleOperations delete task file
//
batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
batchCli.JobOperations.DeleteNodeFile(jobId, taskId, Constants.StandardErrorFileName);
//Ensure delete succeeded
TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));
//Delete directory
directory = batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId2, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
Assert.True(directory.IsDirectory);
TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.DeleteNodeFile(jobId, directoryCreationTaskId2, directory2PathOnNode, recursive: false));
batchCli.JobOperations.DeleteNodeFile(jobId, directoryCreationTaskId2, directory2PathOnNode, recursive: true);
Assert.Null(batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId2, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameTwo)));
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void TestGetNodeFileByTask()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
JobOperations jobOperations = batchCli.JobOperations;
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-" + nameof(TestGetNodeFileByTask);
try
{
//
// Create the job
//
CloudJob job = jobOperations.CreateJob(jobId, new PoolInformation());
job.PoolInformation = new PoolInformation()
{
PoolId = this.poolFixture.PoolId
};
this.testOutputHelper.WriteLine("Initial job schedule commit()");
job.Commit();
//
// Wait for the job
//
this.testOutputHelper.WriteLine("Waiting for job");
CloudJob boundJob = jobOperations.GetJob(jobId);
//
// Add task to the job
//
const string taskId = "T1";
const string taskMessage = "This is a test";
this.testOutputHelper.WriteLine("Adding task: {0}", taskId);
CloudTask task = new CloudTask(taskId, string.Format("cmd /c echo {0}", taskMessage));
boundJob.AddTask(task);
//
// Wait for the task to complete
//
this.testOutputHelper.WriteLine("Waiting for the task to complete");
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
//Wait for the task state to be running
taskStateMonitor.WaitAll(
jobOperations.ListTasks(jobId),
TaskState.Completed,
TimeSpan.FromSeconds(30));
//Download the data
this.testOutputHelper.WriteLine("Downloading the stdout for the file");
NodeFile file = jobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
string data = file.ReadAsString();
this.testOutputHelper.WriteLine("Data: {0}", data);
Assert.Contains(taskMessage, data);
// Download the data again using the JobOperations read file content helper
data = batchCli.JobOperations.CopyNodeFileContentToString(jobId, taskId, Constants.StandardOutFileName);
this.testOutputHelper.WriteLine("Data: {0}", data);
Assert.Contains(taskMessage, data);
}
finally
{
jobOperations.DeleteJob(jobId);
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void Bug1480491NodeFileFileProperties()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1480491Job-" + TestUtilities.GetMyName();
try
{
const string taskId = "hiWorld";
//
// Create the job
//
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(taskId, "cmd /c echo hello world");
boundJob.AddTask(myTask);
this.testOutputHelper.WriteLine("Initial job commit()");
//
// Wait for task to go to completion
//
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));
const int expectedFileSize = 13; //Magic number based on output generated by the task
//
// NodeFile by task
//
NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
this.testOutputHelper.WriteLine("File {0} has content length: {1}", Constants.StandardOutFileName, file.Properties.ContentLength);
this.testOutputHelper.WriteLine("File {0} has content type: {1}", Constants.StandardOutFileName, file.Properties.ContentType);
this.testOutputHelper.WriteLine("File {0} has creation time: {1}", Constants.StandardOutFileName, file.Properties.CreationTime);
this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", Constants.StandardOutFileName, file.Properties.LastModified);
Assert.Equal(expectedFileSize, file.Properties.ContentLength);
Assert.Equal("text/plain", file.Properties.ContentType);
//
// NodeFile by node
//
CloudTask boundTask = boundJob.GetTask(taskId);
string computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];
ComputeNode computeNode = batchCli.PoolOperations.GetComputeNode(this.poolFixture.PoolId, computeNodeId);
this.testOutputHelper.WriteLine("Task ran on compute node: {0}", computeNodeId);
List <NodeFile> files = computeNode.ListNodeFiles(recursive: true).ToList();
foreach (NodeFile nodeFile in files)
{
this.testOutputHelper.WriteLine("Found file: {0}", nodeFile.Path);
}
string filePathToGet = string.Format("workitems/{0}/{1}/{2}/{3}", jobId, "job-1", taskId, Constants.StandardOutFileName);
file = computeNode.GetNodeFile(filePathToGet);
this.testOutputHelper.WriteLine("File {0} has content length: {1}", filePathToGet, file.Properties.ContentLength);
this.testOutputHelper.WriteLine("File {0} has content type: {1}", filePathToGet, file.Properties.ContentType);
this.testOutputHelper.WriteLine("File {0} has creation time: {1}", filePathToGet, file.Properties.CreationTime);
this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", filePathToGet, file.Properties.LastModified);
Assert.Equal(expectedFileSize, file.Properties.ContentLength);
Assert.Equal("text/plain", file.Properties.ContentType);
}
finally
{
batchCli.JobOperations.DeleteJob(jobId);
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public void Bug1480489NodeFileMissingIsDirectory()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1480489Job-" + TestUtilities.GetMyName();
try
{
// here we show how to use an unbound Job + Commit() to run millions of Tasks
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = this.poolFixture.PoolId
});
unboundJob.Commit();
// Open the new Job as bound.
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "Bug1480489Task", commandline: @"md Bug1480489Directory");
// add the task to the job
boundJob.AddTask(myTask);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(3));
CloudTask myCompletedTask = new List <CloudTask>(boundJob.ListTasks(null))[0];
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
this.testOutputHelper.WriteLine("TaskId: " + myCompletedTask.Id);
this.testOutputHelper.WriteLine("StdOut: ");
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine("StdErr: ");
this.testOutputHelper.WriteLine(stdErr);
this.testOutputHelper.WriteLine("Task Files:");
bool foundAtLeastOneDir = false;
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles())
{
this.testOutputHelper.WriteLine(" Filepath: " + curFile.Path);
this.testOutputHelper.WriteLine(" IsDirectory: " + curFile.IsDirectory.ToString());
// turns out wd is created for each task so use it as sentinal
if (curFile.Path.Equals("wd") && curFile.IsDirectory.HasValue && curFile.IsDirectory.Value)
{
foundAtLeastOneDir = true;
}
}
Assert.True(foundAtLeastOneDir);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}