public void Bug1965363_2384616_Wat7OSVersionFeatures()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
PoolOperations poolOperations = batchCli.PoolOperations;
try
{
this.testOutputHelper.WriteLine("Listing OS Versions:");
// create pool tests
// forget to set CloudServiceConfiguration on Create, get error
{
CloudPool noArgs = poolOperations.CreatePool("Bug1965363ButNoOSFamily-" + TestUtilities.GetMyName(), PoolFixture.VMSize, default(CloudServiceConfiguration), targetDedicatedComputeNodes: 0);
BatchException ex = TestUtilities.AssertThrows <BatchException>(() => noArgs.Commit());
string exStr = ex.ToString();
// we are expecting an exception, assert if the exception is not the correct one.
Assert.Contains("cloudServiceConfiguration", exStr);
}
// create a pool WITH an osFamily
{
string poolIdHOSF = "Bug1965363HasOSF-" + TestUtilities.GetMyName();
try
{
CloudPool hasOSF = poolOperations.CreatePool(poolIdHOSF, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily), targetDedicatedComputeNodes: 0);
hasOSF.Commit();
}
finally
{
poolOperations.DeletePool(poolIdHOSF);
}
}
}
catch (Exception ex)
{
// special case os version beacuse it is a common failure and requires human intervention/editing
// test for expired os version
Assert.DoesNotContain("The specified OS Version does not exists", ex.ToString());
throw;
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Extracts failure details from the BatchException object to create a more informative error message for the user.
/// </summary>
/// <param name="ex">The BatchException object</param>
private void HandleBatchException(BatchException ex)
{
if (ex != null)
{
if (ex.RequestInformation != null && ex.RequestInformation.BatchError != null)
{
WriteExceptionError(new BatchException(ex.RequestInformation, ex.ToString(), ex.InnerException));
}
else
{
WriteExceptionError(ex);
}
}
}
private static CloudPool CreatePool(Settings unzipperSettings, BatchClient client)
{
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
poolId: unzipperSettings.PoolId,
targetDedicated: unzipperSettings.PoolNodeCount,
virtualMachineSize: unzipperSettings.MachineSize,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4"));
pool.MaxTasksPerComputeNode = unzipperSettings.MaxTasksPerNode;
Console.WriteLine("Adding pool {0}", unzipperSettings.PoolId);
try
{
pool.Commit();
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", unzipperSettings.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
catch (BatchException be)
{
if (be.Message.Contains("conflict"))
{
Console.WriteLine("pool already exists");
}
}
return(pool);
}
public async static Task JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = Settings.Default;
AccountSettings accountSettings = AccountSettings.Default;
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "small",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4"));
List <string> files = new List <string>
{
Path.Combine(BatchStartTaskFolderName, BatchStartTaskTelemetryRunnerName),
};
files.AddRange(AIFilesToUpload);
var resourceHelperTask = SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
AIBlobConatinerName,
files);
List <ResourceFile> resourceFiles = resourceHelperTask.Result;
pool.StartTask = new StartTask()
{
CommandLine = string.Format("cmd /c {0}", BatchStartTaskTelemetryRunnerName),
ResourceFiles = resourceFiles
};
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
try
{
await GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool);
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
catch (BatchException be)
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
Console.WriteLine(be.ToString());
Console.WriteLine();
}
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
await unboundJob.CommitAsync();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// Upload application insights assemblies
List <FileToStage> aiStagedFiles = new List <FileToStage>();
foreach (string aiFile in AIFilesToUpload)
{
aiStagedFiles.Add(new FileToStage(aiFile, stagingStorageAccount));
}
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string[] documents = Directory.GetFiles(topNWordsConfiguration.DocumentsRootPath);
await SampleHelpers.UploadResourcesAsync(cloudStorageAccount, BooksContainerName, documents);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(documents.Length);
for (int i = 0; i < documents.Length; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
"https://onbehalfoutput.blob.core.windows.net/" + documents[i],
topNWordsConfiguration.TopWordCount,
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll,
};
foreach (FileToStage stagedFile in aiStagedFiles)
{
task.FilesToStage.Add(stagedFile);
}
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
public void Bug1965363_2384616_Wat7OSVersionFeatures()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
PoolOperations poolOperations = batchCli.PoolOperations;
try
{
this.testOutputHelper.WriteLine("Listing OS Versions:");
/* bug 2384616 ListOsVersions hidden for wat 8
*
* // test ListOSVersion
* foreach (OSVersion curOSV in poolMgr.ListOSVersions())
* {
* this.testOutputHelper.WriteLine("Label: " + curOSV.Label);
* this.testOutputHelper.WriteLine(" Version: " + curOSV.Version);
* this.testOutputHelper.WriteLine(" Family: " + curOSV.Family);
* this.testOutputHelper.WriteLine(" FamilyLabel: " + curOSV.FamilyLabel);
* this.testOutputHelper.WriteLine(" isDefault: " + curOSV.IsDefault);
* this.testOutputHelper.WriteLine(" IsActive: " + curOSV.IsActive);
*
* string expDate;
*
* if (curOSV.ExpirationDate.HasValue)
* {
* expDate = curOSV.ExpirationDate.Value.ToString();
* }
* else
* {
* expDate = "<null/novalue>";
* }
*
* this.testOutputHelper.WriteLine(" ExpirationDate: " + expDate);
* }
*
*/
// create pool tests
// forget to set CloudServiceConfiguration on Create, get error
{
CloudPool noArgs = poolOperations.CreatePool("Bug1965363ButNoOSFamily-" + TestUtilities.GetMyName(), PoolFixture.VMSize, default(CloudServiceConfiguration), targetDedicated: 0);
BatchException ex = TestUtilities.AssertThrows <BatchException>(() => noArgs.Commit());
string exStr = ex.ToString();
// we are expecting an exception, assert if the exception is not the correct one.
Assert.Contains("cloudServiceConfiguration", exStr);
}
// create a pool WITH an osFamily
{
string poolIdHOSF = "Bug1965363HasOSF-" + TestUtilities.GetMyName();
try
{
CloudPool hasOSF = poolOperations.CreatePool(poolIdHOSF, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily), targetDedicated: 0);
hasOSF.Commit();
}
finally
{
poolOperations.DeletePool(poolIdHOSF);
}
}
// TODO: ultimately we will either need to find (via list) a family with more than one version or
// manually update these strings as OS versions are depricated
//See here for other OS versions if this test fails: http://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
const string familyVersion0 = "*";
const string familyVersion1 = "WA-GUEST-OS-4.32_201605-01";
// "UpdatePoolOS" tests (ChangeOSVersion in OM)
// PoolManager
{
string poolIdChangeOSV = "Bug1965363ChangeOSVviaMGR-" + TestUtilities.GetMyName();
try
{
CloudPool unboundPool = poolOperations.CreatePool(
poolIdChangeOSV,
PoolFixture.VMSize,
new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
targetDedicated: 0);
unboundPool.Commit();
// fetch the bound pool
CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);
Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);
// switch to new version
poolOperations.ChangeOSVersion(poolIdChangeOSV, familyVersion1);
// UpdatePoolOS is has latency???
PollForOSVersionChange(boundPool, familyVersion1);
// check to make sure the new value is set
boundPool.Refresh();
Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
}
}
// ICloudPool
{
string poolIdChangeOSV = "Bug1965363ChangeOSVviaPool-" + TestUtilities.GetMyName();
try
{
CloudPool unboundPool = poolOperations.CreatePool(
poolIdChangeOSV,
PoolFixture.VMSize,
new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
targetDedicated: 0);
unboundPool.Commit();
// fetch the bound pool
CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);
Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);
// switch to new version
boundPool.ChangeOSVersion(familyVersion1);
// UpdatePoolOS is has latency???
PollForOSVersionChange(boundPool, familyVersion1);
// check to make sure the new value is set
boundPool.Refresh();
Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
}
}
// autopoolspec tests
{
string jobId = "Bug1965363WIName-" + TestUtilities.GetMyName();
// test not setting osversion
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
AutoPoolSpecification aps = new AutoPoolSpecification();
PoolSpecification ps = new PoolSpecification();
// test unbound set constraint
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);
// test unbound get constraint
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.TargetDedicated = 0; // trivial size for testing purposes
aps.PoolSpecification = ps;
aps.PoolLifetimeOption = PoolLifetimeOption.Job;
unboundJob.PoolInformation.AutoPoolSpecification = aps;
// commit to test validation
unboundJob.Commit();
// get bound job
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// test bound get constraints
this.testOutputHelper.WriteLine(" OSFamily == " +
boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
string targetOSVersion =
boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion;
if (string.IsNullOrEmpty(targetOSVersion))
{
targetOSVersion = "<null or empty";
}
this.testOutputHelper.WriteLine(" TargetOSVersion == " + targetOSVersion);
}
finally
{
// cleanup
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
{
string jobScheduleId = "Bug1965363WINameSettingAndChanging-" + TestUtilities.GetMyName();
// test setting osversion
try
{
AutoPoolSpecification aps = new AutoPoolSpecification();
PoolSpecification ps = new PoolSpecification();
CloudJobSchedule unboundJobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(
jobScheduleId,
new Schedule()
{
RecurrenceInterval = TimeSpan.FromDays(7)
},
new JobSpecification(new PoolInformation()
{
AutoPoolSpecification = aps
}));
// test unbound set constraint
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0);
// test unbound get constraint
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.TargetOSVersion == " + ps.CloudServiceConfiguration.TargetOSVersion);
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.TargetDedicated = 0; // trivial size for testing purposes
aps.PoolSpecification = ps;
aps.PoolLifetimeOption = PoolLifetimeOption.Job;
unboundJobSchedule.Commit();
// get bound job schedule
CloudJobSchedule boundJobSchedule = batchCli.JobScheduleOperations.GetJobSchedule(jobScheduleId);
// test bound get constraints
this.testOutputHelper.WriteLine(" OSFamily == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
this.testOutputHelper.WriteLine(" TargetOSVersion == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
// assert the value is as set above
Assert.Equal(familyVersion0, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
// change values
const string altFamily = "3";
const string altOSVersion = "WA-GUEST-OS-3.39_201605-01";
// change values on the bound PUS
PoolSpecification boundPS = boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification;
boundPS.CloudServiceConfiguration = new CloudServiceConfiguration(altFamily, altOSVersion);
// flush changes
boundJobSchedule.Commit();
// confirm changes took
boundJobSchedule.Refresh();
Assert.Equal(altFamily, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
Assert.Equal(altOSVersion, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
}
finally
{
// cleanup
TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jobScheduleId).Wait();
}
}
}
}
catch (Exception ex)
{
// special case os version beacuse it is a common failure and requires human intervention/editing
// test for expired os version
Assert.DoesNotContain("The specified OS Version does not exists", ex.ToString());
throw;
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(configuration.BatchServiceUrl, configuration.BatchAccountName, configuration.BatchAccountKey)))
{
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(configuration.PoolId, targetDedicated: configuration.PoolSize, osFamily: "4", virtualMachineSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolId);
try
{
pool.Commit();
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", configuration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
}
try
{
Console.WriteLine("Creating job: " + configuration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = configuration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = configuration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(configuration.NumberOfTasks);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(configuration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(configuration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolId);
client.PoolOperations.DeletePool(configuration.PoolId);
}
//Delete the job that we created
if (configuration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", configuration.JobId);
client.JobOperations.DeleteJob(configuration.JobId);
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
}
/// <summary>
/// Extracts failure details from the BatchException object to create a more informative error message for the user.
/// </summary>
/// <param name="ex">The BatchException object</param>
private void HandleBatchException(BatchException ex)
{
if (ex != null)
{
if (ex.RequestInformation != null && ex.RequestInformation.BatchError != null)
{
WriteExceptionError(new BatchException(ex.RequestInformation, ex.ToString(), ex.InnerException));
}
else
{
WriteExceptionError(ex);
}
}
}