/// <summary>
/// Loads the configuration from the App.Config file
/// </summary>
/// <returns></returns>
public static TopNWordsConfiguration LoadConfigurationFromAppConfig()
{
TopNWordsConfiguration configuration = new TopNWordsConfiguration();
configuration.BatchServiceUrl = ConfigurationManager.AppSettings["BatchServiceUrl"];
configuration.BatchAccountName = ConfigurationManager.AppSettings["Account"];
configuration.BatchAccountKey = ConfigurationManager.AppSettings["Key"];
configuration.NumberOfTasks = Int32.Parse(ConfigurationManager.AppSettings["NumTasks"]);
configuration.PoolSize = Int32.Parse(ConfigurationManager.AppSettings["PoolSize"]);
configuration.NumberOfTopWords = Int32.Parse(ConfigurationManager.AppSettings["NumTopWords"]);
configuration.PoolName = ConfigurationManager.AppSettings["PoolName"];
configuration.ShouldCreatePool = string.IsNullOrEmpty(configuration.PoolName);
if (configuration.ShouldCreatePool)
{
configuration.PoolName = "TopNWordsPool" + DateTime.Now.ToString("_yyMMdd_HHmmss_") + Guid.NewGuid().ToString("N");
}
configuration.WorkItemName = "TopNWordsWi" + DateTime.Now.ToString("_yyMMdd_HHmmss_") + Guid.NewGuid().ToString("N");
configuration.StorageAccountName = ConfigurationManager.AppSettings["StorageAccountName"];
configuration.StorageAccountKey = ConfigurationManager.AppSettings["StorageAccountKey"];
configuration.StorageAccountBlobEndpoint = ConfigurationManager.AppSettings["StorageAccountBlobEndpoint"];
configuration.BookFileName = ConfigurationManager.AppSettings["BookFileName"];
configuration.ShouldDeleteWorkItem = bool.Parse(ConfigurationManager.AppSettings["DeleteWorkitem"]);
configuration.ShouldDeleteContainer = bool.Parse(ConfigurationManager.AppSettings["DeleteContainer"]);
return(configuration);
}
/// <summary>
/// Upload a text file to a cloud blob.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="fileName">The name of the file to upload</param>
/// <returns>The URI of the blob.</returns>
private static string UploadBookFileToCloudBlob(TopNWordsConfiguration configuration, string fileName)
{
CloudBlobClient client = GetCloudBlobClient(configuration.StorageAccountName, configuration.StorageAccountKey);
//Create the "books" container if it doesn't exist.
CloudBlobContainer container = client.GetContainerReference(booksContainerName);
container.CreateIfNotExists();
//Upload the blob.
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
blob.UploadFromFile(fileName, FileMode.Open);
return(blob.Uri.ToString());
}
/// <summary>
/// Upload a text file to a cloud blob.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="fileName">The name of the file to upload</param>
/// <returns>The URI of the blob.</returns>
private static string UploadBookFileToCloudBlob(TopNWordsConfiguration configuration, string fileName)
{
StorageCredentials cred = new StorageCredentials(configuration.StorageAccountName, configuration.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(cred, true);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
//Create the "books" container if it doesn't exist.
CloudBlobContainer container = client.GetContainerReference("books");
container.CreateIfNotExists();
//Upload the blob.
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
blob.UploadFromFile(fileName, System.IO.FileMode.Open);
return(blob.Uri.ToString());
}
/// <summary>
/// Delete the containers in Azure Storage which are created by this sample.
/// </summary>
private static void DeleteContainers(TopNWordsConfiguration configuration, string fileStagingContainer)
{
CloudBlobClient client = GetCloudBlobClient(configuration.StorageAccountName, configuration.StorageAccountKey);
//Delete the books container
CloudBlobContainer container = client.GetContainerReference(booksContainerName);
Console.WriteLine("Deleting container: " + booksContainerName);
container.DeleteIfExists();
//Delete the file staging container
if (!string.IsNullOrEmpty(fileStagingContainer))
{
container = client.GetContainerReference(fileStagingContainer);
Console.WriteLine("Deleting container: {0}", fileStagingContainer);
container.DeleteIfExists();
}
}
/// <summary>
/// Delete the containers in Azure Storage which are created by this sample.
/// </summary>
/// <param name="configuration"></param>
/// <param name="fileStagingContainer"></param>
private static void DeleteContainers(TopNWordsConfiguration configuration, string fileStagingContainer)
{
StorageCredentials cred = new StorageCredentials(configuration.StorageAccountName, configuration.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(cred, true);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
//Delete the books container
CloudBlobContainer container = client.GetContainerReference("books");
Console.WriteLine("Deleting container: {0}", "books");
container.DeleteIfExists();
//Delete the file staging container
if (!string.IsNullOrEmpty(fileStagingContainer))
{
container = client.GetContainerReference(fileStagingContainer);
Console.WriteLine("Deleting container: {0}", fileStagingContainer);
container.DeleteIfExists();
}
}
/// <summary>
/// Loads the configuration from the App.Config file
/// </summary>
/// <returns></returns>
public static TopNWordsConfiguration LoadConfigurationFromAppConfig()
{
TopNWordsConfiguration configuration = new TopNWordsConfiguration();
configuration.BatchServiceUrl = ConfigurationManager.AppSettings["BatchServiceUrl"];
configuration.BatchAccountName = ConfigurationManager.AppSettings["BatchAccount"];
configuration.BatchAccountKey = ConfigurationManager.AppSettings["BatchKey"];
configuration.NumberOfTasks = Int32.Parse(ConfigurationManager.AppSettings["NumTasks"]);
configuration.PoolSize = Int32.Parse(ConfigurationManager.AppSettings["PoolSize"]);
configuration.NumberOfTopWords = Int32.Parse(ConfigurationManager.AppSettings["NumTopWords"]);
configuration.PoolId = ConfigurationManager.AppSettings["PoolId"];
configuration.ShouldCreatePool = string.IsNullOrEmpty(configuration.PoolId);
if (configuration.ShouldCreatePool)
{
configuration.PoolId = "TopNWordsPool" + DateTime.Now.ToString("_yyMMdd_HHmmss_") + Guid.NewGuid().ToString("N");
}
configuration.JobId = "TopNWordsJob" + DateTime.Now.ToString("_yyMMdd_HHmmss_") + Guid.NewGuid().ToString("N");
configuration.StorageAccountName = ConfigurationManager.AppSettings["StorageAccountName"];
configuration.StorageAccountKey = ConfigurationManager.AppSettings["StorageAccountKey"];
configuration.StorageAccountBlobEndpoint = ConfigurationManager.AppSettings["StorageAccountBlobEndpoint"];
configuration.BookFileName = ConfigurationManager.AppSettings["BookFileName"];
configuration.ShouldDeleteJob = bool.Parse(ConfigurationManager.AppSettings["DeleteJob"]);
configuration.ShouldDeleteContainer = bool.Parse(ConfigurationManager.AppSettings["DeleteContainer"]);
return configuration;
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(configuration.BatchServiceUrl, configuration.BatchAccountName, configuration.BatchAccountKey)))
{
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(configuration.PoolId, targetDedicated: configuration.PoolSize, osFamily: "4", virtualMachineSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolId);
try
{
pool.Commit();
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", configuration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
}
try
{
Console.WriteLine("Creating job: " + configuration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = configuration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = configuration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(configuration.NumberOfTasks);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(configuration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(configuration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolId);
client.PoolOperations.DeletePool(configuration.PoolId);
}
//Delete the job that we created
if (configuration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", configuration.JobId);
client.JobOperations.DeleteJob(configuration.JobId);
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
}
/// <summary>
/// Upload a text file to a cloud blob.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="fileName">The name of the file to upload</param>
/// <returns>The URI of the blob.</returns>
private static string UploadBookFileToCloudBlob(TopNWordsConfiguration configuration, string fileName)
{
CloudBlobClient client = GetCloudBlobClient(configuration.StorageAccountName, configuration.StorageAccountKey);
//Create the "books" container if it doesn't exist.
CloudBlobContainer container = client.GetContainerReference(booksContainerName);
container.CreateIfNotExists();
//Upload the blob.
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
blob.UploadFromFile(fileName, FileMode.Open);
return blob.Uri.ToString();
}
/// <summary>
/// Delete the containers in Azure Storage which are created by this sample.
/// </summary>
private static void DeleteContainers(TopNWordsConfiguration configuration, string fileStagingContainer)
{
CloudBlobClient client = GetCloudBlobClient(configuration.StorageAccountName, configuration.StorageAccountKey);
//Delete the books container
CloudBlobContainer container = client.GetContainerReference(booksContainerName);
Console.WriteLine("Deleting container: " + booksContainerName);
container.DeleteIfExists();
//Delete the file staging container
if (!string.IsNullOrEmpty(fileStagingContainer))
{
container = client.GetContainerReference(fileStagingContainer);
Console.WriteLine("Deleting container: {0}", fileStagingContainer);
container.DeleteIfExists();
}
}
/// <summary>
/// Delete the containers in Azure Storage which are created by this sample.
/// </summary>
/// <param name="configuration"></param>
/// <param name="fileStagingContainer"></param>
private static void DeleteContainers(TopNWordsConfiguration configuration, string fileStagingContainer)
{
StorageCredentials cred = new StorageCredentials(configuration.StorageAccountName, configuration.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(cred, true);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
//Delete the books container
CloudBlobContainer container = client.GetContainerReference("books");
Console.WriteLine("Deleting container: {0}", "books");
container.DeleteIfExists();
//Delete the file staging container
if (!string.IsNullOrEmpty(fileStagingContainer))
{
container = client.GetContainerReference(fileStagingContainer);
Console.WriteLine("Deleting container: {0}", fileStagingContainer);
container.DeleteIfExists();
}
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
//OSFamily 4 == OS 2012 R2
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolName);
pool.Commit();
}
}
try
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
//Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
taskSubmissionHelper.WorkItemName = configuration.WorkItemName;
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the TaskSubmissionHelper's container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
taskSubmissionHelper.AddTask(task);
}
//Commit all the tasks to the Batch Service.
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
{
SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
//Get the job to monitor status.
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.Write("Waiting for tasks to complete ...");
// Wait 1 minute for all tasks to reach the completed state
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("Done.");
foreach (ICloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
pm.DeletePool(configuration.PoolName);
}
}
//Delete the workitem that we created
if (configuration.ShouldDeleteWorkItem)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
wm.DeleteWorkItem(configuration.WorkItemName);
}
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
