public ParallelAPSIM(
Storage.StorageCredentials storageCredentials,
Batch.BatchCredentials batchCredentials,
PoolSettings poolSettings)
{
_storageAccount = new CloudStorageAccount(
new StorageCredentials(
storageCredentials.Account,
storageCredentials.Key),
true);
_batchClient = BatchClient.Open(
new BatchSharedKeyCredentials(
batchCredentials.Url,
batchCredentials.Account,
batchCredentials.Key));
_batchClient.CustomBehaviors.Add(
RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 10));
_fileUploader = new FileUploader(_storageAccount);
_storageCredentials = storageCredentials;
_batchCredentials = batchCredentials;
_poolSettings = poolSettings;
}
public JobPurgeTimer(RetryPolicyProvider retryPolicyProvider, ManagerConfiguration managerConfiguration) : base(managerConfiguration.PurgeJobsIntervalHours * 60 * 60 * 1000)
{
_managerConfiguration = managerConfiguration;
_retryPolicy = retryPolicyProvider.GetPolicy();
_connectionString = managerConfiguration.ConnectionString;
Elapsed += PurgeTimer_elapsed;
}
public async Task BatchRequestWithShortClientSideTimeoutAndRetries()
{
const int maxRetries = 5;
int actualRequestCount = 0;
//The ThrowsAnyAsync must be outside the RunTestAsync or it will deadlock
await Assert.ThrowsAnyAsync <OperationCanceledException>(async() =>
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
client.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromMilliseconds(250), maxRetries));
List <BatchClientBehavior> customBehaviors = new List <BatchClientBehavior>();
customBehaviors.Add(new RequestInterceptor((req) =>
{
//Set the timeout to something small so it is guaranteed to expire before the service has responded
req.Timeout = TimeSpan.FromMilliseconds(25);
var castRequest = (JobGetBatchRequest)req;
Func <CancellationToken, Task <AzureOperationResponse <Microsoft.Azure.Batch.Protocol.Models.CloudJob, Microsoft.Azure.Batch.Protocol.Models.JobGetHeaders> > > oldFunc = castRequest.ServiceRequestFunc;
castRequest.ServiceRequestFunc = async(token) =>
{
actualRequestCount++; //Count the number of calls to the func
return(await oldFunc(token).ConfigureAwait(false));
};
}));
await client.JobOperations.GetJobAsync("Foo", additionalBehaviors: customBehaviors).ConfigureAwait(false);
}
},
TestTimeout));
Assert.Equal(maxRetries, actualRequestCount - 1); //Ensure that the number of retries is as expected
}
public async Task <bool> Deploy(TestJob testJob)
{
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - Deploy {testJob.BatchJobId}", "RunningLog");
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(BatchServiceUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
try
{
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - Create Pool ({testJob.NumofVm} core)", "RunningLog");
await CreatePoolIfNotExistAsync(batchClient, StorageAccount, testJob);
Messenger.Default.Send(
new DeployStatusUpdateMessage()
{
Phase = DeployPhase.PoolCreated,
Status = PhaseStatus.Succeeded
}, "DeployStatus");
}
catch (Exception ex)
{
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - delete test due to {ex.Message}", "RunningLog");
Messenger.Default.Send(
new DeployStatusUpdateMessage()
{
Phase = DeployPhase.PoolCreated,
Status = PhaseStatus.Failed
}, "DeployStatus");
await batchClient.DeleteTest(testJob);
}
try
{
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - Submit job", "RunningLog");
await SubmitJobIfNotExistAsync(batchClient, StorageAccount, testJob);
Messenger.Default.Send(
new DeployStatusUpdateMessage()
{
Phase = DeployPhase.JobCreated,
Status = PhaseStatus.Succeeded
}, "DeployStatus");
}
catch (Exception ex)
{
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - delete test due to {ex.Message}", "RunningLog");
Messenger.Default.Send(
new DeployStatusUpdateMessage()
{
Phase = DeployPhase.JobCreated,
Status = PhaseStatus.Failed
}, "DeployStatus");
await batchClient.DeleteTest(testJob);
}
}
return(true);
}
public JobPurgeTimer(RetryPolicyProvider retryPolicyProvider, ManagerConfiguration managerConfiguration) : base(managerConfiguration.PurgeJobsIntervalHours*60*60*1000)
{
_managerConfiguration = managerConfiguration;
_retryPolicy = retryPolicyProvider.GetPolicy();
_connectionString = managerConfiguration.ConnectionString;
Elapsed += PurgeTimer_elapsed;
Purge();
}
private static BatchClient PrepareConnection()
{
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
Environment.GetEnvironmentVariable("BATCHCONNECTIONSTRING"),
Environment.GetEnvironmentVariable("BATCHNAME"),
Environment.GetEnvironmentVariable("BATCHPASSWORD"));
// Get an instance of the BatchClient for a given Azure Batch account.
BatchClient batchClient = BatchClient.Open(credentials);
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
return(batchClient);
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.configurationSettings.BatchServiceUrl,
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = null;
// Track the containers which are created as part of job submission so that we can clean them up later.
HashSet <string> blobContainerNames = null;
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient);
// Submit the job
jobId = CreateJobId("SimpleJob");
blobContainerNames = await this.SubmitJobAsync(batchClient, jobId);
// Print out the status of the pools/jobs under this account
await ListJobsAsync(batchClient);
await ListPoolsAsync(batchClient);
// Wait for the job to complete
await this.WaitForJobAndPrintOutputAsync(batchClient, jobId);
}
finally
{
// Delete the pool (if configured) and job
// TODO: In C# 6 we can await here instead of .Wait()
this.CleanupResourcesAsync(batchClient, jobId, blobContainerNames).Wait();
}
}
}
public static async Task <BatchClient> OpenBatchClientAsync(BatchSharedKeyCredentials sharedKeyCredentials, bool addDefaultRetryPolicy = true)
{
BatchClient client = await BatchClient.OpenAsync(sharedKeyCredentials);
//Force us to get exception if the server returns something we don't expect
//TODO: To avoid including this test assembly via "InternalsVisibleTo" we resort to some reflection trickery... maybe this property
//TODO: should just be public?
//TODO: Disabled for now because the swagger spec does not accurately reflect all properties returned by the server
//SetDeserializationSettings(client);
//Set up some common stuff like a retry policy
if (addDefaultRetryPolicy)
{
client.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 5));
}
return(client);
}
private static void ConfigureServices(IServiceCollection services, IConfiguration configuration)
{
services.AddTransient <TvMazeScraper>();
services.AddTransient <TvMazeRepository>();
services.AddTransient <TvMazeApiClient>();
// TODO: investigate if it blocks the whole HttpClient instance
// TODO: investigate console app responsiveness when waiting
services.AddHttpClient <TvMazeApiClient>()
.AddPolicyHandler(RetryPolicyProvider.Get());
services.AddDbContext <TvMazeContext>(options =>
options.UseSqlServer(configuration.GetConnectionString("TvMaze"))
);
services.AddLogging(builder => builder.AddConsole());
}
/// <summary>
/// Submits a job to the Azure Batch service, and waits for it to complete
/// </summary>
private static async Task HelloWorldAsync(AccountSettings accountSettings, Settings helloWorldConfigurationSettings)
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(helloWorldConfigurationSettings.ToString());
Console.WriteLine(accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = GettingStartedCommon.CreateJobId("HelloWorldJob");
try
{
// Submit the job
await SubmitJobAsync(batchClient, helloWorldConfigurationSettings, jobId);
// Wait for the job to complete
await WaitForJobAndPrintOutputAsync(batchClient, jobId);
}
finally
{
// Delete the job to ensure the tasks are cleaned up
if (!string.IsNullOrEmpty(jobId) && helloWorldConfigurationSettings.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", jobId);
batchClient.JobOperations.DeleteJob(jobId);
}
}
}
}
public JobManager(
BatchCredentials batchCredentials,
StorageCredentials storageCredentials,
ITaskProvider taskProvider)
{
_batchClient = BatchClient.Open(
new BatchSharedKeyCredentials(
batchCredentials.Url,
batchCredentials.Account,
batchCredentials.Key));
_batchClient.CustomBehaviors.Add(
RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 10));
_taskProvider = taskProvider;
_storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
storageCredentials.Account,
storageCredentials.Key),
true);
}
public JobOutputMonitor(
Storage.StorageCredentials storageCredentials,
Batch.BatchCredentials batchCredentials)
{
_storageAccount = new CloudStorageAccount(
new StorageCredentials(
storageCredentials.Account,
storageCredentials.Key),
true);
_blobClient = _storageAccount.CreateCloudBlobClient();
_blobClient.DefaultRequestOptions.RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(3), 10);
_batchClient = BatchClient.Open(
new BatchSharedKeyCredentials(
batchCredentials.Url,
batchCredentials.Account,
batchCredentials.Key));
_batchClient.CustomBehaviors.Add(
RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 10));
}
private static async Task CreateTasksAsync()
{
//1. Connect with you Azure Batch account
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
Settings.Default.BatchURL,
Settings.Default.AccountName,
Settings.Default.AccountKey);
using (var batchClient = await BatchClient.OpenAsync(credentials))
{
//2. Add a retry policy
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
//3. Create a job id
var jobId = string.Format("MyTasks-{0}", DateTime.Now.ToString("yyyyMMdd-HHmmss"));
try
{
//4. Submit the job
await SubmitJobAsync(batchClient, jobId);
//5. Wait for the job to complete
await WaitForJobAndPrintOutputAsync(batchClient, jobId);
}
finally
{
Console.WriteLine("Press enter to delete the job");
Console.ReadLine();
if (!string.IsNullOrEmpty(jobId))
{
Console.WriteLine("Deleting job: {0}", jobId);
batchClient.JobOperations.DeleteJob(jobId);
}
}
}
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequestMessage req, TraceWriter log)
{
log.Info("Batch Job Requested...");
try
{
var settings = JobSettings.FromAppSettings();
using (var batchClient = await BatchClient.OpenAsync(new BatchSharedKeyCredentials(settings.BatchAccountUrl, settings.BatchAccountName, settings.BatchAccountKey)))
{
// Add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry.
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(settings.RetryDeltaBackoff), settings.RetryMaxCount));
var jobId = $"BlobMover-{ Guid.NewGuid().ToString() }";
var queryArgs = req.GetQueryNameValuePairs();
var container = queryArgs.FirstOrDefault(kvp => String.Equals(kvp.Key, "container", StringComparison.InvariantCultureIgnoreCase)).Value;
var blob = queryArgs.FirstOrDefault(kvp => String.Equals(kvp.Key, "blob", StringComparison.InvariantCultureIgnoreCase)).Value;
await BlobEventHook.SubmitJobAsync(settings, batchClient, container, blob, jobId, log);
log.Info("Batch Job Created.");
return(req.CreateResponse(HttpStatusCode.OK, nameof(HttpStatusCode.OK)));
}
}
catch (Exception ex)
{
log.Info("");
log.Error("An error occurred while submitting the job", ex);
log.Info("");
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, ex));
}
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
System.Net.ServicePointManager.DefaultConnectionLimit = 20;
// Get an instance of the BatchClient for a given Azure Batch account.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchUrl, BatchAccount, BatchKey);
using (BatchClient client = BatchClient.Open(cred))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
client.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
ListPools(client);
ListJobs(client);
CreatePoolIfNeeded(client, PoolId);
AddJobTwoTasks(client, PoolId);
ListPools(client);
ListJobs(client);
AddTasksWithFileStaging(client, PoolId);
ListPools(client);
ListJobs(client);
SubmitLargeNumberOfTasks(client, PoolId);
ListPools(client);
ListJobs(client);
}
Console.WriteLine("Press return to exit...");
Console.ReadLine();
}
private static async Task ProcessAsync(JobSettings settings, string jobId, string containerName, string blobName)
{
// Simulate splitting of the target blob into chunks by just making some copies of the source blob.
var client = CloudStorageAccount.Parse(settings.BatchBlobStorageConnection).CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blob = (CloudBlockBlob)container.GetBlobReferenceFromServer(blobName);
var leaseDuration = TimeSpan.FromSeconds(60);
var leaseName = await blob.AcquireLeaseAsync(leaseDuration).ConfigureAwait(false);
var lease = AccessCondition.GenerateLeaseCondition(leaseName);
var copies = await Task.WhenAll(Enumerable.Range(0, 5).Select(index => EntryPoint.CopyBlobAsync(container, blob, $"{ Path.GetFileNameWithoutExtension(blobName) }-{ index }{ Path.GetExtension(blobName) }", lease, leaseDuration))).ConfigureAwait(false);
blob.Delete(DeleteSnapshotsOption.IncludeSnapshots, lease);
// Create tasks for the job that will process each chunk. In this case, that will be one of the copies that we
// made to simulate it.
using (var batchClient = await BatchClient.OpenAsync(new BatchSharedKeyCredentials(settings.BatchAccountUrl, settings.BatchAccountName, settings.BatchAccountKey)))
{
// Add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry.
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(settings.RetryDeltaBackoff), settings.RetryMaxCount));
try
{
// Create tasks with file references here. I'm reusing the existing resource list, since they've already been staged.
// For input, in our case, we're letting the task access the blob directly. You could also use file staging here, if you'd rather the task not have
// knowledge that it is working on a blob directly. However, since you most likely want to process it for redaction and then upload the result,
// it seems like a better idea to work through storage for both.
var moverPath = $"{ typeof(BlobMover.EntryPoint).Assembly.GetName().Name }.exe";
var jobResources = batchClient.JobOperations.GetJob(jobId)?.JobManagerTask.ResourceFiles;
jobResources.Select(thing => { Console.WriteLine(thing); return(thing); });
var copyTasks = copies
.Where(result => result.Created)
.Select(result => new CloudTask(Path.GetFileNameWithoutExtension(result.Name), $"{ moverPath } { containerName } { result.Name } {settings.BatchBlobStorageConnection }")
{
ResourceFiles = jobResources
});
await batchClient.JobOperations.AddTaskAsync(jobId, copyTasks).ConfigureAwait(false);
// Wait for all of the work associated with processing the chunks to complete.
await EntryPoint.WaitForChildTasksAsync(batchClient, jobId, TimeSpan.FromHours(2)).ConfigureAwait(false);
// This is where we would create the final tasks to process the results for each of the
// processed chunks. After, we would have to perform another WaitForTasksAsync to ensure
// that it is complete before cleaning up.
//
// Because we just simulated work by copying the fake chunk blobs, I skipped those steps. The code
// would be virtually identical to the other code in the try block above.
//
// Alternatively, you could perform the final processing directly here.
}
catch (Exception ex)
{
// Surfacing information from failed tasks can be a challenge. I suspct that there are more efficient and easier ways to do so, but
// for my purposes, a rudimentary capture and upload to blob store was very helpful.
//
// NOTE: This catch block is doing a bunch of things, and things that could fail. It goes without saying, this isn't best
// practice.
var outblob = container.GetBlockBlobReference("task-errors.txt");
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
writer.WriteLine(ex.GetType().Name);
writer.WriteLine(ex.Message);
writer.WriteLine();
writer.WriteLine(ex.StackTrace);
writer.WriteLine();
if (ex.InnerException != null)
{
writer.WriteLine(ex.InnerException.GetType().Name);
writer.WriteLine(ex.InnerException.Message);
writer.WriteLine();
writer.WriteLine(ex.InnerException.StackTrace);
writer.WriteLine();
}
writer.Flush();
memStream.Position = 0;
outblob.UploadFromStream(memStream);
writer.Close();
memStream.Close();
}
await batchClient.JobOperations.TerminateJobAsync(jobId).ConfigureAwait(false);
throw;
}
finally
{
// Clean the resource container used for job file storage.
//
// If we used file staging rather than blob storage access to seed the individual processing jobs, we'd have to clean those
// here as well. Those are a bit awkward to get; you have to create a bag to hold and discover them when the task was added
// to the job. See line 205 of https://github.com/Azure/azure-batch-samples/blob/master/CSharp/GettingStarted/02_PoolsAndResourceFiles/JobSubmitter/JobSubmitter.cs
await client.GetContainerReference(settings.JobResourceContainerName).DeleteIfExistsAsync().ConfigureAwait(false);
// Delete the job to ensure the tasks are cleaned up
if ((!String.IsNullOrEmpty(jobId)) && (settings.ShouldDeleteJob))
{
await batchClient.JobOperations.DeleteJobAsync(jobId).ConfigureAwait(false);
}
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.jobManagerSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
string jobId = null;
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job manager to complete
CloudTask jobManagerTask = await batchClient.JobOperations.GetTaskAsync(jobId, JobManagerTaskId);
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, new List <CloudTask> {
jobManagerTask
}, TimeSpan.FromMinutes(10));
}
finally
{
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.jobManagerSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.jobManagerSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.jobManagerSettings.PoolId);
}
await SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete);
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.poolsAndResourceFileSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
// Delete the blob containers which contain the task input files since we no longer need them
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = null;
// Track the containers which are created as part of job submission so that we can clean them up later.
HashSet <string> blobContainerNames = new HashSet <string>();
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
blobContainerNames = await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job to complete
List <CloudTask> tasks = await batchClient.JobOperations.ListTasks(jobId).ToListAsync();
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, tasks, TimeSpan.FromMinutes(10));
}
finally
{
// Delete the pool (if configured) and job
// TODO: In C# 6 we can await here instead of .Wait()
// Delete Azure Storage container data
SampleHelpers.DeleteContainersAsync(cloudStorageAccount, blobContainerNames).Wait();
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.poolsAndResourceFileSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.poolsAndResourceFileSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.poolsAndResourceFileSettings.PoolId);
}
SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete).Wait();
}
}
}
public JobRepositoryCommandExecuter(RetryPolicyProvider retryPolicyProvider, HalfNodesAffinityPolicy halfNodesAffinityPolicy)
{
_halfNodesAffinityPolicy = halfNodesAffinityPolicy;
_retryPolicy = retryPolicyProvider.GetPolicy();
}
public JobPurgeTimerFake(RetryPolicyProvider retryPolicyProvider, ManagerConfiguration managerConfiguration) : base(retryPolicyProvider, managerConfiguration)
{
}
public JobSubmitter(BatchClient batchClient)
{
this.batchClient = batchClient;
this.batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
}
