/// <summary>
/// Initializes a new instance of the <see cref="OutputFileDestination"/> class.
/// </summary>
/// <param name="container">A location in Azure blob storage to which files are uploaded.</param>
public OutputFileDestination(OutputFileBlobContainerDestination container)
{
this.Container = container;
}
// AddTasks(): Creates tasks to process each of the specified input files, and submits them
// to the specified job for execution.
// batchClient: A BatchClient object.
// jobId: The ID of the job to which the tasks are added.
// inputFiles: A collection of ResourceFile objects representing the input files
// to be processed by the tasks executed on the compute nodes.
// outputContainerSasUrl: The shared access signature URL for the Azure Storage
// container that will hold the output files that the tasks create.
// Returns: A collection of the submitted cloud tasks.
private static List <CloudTask> AddTasks(BatchClient batchClient, string jobId, List <ResourceFile> inputFiles, string outputContainerSasUrl)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks added to the job:
List <CloudTask> tasks = new List <CloudTask>();
// Create each task. The start task copies the application executable (ffmpeg.exe) to the
// node's shared directory, so the cloud tasks can access this application via the shared
// directory on whichever node each task runs.
foreach (ResourceFile inputFile in inputFiles)
{
// Assign a task ID for each iteration
string taskId = "task_" + inputFiles.IndexOf(inputFile);
// Define task command line to convert the video format from MP4 to MPEG-1 using ffmpeg.
// Note that ffmpeg syntax specifies the format conversion using the file extension of
// the input file and the output file respectively.
/*
* // %% REPLACE THIS BLOCK WITH DAN'S OUTPUT FILE FORMATTING
* string inputFilePath = inputFile.FilePath;
* string outputFileName = String.Format("{0}{1}",
* System.IO.Path.GetFileNameWithoutExtension(inputFilePath),
* ".mpeg");
* string outputFilePath = outputContainerSasUrl + outputFileName;
*/
// %% NEW OUTPUT FILE FORMATTING:
// Define paths for input and output files
string inputMediaFile = inputFile.FilePath;
string outputMediaFile = String.Format("{0}{1}",
System.IO.Path.GetFileNameWithoutExtension(inputMediaFile),
".mpeg");
// Format the task command line
string taskCommandLine = String.Format("cmd /c %AZ_BATCH_NODE_SHARED_DIR%\\ffmpeg.exe -i {0} {1}", inputMediaFile, outputMediaFile);
// Create a cloud task (with the task ID and command line) and give it a list of input files
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFile
};
// Send the result to the task's list of output files
List <OutputFile> outputFileList = new List <OutputFile>();
OutputFileBlobContainerDestination outputContainer = new OutputFileBlobContainerDestination(outputContainerSasUrl);
OutputFile outputFile = new OutputFile(outputMediaFile,
new OutputFileDestination(outputContainer),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskSuccess));
outputFileList.Add(outputFile);
task.OutputFiles = outputFileList;
// Add the cloud task to the task list
tasks.Add(task);
}
// Call BatchClient.JobOperations.AddTask() to add the tasks as a collection rather than making a
// separate call for each. Bulk task submission helps to ensure efficient underlying API
// calls to the Batch service. calls AddTaskAsync() so the add operation doesn't hang up program execution.
batchClient.JobOperations.AddTaskAsync(jobId, tasks).Wait();
return(tasks);
}
public async Task RunTaskAndUploadFiles_FilesAreSuccessfullyUploaded()
{
async Task test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "RunTaskAndUploadFiles-" + TestUtilities.GetMyName();
string containerName = "runtaskanduploadfiles";
StagingStorageAccount storageAccount = TestUtilities.GetStorageCredentialsFromEnvironment();
BlobServiceClient blobClient = BlobUtilities.GetBlobServiceClient(storageAccount);
BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, blobClient, storageAccount);
try
{
// Create container and writeable SAS
containerClient.CreateIfNotExists();
string sasUri = BlobUtilities.GetWriteableSasUri(containerClient, storageAccount);
CloudJob createJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolFixture.PoolId
});
createJob.Commit();
const string blobPrefix = "foo/bar";
const string taskId = "simpletask";
OutputFileBlobContainerDestination containerDestination = new OutputFileBlobContainerDestination(sasUri, blobPrefix);
containerDestination.UploadHeaders = new List <HttpHeader> {
new HttpHeader("x-ms-blob-content-type", "test-type")
};
OutputFileDestination destination = new OutputFileDestination(containerDestination);
OutputFileUploadOptions uploadOptions = new OutputFileUploadOptions(uploadCondition: OutputFileUploadCondition.TaskCompletion);
CloudTask unboundTask = new CloudTask(taskId, "echo test")
{
OutputFiles = new List <OutputFile>
{
new OutputFile(@"../*.txt", destination, uploadOptions)
}
};
batchCli.JobOperations.AddTask(jobId, unboundTask);
IPagedEnumerable <CloudTask> tasks = batchCli.JobOperations.ListTasks(jobId);
TaskStateMonitor monitor = batchCli.Utilities.CreateTaskStateMonitor();
monitor.WaitAll(tasks, TaskState.Completed, TimeSpan.FromMinutes(1));
// Ensure that the correct files got uploaded
List <BlobItem> blobs = containerClient.GetAllBlobs();
Assert.Equal(4, blobs.Count()); //There are 4 .txt files created, stdout, stderr, fileuploadout, and fileuploaderr
foreach (BlobItem blob in blobs)
{
Assert.StartsWith(blobPrefix, blob.Name);
Assert.Equal("test-type", blob.Properties.ContentType); // Ensure test Upload header was applied to blob.
}
}
finally
{
await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).ConfigureAwait(false);
containerClient.DeleteIfExists();
}
}
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
private static async Task TaskCreation(BatchClient p_batchClient, string p_jobId)
{
Console.WriteLine("Creating the Task");
string taskId = "demotask";
string in_container_name = "input";
string out_container_name = "output";
string l_blobName = "sample.mp4";
string outputfile = "audio.aac";
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
demo_storageAccountName, demo_storageAccountKey);
CloudStorageAccount l_storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient l_blobClient = l_storageAccount.CreateCloudBlobClient();
CloudBlobContainer in_container = l_blobClient.GetContainerReference(in_container_name);
CloudBlobContainer out_container = l_blobClient.GetContainerReference(out_container_name);
SharedAccessBlobPolicy i_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
};
SharedAccessBlobPolicy o_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Write
};
string in_sasToken = in_container.GetSharedAccessSignature(i_sasConstraints);
string in_containerSasUrl = String.Format("{0}{1}", in_container.Uri, in_sasToken);
string out_sasToken = out_container.GetSharedAccessSignature(o_sasConstraints);
string out_containerSasUrl = String.Format("{0}{1}", out_container.Uri, out_sasToken);
ResourceFile inputFile = ResourceFile.FromStorageContainerUrl(in_containerSasUrl);
List <ResourceFile> file = new List <ResourceFile>();
file.Add(inputFile);
string appPath = String.Format("%AZ_BATCH_APP_PACKAGE_{0}#{1}%", demo_packageid, demo_packageversion);
string taskCommandLine = String.Format("cmd /c {0}\\ffmpeg.exe -i {1} -vn -acodec copy audio.aac", appPath, l_blobName);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = file;
// Setting the output file location
List <OutputFile> outputFileList = new List <OutputFile>();
OutputFileBlobContainerDestination outputContainer = new OutputFileBlobContainerDestination(out_containerSasUrl);
OutputFile outputFile = new OutputFile(outputfile,
new OutputFileDestination(outputContainer),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskSuccess));
outputFileList.Add(outputFile);
task.OutputFiles = outputFileList;
await p_batchClient.JobOperations.AddTaskAsync(p_jobId, task);
}
