public static async Task <int> StoreOutput(Func <Task <int> > batchOperation)
        {
            if (!StoreBatchOutput)
            {
                return(await ExecTask(batchOperation));
            }

            var batch            = ServiceProvider.GetService <IOptions <BatchConfiguration> >().Value;
            var stdoutFlushDelay = TimeSpan.FromSeconds(5);
            var stdoutFreqDelay  = TimeSpan.FromSeconds(30);
            var logFilePath      = Path.Combine(batch.TaskPath, OutputFileName);

            var credentials    = new StorageCredentials(batch.StorageName, batch.StorageKey);
            var storageAccount = new CloudStorageAccount(credentials, true);
            var taskStorage    = new TaskOutputStorage(storageAccount, batch.JobId, batch.TaskId);

            // The stdout.txt blob in Storage every stdoutFreqDelay (30 seconds) while the task code runs.
            using (await taskStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, logFilePath, OutputFileName, stdoutFreqDelay))
            {
                int result = await ExecTask(batchOperation);

                await Task.Delay(stdoutFlushDelay);

                return(result);
            }
        }
        public async Task IfAFileIsSavedTracked_ThenChangesArePersisted()
        {
            var file = Path.GetTempFileName();

            try
            {
                var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
                using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked1.txt", TimeSpan.FromMilliseconds(10)))
                {
                    await Task.Delay(TimeSpan.FromMilliseconds(30));

                    File.AppendAllLines(file, new[] { "Line 1" });
                    await Task.Delay(TimeSpan.FromMilliseconds(20));

                    File.AppendAllLines(file, new[] { "Line 2" });
                    await Task.Delay(TimeSpan.FromMilliseconds(20));

                    File.AppendAllLines(file, new[] { "Line 3" });
                }

                var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked1.txt");

                var blobContent = await blob.ReadAsByteArrayAsync();

                var originalContent = File.ReadAllBytes(file);

                Assert.Equal(originalContent, blobContent);
            }
            finally
            {
                File.Delete(file);
            }
        }
Beispiel #3
0
        public static async Task Main(string[] args)
        {
            Console.WriteLine("Starting testbatchapp exe in Cloud Task");
            string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID");

            // Obtain the custom environment variable we set in the client application
            string            jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL");
            TaskOutputStorage taskStorage     = new TaskOutputStorage(new Uri(jobContainerUrl), taskId);

            string index = args[0];
            string storageAccountName = "emergentai";
            string storageAccountKey  = "FqOL31fHPNRhRW3jpLAeL8FF+sfyZqYEtUqzMpJo257CCSIleJKusGihiNxOqqwV4b/JSq+srU23S5C2Ao9dRw==";

            // open the cloud blob that contains the input file
            int number = ReadDataFromBlobStorage(index, storageAccountName, storageAccountKey);

            // add some data to the loaded input file
            number = taskETL(number);

            // write back to blob ready for next task
            WriteDataToBlobStorage(index, number, storageAccountName, storageAccountKey);

            Thread.Sleep(5000);

            // finally output to task storage for exit data reporting
            await WriteFinalOutput(number, taskId, taskStorage, index);

            Thread.Sleep(5000);

            Console.WriteLine("Finishing testbatchapp Cloud Task");
        }
Beispiel #4
0
        private static async Task WriteFinalOutput(int number, string taskId, TaskOutputStorage taskStorage, string index)
        {
            Console.WriteLine("Write output to task storage from EXE");
            using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
                       TaskOutputKind.TaskLog,
                       RootDir("stdout.txt"),
                       "stdout.txt",
                       TimeSpan.FromSeconds(15)))
            {
                Console.WriteLine("Dump output to file from EXE - start");

                string outputFile = "results.txt";

                using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
                {
                    output.WriteLine($"final task {taskId}");
                    output.WriteLine($"element: {number}");
                }

                // Persist the task output to Azure Storage
                Task.WaitAll(taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile));

                // We are tracking the disk file to save our standard output, but the node agent may take
                // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
                await Task.Delay(stdoutFlushDelay);

                Console.WriteLine("Dump output to file from EXE - finish");
            }
        }
        public async Task IfAUserAttemptsToWriteOutsideTheContainerByBypassingTheUpChecker_ThenTheWriteFails()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            var ex = await Assert.ThrowsAsync <StorageException>(async() =>
                                                                 await taskOutputStorage.SaveAsyncImpl(TaskOutputKind.TaskIntermediate, FileSubfolder("File\\Under\\Further"), @"Under\..\..\..\..\TestTextForFarOutsideWorkingDirectory.txt")
                                                                 );

            Assert.Equal(404, ex.RequestInformation.HttpStatusCode);
        }
        public async Task IfAFileWithAMultiLevelPathIsSaved_ThenItAppearsInTheList()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsyncImpl(TaskOutputKind.TaskPreview, FileBase, "File\\Under\\TestText2.txt");

            var blobs = taskOutputStorage.ListOutputs(TaskOutputKind.TaskPreview).ToList();

            Assert.NotEqual(0, blobs.Count);
            Assert.Contains(blobs, b => b.Uri.AbsoluteUri.EndsWith($"{_jobId}/{_taskId}/$TaskPreview/File/Under/TestText2.txt"));
        }
        public async Task IfAFileIsSaved_ThenItAppearsInTheList()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsyncImpl(TaskOutputKind.TaskPreview, FileBase, "TestText1.txt");

            var blobs = taskOutputStorage.ListOutputs(TaskOutputKind.TaskPreview).ToList();

            Assert.NotEmpty(blobs);
            Assert.Contains(blobs, b => b.Uri.AbsoluteUri.EndsWith($"{_jobId}/{_taskId}/$TaskPreview/TestText1.txt"));
        }
        public async Task IfAFileIsSavedWithAnExplicitMultiLevelPath_ThenItAppearsInTheList()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsync(TaskOutputKind.TaskPreview, FilePath("TestText1.txt"), "File/In/The/Depths/TestText3.txt");

            var blobs = taskOutputStorage.ListOutputs(TaskOutputKind.TaskPreview).ToList();

            Assert.NotEmpty(blobs);
            Assert.Contains(blobs, b => b.Uri.AbsoluteUri.EndsWith($"{_jobId}/{_taskId}/$TaskPreview/File/In/The/Depths/TestText3.txt"));
        }
        public async Task IfAFileIsSavedWithAnExplicitPath_ThenItAppearsInTheList()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsync(TaskOutputKind.TaskPreview, FilePath("TestText1.txt"), "RenamedTestText1.txt");

            var blobs = taskOutputStorage.ListOutputs(TaskOutputKind.TaskPreview).ToList();

            Assert.NotEqual(0, blobs.Count);
            Assert.Contains(blobs, b => b.Uri.AbsoluteUri.EndsWith($"{_jobId}/{_taskId}/$TaskPreview/RenamedTestText1.txt"));
        }
        public async Task IfARetryPolicyIsSpecifiedInTheStorageAccountConstructor_ThenItIsUsed()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId, new LinearRetry(TimeSpan.FromSeconds(5), 4));
            await taskOutputStorage.SaveAsync(TaskOutputKind.TaskOutput, FilePath("TestText1.txt"), "SavedWithLinearRetry1.txt");

            var output = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskOutput, "SavedWithLinearRetry1.txt");

            var blob          = output.CloudBlob;
            var storageClient = blob.ServiceClient;

            Assert.IsType <LinearRetry>(storageClient.DefaultRequestOptions.RetryPolicy);
        }
        public async Task IfAFileIsSavedWithAPathOutsideTheWorkingDirectory_ThenTheUpPartsOfThePathAreStripped()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsyncImpl(TaskOutputKind.TaskIntermediate, FileSubfolder("File"), @"..\TestTextForOutsideWorkingDirectory.txt");

            var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskIntermediate, "TestTextForOutsideWorkingDirectory.txt");

            var blobContent = await blob.ReadAsByteArrayAsync();

            var originalContent = File.ReadAllBytes(FilePath("TestTextForOutsideWorkingDirectory.txt"));

            Assert.Equal(originalContent, blobContent);
        }
        public async Task IfAFileIsSavedWithAMultiLevelPath_ThenItCanBeGot()
        {
            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsync(TaskOutputKind.TaskPreview, FilePath("TestText1.txt"), "This/File/Is/Gettable.txt");

            var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskPreview, "This/File/Is/Gettable.txt");

            var blobContent = await blob.ReadAsByteArrayAsync();

            var originalContent = File.ReadAllBytes(FilePath("TestText1.txt"));

            Assert.Equal(originalContent, blobContent);
        }
        public async Task IfTextIsSaved_ThenItCanBeGot()
        {
            var sampleXml = "<document><empty /></document>";

            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveTextAsync(TaskOutputKind.TaskOutput, sampleXml, "TextNotFromFile.xml");

            var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskOutput, "TextNotFromFile.xml");

            var blobContent = Encoding.UTF8.GetString(await blob.ReadAsByteArrayAsync());

            Assert.Equal(sampleXml, blobContent);
        }
        public async Task IfAFileIsSaved_UsingThePublicMethod_ThenTheCurrentDirectoryIsInferred()
        {
            // To avoid needing to mess with the process working directory, relative path tests
            // normally go through the internal SaveAsyncImpl method.  This test verifies that
            // the public SaveAsync method forwards the appropriate directory to SaveAsyncImpl.

            Assert.True(File.Exists(FilePath("TestText1.txt")), "Current directory is not what was expected - cannot verify current directory inference");

            var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
            await taskOutputStorage.SaveAsync(TaskOutputKind.TaskPreview, FilePath("TestText1.txt"));

            var blobs = taskOutputStorage.ListOutputs(TaskOutputKind.TaskPreview).ToList();

            Assert.NotEqual(0, blobs.Count);
            Assert.Contains(blobs, b => b.Uri.AbsoluteUri.EndsWith($"{_jobId}/{_taskId}/$TaskPreview/TestText1.txt"));
        }
        public async Task IfARetryPolicyIsSpecifiedInTheContainerUrlConstructor_ThenItIsUsed()
        {
            using (var batchClient = await BatchClient.OpenAsync(new FakeBatchServiceClient()))
            {
                var job       = batchClient.JobOperations.CreateJob(_jobId, null);
                var container = job.GetOutputStorageContainerUrl(StorageAccount, TimeSpan.FromMinutes(2));

                var taskOutputStorage = new TaskOutputStorage(new Uri(container), _taskId, new LinearRetry(TimeSpan.FromSeconds(5), 4));
                await taskOutputStorage.SaveAsync(TaskOutputKind.TaskOutput, FilePath("TestText1.txt"), "SavedWithLinearRetry2.txt");

                var output = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskOutput, "SavedWithLinearRetry2.txt");

                var blob          = output.CloudBlob;
                var storageClient = blob.ServiceClient;
                Assert.IsType <LinearRetry>(storageClient.DefaultRequestOptions.RetryPolicy);
            }
        }
        public void GetTaskOutputStoragePathReturnsExpectedValue()
        {
            const string taskId      = "test-task";
            var          taskStorage = new TaskOutputStorage(new Uri("http://example.test/"), taskId);

            var taskLogPath = taskStorage.GetOutputStoragePath(TaskOutputKind.TaskLog);

            Assert.Equal($"{taskId}/${TaskOutputKind.TaskLog.ToString()}/", taskLogPath);

            taskLogPath = taskStorage.GetOutputStoragePath(TaskOutputKind.TaskOutput);
            Assert.Equal($"{taskId}/${TaskOutputKind.TaskOutput.ToString()}/", taskLogPath);

            taskLogPath = taskStorage.GetOutputStoragePath(TaskOutputKind.TaskPreview);
            Assert.Equal($"{taskId}/${TaskOutputKind.TaskPreview.ToString()}/", taskLogPath);

            taskLogPath = taskStorage.GetOutputStoragePath(TaskOutputKind.Custom("foo"));
            Assert.Equal($"{taskId}/${TaskOutputKind.Custom("foo").ToString()}/", taskLogPath);
        }
        public async Task IfATrackedFileIsIsUseWhenItIsDueToBeFlushed_ThenNoErrorOccursAndChangesArePersisted()
        {
            var file = Path.GetTempFileName();

            try
            {
                var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
                using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked2.txt", TimeSpan.FromMilliseconds(5)))
                {
                    using (var writer = File.AppendText(file))
                    {
                        for (int i = 0; i < 100; ++i)
                        {
                            await Task.Delay(TimeSpan.FromMilliseconds(3));

                            await writer.WriteLineAsync($"Line {i}");

                            await Task.Delay(TimeSpan.FromMilliseconds(3));
                        }
                    }
                    using (var writer = File.AppendText(file))
                    {
                        for (int i = 0; i < 100; ++i)
                        {
                            await writer.WriteLineAsync($"Line {i + 100}");

                            await Task.Delay(TimeSpan.FromMilliseconds(2));
                        }
                    }
                }

                var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked2.txt");

                var blobContent = await blob.ReadAsByteArrayAsync();

                var originalContent = File.ReadAllBytes(file);

                Assert.Equal(originalContent, blobContent);
            }
            finally
            {
                File.Delete(file);
            }
        }
        public async void GetTaskResult(Guid guid)
        {
            CloudTask task = await GetTask(guid);

            TaskOutputStorage output = task.OutputStorage(DatabaseConnection.account);
        }
Beispiel #19
0
        public static async Task<int> RunTaskAsync()
        {
            // Obtain service-defined environment variables
            string jobId = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID");
            string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID");

            // Obtain the custom environment variable we set in the client application
            string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL");

            // The task will use the TaskOutputStorage to store both its output and log updates
            TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId);

            // The primary task logic is wrapped in a using statement that sends updates to the
            // stdout.txt blob in Storage every 15 seconds while the task code runs.
            using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
                TaskOutputKind.TaskLog, 
                RootDir("stdout.txt"), 
                "stdout.txt", 
                TimeSpan.FromSeconds(15)))
            {
                string outputFile = $"results_{taskId}.txt";
                string summaryFile = $"summary_{taskId}.txt";

                using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
                {
                    using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile)))
                    {
                        output.WriteLine($"# Task {taskId}");

                        const int runCount = 1000000;
                        int[] results = new int[runCount];
                        double resultTotal = 0;

                        for (int i = 0; i < runCount; ++i)
                        {
                            int runResult = PerformSingleRunMonteCarloSimulation();
                            output.WriteLine($"{i}, {runResult}");
                            results[i] = runResult;
                            resultTotal += runResult;

                            if (i % 5000 == 0)
                            {
                                Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}");
                            }
                        }

                        double mean = resultTotal / runCount;
                        double stddev = Math.Sqrt((from r in results
                                                let d = r - mean
                                                select d * d).Average());

                        summary.WriteLine($"Task:      {taskId}");
                        summary.WriteLine($"Run count: {runCount}");
                        summary.WriteLine($"Mean:      {mean}");
                        summary.WriteLine($"Std dev:   {stddev}");
                    }
                }

                // Persist the task output to Azure Storage
                Task.WaitAll(
                    taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile),
                    taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile)
                    );

                // We are tracking the disk file to save our standard output, but the node agent may take
                // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
                await Task.Delay(stdoutFlushDelay);

                return 0;
            }
        }
        static void Main(string[] args)
        {
            /*
             * Variables set by environment variables.  If the environment variable does not exist an alternate value is defined under
             * videoProcessor/Properties/Settings.settings.  Double click that item in Solution Explorer if you want to put in persistent values.
             */
            var sbConnection        = getEnvironmentVariable("SB_CONNECT", Properties.Settings.Default.SB_CONNECT);
            var videoSAS            = getEnvironmentVariable("VIDEO_STORAGE_SAS", Properties.Settings.Default.VIDEO_STORAGE_SAS);
            var videoAccountName    = getEnvironmentVariable("VIDEO_STORAGE_ACCOUNT", Properties.Settings.Default.VIDEO_STORAGE_ACCOUNT);
            var batchStorageAccount = getEnvironmentVariable("BATCH_STORAGE_ACCOUNT", Properties.Settings.Default.BATCH_STORAGE_ACCOUNT);
            var batchStorageKey     = getEnvironmentVariable("BATCH_STORAGE_KEY", Properties.Settings.Default.BATCH_STORAGE_KEY);
            var batchURI            = getEnvironmentVariable("BATCH_URI", Properties.Settings.Default.BATCH_URI);
            var batchName           = getEnvironmentVariable("BATCH_ACCOUNT_NAME", Properties.Settings.Default.BATCH_ACCOUNT_NAME);
            var batchKey            = getEnvironmentVariable("BATCH_KEY", Properties.Settings.Default.BATCH_KEY);

            var development = getEnvironmentBoolean("DEVELOPMENT", Properties.Settings.Default.DEVELOPMENT);

            //This data does not come from user defined environment variables.
            var taskOutputFolder = getEnvironmentVariable("AZ_BATCH_TASK_DIR", "d:\\dump");
            var jobId            = getEnvironmentVariable("AZ_BATCH_JOB_ID", "DEMOJOB");
            var taskId           = getEnvironmentVariable("AZ_BATCH_TASK_ID", "DEMOTASK");
            var taskOutputFile   = taskOutputFolder + @"\stdout.log";
            var errOutputFile    = taskOutputFolder + @"\stderr.log";

            if (development)
            {
                if (!Directory.Exists(taskOutputFolder))
                {
                    Directory.CreateDirectory(taskOutputFolder);
                }
            }

            //Redirect console output to the stdOut and stddError files
            using (var taskOutput = new StreamWriter(taskOutputFile))
            {
                var output = Console.Out;
                if (!development)
                {
                    Console.SetOut(taskOutput);
                }
                using (var taskErrors = new StreamWriter(errOutputFile))
                {
                    var errorOutput = Console.Error;
                    if (!development)
                    {
                        Console.SetError(taskErrors);
                    }
                    //wrap the rest of the operation in try/catch and log errors to batch log output
                    try
                    {
                        //Get Service bus queue.  Raise an error if the queue does not exist.
                        var queueName = "videoprocess";
                        var sbEnv     = Microsoft.ServiceBus.NamespaceManager.CreateFromConnectionString(sbConnection);
                        if (!sbEnv.QueueExists(queueName))
                        {
                            throw new ArgumentException("The \"videoprocess\" queue does not exist.");
                        }
                        var sbClient = QueueClient.CreateFromConnectionString(sbConnection, queueName);

                        //Set up the output container in the same storage account as the video
                        var resultsContainerName = "results";
                        sbClient.PrefetchCount = 5;
                        var             videoCred  = new StorageCredentials(videoSAS);
                        string          endpoint   = $"https://{videoAccountName}.blob.core.windows.net/";
                        CloudBlobClient blobClient = new CloudBlobClient(new Uri(endpoint), videoCred);
                        var             container  = blobClient.GetContainerReference(resultsContainerName);
                        container.CreateIfNotExists();


                        //Process the messages in the videoprocess queue.  Create a dummy file in the output container for each message.
                        var emptyCount  = 0;
                        var messageWait = new TimeSpan(0, 0, 2);
                        var message     = sbClient.Receive(messageWait);
                        var cleanRun    = true;
                        //This processes two empty receives.  This should get around any timing issues.
                        while ((message != null) || (emptyCount < 2))
                        {
                            if (message != null)
                            {
                                try
                                {
                                    var fileName = message.GetBody <string>();
                                    Console.WriteLine($"Message: {fileName}");
                                    var blob = container.GetBlockBlobReference(fileName);
                                    blob.UploadText("File successfully processed");
                                    message.Complete();
                                }
                                catch (Exception exPerMessage)
                                {
                                    Console.Error.WriteLine($"Error processing message {message.MessageId}: {exPerMessage.Message}");
                                    cleanRun = false;
                                }
                            }
                            else
                            {
                                emptyCount++;
                            }
                            message = sbClient.Receive(messageWait);
                        }
                        if (cleanRun)
                        {
                            Console.WriteLine("Successfully processed all messages");
                        }
                        else
                        {
                            Console.WriteLine("There were errors writing the files");
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.Error.WriteLine($"An error occurred processing the messages: {ex.Message}");
                    }
                    if (!development)
                    {
                        Console.SetError(errorOutput);
                    }
                    taskErrors.Flush();
                    taskErrors.Close();
                }
                if (!development)
                {
                    Console.SetOut(output);
                }
                taskOutput.Flush();
                taskOutput.Close();
            }

            //Try to write the output to the batch output using the library.  If that doesn't work (testing) then write to the console
            if (!development)
            {
                try
                {
                    var linkedStorageAccount = new CloudStorageAccount(new StorageCredentials(batchStorageAccount, batchStorageKey), true);

                    var taskOutputStorage = new TaskOutputStorage(linkedStorageAccount, jobId, taskId);
                    //This needs a task because TaskOutputStorage is completely async
                    var t = Task.Run(async() =>
                    {
                        //Set up output storage
                        BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(batchURI,
                                                                                       batchName,
                                                                                       batchKey);

                        using (BatchClient batchClient = BatchClient.Open(cred))
                        {
                            // Create the blob storage container for the outputs.
                            await batchClient.JobOperations.GetJob(jobId).PrepareOutputStorageAsync(linkedStorageAccount);
                        }
                        await taskOutputStorage.SaveAsync(TaskOutputKind.TaskLog, taskOutputFile, Path.GetFileName(taskOutputFile));
                        await taskOutputStorage.SaveAsync(TaskOutputKind.TaskLog, errOutputFile, Path.GetFileName(errOutputFile));
                    });
                    t.Wait();
                }
                catch (AggregateException ae)
                {
                    var errorText = new StringBuilder();
                    foreach (Exception ex in ae.InnerExceptions)
                    {
                        errorText.AppendLine(ex.Message);
                    }
                    Console.Error.Write(errorText.ToString());
                    Console.WriteLine("Could not write output to batch storage. Check the standard error output for details.  Processing information: \n {0}", File.ReadAllText(taskOutputFile));
                }
            }
            else
            {
                Console.WriteLine("Done. Press any key to continue");
                Console.ReadLine();
            }
        }
Beispiel #21
0
        public static async Task <int> RunTaskAsync()
        {
            // Obtain service-defined environment variables
            string jobId  = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID");
            string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID");

            // Obtain the custom environment variable we set in the client application
            string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL");

            // The task will use the TaskOutputStorage to store both its output and log updates
            TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId);

            // The primary task logic is wrapped in a using statement that sends updates to the
            // stdout.txt blob in Storage every 15 seconds while the task code runs.
            using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
                       TaskOutputKind.TaskLog,
                       RootDir("stdout.txt"),
                       "stdout.txt",
                       TimeSpan.FromSeconds(15)))
            {
                string outputFile  = $"results_{taskId}.txt";
                string summaryFile = $"summary_{taskId}.txt";

                using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
                {
                    using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile)))
                    {
                        output.WriteLine($"# Task {taskId}");

                        const int runCount    = 1000000;
                        int[]     results     = new int[runCount];
                        double    resultTotal = 0;

                        for (int i = 0; i < runCount; ++i)
                        {
                            int runResult = PerformSingleRunMonteCarloSimulation();
                            output.WriteLine($"{i}, {runResult}");
                            results[i]   = runResult;
                            resultTotal += runResult;

                            if (i % 5000 == 0)
                            {
                                Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}");
                            }
                        }

                        double mean   = resultTotal / runCount;
                        double stddev = Math.Sqrt((from r in results
                                                   let d = r - mean
                                                           select d * d).Average());

                        summary.WriteLine($"Task:      {taskId}");
                        summary.WriteLine($"Run count: {runCount}");
                        summary.WriteLine($"Mean:      {mean}");
                        summary.WriteLine($"Std dev:   {stddev}");
                    }
                }

                // Persist the task output to Azure Storage
                Task.WaitAll(
                    taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile),
                    taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile)
                    );

                // We are tracking the disk file to save our standard output, but the node agent may take
                // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
                await Task.Delay(stdoutFlushDelay);

                return(0);
            }
        }