public async Task IfAFileIsSavedTracked_ThenChangesArePersisted() { var file = Path.GetTempFileName(); try { var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId); using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked1.txt", TimeSpan.FromMilliseconds(10))) { await Task.Delay(TimeSpan.FromMilliseconds(30)); File.AppendAllLines(file, new[] { "Line 1" }); await Task.Delay(TimeSpan.FromMilliseconds(20)); File.AppendAllLines(file, new[] { "Line 2" }); await Task.Delay(TimeSpan.FromMilliseconds(20)); File.AppendAllLines(file, new[] { "Line 3" }); } var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked1.txt"); var blobContent = await blob.ReadAsByteArrayAsync(); var originalContent = File.ReadAllBytes(file); Assert.Equal(originalContent, blobContent); } finally { File.Delete(file); } }
public static async Task <int> StoreOutput(Func <Task <int> > batchOperation) { if (!StoreBatchOutput) { return(await ExecTask(batchOperation)); } var batch = ServiceProvider.GetService <IOptions <BatchConfiguration> >().Value; var stdoutFlushDelay = TimeSpan.FromSeconds(5); var stdoutFreqDelay = TimeSpan.FromSeconds(30); var logFilePath = Path.Combine(batch.TaskPath, OutputFileName); var credentials = new StorageCredentials(batch.StorageName, batch.StorageKey); var storageAccount = new CloudStorageAccount(credentials, true); var taskStorage = new TaskOutputStorage(storageAccount, batch.JobId, batch.TaskId); // The stdout.txt blob in Storage every stdoutFreqDelay (30 seconds) while the task code runs. using (await taskStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, logFilePath, OutputFileName, stdoutFreqDelay)) { int result = await ExecTask(batchOperation); await Task.Delay(stdoutFlushDelay); return(result); } }
private static async Task WriteFinalOutput(int number, string taskId, TaskOutputStorage taskStorage, string index) { Console.WriteLine("Write output to task storage from EXE"); using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync( TaskOutputKind.TaskLog, RootDir("stdout.txt"), "stdout.txt", TimeSpan.FromSeconds(15))) { Console.WriteLine("Dump output to file from EXE - start"); string outputFile = "results.txt"; using (StreamWriter output = File.CreateText(WorkingDir(outputFile))) { output.WriteLine($"final task {taskId}"); output.WriteLine($"element: {number}"); } // Persist the task output to Azure Storage Task.WaitAll(taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile)); // We are tracking the disk file to save our standard output, but the node agent may take // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up. await Task.Delay(stdoutFlushDelay); Console.WriteLine("Dump output to file from EXE - finish"); } }
public async Task IfATrackedFileIsIsUseWhenItIsDueToBeFlushed_ThenNoErrorOccursAndChangesArePersisted() { var file = Path.GetTempFileName(); try { var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId); using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked2.txt", TimeSpan.FromMilliseconds(5))) { using (var writer = File.AppendText(file)) { for (int i = 0; i < 100; ++i) { await Task.Delay(TimeSpan.FromMilliseconds(3)); await writer.WriteLineAsync($"Line {i}"); await Task.Delay(TimeSpan.FromMilliseconds(3)); } } using (var writer = File.AppendText(file)) { for (int i = 0; i < 100; ++i) { await writer.WriteLineAsync($"Line {i + 100}"); await Task.Delay(TimeSpan.FromMilliseconds(2)); } } } var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked2.txt"); var blobContent = await blob.ReadAsByteArrayAsync(); var originalContent = File.ReadAllBytes(file); Assert.Equal(originalContent, blobContent); } finally { File.Delete(file); } }
public async Task CannotPassANullFilePathWhenSavingTracked() { var ex = await Assert.ThrowsAsync <ArgumentNullException>(() => _storage.SaveTrackedAsync(null)); Assert.Equal("relativePath", ex.ParamName); }
public static async Task<int> RunTaskAsync() { // Obtain service-defined environment variables string jobId = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID"); string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID"); // Obtain the custom environment variable we set in the client application string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL"); // The task will use the TaskOutputStorage to store both its output and log updates TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId); // The primary task logic is wrapped in a using statement that sends updates to the // stdout.txt blob in Storage every 15 seconds while the task code runs. using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync( TaskOutputKind.TaskLog, RootDir("stdout.txt"), "stdout.txt", TimeSpan.FromSeconds(15))) { string outputFile = $"results_{taskId}.txt"; string summaryFile = $"summary_{taskId}.txt"; using (StreamWriter output = File.CreateText(WorkingDir(outputFile))) { using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile))) { output.WriteLine($"# Task {taskId}"); const int runCount = 1000000; int[] results = new int[runCount]; double resultTotal = 0; for (int i = 0; i < runCount; ++i) { int runResult = PerformSingleRunMonteCarloSimulation(); output.WriteLine($"{i}, {runResult}"); results[i] = runResult; resultTotal += runResult; if (i % 5000 == 0) { Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}"); } } double mean = resultTotal / runCount; double stddev = Math.Sqrt((from r in results let d = r - mean select d * d).Average()); summary.WriteLine($"Task: {taskId}"); summary.WriteLine($"Run count: {runCount}"); summary.WriteLine($"Mean: {mean}"); summary.WriteLine($"Std dev: {stddev}"); } } // Persist the task output to Azure Storage Task.WaitAll( taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile), taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile) ); // We are tracking the disk file to save our standard output, but the node agent may take // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up. await Task.Delay(stdoutFlushDelay); return 0; } }
public static async Task <int> RunTaskAsync() { // Obtain service-defined environment variables string jobId = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID"); string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID"); // Obtain the custom environment variable we set in the client application string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL"); // The task will use the TaskOutputStorage to store both its output and log updates TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId); // The primary task logic is wrapped in a using statement that sends updates to the // stdout.txt blob in Storage every 15 seconds while the task code runs. using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync( TaskOutputKind.TaskLog, RootDir("stdout.txt"), "stdout.txt", TimeSpan.FromSeconds(15))) { string outputFile = $"results_{taskId}.txt"; string summaryFile = $"summary_{taskId}.txt"; using (StreamWriter output = File.CreateText(WorkingDir(outputFile))) { using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile))) { output.WriteLine($"# Task {taskId}"); const int runCount = 1000000; int[] results = new int[runCount]; double resultTotal = 0; for (int i = 0; i < runCount; ++i) { int runResult = PerformSingleRunMonteCarloSimulation(); output.WriteLine($"{i}, {runResult}"); results[i] = runResult; resultTotal += runResult; if (i % 5000 == 0) { Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}"); } } double mean = resultTotal / runCount; double stddev = Math.Sqrt((from r in results let d = r - mean select d * d).Average()); summary.WriteLine($"Task: {taskId}"); summary.WriteLine($"Run count: {runCount}"); summary.WriteLine($"Mean: {mean}"); summary.WriteLine($"Std dev: {stddev}"); } } // Persist the task output to Azure Storage Task.WaitAll( taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile), taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile) ); // We are tracking the disk file to save our standard output, but the node agent may take // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up. await Task.Delay(stdoutFlushDelay); return(0); } }