public async Task IfAFileIsSavedTracked_ThenChangesArePersisted()
{
var file = Path.GetTempFileName();
try
{
var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked1.txt", TimeSpan.FromMilliseconds(10)))
{
await Task.Delay(TimeSpan.FromMilliseconds(30));
File.AppendAllLines(file, new[] { "Line 1" });
await Task.Delay(TimeSpan.FromMilliseconds(20));
File.AppendAllLines(file, new[] { "Line 2" });
await Task.Delay(TimeSpan.FromMilliseconds(20));
File.AppendAllLines(file, new[] { "Line 3" });
}
var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked1.txt");
var blobContent = await blob.ReadAsByteArrayAsync();
var originalContent = File.ReadAllBytes(file);
Assert.Equal(originalContent, blobContent);
}
finally
{
File.Delete(file);
}
}
public static async Task <int> StoreOutput(Func <Task <int> > batchOperation)
{
if (!StoreBatchOutput)
{
return(await ExecTask(batchOperation));
}
var batch = ServiceProvider.GetService <IOptions <BatchConfiguration> >().Value;
var stdoutFlushDelay = TimeSpan.FromSeconds(5);
var stdoutFreqDelay = TimeSpan.FromSeconds(30);
var logFilePath = Path.Combine(batch.TaskPath, OutputFileName);
var credentials = new StorageCredentials(batch.StorageName, batch.StorageKey);
var storageAccount = new CloudStorageAccount(credentials, true);
var taskStorage = new TaskOutputStorage(storageAccount, batch.JobId, batch.TaskId);
// The stdout.txt blob in Storage every stdoutFreqDelay (30 seconds) while the task code runs.
using (await taskStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, logFilePath, OutputFileName, stdoutFreqDelay))
{
int result = await ExecTask(batchOperation);
await Task.Delay(stdoutFlushDelay);
return(result);
}
}
private static async Task WriteFinalOutput(int number, string taskId, TaskOutputStorage taskStorage, string index)
{
Console.WriteLine("Write output to task storage from EXE");
using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
TaskOutputKind.TaskLog,
RootDir("stdout.txt"),
"stdout.txt",
TimeSpan.FromSeconds(15)))
{
Console.WriteLine("Dump output to file from EXE - start");
string outputFile = "results.txt";
using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
{
output.WriteLine($"final task {taskId}");
output.WriteLine($"element: {number}");
}
// Persist the task output to Azure Storage
Task.WaitAll(taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile));
// We are tracking the disk file to save our standard output, but the node agent may take
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
await Task.Delay(stdoutFlushDelay);
Console.WriteLine("Dump output to file from EXE - finish");
}
}
public async Task IfATrackedFileIsIsUseWhenItIsDueToBeFlushed_ThenNoErrorOccursAndChangesArePersisted()
{
var file = Path.GetTempFileName();
try
{
var taskOutputStorage = new TaskOutputStorage(StorageAccount, _jobId, _taskId);
using (await taskOutputStorage.SaveTrackedAsync(TaskOutputKind.TaskLog, file, "Tracked2.txt", TimeSpan.FromMilliseconds(5)))
{
using (var writer = File.AppendText(file))
{
for (int i = 0; i < 100; ++i)
{
await Task.Delay(TimeSpan.FromMilliseconds(3));
await writer.WriteLineAsync($"Line {i}");
await Task.Delay(TimeSpan.FromMilliseconds(3));
}
}
using (var writer = File.AppendText(file))
{
for (int i = 0; i < 100; ++i)
{
await writer.WriteLineAsync($"Line {i + 100}");
await Task.Delay(TimeSpan.FromMilliseconds(2));
}
}
}
var blob = await taskOutputStorage.GetOutputAsync(TaskOutputKind.TaskLog, "Tracked2.txt");
var blobContent = await blob.ReadAsByteArrayAsync();
var originalContent = File.ReadAllBytes(file);
Assert.Equal(originalContent, blobContent);
}
finally
{
File.Delete(file);
}
}
public async Task CannotPassANullFilePathWhenSavingTracked()
{
var ex = await Assert.ThrowsAsync <ArgumentNullException>(() => _storage.SaveTrackedAsync(null));
Assert.Equal("relativePath", ex.ParamName);
}
public static async Task<int> RunTaskAsync()
{
// Obtain service-defined environment variables
string jobId = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID");
string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID");
// Obtain the custom environment variable we set in the client application
string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL");
// The task will use the TaskOutputStorage to store both its output and log updates
TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId);
// The primary task logic is wrapped in a using statement that sends updates to the
// stdout.txt blob in Storage every 15 seconds while the task code runs.
using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
TaskOutputKind.TaskLog,
RootDir("stdout.txt"),
"stdout.txt",
TimeSpan.FromSeconds(15)))
{
string outputFile = $"results_{taskId}.txt";
string summaryFile = $"summary_{taskId}.txt";
using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
{
using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile)))
{
output.WriteLine($"# Task {taskId}");
const int runCount = 1000000;
int[] results = new int[runCount];
double resultTotal = 0;
for (int i = 0; i < runCount; ++i)
{
int runResult = PerformSingleRunMonteCarloSimulation();
output.WriteLine($"{i}, {runResult}");
results[i] = runResult;
resultTotal += runResult;
if (i % 5000 == 0)
{
Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}");
}
}
double mean = resultTotal / runCount;
double stddev = Math.Sqrt((from r in results
let d = r - mean
select d * d).Average());
summary.WriteLine($"Task: {taskId}");
summary.WriteLine($"Run count: {runCount}");
summary.WriteLine($"Mean: {mean}");
summary.WriteLine($"Std dev: {stddev}");
}
}
// Persist the task output to Azure Storage
Task.WaitAll(
taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile),
taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile)
);
// We are tracking the disk file to save our standard output, but the node agent may take
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
await Task.Delay(stdoutFlushDelay);
return 0;
}
}
public static async Task <int> RunTaskAsync()
{
// Obtain service-defined environment variables
string jobId = Environment.GetEnvironmentVariable("AZ_BATCH_JOB_ID");
string taskId = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_ID");
// Obtain the custom environment variable we set in the client application
string jobContainerUrl = Environment.GetEnvironmentVariable("JOB_CONTAINER_URL");
// The task will use the TaskOutputStorage to store both its output and log updates
TaskOutputStorage taskStorage = new TaskOutputStorage(new Uri(jobContainerUrl), taskId);
// The primary task logic is wrapped in a using statement that sends updates to the
// stdout.txt blob in Storage every 15 seconds while the task code runs.
using (ITrackedSaveOperation stdout = await taskStorage.SaveTrackedAsync(
TaskOutputKind.TaskLog,
RootDir("stdout.txt"),
"stdout.txt",
TimeSpan.FromSeconds(15)))
{
string outputFile = $"results_{taskId}.txt";
string summaryFile = $"summary_{taskId}.txt";
using (StreamWriter output = File.CreateText(WorkingDir(outputFile)))
{
using (StreamWriter summary = File.CreateText(WorkingDir(summaryFile)))
{
output.WriteLine($"# Task {taskId}");
const int runCount = 1000000;
int[] results = new int[runCount];
double resultTotal = 0;
for (int i = 0; i < runCount; ++i)
{
int runResult = PerformSingleRunMonteCarloSimulation();
output.WriteLine($"{i}, {runResult}");
results[i] = runResult;
resultTotal += runResult;
if (i % 5000 == 0)
{
Console.WriteLine($"{DateTime.UtcNow}: Processing... done {i}");
}
}
double mean = resultTotal / runCount;
double stddev = Math.Sqrt((from r in results
let d = r - mean
select d * d).Average());
summary.WriteLine($"Task: {taskId}");
summary.WriteLine($"Run count: {runCount}");
summary.WriteLine($"Mean: {mean}");
summary.WriteLine($"Std dev: {stddev}");
}
}
// Persist the task output to Azure Storage
Task.WaitAll(
taskStorage.SaveAsync(TaskOutputKind.TaskOutput, outputFile),
taskStorage.SaveAsync(TaskOutputKind.TaskPreview, summaryFile)
);
// We are tracking the disk file to save our standard output, but the node agent may take
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
await Task.Delay(stdoutFlushDelay);
return(0);
}
}
