private APSIMJob BuildAPSIMJobAndStageFiles(
Guid jobId,
string apsimAppPkgVersion,
string displayName,
string modelZipFile,
CancellationToken ct)
{
var job = new APSIMJob
{
DisplayName = displayName,
StorageCredentials = _storageCredentials,
BatchCredentials = _batchCredentials,
PoolSettings = _poolSettings,
ApsimApplicationPackageVersion = apsimAppPkgVersion,
};
ct.ThrowIfCancellationRequested();
job.ModelZipFileSas = _fileUploader.UploadFile(
modelZipFile,
jobId.ToString(),
Path.GetFileName(modelZipFile), ct);
ct.ThrowIfCancellationRequested();
return(job);
}
/// <summary>
/// Returns the APSIM ZIP file and helpers like AzCopy and 7zip.
/// </summary>
private static IEnumerable <ResourceFile> GetResourceFiles(APSIMJob job, CloudBlobClient blobClient)
{
yield return(new ResourceFile(job.ModelZipFileSas, BatchConstants.MODEL_ZIPFILE_NAME));
var toolsRef = blobClient.GetContainerReference("tools");
foreach (CloudBlockBlob listBlobItem in toolsRef.ListBlobs())
{
var sas = listBlobItem.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
SharedAccessStartTime = DateTime.UtcNow.AddHours(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddMonths(2),
Permissions = SharedAccessBlobPermissions.Read,
});
yield return(new ResourceFile(listBlobItem.Uri.AbsoluteUri + sas, listBlobItem.Name));
}
var apsimRef = blobClient.GetContainerReference("apsim");
foreach (CloudBlockBlob listBlobItem in apsimRef.ListBlobs())
{
if (listBlobItem.Name.ToLower().Contains(job.ApsimApplicationPackageVersion.ToLower()))
{
var sas = listBlobItem.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
SharedAccessStartTime = DateTime.UtcNow.AddHours(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddMonths(2),
Permissions = SharedAccessBlobPermissions.Read,
});
yield return(new ResourceFile(listBlobItem.Uri.AbsoluteUri + sas, listBlobItem.Name));
}
}
}
public static JobPreparationTask ToJobPreparationTask(this APSIMJob job, Guid jobId, CloudBlobClient blobClient)
{
return(new JobPreparationTask
{
CommandLine = "cmd.exe /c jobprep.cmd",
ResourceFiles = GetResourceFiles(job, blobClient).ToList(),
WaitForSuccess = true,
});
}
public static JobReleaseTask ToJobReleaseTask(this APSIMJob job, Guid jobId)
{
return(new JobReleaseTask
{
//CommandLine = "cmd.exe /c rmdir /s /q " + BatchConstants.GetJobInputPath(jobId),
//CommandLine = "cmd.exe /c echo test > " + BatchConstants.GetJobInputPath(jobId) + "\\test.txt",
//CommandLine = "cmd.exe /c jobrelease.cmd " + BatchConstants.GetJobInputPath(jobId),
//CommandLine = "cmd.exe /c jobrelease.cmd",
CommandLine = "cmd.exe /c jobrelease.cmd " + job.StorageCredentials.Key,
//CommandLine = "cmd.exe /c md c:\temp && echo test > c:\temp\test.stdout",
});
}
private static IEnumerable <ResourceFile> GetResourceFiles(APSIMJob job, CloudBlobClient blobClient)
{
var toolsRef = blobClient.GetContainerReference("jobmanager");
foreach (CloudBlockBlob listBlobItem in toolsRef.ListBlobs())
{
var sas = listBlobItem.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
SharedAccessStartTime = DateTime.UtcNow.AddHours(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddMonths(2),
Permissions = SharedAccessBlobPermissions.Read,
});
yield return(new ResourceFile(listBlobItem.Uri.AbsoluteUri + sas, listBlobItem.Name));
}
}
public void SubmitJob(Guid jobId, APSIMJob job, bool shouldSubmitTasks, bool autoScale)
{
try
{
var cloudJob = _batchClient.JobOperations.CreateJob(jobId.ToString(), GetPoolInfo(job.PoolSettings));
cloudJob.DisplayName = job.DisplayName;
cloudJob.JobPreparationTask = job.ToJobPreparationTask(jobId, _storageAccount.CreateCloudBlobClient());
cloudJob.JobReleaseTask = job.ToJobReleaseTask(jobId);
cloudJob.JobManagerTask = job.ToJobManagerTask(jobId, _storageAccount.CreateCloudBlobClient(), shouldSubmitTasks, autoScale);
cloudJob.Commit();
}
catch (AggregateException e)
{
throw ExceptionHelper.UnwrapAggregateException(e);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public static JobManagerTask ToJobManagerTask(this APSIMJob job, Guid jobId, CloudBlobClient blobClient, bool shouldSubmitTasks, bool autoScale)
{
var cmd = string.Format("cmd.exe /c {0} job-manager {1} {2} {3} {4} {5} {6} {7} {8} {9}",
BatchConstants.GetJobManagerPath(jobId),
job.BatchCredentials.Url,
job.BatchCredentials.Account,
job.BatchCredentials.Key,
job.StorageCredentials.Account,
job.StorageCredentials.Key,
jobId,
BatchConstants.GetModelPath(jobId),
shouldSubmitTasks,
autoScale);
return(new JobManagerTask
{
CommandLine = cmd,
DisplayName = "Job manager task",
KillJobOnCompletion = true,
Id = BatchConstants.JOB_MANAGER_NAME,
RunExclusive = false,
ResourceFiles = GetResourceFiles(job, blobClient).ToList(),
});
}
/// <summary>
/// Handles the bulk of the work for submitting the job to the cloud.
/// Zips up ApsimX (if necessary), uploads tools and ApsimX,
/// </summary>
/// <param name="e">Event arg, containing the job parameters</param>
private void SubmitJob_DoWork(object o, DoWorkEventArgs e)
{
JobParameters jp = (JobParameters)e.Argument;
jp.JobId = Guid.NewGuid();
jp.CoresPerProcess = 1;
jp.JobManagerShouldSubmitTasks = true;
jp.AutoScale = true;
jp.PoolMaxTasksPerVM = 16;
string tmpZip = "";
SetAzureMetaData("job-" + jp.JobId, "Owner", Environment.UserName.ToLower());
// if jp.ApplicationPackagePath is a directory it will need to be zipped up
if (Directory.Exists(jp.ApplicationPackagePath))
{
view.Status = "Zipping APSIM";
tmpZip = Path.Combine(Path.GetTempPath(), "Apsim-tmp-X-" + Environment.UserName.ToLower() + ".zip");
if (File.Exists(tmpZip))
{
File.Delete(tmpZip);
}
if (CreateApsimXZip(jp.ApplicationPackagePath, tmpZip) > 0)
{
view.Status = "Cancelled";
return;
}
jp.ApplicationPackagePath = tmpZip;
jp.ApplicationPackageVersion = Path.GetFileName(tmpZip).Substring(Path.GetFileName(tmpZip).IndexOf('-') + 1);
}
// add current job to the list of jobs
// TODO : do we actually need/use the APSIMJob class?
APSIMJob job = new APSIMJob(jp.JobDisplayName, "", jp.ApplicationPackagePath, jp.ApplicationPackageVersion, jp.Recipient, batchAuth, storageAuth, PoolSettings.FromConfiguration());
job.PoolInfo.MaxTasksPerVM = jp.PoolMaxTasksPerVM;
job.PoolInfo.VMCount = jp.PoolVMCount;
// upload tools such as 7zip, AzCopy, CMail, etc.
view.Status = "Checking tools";
string executableDirectory = GetExecutableDirectory();
string toolsDir = Path.Combine(executableDirectory, "tools");
if (!Directory.Exists(toolsDir))
{
ShowErrorMessage("Tools Directory not found: " + toolsDir);
}
foreach (string filePath in Directory.EnumerateFiles(toolsDir))
{
UploadFileIfNeeded("tools", filePath);
}
if (jp.Recipient.Length > 0)
{
try
{
// Store a config file into the job directory that has the e-mail config
string tmpConfig = Path.Combine(Path.GetTempPath(), settingsFileName);
using (StreamWriter file = new StreamWriter(tmpConfig))
{
file.WriteLine("EmailRecipient=" + jp.Recipient);
file.WriteLine("EmailSender=" + AzureSettings.Default["EmailSender"]);
file.WriteLine("EmailPW=" + AzureSettings.Default["EmailPW"]);
}
UploadFileIfNeeded("job-" + jp.JobId, tmpConfig);
File.Delete(tmpConfig);
}
catch (Exception err)
{
ShowError(new Exception("Error writing to settings file; you may not receive an email upon job completion: ", err));
}
}
// upload job manager
UploadFileIfNeeded("jobmanager", Path.Combine(executableDirectory, "azure-apsim.exe"));
// upload apsim
view.Status = "Uploading APSIM Next Generation";
UploadFileIfNeeded("apsim", jp.ApplicationPackagePath);
// generate model files
view.Status = "Generating model files";
if (!Directory.Exists(jp.ModelPath))
{
Directory.CreateDirectory(jp.ModelPath);
}
try
{
// copy weather files to models directory to be zipped up
foreach (Models.Weather child in Apsim.ChildrenRecursively(model).OfType <Models.Weather>())
{
if (Path.GetDirectoryName(child.FullFileName) != Path.GetDirectoryName(presenter.ApsimXFile.FileName))
{
presenter.MainPresenter.ShowError("Weather file must be in the same directory as .apsimx file: " + child.FullFileName);
view.Status = "Cancelled";
return;
}
string sourceFile = child.FullFileName;
string destFile = Path.Combine(jp.ModelPath, child.FileName);
if (!File.Exists(destFile))
{
File.Copy(sourceFile, destFile);
}
;
}
// Generate .apsimx files, and if any errors are encountered, abort the job submission process.
if (!presenter.GenerateApsimXFiles(model, jp.ModelPath))
{
view.Status = "Cancelled";
return;
}
}
catch (Exception err)
{
presenter.MainPresenter.ShowError(err);
return;
}
tmpZip = "";
// zip up models directory
if (Directory.Exists(jp.ModelPath)) // this test may be unnecessary
{
tmpZip = GetTempFileName("Model-", ".zip", true);
ZipFile.CreateFromDirectory(jp.ModelPath, tmpZip, CompressionLevel.Fastest, false);
jp.ModelPath = tmpZip;
}
// upload models
view.Status = "Uploading models";
job.ModelZipFileSas = uploader.UploadFile(jp.ModelPath, jp.JobId.ToString(), Path.GetFileName(jp.ModelPath));
// clean up temporary model files
if (File.Exists(tmpZip))
{
File.Delete(tmpZip);
}
if (!jp.SaveModelFiles)
{
if (Directory.Exists(jp.ModelPath))
{
Directory.Delete(jp.ModelPath);
}
}
view.Status = "Submitting Job";
// submit job
try
{
CloudJob cloudJob = batchCli.JobOperations.CreateJob(jp.JobId.ToString(), GetPoolInfo(job.PoolInfo));
cloudJob.DisplayName = job.DisplayName;
cloudJob.JobPreparationTask = job.ToJobPreparationTask(jp.JobId, Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount));
cloudJob.JobReleaseTask = job.ToJobReleaseTask(jp.JobId, Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount));
cloudJob.JobManagerTask = job.ToJobManagerTask(jp.JobId, Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount), jp.JobManagerShouldSubmitTasks, jp.AutoScale);
cloudJob.Commit();
}
catch (Exception err)
{
ShowError(err);
}
view.Status = "Job Successfully submitted";
if (jp.AutoDownload)
{
AzureResultsDownloader dl = new AzureResultsDownloader(jp.JobId, jp.JobDisplayName, jp.OutputDir, null, true, jp.Summarise, true, true, true);
dl.DownloadResults(true);
}
}
/// <summary>
/// Handles the bulk of the work for submitting the job to the cloud.
/// Zips up ApsimX (if necessary), uploads tools and ApsimX,
/// </summary>
/// <param name="e">Event arg, containing the job parameters</param>
private void SubmitJob_DoWork(object o, DoWorkEventArgs e)
{
JobParameters jp = (JobParameters)e.Argument;
jp.JobId = Guid.NewGuid();
jp.CoresPerProcess = 1;
jp.JobManagerShouldSubmitTasks = true;
jp.AutoScale = true;
jp.PoolMaxTasksPerVM = 16;
string tmpZip = "";
SetAzureMetaData("job-" + jp.JobId, "Owner", Environment.UserName.ToLower());
// if jp.ApplicationPackagePath is a directory it will need to be zipped up
if (Directory.Exists(jp.ApplicationPackagePath))
{
view.Status = "Zipping APSIM";
tmpZip = Path.Combine(Path.GetTempPath(), "Apsim-tmp-X-" + Environment.UserName.ToLower() + ".zip");
if (File.Exists(tmpZip))
{
File.Delete(tmpZip);
}
if (createApsimXZip(jp.ApplicationPackagePath, tmpZip) > 0)
{
ShowError("Error zipping up Apsim");
}
jp.ApplicationPackagePath = tmpZip;
jp.ApplicationPackageVersion = Path.GetFileName(tmpZip).Substring(Path.GetFileName(tmpZip).IndexOf('-') + 1);
}
// add current job to the list of jobs
// TODO : do we actually need/use the APSIMJob class?
APSIMJob job = new APSIMJob(jp.JobDisplayName, "", jp.ApplicationPackagePath, jp.ApplicationPackageVersion, jp.Recipient, batchCredentials, storageCredentials, PoolSettings.FromConfiguration());
job.PoolInfo.MaxTasksPerVM = jp.PoolMaxTasksPerVM;
job.PoolInfo.VMCount = jp.PoolVMCount;
// upload tools such as 7zip, AzCopy, CMail, etc.
view.Status = "Checking tools";
string executableDirectory = GetExecutableDirectory();
string toolsDir = Path.Combine(executableDirectory, "tools");
if (!Directory.Exists(toolsDir))
{
ShowError("Tools Directory not found: " + toolsDir);
}
foreach (string filePath in Directory.EnumerateFiles(toolsDir))
{
UploadFileIfNeeded("tools", filePath);
}
if (jp.Recipient.Length > 0)
{
try
{
// Store a config file into the job directory that has the e-mail config
string tmpConfig = Path.Combine(Path.GetTempPath(), SETTINGS_FILENAME);
using (StreamWriter file = new StreamWriter(tmpConfig))
{
file.WriteLine("EmailRecipient=" + jp.Recipient);
file.WriteLine("EmailSender=" + AzureSettings.Default["EmailSender"]);
file.WriteLine("EmailPW=" + AzureSettings.Default["EmailPW"]);
}
UploadFileIfNeeded("job-" + jp.JobId, tmpConfig);
File.Delete(tmpConfig);
}
catch (Exception ex)
{
ShowError("Error writing to settings file; you may not receive an email upon job completion: " + ex.ToString());
}
}
// upload job manager
UploadFileIfNeeded("jobmanager", Path.Combine(executableDirectory, "azure-apsim.exe"));
// upload apsim
view.Status = "Uploading APSIM Next Generation";
UploadFileIfNeeded("apsim", jp.ApplicationPackagePath);
// create models
// TODO : show error message if other files already exist in model output directory?
// may be necessary if using a user-selected directory
string path = "";
// generate model files
view.Status = "Generating model files";
if (!Directory.Exists(jp.ModelPath))
{
Directory.CreateDirectory(jp.ModelPath);
}
// generate xml
string xml = "";
foreach (Simulation sim in Runner.AllSimulations(model))
{
path = Path.Combine(jp.ModelPath, sim.Name + ".apsimx");
// if weather file is not in the same directory as the .apsimx file, display an error then abort
foreach (var child in sim.Children)
{
if (child is Models.Weather)
{
string childPath = sim.Children.OfType <Models.Weather>().ToList()[0].FileName;
if (Path.GetDirectoryName(childPath) != "")
{
ShowError(childPath + " must be in the same directory as the .apsimx file" + (sim.FileName != null ? " (" + Path.GetDirectoryName(sim.FileName) + ")" : ""));
view.Status = "Cancelled";
return;
}
string sourcePath = sim.Children.OfType <Models.Weather>().ToList()[0].FullFileName;
string destPath = Path.Combine(jp.ModelPath, sim.Children.OfType <Models.Weather>().ToList()[0].FileName);
if (!File.Exists(destPath))
{
File.Copy(sourcePath, destPath);
}
}
}
xml = Apsim.Serialise(sim);
// delete model file if it already exists
if (File.Exists(path))
{
File.Delete(path);
}
string pattern = @"<\?xml[^<>]+>";
System.Text.RegularExpressions.Regex rx = new System.Text.RegularExpressions.Regex(pattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
var matches = rx.Matches(xml);
xml = "<Simulations xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" Version=\"23\">\r\n<Name>" + sim.Name + "</Name>\r\n" + xml.Replace(matches[0].ToString(), "") + "<DataStore><Name>DataStore</Name><AutoExport>false</AutoExport><MaximumResultsPerPage>0</MaximumResultsPerPage></DataStore><ExplorerWidth>296</ExplorerWidth></Simulations>";
// write xml to file
using (FileStream fs = File.Create(path))
{
Byte[] info = new UTF8Encoding(true).GetBytes(xml);
fs.Write(info, 0, info.Length);
}
}
tmpZip = "";
// zip up models directory
if (Directory.Exists(jp.ModelPath)) // this test may be unnecessary
{
tmpZip = GetTempFileName("Model-", ".zip", true);
ZipFile.CreateFromDirectory(jp.ModelPath, tmpZip, CompressionLevel.Fastest, false);
jp.ModelPath = tmpZip;
}
// upload models
view.Status = "Uploading models";
job.ModelZipFileSas = uploader.UploadFile(jp.ModelPath, jp.JobId.ToString(), Path.GetFileName(jp.ModelPath));
// clean up temporary model files
if (File.Exists(tmpZip))
{
File.Delete(tmpZip);
}
if (!jp.SaveModelFiles)
{
if (Directory.Exists(jp.ModelPath))
{
Directory.Delete(jp.ModelPath);
}
}
view.Status = "Submitting Job";
// submit job
try
{
CloudJob cloudJob = batchClient.JobOperations.CreateJob(jp.JobId.ToString(), GetPoolInfo(job.PoolInfo));
cloudJob.DisplayName = job.DisplayName;
cloudJob.JobPreparationTask = job.ToJobPreparationTask(jp.JobId, storageAccount.CreateCloudBlobClient());
cloudJob.JobReleaseTask = job.ToJobReleaseTask(jp.JobId, storageAccount.CreateCloudBlobClient());
cloudJob.JobManagerTask = job.ToJobManagerTask(jp.JobId, storageAccount.CreateCloudBlobClient(), jp.JobManagerShouldSubmitTasks, jp.AutoScale);
cloudJob.Commit();
} catch (Exception ex)
{
ShowError(ex.ToString());
}
view.Status = "Job Successfully submitted";
// in theory, instantiating an AzureResultsDownloader here and passing in this job's ID and name, then calling its DownloadResults() method would allow
// for automatic job downloading as soon as the job is finished. The trouble is it needs an AzureJobDisplayPresenter (for progress bar updating, etc).
// explorerPresenter.MainPresenter.ShowMessage("Job Successfully submitted", Simulation.ErrorLevel.Information);
// AzureResultsDownloader dl = new AzureResultsDownloader(jp.JobId, jp.JobDisplayName, jp.OutputDir, this.explorerPresenter, true, false, false);
}
