/// <summary>
/// Where the magic happens (program enters here)
/// </summary>
/// <param name="args"></param>
public static void Main(string[] args)
{
#region select input data
/* // Ask the user enter the location of a parameter file
Console.WriteLine("Location of parameter file is " + Properties.Settings.Default.setupParams);
Console.WriteLine("Enter new location or press <Enter> if you want to keep it");
var newParamsFile = Console.ReadLine();
// if a new parameter location is given then set the Properties.Settings value holding the current location of the params
// file to the new loctaion, given by the user.
if ( newParamsFile != String.Empty ){
Properties.Settings.Default.setupParams = newParamsFile;
Properties.Settings.Default.Save();
} */
if (args.Length != 1)
{
string argString = null;
foreach (string arg in args)
{
argString += " "+arg;
}
throw new ArgumentException("GWydiR does not accept " + argString + " as parameters, length : "+args.Length+", please input with the path to a parameter file");
}
// read arguments into properties for param file location
Properties.Settings.Default.setupParams = args[0];
// read in params from file at location in Properties.Settings.setupParams
var paramsData = readParams();
string doUpload = "";
string dataKeyFile = "";
// if the key doUpload is present
if ( paramsData.ContainsKey("doUpload")){
// get the data given in the params file for param 'doUpload'
doUpload = paramsData["doUpload"];
// if a key 'dataKeyFile' is peresent in the params data
if ( paramsData.ContainsKey("dataKeyFile")){
// Get the location of that dataKeyFile file
dataKeyFile = paramsData["dataKeyFile"];
}
// else the user has failed to submit the required data
else{
Console.Write("If you wish to upload the R environment, you must set the access to mass store via the dataKeyFile");
Console.Write("Press Return to Exit");
var leaving = Console.ReadLine();
Environment.Exit(-1);
}
}
// Here the locations of files used in this program are intialised from the params file given.
#region initilaise file locations
string RFileName = getParamVariable(paramsData,"RFileName",@"");
string userZipFileName = getParamVariable(paramsData,"userZipFileName","");
string csvFileName = getParamVariable(paramsData,"csvFileName",@"C:\Users\hugh\list.txt");
string outputRoot = getParamVariable(paramsData,"outputRoot",@"test");
string appKeyFile = getParamVariable(paramsData,"appKeyFile",@"");
string myApplicationName = getParamVariable(paramsData,"ApplicationURL",@"");
string serviceURL = getParamVariable(paramsData,"serviceURL",@"");
UserDataStoreConnectionString = getConnectionString(appKeyFile);
#endregion
#endregion
#region Upload application zip
// test to see if the value of doUpload is the character y
if ( doUpload.Equals("y") ){
// if yes, inform the user that an uplod is begining
Console.Write("Uploading application ZIP.. ");
Reference appReference = null;
Reference descReference = null;
// upload and install application - creates application package and description
UploadApplication(UserDataStoreConnectionString, myApplicationName, dataKeyFile, out appReference, out descReference);
}
#endregion
#region Set up containers for input, output etc.
// initialise an account object from a data stroe connection string
var account = CloudStorageAccount.Parse(UserDataStoreConnectionString);
// retrieve a blob client from that account
var blobClient = account.CreateCloudBlobClient();
// get a container from the blob (containers store data like flat directories)
CloudBlobContainer appDataContainer = blobClient.GetContainerReference("applicationcontainer");
// if the requested container does not exist, create it
appDataContainer.CreateIfNotExist();
// Apply the same as above.
var blobContainer = blobClient.GetContainerReference("testcontainer");
blobContainer.CreateIfNotExist();
// retrieve the jobs to be run from a csv file, location given in params file
var items = determineJobs(csvFileName);
foreach ( string item in items ){
// The job description tells the GenericWorker values for arguments and which files need to be downloaded before and uploaded after execution
VENUSJobDescription mySimpleJobDescription = new VENUSJobDescription();
mySimpleJobDescription.ApplicationIdentificationURI = myApplicationName; // This needs to be the same URI as used when creating the application description
mySimpleJobDescription.CustomerJobID = outputRoot + " Job " + item + DateTime.Now.ToLocalTime().ToString();
mySimpleJobDescription.JobName = outputRoot + " Job " + item;
// The GenericWorker needs to know where to find the application and description
mySimpleJobDescription.AppPkgReference = new Reference(new AzureBlobReference(appDataContainer.GetBlobReference(HttpUtility.UrlEncode(mySimpleJobDescription.ApplicationIdentificationURI) + "_App"), UserDataStoreConnectionString));
mySimpleJobDescription.AppDescReference = new Reference(new AzureBlobReference(appDataContainer.GetBlobReference(HttpUtility.UrlEncode(mySimpleJobDescription.ApplicationIdentificationURI) + "_Desc"), UserDataStoreConnectionString));
// The application description says, that a several parameters are needed to execute the application.
// So let's fill them with values.
// First set up the R script for upload
createArgument(RFileName, "RFile", mySimpleJobDescription, blobContainer);
// Then the list of GSE's
string jobFileName = item + @".csv";
writeJobToFile(item, jobFileName);
createArgument(jobFileName, "GSEs", mySimpleJobDescription, blobContainer);
// Now the output file - we generate a unique file name for this.
var nameOut = outputRoot + item + DateTime.Now.ToBinary() + ".log";
createArgument(jobFileName, "OutputFile", mySimpleJobDescription, blobContainer,nameOut: nameOut);
// Finally an optional zip file of additional material
if (!userZipFileName.Equals(""))
{
createArgument(userZipFileName, "userZip", mySimpleJobDescription, blobContainer);
}
Console.WriteLine("Done");
#region Submit job
Console.WriteLine("Submitting job for item " + item);
Func<GenericWorkerJobManagementClient> CreateUnprotectedClient = () =>
{
var svcUrl = serviceURL + "JobSubmission/Service.svc";
//ConfigurationManager.AppSettings["Microsoft.EMIC.Cloud.GenericWorker.URL"];
Console.WriteLine(string.Format("Submitting all jobs to {0}", svcUrl));
return GenericWorkerJobManagementClient.CreateUnprotectedClient(svcUrl);
};
var submissionPortal = CreateUnprotectedClient();
// The order of job submissions does not matter, because the jobs are data driven
//
submissionPortal.SubmitVENUSJob(mySimpleJobDescription);
}
#endregion
#endregion
/* Console.WriteLine("Done");
// var allJobs = submissionPortal.GetAllJobs();
Console.WriteLine();
Console.WriteLine("Press enter to quit");
Console.ReadLine();
*/
}
// Adds a new command file to a jod description after uploading a file to a blob
private static void createArgument(string FileName, string commandName, VENUSJobDescription mySimpleJobDescription, CloudBlobContainer blobContainer, string nameOut="")
{
var commandBlob = blobContainer.GetBlobReference(FileName);
commandBlob.UploadFile(FileName);
var commandFile = new AzureArgumentSingleReference();
commandFile.Name = commandName; // This has to be the same name as in the application description
if ( nameOut == "" ){
commandFile.DataAddress = commandBlob.Uri.AbsoluteUri;
}
else{
commandFile.DataAddress = blobContainer.GetBlobReference(nameOut).Uri.AbsoluteUri;
}
commandFile.ConnectionString = UserDataStoreConnectionString;
mySimpleJobDescription.JobArgs.Add(commandFile);
}