/// <summary>
/// Uploads a CSV results file to shared AWS storage, allowing access to the results by other system components.
/// </summary>
/// <param name="filePath">Path of the file to upload.</param>
/// <param name="bucketName">Bucket to upload to.</param>
/// <param name="keyName">Storage key - used for retrieval. ".csv" is appended to this.</param>
public static void AWSUpload(string filePath, string bucketName, string keyName)
{
try
{
TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(Amazon.RegionEndpoint.EUWest1));
fileTransferUtility.Upload(filePath, bucketName, keyName + ".csv");
Console.WriteLine("Upload Completed");
UploadQueue.Upload();
}
catch (AmazonS3Exception amazonS3Exception)
{
AWS.AWSerror(amazonS3Exception);
}
}
/// <summary>
/// <para>Attempts to execute the given Job.</para>
/// <para>Output is stored line-by-line in "results.csv" in the App_Data/{id} folder for the job.</para>
/// </summary>
/// <param name="fileName">Name of executable to run from the Job's .zip.</param>
/// <param name="jobId">Job's ID.</param>
public void RunTask(string fileName, int jobId)
{
StoredJob job = ProcessManager.GetJob(jobId);
WorkArray[] Images = job.Images;
string filePath = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/Extracted/" + fileName);
string ImagePath = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/Images/");
string output = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/results.csv");
// Validate each image
foreach (WorkArray img in Images)
{
if (!ValidateImageName(img.Image1.Key) || !ValidateImageName(img.Image2.Key))
{
Debug.WriteLine("Invalid image hash " + img.Image1.Key + "or" + img.Image2.Key + ". Aborting executable launch.");
return;
}
}
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.FileName = filePath;
startInfo.UseShellExecute = false;
startInfo.RedirectStandardOutput = true;
startInfo.RedirectStandardInput = false;
startInfo.RedirectStandardError = true;
startInfo.CreateNoWindow = false;
//Set up header for file
var w = new StreamWriter(output);
var line = string.Format("{0},{1},{2},{3}", "Image1", "Image2", "Result", "Errors");
w.WriteLine(line);
w.Flush();
try
{
for (int i = 0; i < Images.Length; i++)
{
while (GlobalQueue.QueueSize(jobId) == 0 || job.Paused == true)
{
// Hang till there's stuff on the queue to process
//Add sleep or wait
}
if (job.Stopped)
{
if (i > 0) //If i == 0 then RunningTasks hasn't been increased yet
{
JobQueue.RunningTasks -= 1;
}
w.Close();
//Maybe clean job of system
return;
}
if (i == 0)
{
Debug.WriteLine("Job + " + job.JobId + " started");
job.Started = true;
}
job.BatchIndex = i; // Helps to give the progress of the code
Tuple <string, string> arguments = GlobalQueue.RemoveFromQueue(jobId);
// Generate the image arguments
startInfo.Arguments = ImagePath + arguments.Item1 + " " + ImagePath + arguments.Item2 + " ";
using (job.exeProcess = Process.Start(startInfo))
{
string strOut = job.exeProcess.StandardOutput.ReadToEnd();
string strErr = job.exeProcess.StandardError.ReadToEnd();
job.exeProcess.WaitForExit();
// Save the results
job.ExitCode = job.exeProcess.ExitCode;
//Write to csv files
string first = Images[i].Image1.Key;
string second = Images[i].Image2.Key;
//Check that there are no commas in strings otherwise cause error
String[] strOutArray = strOut.Split(',');
String[] strErrArray = strErr.Split(',');
strOut = String.Join("", strOutArray);
strErr = String.Join("", strErrArray);
line = string.Format("{0},{1},{2},{3}", first, second, strOut, strErr).Trim();
w.WriteLine(line);
w.Flush();
}
}
w.Close();
job.Completed = true; //Signifies that the job is now complete
//Given upload destination is currently the jobId
JobQueue.RunningTasks -= 1; // Frees up space for the next running executable
Action b = delegate { UploadQueue.AddToQueue(output, "citizen.science.image.storage.public", job.JobId.ToString()); };
b();
Debug.WriteLine("Uploaded:" + jobId);
JobQueue.AllocateJobs();
}
catch (Exception e)
{
Debug.WriteLine("Job execution failed: " + e.Message);
}
}
