예제 #1
0
 /// <summary>
 /// Uploads a CSV results file to shared AWS storage, allowing access to the results by other system components.
 /// </summary>
 /// <param name="filePath">Path of the file to upload.</param>
 /// <param name="bucketName">Bucket to upload to.</param>
 /// <param name="keyName">Storage key - used for retrieval. ".csv" is appended to this.</param>
 public static void AWSUpload(string filePath, string bucketName, string keyName)
 {
     try
     {
         TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(Amazon.RegionEndpoint.EUWest1));
         fileTransferUtility.Upload(filePath, bucketName, keyName + ".csv");
         Console.WriteLine("Upload Completed");
         UploadQueue.Upload();
     }
     catch (AmazonS3Exception amazonS3Exception)
     {
         AWS.AWSerror(amazonS3Exception);
     }
 }
예제 #2
0
        /// <summary>
        /// <para>Attempts to execute the given Job.</para>
        /// <para>Output is stored line-by-line in "results.csv" in the App_Data/{id} folder for the job.</para>
        /// </summary>
        /// <param name="fileName">Name of executable to run from the Job's .zip.</param>
        /// <param name="jobId">Job's ID.</param>
        public void RunTask(string fileName, int jobId)
        {
            StoredJob job = ProcessManager.GetJob(jobId);

            WorkArray[] Images    = job.Images;
            string      filePath  = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/Extracted/" + fileName);
            string      ImagePath = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/Images/");
            string      output    = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data/Jobs/" + jobId + "/results.csv");

            // Validate each image
            foreach (WorkArray img in Images)
            {
                if (!ValidateImageName(img.Image1.Key) || !ValidateImageName(img.Image2.Key))
                {
                    Debug.WriteLine("Invalid image hash " + img.Image1.Key + "or" + img.Image2.Key + ". Aborting executable launch.");
                    return;
                }
            }

            ProcessStartInfo startInfo = new ProcessStartInfo();

            startInfo.FileName               = filePath;
            startInfo.UseShellExecute        = false;
            startInfo.RedirectStandardOutput = true;
            startInfo.RedirectStandardInput  = false;
            startInfo.RedirectStandardError  = true;
            startInfo.CreateNoWindow         = false;

            //Set up header for file
            var w    = new StreamWriter(output);
            var line = string.Format("{0},{1},{2},{3}", "Image1", "Image2", "Result", "Errors");

            w.WriteLine(line);
            w.Flush();

            try
            {
                for (int i = 0; i < Images.Length; i++)
                {
                    while (GlobalQueue.QueueSize(jobId) == 0 || job.Paused == true)
                    {
                        // Hang till there's stuff on the queue to process

                        //Add sleep or wait
                    }
                    if (job.Stopped)
                    {
                        if (i > 0) //If i == 0 then RunningTasks hasn't been increased yet
                        {
                            JobQueue.RunningTasks -= 1;
                        }
                        w.Close();
                        //Maybe clean job of system
                        return;
                    }
                    if (i == 0)
                    {
                        Debug.WriteLine("Job + " + job.JobId + " started");
                        job.Started = true;
                    }
                    job.BatchIndex = i; // Helps to give the progress of the code
                    Tuple <string, string> arguments = GlobalQueue.RemoveFromQueue(jobId);
                    // Generate the image arguments
                    startInfo.Arguments = ImagePath + arguments.Item1 + " " + ImagePath + arguments.Item2 + " ";

                    using (job.exeProcess = Process.Start(startInfo))
                    {
                        string strOut = job.exeProcess.StandardOutput.ReadToEnd();
                        string strErr = job.exeProcess.StandardError.ReadToEnd();
                        job.exeProcess.WaitForExit();

                        // Save the results
                        job.ExitCode = job.exeProcess.ExitCode;

                        //Write to csv files
                        string first  = Images[i].Image1.Key;
                        string second = Images[i].Image2.Key;
                        //Check that there are no commas in strings otherwise cause error
                        String[] strOutArray = strOut.Split(',');
                        String[] strErrArray = strErr.Split(',');
                        strOut = String.Join("", strOutArray);
                        strErr = String.Join("", strErrArray);
                        line   = string.Format("{0},{1},{2},{3}", first, second, strOut, strErr).Trim();
                        w.WriteLine(line);
                        w.Flush();
                    }
                }
                w.Close();
                job.Completed = true;       //Signifies that the job is now complete
                //Given upload destination is currently the jobId
                JobQueue.RunningTasks -= 1; // Frees up space for the next running executable
                Action b = delegate { UploadQueue.AddToQueue(output, "citizen.science.image.storage.public", job.JobId.ToString()); };
                b();
                Debug.WriteLine("Uploaded:" + jobId);
                JobQueue.AllocateJobs();
            }
            catch (Exception e)
            {
                Debug.WriteLine("Job execution failed: " + e.Message);
            }
        }