/// <summary> /// Gets a <see cref="FileDef"/> from <paramref name="fileDefs"/> that matches <paramref name="name"/>. /// </summary> /// <param name="fileDefs">The file defs.</param> /// <param name="name">The name.</param> /// <returns></returns> public static FileDef GetNamedFileDef(FileDefCollection fileDefs, string name) { if (fileDefs == null) { throw new ArgumentNullException("fileDefs"); } int index = fileDefs.IndexOf(name); if (index == -1) { throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, "No FileDef in the given collection has name '{0}'.", name), "name"); } return(fileDefs[index]); }
static void Main(string[] args) { try { // before using the ArgCollection class, let the DigipedeClient grab its args. DigipedeClient client = new DigipedeClient(); args = client.ProcessArguments(args, true); ArgCollection argCollection = ArgCollection.GetInstance(args); // show help, if requested if (argCollection.ExtractOptionalFlag("help")) { Console.WriteLine(""); Console.WriteLine(UsageMessage); Console.WriteLine(HelpMessage); return; } // fail if the DigipedeClient doesn't have the args SpecialFunctions.CheckCondition(client.IsInitialized, "Digipede Client didn't initialize."); string keepTestName = argCollection.ExtractOptional("keepTest", "AlwaysKeep"); string skipRowIndexFileNameOrNull = argCollection.ExtractOptional <string>("skipRowIndexFile", null); string optimizerName = argCollection.ExtractOptional("optimizer", "BrentThenGrid"); argCollection.CheckNoMoreOptions(); int pieceCount = argCollection.ExtractNext <int>("pieceCount"); //int taskPerJobCount = argCollection.ExtractNext<int>("taskPerJobCount"); -- we're not using this -- keep it in case we want to put it back string treeFileName = argCollection.ExtractNext <string>("treeFile"); string predictorFileName = argCollection.ExtractNext <string>("predictorFile"); string targetFileName = argCollection.ExtractNext <string>("targetFile"); string leafDistributionName = argCollection.ExtractNext <string>("leafDistribution"); string nullDataGeneratorName = argCollection.ExtractNext <string>("nullDataGenerator"); string niceName = argCollection.ExtractNext <string>("niceName"); string outputDirectoryName = argCollection.ExtractNext <string>("outputDirectory"); RangeCollection nullIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("nullIndexRange"); SpecialFunctions.CheckCondition(nullIndexRangeCollection.IsBetween(-1, int.MaxValue), "nullIndex must be at least -1"); argCollection.CheckThatEmpty(); Directory.CreateDirectory(outputDirectoryName); // Define a JobTemplate for PhyloD. JobTemplate jobTemplate = CreateJobTemplate(); // Require 32 bit (ensures we use WOW64 on 64-bit machines) since SpecialFunctions.dll built for x86. jobTemplate.Control.UseWow64On64Bit = true; // allow task failures (all but one failure will result in job success) jobTemplate.JobDefaults.MaxFailures = pieceCount - 1; // allow multiple concurrent tasks (one for each core); each isolated in its own process. jobTemplate.Control.Concurrency = ApplicationConcurrency.MultiplePerCore; jobTemplate.Control.ProcessHostingOptions = HostingOptions.ManySingleUse; // create a Job based on that JobTemplate Job job = jobTemplate.NewJob(); // add job-specific data / files FileDefCollection fileDefs = job.FileDefs; // files Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).RemoteName = treeFileName; Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).RemoteName = predictorFileName; Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).RemoteName = targetFileName; // skipRowIndex file is more complicated because it may not exist, but the JobTemplate requires it. FileDef fileDef = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName); if (skipRowIndexFileNameOrNull == null || skipRowIndexFileNameOrNull == "null") { // stream an empty file fileDef.Stream = new MemoryStream(0); } else { // stream the actual file fileDef.LocalName = skipRowIndexFileNameOrNull; } // Create the tasks for the template for (int pieceIndex = 0; pieceIndex < pieceCount; pieceIndex++) { // Create a Task for this piece Task task = job.NewTask(); // Create an InputData object to encapsulate all input data in one place InputData inputData = new InputData(optimizerName, keepTestName, leafDistributionName, nullDataGeneratorName, niceName, outputDirectoryName, pieceIndex, pieceCount, nullIndexRangeCollection.ToString()); // create a Worker for this task task.Worker = new PhyloDWorker(inputData); } // Wire events to catch result data. Note that retrieving data isn't necessary here -- // data can be requested in a server call from another process. job.TaskCompleted += job_TaskCompleted; // Write an event to catch any monitoring errors client.MonitoringError += client_MonitoringError; // submit the job SubmissionResult sr = client.SubmitJob(jobTemplate, job); Console.WriteLine("Submitted job {0} with {1} tasks.", sr.JobId, job.Tasks.Count); // wait for the result JobStatusSummary jss = client.WaitForJobWithStatus(sr.JobId); Console.WriteLine("Job finished with status of {0}", jss.Status); } catch (Exception exception) { Console.WriteLine(""); Console.WriteLine(exception.Message); if (exception.InnerException != null) { Console.WriteLine(exception.InnerException.Message); } Console.WriteLine(""); Console.WriteLine(UsageMessage); throw; } }
/// <summary> /// Does the work. /// </summary> public override void DoWork() { // get our input data and null the field to make sure we don't serialize it back InputData inputData = mInputData; mInputData = null; // get the job-specific names of input files FileDefCollection fileDefs = Job.FileDefs; string treeFileName = Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).LocalName; string predictorFileName = Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).LocalName; string targetFileName = Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).LocalName; string skipRowIndexFileName = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName).LocalName; // construct RangeCollections RangeCollection pieceIndexRangeCollection = RangeCollection.Parse(inputData.PieceIndexRange); RangeCollection nullIndexRangeCollection = RangeCollection.Parse(inputData.NullIndexRange); RangeCollection skipRowIndexRangeCollection; FileInfo fileInfo = new FileInfo(skipRowIndexFileName); if (fileInfo.Length > 0) { skipRowIndexRangeCollection = RangeCollection.Parse(File.ReadAllText(skipRowIndexFileName)); } else { skipRowIndexRangeCollection = null; } // do the rest PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null); ModelScorer modelScorer = ModelScorer.GetInstance(aPhyloTree, inputData.LeafDistributionName, inputData.OptimizerName); ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(inputData.LeafDistributionName, modelScorer); KeepTest <Dictionary <string, string> > keepTest = KeepTest <Dictionary <string, string> > .GetInstance(null, inputData.KeepTestName); PhyloDDriver driver = PhyloDDriver.GetInstance(); // create a name for the temporary job sandbox. This directory gets created by driver.Run(...) string agentOutputDirectoryName = Path.Combine(Environment.CurrentDirectory, String.Format(CultureInfo.InvariantCulture, "{0}.{1}", Job.JobId, Task.TaskId)); // save the standard out and standard error in memory streams using (MemoryStream streamOut = new MemoryStream(), streamError = new MemoryStream()) { try { // redirect the outputs using ( StreamWriter writerOut = new StreamWriter(streamOut), writerError = new StreamWriter(streamError)) { Console.SetOut(writerOut); Console.SetError(writerError); try { // run the model string outputFileName = driver.Run( modelEvaluator, predictorFileName, targetFileName, inputData.LeafDistributionName, inputData.NullDataGeneratorName, keepTest, skipRowIndexRangeCollection, inputData.NiceName, agentOutputDirectoryName, pieceIndexRangeCollection, inputData.PieceCount, nullIndexRangeCollection, inputData.OptimizerName); // this is the expected output file name -- save this so it can be written on the master side with the same name. mOutputFileName = Path.GetFileName(outputFileName); mLocalOutputFileName = Path.Combine(inputData.LocalOutputDirectoryName, mOutputFileName); // get the output data string fullOutputPath = Path.Combine(agentOutputDirectoryName, mOutputFileName); if (!File.Exists(fullOutputPath)) { TaskResult.FailureReason = TaskFailureReason.MissingOutput; TaskResult.FailureMessage = String.Format(CultureInfo.CurrentCulture, "Cannot find output file '{0}'", targetFileName); TaskResult.Status = TaskAssignmentStatus.Failed; } using (StreamReader outputData = new StreamReader(fullOutputPath)) { mOutputData = outputData.ReadToEnd(); } } finally { // this finally is to make sure we delete the folder // get rid of the sandbox Directory.Delete(agentOutputDirectoryName, true); } } } finally { // this finally is to make sure we get console output Encoding encoding = Encoding.Default; TaskResult.StandardOutput = encoding.GetString(streamOut.GetBuffer()); TaskResult.StandardError = encoding.GetString(streamError.GetBuffer()); } } }