/// <summary>
/// Gets a <see cref="FileDef"/> from <paramref name="fileDefs"/> that matches <paramref name="name"/>.
/// </summary>
/// <param name="fileDefs">The file defs.</param>
/// <param name="name">The name.</param>
/// <returns></returns>
public static FileDef GetNamedFileDef(FileDefCollection fileDefs, string name)
{
if (fileDefs == null)
{
throw new ArgumentNullException("fileDefs");
}
int index = fileDefs.IndexOf(name);
if (index == -1)
{
throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, "No FileDef in the given collection has name '{0}'.", name),
"name");
}
return(fileDefs[index]);
}
static void Main(string[] args)
{
try {
// before using the ArgCollection class, let the DigipedeClient grab its args.
DigipedeClient client = new DigipedeClient();
args = client.ProcessArguments(args, true);
ArgCollection argCollection = ArgCollection.GetInstance(args);
// show help, if requested
if (argCollection.ExtractOptionalFlag("help"))
{
Console.WriteLine("");
Console.WriteLine(UsageMessage);
Console.WriteLine(HelpMessage);
return;
}
// fail if the DigipedeClient doesn't have the args
SpecialFunctions.CheckCondition(client.IsInitialized, "Digipede Client didn't initialize.");
string keepTestName = argCollection.ExtractOptional("keepTest", "AlwaysKeep");
string skipRowIndexFileNameOrNull = argCollection.ExtractOptional <string>("skipRowIndexFile", null);
string optimizerName = argCollection.ExtractOptional("optimizer", "BrentThenGrid");
argCollection.CheckNoMoreOptions();
int pieceCount = argCollection.ExtractNext <int>("pieceCount");
//int taskPerJobCount = argCollection.ExtractNext<int>("taskPerJobCount"); -- we're not using this -- keep it in case we want to put it back
string treeFileName = argCollection.ExtractNext <string>("treeFile");
string predictorFileName = argCollection.ExtractNext <string>("predictorFile");
string targetFileName = argCollection.ExtractNext <string>("targetFile");
string leafDistributionName = argCollection.ExtractNext <string>("leafDistribution");
string nullDataGeneratorName = argCollection.ExtractNext <string>("nullDataGenerator");
string niceName = argCollection.ExtractNext <string>("niceName");
string outputDirectoryName = argCollection.ExtractNext <string>("outputDirectory");
RangeCollection nullIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("nullIndexRange");
SpecialFunctions.CheckCondition(nullIndexRangeCollection.IsBetween(-1, int.MaxValue), "nullIndex must be at least -1");
argCollection.CheckThatEmpty();
Directory.CreateDirectory(outputDirectoryName);
// Define a JobTemplate for PhyloD.
JobTemplate jobTemplate = CreateJobTemplate();
// Require 32 bit (ensures we use WOW64 on 64-bit machines) since SpecialFunctions.dll built for x86.
jobTemplate.Control.UseWow64On64Bit = true;
// allow task failures (all but one failure will result in job success)
jobTemplate.JobDefaults.MaxFailures = pieceCount - 1;
// allow multiple concurrent tasks (one for each core); each isolated in its own process.
jobTemplate.Control.Concurrency = ApplicationConcurrency.MultiplePerCore;
jobTemplate.Control.ProcessHostingOptions = HostingOptions.ManySingleUse;
// create a Job based on that JobTemplate
Job job = jobTemplate.NewJob();
// add job-specific data / files
FileDefCollection fileDefs = job.FileDefs;
// files
Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).RemoteName = treeFileName;
Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).RemoteName = predictorFileName;
Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).RemoteName = targetFileName;
// skipRowIndex file is more complicated because it may not exist, but the JobTemplate requires it.
FileDef fileDef = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName);
if (skipRowIndexFileNameOrNull == null || skipRowIndexFileNameOrNull == "null")
{
// stream an empty file
fileDef.Stream = new MemoryStream(0);
}
else
{
// stream the actual file
fileDef.LocalName = skipRowIndexFileNameOrNull;
}
// Create the tasks for the template
for (int pieceIndex = 0; pieceIndex < pieceCount; pieceIndex++)
{
// Create a Task for this piece
Task task = job.NewTask();
// Create an InputData object to encapsulate all input data in one place
InputData inputData = new InputData(optimizerName, keepTestName, leafDistributionName, nullDataGeneratorName,
niceName, outputDirectoryName, pieceIndex, pieceCount, nullIndexRangeCollection.ToString());
// create a Worker for this task
task.Worker = new PhyloDWorker(inputData);
}
// Wire events to catch result data. Note that retrieving data isn't necessary here --
// data can be requested in a server call from another process.
job.TaskCompleted += job_TaskCompleted;
// Write an event to catch any monitoring errors
client.MonitoringError += client_MonitoringError;
// submit the job
SubmissionResult sr = client.SubmitJob(jobTemplate, job);
Console.WriteLine("Submitted job {0} with {1} tasks.", sr.JobId, job.Tasks.Count);
// wait for the result
JobStatusSummary jss = client.WaitForJobWithStatus(sr.JobId);
Console.WriteLine("Job finished with status of {0}", jss.Status);
}
catch (Exception exception)
{
Console.WriteLine("");
Console.WriteLine(exception.Message);
if (exception.InnerException != null)
{
Console.WriteLine(exception.InnerException.Message);
}
Console.WriteLine("");
Console.WriteLine(UsageMessage);
throw;
}
}
/// <summary>
/// Does the work.
/// </summary>
public override void DoWork()
{
// get our input data and null the field to make sure we don't serialize it back
InputData inputData = mInputData;
mInputData = null;
// get the job-specific names of input files
FileDefCollection fileDefs = Job.FileDefs;
string treeFileName = Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).LocalName;
string predictorFileName = Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).LocalName;
string targetFileName = Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).LocalName;
string skipRowIndexFileName = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName).LocalName;
// construct RangeCollections
RangeCollection pieceIndexRangeCollection = RangeCollection.Parse(inputData.PieceIndexRange);
RangeCollection nullIndexRangeCollection = RangeCollection.Parse(inputData.NullIndexRange);
RangeCollection skipRowIndexRangeCollection;
FileInfo fileInfo = new FileInfo(skipRowIndexFileName);
if (fileInfo.Length > 0)
{
skipRowIndexRangeCollection = RangeCollection.Parse(File.ReadAllText(skipRowIndexFileName));
}
else
{
skipRowIndexRangeCollection = null;
}
// do the rest
PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null);
ModelScorer modelScorer =
ModelScorer.GetInstance(aPhyloTree, inputData.LeafDistributionName, inputData.OptimizerName);
ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(inputData.LeafDistributionName, modelScorer);
KeepTest <Dictionary <string, string> > keepTest =
KeepTest <Dictionary <string, string> > .GetInstance(null, inputData.KeepTestName);
PhyloDDriver driver = PhyloDDriver.GetInstance();
// create a name for the temporary job sandbox. This directory gets created by driver.Run(...)
string agentOutputDirectoryName =
Path.Combine(Environment.CurrentDirectory, String.Format(CultureInfo.InvariantCulture, "{0}.{1}", Job.JobId, Task.TaskId));
// save the standard out and standard error in memory streams
using (MemoryStream streamOut = new MemoryStream(), streamError = new MemoryStream())
{
try
{
// redirect the outputs
using (
StreamWriter writerOut = new StreamWriter(streamOut),
writerError = new StreamWriter(streamError))
{
Console.SetOut(writerOut);
Console.SetError(writerError);
try
{
// run the model
string outputFileName = driver.Run(
modelEvaluator,
predictorFileName, targetFileName,
inputData.LeafDistributionName, inputData.NullDataGeneratorName,
keepTest, skipRowIndexRangeCollection,
inputData.NiceName,
agentOutputDirectoryName,
pieceIndexRangeCollection, inputData.PieceCount,
nullIndexRangeCollection,
inputData.OptimizerName);
// this is the expected output file name -- save this so it can be written on the master side with the same name.
mOutputFileName = Path.GetFileName(outputFileName);
mLocalOutputFileName = Path.Combine(inputData.LocalOutputDirectoryName, mOutputFileName);
// get the output data
string fullOutputPath = Path.Combine(agentOutputDirectoryName, mOutputFileName);
if (!File.Exists(fullOutputPath))
{
TaskResult.FailureReason = TaskFailureReason.MissingOutput;
TaskResult.FailureMessage = String.Format(CultureInfo.CurrentCulture, "Cannot find output file '{0}'", targetFileName);
TaskResult.Status = TaskAssignmentStatus.Failed;
}
using (StreamReader outputData = new StreamReader(fullOutputPath))
{
mOutputData = outputData.ReadToEnd();
}
}
finally
{
// this finally is to make sure we delete the folder
// get rid of the sandbox
Directory.Delete(agentOutputDirectoryName, true);
}
}
}
finally
{
// this finally is to make sure we get console output
Encoding encoding = Encoding.Default;
TaskResult.StandardOutput = encoding.GetString(streamOut.GetBuffer());
TaskResult.StandardError = encoding.GetString(streamError.GetBuffer());
}
}
}
