/// <summary>
/// Creates a job template.
/// </summary>
/// <returns></returns>
public static JobTemplate CreateJobTemplate()
{
JobTemplate jobTemplate = JobTemplate.NewWorkerJobTemplate(Assembly.GetExecutingAssembly());
// add job-specific input files to the template. These will get streamed to the Server and downloaded to the Agents.
// tree file
FileDef fd = jobTemplate.FileDefs.Add();
fd.Name = Constants.TreeFileDefName;
fd.TransferType = TransferType.Streamed;
fd.Relevance = Relevance.JobPlaceholder;
fd.LocalName = "tree$(JobId).txt";
// predictor file
fd = jobTemplate.FileDefs.Add();
fd.Name = Constants.PredictorFileDefName;
fd.TransferType = TransferType.Streamed;
fd.Relevance = Relevance.JobPlaceholder;
fd.LocalName = "predictor$(JobId).txt";
// target file (not an output file!)
fd = jobTemplate.FileDefs.Add();
fd.Name = Constants.TargetFileDefName;
fd.TransferType = TransferType.Streamed;
fd.Relevance = Relevance.JobPlaceholder;
fd.LocalName = "target$(JobId).txt";
// skip row index file (this requires special handling below because it may be null)
fd = jobTemplate.FileDefs.Add();
fd.Name = Constants.SkipRowIndexFileDefName;
fd.TransferType = TransferType.Streamed;
fd.Relevance = Relevance.JobPlaceholder;
fd.LocalName = "skipRowIndex$(JobId).txt";
return(jobTemplate);
}
FileDef Clone(FileDef source)
{
var result = new FileDefUser(source.Name, source.Flags, source.HashValue);
CopyCustomAttributes(source, result);
CopyCustomDebugInfo(source, result);
return(result);
}
private static void DefineTypes(FileDef fileDef, String fileName)
{
ClassDef classDef = new ClassDef(fileDef, $"Class{fileName}");
ClassPropertyDef classPropertyDef = new ClassPropertyDef(classDef, "Test", TypeInt64.Instance, new LiteralInt64(42));
ConstantDef classConstantDef = new ConstantDef(classDef, "ID", TypeText.Instance, new LiteralText("<N/A>"));
InterfaceDef interfaceDef = new InterfaceDef(fileDef, $"Interface{fileName}");
InterfacePropertyDef interfacePropertyDef = new InterfacePropertyDef(interfaceDef, "Test", TypeText.Instance, false, true, true);
EnumerationDef enumerationDef = new EnumerationDef(fileDef, $"Enumeration{fileName}", IntegralType.Int32);
EnumerationMemberDef enumerationMemberDef1 = new EnumerationMemberDef(enumerationDef, "A", new LiteralInt32(1));
EnumerationMemberDef enumerationMemberDef2 = new EnumerationMemberDef(enumerationDef, "A", new LiteralInt32(2));
DecoratorDef decoratorDef = new DecoratorDef(fileDef, $"Decorator{fileName}");
DecoratorPropertyDef decoratorPropertyDef = new DecoratorPropertyDef(decoratorDef, "Value", TypeInt64.Instance, new LiteralInt64(42));
}
public ResourceOptions(Resource resource)
{
ResourceType = resource.ResourceType;
Name = resource.Name ?? UTF8String.Empty;
Attributes = resource.Attributes;
switch (resource.ResourceType)
{
case ResourceType.Embedded:
break;
case ResourceType.AssemblyLinked:
Assembly = ((AssemblyLinkedResource)resource).Assembly;
break;
case ResourceType.Linked:
File = ((LinkedResource)resource).File;
break;
default:
throw new InvalidOperationException();
}
}
public SectionDef AddDef(FileDef file, string sectionName)
{
if (file.Type == FileType.Definition)
{
var parts = sectionName.Split('.');
if (!file.Sections.TryGetValue(parts[0], out SectionDef section))
{
file.Sections[parts[0]] = section = new SectionDef(parts[0]);
}
// add property def
if (parts.Length > 1)
{
var propName = string.Join(".", parts.Skip(1));
if (!section.Properties.ContainsKey(propName))
{
section.Properties[propName] = new PropertyDef();
}
}
return(section);
}
else
{
if (!file.Sections.TryGetValue(sectionName, out SectionDef section))
{
file.Sections[sectionName] = section = new SectionDef(sectionName);
}
if (section.Type == SectionType.List)
{
file.Lists.Add(section);
}
AddWord(sectionName);
return(section);
}
}
public void AddDef(FileDef file, string sectionName, string propertyName, string propertyValue)
{
if (file.Type == FileType.Definition)
{
// process special sections
if (sectionName.IgnoreCaseEqual("applyto"))
{
if (!Definitions.TryGetValue(propertyValue, out HashSet <FileDef> list))
{
Definitions[propertyValue] = list = new HashSet <FileDef>();
}
list.Add(file);
// remove the file from generic def
var genericDefs = Definitions[string.Empty];
if (genericDefs.Contains(file))
{
genericDefs.Remove(file);
}
return;
}
if (sectionName.IgnoreCaseEqual("templates"))
{
Templates[propertyName] = propertyValue.Replace(@"\n", "\r\n");
return;
}
var parts = sectionName.Split('.');
var section = AddDef(file, sectionName);
if (parts.Length > 1)
{
// property metadata
var propName = string.Join(".", parts.Skip(1));
var property = section.Properties[propName];
if (propertyName.IgnoreCaseEqual("optional"))
{
property.Optional = propertyValue.IsTrue();
}
else if (propertyName.IgnoreCaseEqual("applyto"))
{
property.ApplyTo = propertyValue;
}
else if (propertyName.IgnoreCaseEqual("type"))
{
var itemOfPrefix = "itemof:";
if (propertyValue.StartsWith(itemOfPrefix, StringComparison.OrdinalIgnoreCase))
{
property.Type = PropertyType.ItemOf;
property.Lists = propertyValue.Substring(itemOfPrefix.Length).SplitAndTrim().ToArray();
}
else
{
property.Type = propertyValue.ToEnum <PropertyType>();
}
}
else if (propertyName.IgnoreCaseEqual("value"))
{
property.Value = propertyValue;
}
else if (int.TryParse(propertyName, out int _)) // is item
{
property.Values.Add(propertyValue);
}
}
else
{
// section metadata
if (propertyName.IgnoreCaseEqual("optional"))
{
section.Optional = propertyValue.IsTrue();
}
else if (propertyName.IgnoreCaseEqual("type"))
{
section.Type = propertyValue.ToEnum <SectionType>();
}
else if (propertyName.IgnoreCaseEqual("itemof"))
{
section.ItemOf = propertyValue;
}
}
}
else
{
var section = AddDef(file, sectionName);
if (!section.Properties.TryGetValue(propertyName, out PropertyDef property))
{
section.Properties[propertyName] = property = new PropertyDef();
}
property.Value = propertyValue;
AddWord(sectionName);
AddWord(propertyName);
AddPair(propertyName, propertyValue.SplitAndTrim().ToArray());
}
}
private static void DefineFile(NamespaceDef namespaceDef, String fileName)
{
FileDef fileDef = new FileDef(namespaceDef, $"{fileName}.lux");
DefineTypes(fileDef, fileName);
}
static void Main(string[] args)
{
try {
// before using the ArgCollection class, let the DigipedeClient grab its args.
DigipedeClient client = new DigipedeClient();
args = client.ProcessArguments(args, true);
ArgCollection argCollection = ArgCollection.GetInstance(args);
// show help, if requested
if (argCollection.ExtractOptionalFlag("help"))
{
Console.WriteLine("");
Console.WriteLine(UsageMessage);
Console.WriteLine(HelpMessage);
return;
}
// fail if the DigipedeClient doesn't have the args
SpecialFunctions.CheckCondition(client.IsInitialized, "Digipede Client didn't initialize.");
string keepTestName = argCollection.ExtractOptional("keepTest", "AlwaysKeep");
string skipRowIndexFileNameOrNull = argCollection.ExtractOptional <string>("skipRowIndexFile", null);
string optimizerName = argCollection.ExtractOptional("optimizer", "BrentThenGrid");
argCollection.CheckNoMoreOptions();
int pieceCount = argCollection.ExtractNext <int>("pieceCount");
//int taskPerJobCount = argCollection.ExtractNext<int>("taskPerJobCount"); -- we're not using this -- keep it in case we want to put it back
string treeFileName = argCollection.ExtractNext <string>("treeFile");
string predictorFileName = argCollection.ExtractNext <string>("predictorFile");
string targetFileName = argCollection.ExtractNext <string>("targetFile");
string leafDistributionName = argCollection.ExtractNext <string>("leafDistribution");
string nullDataGeneratorName = argCollection.ExtractNext <string>("nullDataGenerator");
string niceName = argCollection.ExtractNext <string>("niceName");
string outputDirectoryName = argCollection.ExtractNext <string>("outputDirectory");
RangeCollection nullIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("nullIndexRange");
SpecialFunctions.CheckCondition(nullIndexRangeCollection.IsBetween(-1, int.MaxValue), "nullIndex must be at least -1");
argCollection.CheckThatEmpty();
Directory.CreateDirectory(outputDirectoryName);
// Define a JobTemplate for PhyloD.
JobTemplate jobTemplate = CreateJobTemplate();
// Require 32 bit (ensures we use WOW64 on 64-bit machines) since SpecialFunctions.dll built for x86.
jobTemplate.Control.UseWow64On64Bit = true;
// allow task failures (all but one failure will result in job success)
jobTemplate.JobDefaults.MaxFailures = pieceCount - 1;
// allow multiple concurrent tasks (one for each core); each isolated in its own process.
jobTemplate.Control.Concurrency = ApplicationConcurrency.MultiplePerCore;
jobTemplate.Control.ProcessHostingOptions = HostingOptions.ManySingleUse;
// create a Job based on that JobTemplate
Job job = jobTemplate.NewJob();
// add job-specific data / files
FileDefCollection fileDefs = job.FileDefs;
// files
Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).RemoteName = treeFileName;
Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).RemoteName = predictorFileName;
Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).RemoteName = targetFileName;
// skipRowIndex file is more complicated because it may not exist, but the JobTemplate requires it.
FileDef fileDef = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName);
if (skipRowIndexFileNameOrNull == null || skipRowIndexFileNameOrNull == "null")
{
// stream an empty file
fileDef.Stream = new MemoryStream(0);
}
else
{
// stream the actual file
fileDef.LocalName = skipRowIndexFileNameOrNull;
}
// Create the tasks for the template
for (int pieceIndex = 0; pieceIndex < pieceCount; pieceIndex++)
{
// Create a Task for this piece
Task task = job.NewTask();
// Create an InputData object to encapsulate all input data in one place
InputData inputData = new InputData(optimizerName, keepTestName, leafDistributionName, nullDataGeneratorName,
niceName, outputDirectoryName, pieceIndex, pieceCount, nullIndexRangeCollection.ToString());
// create a Worker for this task
task.Worker = new PhyloDWorker(inputData);
}
// Wire events to catch result data. Note that retrieving data isn't necessary here --
// data can be requested in a server call from another process.
job.TaskCompleted += job_TaskCompleted;
// Write an event to catch any monitoring errors
client.MonitoringError += client_MonitoringError;
// submit the job
SubmissionResult sr = client.SubmitJob(jobTemplate, job);
Console.WriteLine("Submitted job {0} with {1} tasks.", sr.JobId, job.Tasks.Count);
// wait for the result
JobStatusSummary jss = client.WaitForJobWithStatus(sr.JobId);
Console.WriteLine("Job finished with status of {0}", jss.Status);
}
catch (Exception exception)
{
Console.WriteLine("");
Console.WriteLine(exception.Message);
if (exception.InnerException != null)
{
Console.WriteLine(exception.InnerException.Message);
}
Console.WriteLine("");
Console.WriteLine(UsageMessage);
throw;
}
}
