private EvaluationContext GetEvaluationContext(string contextId)
{
XElement contextEl = ConfigRoot.Descendants("evalContext")
.Where(el => el.Attribute("id").Value == contextId).Single();
var ec = new EvaluationContext();
ec.Id = contextEl.Attribute("id").Value;
foreach (XElement evalEl in contextEl.Descendants("evaluator"))
{
Evaluator eval;
Type evalType = Helpers.ResolveType(evalEl.Attribute("class").Value.Inject(Parameters));
if (!typeof(Evaluator).IsAssignableFrom(evalType))
{
throw new WrapRecException(string.Format("Evaluator type '{0}' should inherit class 'WrapRec.Evaluation.Evaluator'", evalType.Name));
}
eval = (Evaluator)evalType.GetConstructor(Type.EmptyTypes).Invoke(null);
eval.SetupParameters = evalEl.Attributes().ToDictionary(a => a.Name.LocalName, a => a.Value.Inject(Parameters));
ec.AddEvaluator(eval);
}
return(ec);
}
private DataContainer GetDataContainer(string containerId)
{
if (DataContainers.ContainsKey(containerId))
{
return(DataContainers[containerId]);
}
XElement dcEl = ConfigRoot.Descendants("dataContainer")
.Where(el => el.Attribute("id").Value.Inject(Parameters) == containerId).Single();
bool allowDup = false;
if (dcEl.Attribute("allowDuplicates") != null && dcEl.Attribute("allowDuplicates").Value == "true")
{
allowDup = true;
}
var container = new DataContainer()
{
AllowDuplicates = allowDup
};
container.Id = containerId;
foreach (string readerId in dcEl.Attribute("dataReaders").Value.Inject(Parameters).Split(','))
{
container.DataReaders.Add(ParseDataReader(readerId));
}
DataContainers.Add(containerId, container);
return(container);
}
private IEnumerable <Model> ParseModelsSet(string modelId)
{
XElement modelEl = ConfigRoot.Descendants("model")
.Where(el => el.Attribute("id").Value == modelId).Single();
Type modelType = Helpers.ResolveType(modelEl.Attribute("class").Value.Inject(Parameters));
if (!typeof(Model).IsAssignableFrom(modelType))
{
throw new WrapRecException(string.Format("Model type '{0}' should inherit class 'WrapRec.Models.Model'", modelType.Name));
}
var allSetupParams = modelEl.Descendants("parameters").Single()
.Attributes().ToDictionary(a => a.Name, a => a.Value.Inject(Parameters));
var paramCartesians = allSetupParams.Select(kv => kv.Value.Inject(Parameters).Split(',').AsEnumerable()).CartesianProduct();
foreach (IEnumerable <string> pc in paramCartesians)
{
var setupParams = allSetupParams.Select(kv => kv.Key)
.Zip(pc, (k, v) => new { Name = k, Value = v })
.ToDictionary(kv => kv.Name.LocalName, kv => kv.Value.Inject(Parameters));
var model = (Model)modelType.GetConstructor(Type.EmptyTypes).Invoke(null);
model.Id = modelId;
model.SetupParameters = setupParams;
yield return(model);
}
}
public static DatasetReader ParseDataReader(string readerId)
{
XElement readerEl = ConfigRoot.Descendants("reader")
.Where(el => el.Attribute("id").Value.Inject(Parameters) == readerId).Single();
FeedbackSlice sliceType = FeedbackSlice.NOT_APPLICABLE;
XAttribute sliceTypeAttr = readerEl.Attribute("sliceType");
if (sliceTypeAttr != null)
{
if (sliceTypeAttr.Value.Inject(Parameters) == "train")
{
sliceType = FeedbackSlice.TRAIN;
}
else if (sliceTypeAttr.Value.Inject(Parameters) == "test")
{
sliceType = FeedbackSlice.TEST;
}
}
DataType dataType = DataType.Other;
XAttribute dataTypeAttr = readerEl.Attribute("dataType");
if (dataTypeAttr != null)
{
dataType = (DataType)Enum.Parse(typeof(DataType), dataTypeAttr.Value.Inject(Parameters), true);
}
DatasetReader reader;
if (readerEl.Attribute("class") != null)
{
Type readerClassType = Helpers.ResolveType(readerEl.Attribute("class").Value.Inject(Parameters));
if (!typeof(DatasetReader).IsAssignableFrom(readerClassType))
{
throw new WrapRecException(string.Format("Reader type '{0}' should inherit class 'WrapRec.Data.DatasetReader'", readerClassType.Name));
}
reader = (DatasetReader)readerClassType.GetConstructor(Type.EmptyTypes).Invoke(null);
}
else
{
reader = new CsvReader();
}
reader.Id = readerId;
reader.Path = readerEl.Attribute("path").Value.Inject(Parameters);
reader.DataType = dataType;
reader.SliceType = sliceType;
reader.SetupParameters = readerEl.Attributes().ToDictionary(a => a.Name.LocalName, a => a.Value.Inject(Parameters));
return(reader);
}
private Split ParseSplit(string splitId)
{
XElement splitEl = ConfigRoot.Descendants("split")
.Where(el => el.Attribute("id").Value.Inject(Parameters) == splitId).Single();
SplitType splitType = (SplitType)Enum.Parse(typeof(SplitType), splitEl.Attribute("type").Value.Inject(Parameters).ToUpper());
DataContainer container = GetDataContainer(splitEl.Attribute("dataContainer").Value.Inject(Parameters));
Split split;
if (splitEl.Attribute("class") != null)
{
Type splitClassType = Helpers.ResolveType(splitEl.Attribute("class").Value.Inject(Parameters));
if (!typeof(Split).IsAssignableFrom(splitClassType))
{
throw new WrapRecException(string.Format("Split type '{0}' should inherit class 'WrapRec.Data.Split'", splitClassType.Name));
}
split = (Split)splitClassType.GetConstructor(Type.EmptyTypes).Invoke(null);
}
else
{
split = new FeedbackSimpleSplit();
}
var setupParams = splitEl.Attributes().ToDictionary(a => a.Name.LocalName, a => a.Value.Inject(Parameters));
split.Id = splitId;
split.Type = splitType;
split.Container = container;
split.SetupParameters = setupParams;
// Splits are required to be Setuped when they are being created to make sure
// the subSplits are being formed (this is necessary for CrossValidation) becuase
// the number of experiments is determined based on the number of SubSplits
split.Setup();
return(split);
}
private void Setup(Dictionary <string, string> overrideParams)
{
try
{
XElement allExpEl = ConfigRoot.Descendants("experiments").Single();
Parameters = ConfigRoot.Descendants("param").ToDictionary(p => p.Attribute("name").Value, p => p.Attribute("value").Value);
if (overrideParams != null)
{
foreach (string name in overrideParams.Keys)
{
if (!Parameters.ContainsKey(name))
{
Console.WriteLine("Parameter {0} is not defined in the configuration file.", name);
Environment.Exit(1);
}
Parameters[name] = overrideParams[name];
}
}
if (allExpEl.Attribute("verbosity") != null)
{
if (allExpEl.Attribute("verbosity").Value.Inject(Parameters).ToLower() == "trace")
{
Logger.Current = NLog.LogManager.GetLogger("traceLogger");
}
else if (allExpEl.Attribute("verbosity").Value.Inject(Parameters).ToLower() == "warn")
{
Logger.Current = NLog.LogManager.GetLogger("warnLogger");
}
}
ResultSeparator = allExpEl.Attribute("separator") != null?allExpEl.Attribute("separator").Value.Replace("\\t", "\t") : ",";
RunParallel = allExpEl.Attribute("parallel") != null && allExpEl.Attribute("parallel").Value == "true" ? true : false;
bool subFolder = allExpEl.Attribute("subFolder") != null && allExpEl.Attribute("subFolder").Value == "true" ? true : false;
string expFolder = subFolder ? DateTime.Now.ToString("wr yyyy-MM-dd HH.mm", CultureInfo.InvariantCulture) : "";
string rootPath = allExpEl.Attribute("resultsFolder") != null?allExpEl.Attribute("resultsFolder").Value.Inject(Parameters)
: Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
ResultsFolder = Directory.CreateDirectory(Path.Combine(rootPath, expFolder)).FullName;
JointFile = allExpEl.Attribute("jointResults") != null?
Path.Combine(ResultsFolder, allExpEl.Attribute("jointResults").Value.Inject(Parameters)) : "";
XAttribute runAttr = allExpEl.Attribute("run");
IEnumerable <XElement> expEls = allExpEl.Descendants("experiment");
if (runAttr != null)
{
var expIds = runAttr.Value.Inject(Parameters).Split(',');
_resultWriters = expIds.ToDictionary(eId => eId, eId => new StreamWriter(Path.Combine(ResultsFolder, eId + ".csv")));
_statWriters = expIds.ToDictionary(eId => eId, eId => new StreamWriter(Path.Combine(ResultsFolder, eId + ".splits.csv")));
_errorWriters = expIds.ToDictionary(eId => eId, eId => new StreamWriter(Path.Combine(ResultsFolder, eId + ".err.txt")));
_loggedSplits = expIds.ToDictionary(eId => eId, eId => new List <string>());
_loggedModels = expIds.ToDictionary(eId => eId, eId => new List <string>());
expEls = expEls.Where(el => expIds.Contains(el.Attribute("id").Value.Inject(Parameters)));
ExperimentIds = expEls.Select(el => el.Attribute("id").Value.Inject(Parameters)).ToArray();
}
Logger.Current.Info("Resolving experiments...");
Experiments = expEls.SelectMany(el => ParseExperiments(el));
}
catch (Exception ex)
{
Logger.Current.Error("Error in setuping experiments or parsing the configuration file: {0}\n{1}\n", ex.Message, ex.StackTrace);
//Environment.Exit(1);
}
}