/// <summary>
/// Saves the model data set.
/// </summary>
/// <returns></returns>
public WeightingModelDataSet SaveModelDataSet(ILogBuilder log)
{
WeightingModelDataSet output = new WeightingModelDataSet();
output.modelData.Add(LocalFunction.SaveModelData());
foreach (FeatureWeightFactor gf in GlobalFactors)
{
output.modelData.Add(gf.GlobalFunction.SaveModelData());
}
return(output);
}
/// <summary>
/// Prepares the specified context.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="log">The log.</param>
/// <exception cref="ArgumentException">context</exception>
public override void Prepare(DocumentSelectResult context, ILogBuilder log)
{
String p_m = FeatureWeightModel.GetModelDefinitionFilename(modelDefinitionFile, context.folder);
String p_d = FeatureWeightModel.GetModelDataFilename(modelDefinitionFile, context.folder);
TermWeightModel = FeatureWeightModel.LoadModel(p_m, log);
//if (File.Exists(p_m))
//{
// //TermWeightModel = objectSerialization.loadObjectFromXML<FeatureWeightModel>(p_m, log);
//}
TermWeightModel.Deploy(log);
if (context.spaceModel == null)
{
String msg = "Error: TermWeight factor requires SpaceModel declared in the context for operation";
throw new ArgumentException(msg, nameof(context));
}
if (File.Exists(p_d) && useStoredData)
{
WeightingModelDataSet data = objectSerialization.loadObjectFromXML <WeightingModelDataSet>(p_d, log);
TermWeightModel.LoadModelDataSet(data, log);
if (useSelectedFeatures)
{
SelectedTerms = WeightDictionary.LoadFile(WeightDictionary.GetDictionaryFilename(modelDefinitionFile + "_sf", context.folder), log);
}
}
else
{
TermWeightModel.PrepareTheModel(context.spaceModel, log);
}
if (context.query.isNullOrEmpty())
{
context.query.QueryTerms = context.query.QueryTerms.Trim();
List <String> tkns = context.query.QueryTerms.getTokens(true, true, true, false, 4);
foreach (String tkn in tkns)
{
queryTerms.Add(context.stemmingContext.Stem(tkn));
}
}
}
/// <summary>
/// Loads the specified output filename.
/// </summary>
/// <param name="outputFilename">The output filename.</param>
/// <param name="folder">The folder.</param>
/// <param name="logger">The logger.</param>
/// <param name="loadData">if set to <c>true</c> [load data].</param>
/// <returns></returns>
public static FeatureWeightModel Load(String outputFilename, folderNode folder, ILogBuilder logger, Boolean loadData = true)
{
String p_m = FeatureWeightModel.GetModelDefinitionFilename(outputFilename, folder);
String p_d = FeatureWeightModel.GetModelDataFilename(outputFilename, folder);
FeatureWeightModel output = objectSerialization.loadObjectFromXML <FeatureWeightModel>(p_m, logger);
if (loadData)
{
WeightingModelDataSet dataset = objectSerialization.loadObjectFromXML <WeightingModelDataSet>(p_d, logger);
output.LoadModelDataSet(dataset, logger);
}
return(output);
}
/// <summary>
/// Loads the model data set.
/// </summary>
/// <param name="dataset">The dataset.</param>
public void LoadModelDataSet(WeightingModelDataSet dataset, ILogBuilder log)
{
Dictionary <string, WeightingModelData> dict = dataset.GetDataDictionary();
if (dict.ContainsKey(LocalFunction.shortName))
{
var localData = dict[LocalFunction.shortName];
if (DoUseLocalFunction)
{
LocalFunction.LoadModelData(localData);
}
}
else
{
log.log("Loaded data set contains no data for [" + LocalFunction.shortName + "]");
}
foreach (FeatureWeightFactor gf in GlobalFactors)
{
gf.GlobalFunction.LoadModelData(dict[gf.GlobalFunction.shortName]);
}
}
/// <summary>
/// Prepares the specified context.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="log">The log.</param>
/// <exception cref="ArgumentException">context</exception>
public override void Prepare(DocumentSelectResult context, ILogBuilder log)
{
//context.folder.GetOrFindFiles("*", dictionaryFile + "*.xml");
scoreDictionary = FeatureVectorDictionaryWithDimensions.LoadFile(context.folder, dictionaryFile, log); // WeightDictionary.LoadFile(WeightDictionary.GetDictionaryFilename(dictionaryFile, context.folder), log);
if (scoreDictionary == null)
{
String msg = "Error: Failed to find score dictionary [" + dictionaryFile + "] in " + context.folder.path;
throw new ArgumentException(msg, nameof(context));
}
if (useMachineLearning)
{
#region --------------- PREPARING TERM WEIGHT MODEL
String p_m = FeatureWeightModel.GetModelDefinitionFilename(modelDefinitionFile, context.folder);
String p_d = FeatureWeightModel.GetModelDataFilename(modelDefinitionFile, context.folder);
if (TermWeightModel == null)
{
TermWeightModel = FeatureWeightModel.LoadModel(p_m, log);
}
TermWeightModel.Deploy(log);
if (context.spaceModel == null)
{
String msg = "Error: TermWeight factor requires SpaceModel declared in the context for operation";
throw new ArgumentException(msg, nameof(context));
}
if (File.Exists(p_d) && useStoredData)
{
WeightingModelDataSet data = objectSerialization.loadObjectFromXML <WeightingModelDataSet>(p_d, log);
TermWeightModel.LoadModelDataSet(data, log);
if (useSelectedFeatures)
{
SelectedTerms = WeightDictionary.LoadFile(WeightDictionary.GetDictionaryFilename(modelDefinitionFile + "_sf", context.folder), log);
}
}
else
{
TermWeightModel.PrepareTheModel(context.spaceModel, log);
}
if (SelectedTerms.Count == 0)
{
SelectedTerms = context.selectedFeatures;
}
List <String> sel_tkns = new List <String>();
sel_tkns.AddRange(SelectedTerms.index.Values.Select(x => x.name));
if (!sel_tkns.Any())
{
sel_tkns.AddRange(context.spaceModel.terms_known_label.GetTokens());
}
#endregion
fvConstructor.Deploy(featureMethod.constructor, sel_tkns);
classifier = featureMethod.classifierSettings.GetClassifier();
sc_id = scoreDictionary.GetVectorsWithLabelID(null, criterion).ToNameVsLabelID();
List <FeatureVectorWithLabelID> trainingSet = new List <FeatureVectorWithLabelID>();
foreach (var item in context.items)
{
if (sc_id.ContainsKey(item.AssignedID))
{
WeightDictionary dc_vec = TermWeightModel.GetWeights(sel_tkns, item.spaceDocument, context.spaceModel);
var n_vec = fvConstructor.ConstructFeatureVector(dc_vec, item.AssignedID);
FeatureVectorWithLabelID id_vec = new FeatureVectorWithLabelID(n_vec, sc_id[item.AssignedID]);
trainingSet.Add(id_vec);
}
}
log.log("Training [" + classifier.name + "] with [" + sc_id.Count + "] feature vectors.");
classifier.DoTraining(trainingSet, log);
}
}
