public T GetKnowledgeInstance <T>(IDocumentSetClass setClass, kFoldValidationCase validationCase, ILogBuilder logger) where T : class, IWebFVExtractorKnowledge, new()
{
T knowledge = GetKnowledgeInstance <T>("class_" + setClass.name, validationCase, WebFVExtractorKnowledgeType.aboutCategory, logger);
knowledge.relatedItemPureName = setClass.name;
return(knowledge);
}
/// <summary>
/// Performing post processing of FV knowledge
/// </summary>
/// <param name="validationCase">The validation case.</param>
/// <param name="tools">The tools.</param>
/// <param name="logger">The logger.</param>
public override void DoFVPostProcessing(kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger)
{
List <lemmaSemanticCloud> clouds = new List <lemmaSemanticCloud>();
foreach (var docClass in validationCase.context.classes.GetClasses())
{
var knowledge = validationCase.knowledgeLibrary.GetKnowledgeInstance <semanticFVExtractorKnowledge>(docClass, validationCase, logger);
knowledge.semanticCloudFiltered = knowledge.semanticCloud.Clone();
clouds.Add(knowledge.semanticCloudFiltered);
knowledge.semanticCloud.className = docClass.name;
knowledge.semanticCloudFiltered.className = docClass.name + "flt";
if (settings.semanticCloudFilter.isActive)
{
knowledge.semanticCloudFiltered.description = "Semantic cloud filtered with cloud matrix";
}
else
{
knowledge.semanticCloudFiltered.description = "Semantic cloud filter is off - this is initial cloud";
}
}
if (settings.semanticCloudFilter.isActive)
{
logger.log(validationCase.name + ": Cloud matrix creation starts...");
cloudMatrix matrix = new cloudMatrix(validationCase.name, "Cloud overlap matrix of [" + clouds.Count + "] for fold [" + validationCase.name + "] of experiment [" + validationCase.context.setup.name + "]");
matrix.build(clouds, logger);
matrix.TransformClouds(settings.semanticCloudFilter, logger);
if (tools.operation.doMakeGraphForClassClouds)
{
foreach (var cloud in clouds)
{
if (tools.operation.doUseSimpleGraphs)
{
cloud.GetSimpleGraph(true).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite));
}
else
{
var converter = lemmaSemanticCloud.GetDGMLConverter();
converter.ConvertToDMGL(cloud).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite));
}
}
}
//logger.log(validationCase.name + ": Cloud matrix report creation ...");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_initial");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_initial");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_reduced");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_reduced");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.maxCloudFrequency | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_CF_norm_reduced");
// logger.log(validationCase.name + ": Cloud matrix report done.");
}
else
{
logger.log(validationCase.name + ": Cloud matrix is not active");
}
}
protected T CreateKnowledgeInstance <T>(String name, kFoldValidationCase validationCase, WebFVExtractorKnowledgeType type, ILogBuilder logger) where T : class, IWebFVExtractorKnowledge, new()
{
folderNode folder = validationCase.caseFolder;
if (type == WebFVExtractorKnowledgeType.aboutDocumentSet)
{
folder = validationCollection.caseFolder;
if (DoShareCaseKnowledge)
{
folder = ExperimentSharedCasesFolder;
}
}
T knowledge = new T();
knowledge.name = name;
knowledge.type = type;
try
{
knowledge.Deploy(folder, logger);
} catch (aceGeneralException ex)
{
validationCase.context.errorNotes.LogException("Create Knowledge Instance (" + typeof(T).Name + ") error for [" + name + "] in fold [" + validationCase.name + "]:" + ex.title, ex);
}
return(knowledge);
}
public void SaveKnowledgeInstancesForClasses(kFoldValidationCase validationCase, ILogBuilder logger)
{
logger.log("Saving knowledge objects for [" + validationCase.name + "] classes");
foreach (IWebFVExtractorKnowledge knowledge in registry[validationCase].Values)
{
knowledge.OnBeforeSave();
}
logger.log("Knowledge saved for [" + validationCase.name + "] classes");
}
public void SaveCaseKnowledge <T>(DocumentSetCase setCase, kFoldValidationCase validationCase, ILogBuilder logger) where T : class, IWebFVExtractorKnowledge, new()
{
IWebFVExtractorKnowledge knowledge = GetKnowledgeInstance <T>(setCase, validationCase, logger);
if (!savedKnowledge.Contains(knowledge))
{
lock (savedKnowledgeLock)
{
if (!savedKnowledge.Contains(knowledge))
{
savedKnowledge.Add(knowledge);
knowledge.OnBeforeSave();
}
}
}
}
/// <summary>
/// Deploys the specified v case coll.
/// </summary>
/// <param name="vCaseColl">The v case coll.</param>
/// <param name="validationCase">The validation case.</param>
/// <param name="subjects">The subjects.</param>
/// <param name="classes">The classes.</param>
public void deploy(validationCaseCollection vCaseColl, kFoldValidationCase _validationCase, IEnumerable <IPipelineTaskSubject> subjects, DocumentSetClasses classes)
{
lock (deployLock)
{
validation = vCaseColl;
validationCase = _validationCase;
Thread.Sleep(100);
var sites = subjects.ToSubjectTokenType <pipelineTaskMCSiteSubject>().ToList();
// var sitesCopy = sites.ToList();
sites = vCaseColl.FilterSites(sites);
foreach (pipelineTaskMCSiteSubject site in sites)
{
DocumentSetCase sCase = new DocumentSetCase();
sCase.subject = site;
sCase.data = new WebClassifierResultSet(classes, validationCase.context, validationCase.extractor.settings);
Add(sCase);
}
}
}
public override void DoFVPostProcessing(kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger)
{
}
public override tfidfFVExtractorKnowledge DoFVExtractionForClassViaCases(validationCaseCollection vCaseColl, IDocumentSetClass documentSetClass, kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger)
{
throw new NotImplementedException();
}
/// <summary>
/// Gets the knowledge instance.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="name">The name.</param>
/// <param name="validationCase">The validation case.</param>
/// <param name="type">The type.</param>
/// <param name="logger">The logger.</param>
/// <returns></returns>
protected T GetKnowledgeInstance <T>(String name, kFoldValidationCase validationCase, WebFVExtractorKnowledgeType type, ILogBuilder logger) where T : class, IWebFVExtractorKnowledge, new()
{
IWebFVExtractorKnowledge knowledge = null;
Boolean found = false;
if (type == WebFVExtractorKnowledgeType.aboutCategory)
{
if (!registry[validationCase].TryGetValue(name, out knowledge))
{
lock (createKnowledgeLock)
{
if (!registry[validationCase].ContainsKey(name))
{
statistics[name] = 0;
knowledge = CreateKnowledgeInstance <T>(name, validationCase, type, logger);
registry[validationCase][name] = knowledge;
}
else
{
knowledge = registry[validationCase][name];
}
}
}
else
{
found = true;
}
}
else
{
if (!registryForCases.TryGetValue(name, out knowledge))
{
lock (createCaseKnowledgeLock)
{
if (!registryForCases.ContainsKey(name))
{
statistics[name] = 0;
knowledge = CreateKnowledgeInstance <T>(name, validationCase, type, logger);
registryForCases[name] = knowledge;
}
else
{
knowledge = registryForCases[name];
}
}
}
else
{
found = true;
}
}
if (doKeepStatisticsOnAccess)
{
if (found)
{
statistics[name]++;
}
}
return(knowledge as T);
}
///// <summary>
///// Report on kFold case
///// </summary>
///// <returns></returns>
//public Dictionary<IWebPostClassifier, DocumentSetCaseCollectionReport> GetCaseReport()
//{
// Dictionary<IWebPostClassifier, DocumentSetCaseCollectionReport> reports = new Dictionary<IWebPostClassifier, DocumentSetCaseCollectionReport>();
// foreach (var classifier in validationCase.context.setup.classifiers)
// {
// DocumentSetCaseCollectionReport output = new DocumentSetCaseCollectionReport(validationCase.name + "_" + classifier.name + "_" + );
// output.kFoldCase = ;
// output.Classifier = classifier.name;
// classificationEvalMetricSet metrics = GetMetrics(classifier);
// foreach (KeyValuePair<int, DocumentSetCaseCollection> pair in this)
// {
// String className = pair.Value.validation.className;
// foreach (var setCase in pair.Value)
// {
// var assocClass = setCase.data[classifier].selected;
// if (assocClass != null)
// {
// if (pair.Value.rightClassID == assocClass.classID)
// {
// output.Correct++;
// }
// else
// {
// output.Wrong++;
// }
// }
// output.Targets++;
// }
// }
// output.Precision = metrics.GetPrecision();
// output.Recall = metrics.GetRecall();
// output.F1measure = metrics.GetF1();
// reports.Add(classifier, output);
// }
// return reports;
//}
public DocumentSetCaseCollectionSet(kFoldValidationCase _validationCase, DocumentSetClasses documentSetClasses)
{
classCollection = documentSetClasses;
validationCase = _validationCase;
}
public override semanticFVExtractorKnowledge DoFVExtractionForClassViaCases(validationCaseCollection vCaseColl, IDocumentSetClass documentSetClass, kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger)
{
semanticFVExtractorKnowledge knowledge = vCaseColl.kFoldMaster.knowledgeLibrary.GetKnowledgeInstance <semanticFVExtractorKnowledge>(documentSetClass, vCaseColl.kFoldCase, logger);
knowledge.SetRebuild(!tools.DoUseExistingKnowledge);
if (knowledge.ShouldBuildAny())
{
DocumentSetCaseCollection dSetCol = new DocumentSetCaseCollection(documentSetClass);
var context = tools.context.pipelineCollection.GetContext(tools, documentSetClass);
//var sites = context.exitByType[typeof(pipelineTaskMCSiteSubject)].ConvertList<IPipelineTaskSubject, pipelineTaskMCSiteSubject>().ToList();
var sites = context.exitByType[typeof(pipelineTaskMCSiteSubject)].ToList();
List <pipelineTaskMCSiteSubject> ISites = sites.ConvertList <IPipelineTaskSubject, pipelineTaskMCSiteSubject>().ToList();
List <pipelineTaskMCSiteSubject> fSites = vCaseColl.FilterSites(ISites);
dSetCol.deploy(vCaseColl, validationCase, fSites, classes);
List <webLemmaTermTable> tables = new List <webLemmaTermTable>();
//List<webLemmaTermTable> chunkTables = new List<webLemmaTermTable>();
foreach (DocumentSetCase vc in dSetCol)
{
semanticFVExtractorKnowledge cKnowledge = vCaseColl.kFoldMaster.knowledgeLibrary.GetKnowledgeInstance <semanticFVExtractorKnowledge>(vc, validationCase, logger);
DoMakeKnowledgeForCase(vc, tools, dSetCol, logger);
tables.Add(cKnowledge.WLTableOfIndustryClass);
}
var tbl = tables.GetMergedLemmaTable(knowledge.name, logger);
termTableConstructor.recompute(knowledge.WLTableOfIndustryClass, logger, false, tbl.GetList());
DoMakeKnowledge(fSites, tools, knowledge, logger);
}
// SetKnowledge(knowledge);
//knowledge.OnBeforeSave();
logger.log("[ALTPROC] Feature Extraction by [" + name + "][" + vCaseColl.kFoldCase.name + "][" + documentSetClass.name + "] done for " + vCaseColl.className);
return(knowledge);
}
public void AddNote(String note, kFoldValidationCase validationCase)
{
notes[validationCase.name].Add(note);
log(validationCase.name + "> " + note);
}
public List<String> GetNotes(kFoldValidationCase validationCase)
{
return notes[validationCase.name].ToList();
}
