/// <summary>
/// Loads the specified folder.
/// </summary>
/// <param name="folder">The folder - to be scanned for experiment data</param>
/// <param name="logger">The logger.</param>
public void Load(folderNode _folder, ILogBuilder logger)
{
folder = _folder;
try
{
var experimentFiles = folder.findFiles("experimentSetup.xml", SearchOption.AllDirectories, false);
foreach (var path in experimentFiles)
{
try
{
var exp = new secondaryReportOnExperiment(path, logger, folder);
items.Add(exp);
allItems.AddRange(exp.items);
allTopItems.Add(exp.topPerformer);
// logger.log("Experiment report processed [" + path + "] : [" + allItems.Count + "]");
}
catch (Exception ex)
{
logger.log(ex.LogException("secondaryReport.Load", "SECREP_LOAD=>" + path));
}
}
}
catch (Exception ex)
{
logger.log(ex.LogException("secondaryReport.Load", "SECREP_LOAD"));
}
}
public void Save(ILogBuilder loger = null)
{
try
{
setContentLines(contentLines.ToList());
if (loger != null)
{
if (!info.Exists)
{
loger.log("File [" + path + "] not saved since the content is empty.");
}
else
{
loger.log("File [" + path + "] saved. Size: " + imbStringFormats.getKByteCountFormated(getByteSize()) + " - lines: " + contentLines.Count());
}
}
Accept();
}
catch (Exception ex)
{
if (loger != null)
{
loger.log("File [" + path + "] not saved due exception: " + ex.Message);
}
}
}
/// <summary>
/// Applies the index of the manual page --- loads external, manually authored truth index
/// </summary>
/// <param name="manualPageIndex">Index of the manual page.</param>
/// <param name="loger">The loger.</param>
/// <param name="callDomainIndexRecheck">if set to <c>true</c> [call domain index recheck].</param>
public void ApplyManualPageIndex(string manualPageIndex = "pageIndex_revision.xlsx", ILogBuilder loger = null, bool callDomainIndexRecheck = true)
{
string path = folder.pathFor(manualPageIndex);
if (File.Exists(path))
{
DataTable manualIndex = path.deserializeDataTable(dataTableExportEnum.excel, folder);
int loaded = pageIndexTable.Load(manualIndex, loger, objectTableUpdatePolicy.overwrite);
if (loger != null)
{
loger.log("Page index entries updated: " + loaded);
}
if (callDomainIndexRecheck)
{
Recheck(loger);
}
isFullTrustMode = true;
}
else
{
loger.log("Excel file not found on: " + path);
}
}
/// <summary>
/// Loads the semantic clouds of the specified FVE
/// </summary>
/// <param name="fve">The fve.</param>
/// <param name="logger">The logger.</param>
public void LoadSemanticClouds(semanticFVExtractor fve, ILogBuilder logger)
{
DirectoryInfo di = folder;
var dirs = di.GetDirectories(fve.name);
if (!dirs.Any())
{
logger.log("Failed to find subfolder for FVE [" + fve.name + "]");
}
else
{
DirectoryInfo dir = dirs.First();
folderNode fveFolder = dir;
var allCloudFiles = fveFolder.findFiles("*Cloud.xml", SearchOption.AllDirectories);
Int32 cl = 0;
foreach (String cloudFile in allCloudFiles)
{
if (cloudFile.Contains("General") || cloudFile.Contains("SharedKnowledge"))
{
}
else
{
semanticClouds.Add(fve.name, objectSerialization.loadObjectFromXML <lemmaSemanticCloud>(cloudFile, logger));
cl++;
}
}
logger.log("Semantic clouds loaded [" + cl + "] for " + fve.name);
}
}
/// <summary>
/// Loads the bin.
/// </summary>
/// <param name="folder">The folder.</param>
/// <param name="logger">The logger.</param>
/// <param name="filenamePrefix">The filename prefix.</param>
public Boolean LoadBin(folderNode folder, ILogBuilder logger, String filenamePrefix = "lexicResource")
{
var files = folder.findFiles(filenamePrefix + "_*.bin", SearchOption.TopDirectoryOnly);
Int32 c = 0;
foreach (var pair in files)
{
String filename = Path.GetFileNameWithoutExtension(pair);
String letter = filename.Replace(filenamePrefix + "_", "");
ConcurrentDictionary <String, lexicInflection> dict = Accord.IO.Serializer.Load <ConcurrentDictionary <String, lexicInflection> >(pair);
if (dict.Any())
{
c++;
logger.log("File [" + filename + "] loaded --> index [" + letter + "]");
if (items.ContainsKey(letter))
{
items[letter].Clear();
items[letter].AddRange(dict);
}
else
{
items.TryAdd(letter, dict);
}
}
}
if (c > 0)
{
logger.log("[" + c + "] lexic files loaded from [" + folder.path + "]");
return(true);
}
return(false);
}
/// <summary>
/// Loads the table from the specified filepath
/// </summary>
/// <param name="filepath">The filepath.</param>
public void Load(string filepath, ILogBuilder loger)
{
sourceFile = new fileunit(filepath, true);
int cl = sourceFile.contentLines.Count;
loger.log("Lexicon terms coding twins definitions: " + cl);
int lind = 0;
int lmax = cl / 20;
lind = lmax;
LoadCount = 0;
foreach (string ln in sourceFile.contentLines)
{
lind--;
SetEntryFromString(ln);
LoadCount++;
if (lind <= 0)
{
lind = lmax;
loger.log("Coding twins loaded: " + LoadCount);
}
}
loger.log("Coding twins completly loaded: " + LoadCount);
}
/// <summary>
/// Performing post processing of FV knowledge
/// </summary>
/// <param name="validationCase">The validation case.</param>
/// <param name="tools">The tools.</param>
/// <param name="logger">The logger.</param>
public override void DoFVPostProcessing(kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger)
{
List <lemmaSemanticCloud> clouds = new List <lemmaSemanticCloud>();
foreach (var docClass in validationCase.context.classes.GetClasses())
{
var knowledge = validationCase.knowledgeLibrary.GetKnowledgeInstance <semanticFVExtractorKnowledge>(docClass, validationCase, logger);
knowledge.semanticCloudFiltered = knowledge.semanticCloud.Clone();
clouds.Add(knowledge.semanticCloudFiltered);
knowledge.semanticCloud.className = docClass.name;
knowledge.semanticCloudFiltered.className = docClass.name + "flt";
if (settings.semanticCloudFilter.isActive)
{
knowledge.semanticCloudFiltered.description = "Semantic cloud filtered with cloud matrix";
}
else
{
knowledge.semanticCloudFiltered.description = "Semantic cloud filter is off - this is initial cloud";
}
}
if (settings.semanticCloudFilter.isActive)
{
logger.log(validationCase.name + ": Cloud matrix creation starts...");
cloudMatrix matrix = new cloudMatrix(validationCase.name, "Cloud overlap matrix of [" + clouds.Count + "] for fold [" + validationCase.name + "] of experiment [" + validationCase.context.setup.name + "]");
matrix.build(clouds, logger);
matrix.TransformClouds(settings.semanticCloudFilter, logger);
if (tools.operation.doMakeGraphForClassClouds)
{
foreach (var cloud in clouds)
{
if (tools.operation.doUseSimpleGraphs)
{
cloud.GetSimpleGraph(true).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite));
}
else
{
var converter = lemmaSemanticCloud.GetDGMLConverter();
converter.ConvertToDMGL(cloud).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite));
}
}
}
//logger.log(validationCase.name + ": Cloud matrix report creation ...");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_initial");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_initial");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_reduced");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_reduced");
// matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.maxCloudFrequency | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_CF_norm_reduced");
// logger.log(validationCase.name + ": Cloud matrix report done.");
}
else
{
logger.log(validationCase.name + ": Cloud matrix is not active");
}
}
/// <summary>
/// Preloads the lexicon.
/// </summary>
/// <param name="loger">The loger.</param>
/// <param name="context">The context.</param>
public void preloadLexicon(ILogBuilder loger, semanticLexiconContext context)
{
loger.log("Preloading Semantic Lexicon");
if (isLexiconPreloaded)
{
loger.log("Semantic Lexicon is already loaded");
return;
}
double ratio = 0;
int i = 0;
int repIndex = lemmaCount / 20;
foreach (ITermLemma lemma in context.TermLemmas)
{
Add(lemma);
i++;
repIndex--;
if (repIndex == 0)
{
repIndex = lemmaCount / 20;
ratio = (double)i / (double)lemmaCount;
loger.log("Loaded [" + i + "] lemmas (" + ratio.ToString("P2") + ")");
}
}
doAllowLexiconQuery = false;
isLexiconPreloaded = true;
loger.log("Semantic Lexicon preload done");
}
/// <summary>
/// Initializes loading of plugins
/// </summary>
/// <param name="output">The output.</param>
protected void loadPlugins(ILogBuilder output)
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
Type pluginType = typeof(T);
foreach (Assembly ass in assemblies)
{
try
{
if (ass.FullName.StartsWith("imb") || AllowNonImbAssemblies)
{
var types = ass.GetTypes();
foreach (Type type in types)
{
if (type.IsInterface || type.IsAbstract)
{
continue;
}
else
{
if (type.GetInterface(pluginType.FullName) != null)
{
registerPlugin(type, type.Namespace, output);
}
}
}
}
} catch (Exception ex)
{
output.log("Assembly [" + ass.FullName + "] type harvest failed");
output.log("Exception: " + ex.Message);
}
}
}
/// <summary>
/// Saves the bin.
/// </summary>
/// <param name="folder">The folder.</param>
/// <param name="logger">The logger.</param>
/// <param name="skipExisting">if set to <c>true</c> [skip existing].</param>
/// <param name="filenamePrefix">The filename prefix.</param>
public Boolean SaveBin(folderNode folder, ILogBuilder logger, Boolean skipExisting = true, String filenamePrefix = "lexicResource")
{
Int32 c = 0;
foreach (var pair in items)
{
String pbin = folder.pathFor(filenamePrefix + "_" + pair.Key + ".bin", imbSCI.Data.enums.getWritableFileMode.none, "Binary serialized lexic entries starting with [" + pair.Key + "]");
if (skipExisting && File.Exists(pbin))
{
logger.log("File [" + pbin + "] exists. Skipping binary serialization");
}
else
{
IFormatter formatter = new BinaryFormatter();
Stream stream = new FileStream(pair.Key, FileMode.Create, FileAccess.Write, FileShare.None);
formatter.Serialize(stream, pair.Value);
stream.Close();
//ConcurrentDictionary<String, lexicInflection> dict = pair.Value;
//dict.Save(pbin);
c++;
}
}
if (c > 0)
{
logger.log("[" + c + "] lexic files serialized to [" + folder.path + "]");
return(true);
}
return(false);
}
/// <summary>
/// Features the vector construction.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="log">The log.</param>
public void FeatureVectorConstruction(OperationContext context, ILogBuilder log)
{
// deploying feature vector space constructor
featureSpaceConstructor.Deploy(constructorSettings, context.vectorSpace);
featureSpaceConstructor.Deploy(constructorSettings, context.SelectedFeatures.GetKeys());
Int32 i = 0;
Int32 s = 100;
foreach (IVector vector in context.vectorSpace.documents)
{
var lab = designateSpaceLabel(context, vector);
FeatureVector fv = featureSpaceConstructor.ConstructFeatureVector(vector);
context.featureVectorByName.Add(vector.name, fv);
context.featureSpace.documents.Add(fv);
context.featureSpace.labelToDocumentAssociations.Add(fv, lab, 1);
if (i % s == 0)
{
Double r = i.GetRatio(context.spaceModel.documents.Count);
log.log("Building feature vectors [" + r.ToString("P2") + "] : [" + i + "/" + context.vectorSpace.documents.Count + "]");
}
i++;
}
log.log("Feature vector construction [" + context.featureSpace.documents.Count + "] done");
//if (context.reportOptions.HasFlag(PlanesReportOptions.report_featureVectors))
//{
// var dt = context.featureSpace.MakeTable(featureSpaceConstructor, "FeatureSpace", "Feature space");
// notes.SaveDataTable(dt, notes.folder_feature);
//}
}
/// <summary>
/// Transforms to fv dictionary.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="TermWeightModel">The term weight model.</param>
/// <param name="function">The function.</param>
/// <returns></returns>
public static FeatureVectorSetDictionary TransformToFVDictionaryAsCategorySimilarity(this DocumentSelectResult context, FeatureWeightModel TermWeightModel, IVectorSimilarityFunction function, ILogBuilder log)
{
log.log("... Category Similarity ...");
List <string> selectedTerms = context.selectedFeatures.GetKeys(); //.entries.Select(x => x.name)?.ToList();
Dictionary <String, WeightDictionary> categoryDictionarties = new Dictionary <string, WeightDictionary>();
foreach (SpaceLabel label in context.spaceModel.labels)
{
Relationship <SpaceLabel, SpaceCategoryModel> categoryModel = context.spaceModel.LabelToCategoryLinks.GetAllRelationships(label).FirstOrDefault();
var c = TermWeightModel.GetWeights(selectedTerms, categoryModel.NodeB, context.spaceModel, label);
categoryDictionarties.Add(label.name, c);
}
FeatureVectorSetDictionary dict = new FeatureVectorSetDictionary();
String domainNameLast = "";
Double total = context.Count;
Int32 i = 0;
Int32 p = (context.Count / 20);
foreach (var entry in context.items)
{
i++;
WeightDictionary documentWeights = TermWeightModel.GetWeights(selectedTerms, entry.spaceDocument, context.spaceModel);
FeatureVector fv = new FeatureVector(entry.AssignedID);
fv.dimensions = new double[context.spaceModel.labels.Count];
Int32 c = 0;
Parallel.ForEach(context.spaceModel.labels, (label) =>
{
var docToClassSimilarity = function.ComputeSimilarity(categoryDictionarties[label.name], documentWeights);
fv.dimensions[context.spaceModel.labels.IndexOf(label)] = docToClassSimilarity;
});
Int32 r = i % p;
if (r == 0)
{
log.Append(" [" + i.GetRatio(context.Count).ToString("P2") + "] ");
}
dict.GetOrAdd(entry.DomainID).Add(fv, -1);
}
foreach (KeyValuePair <string, FeatureVectorWithLabelIDSet> pair in dict)
{
pair.Value.CloseDeploy();
}
log.log("... Preparation done...");
return(dict);
}
/// <summary>
/// Loads from file.
/// </summary>
/// <param name="path">The path.</param>
/// <param name="logger">The logger.</param>
/// <returns></returns>
public static DocumentSelectResult LoadFromFile(String path, ILogBuilder logger)
{
DocumentSelectResult scores = null;
if (path.Trim().isNullOrEmpty())
{
logger.log(" _ NO PATH SPECIFIED _ for DocumentSelectResult.LoadFromFile()");
return(scores);
}
if (!File.Exists(path))
{
logger.log(" _ FILE NOT FOUND _ for DocumentSelectResult.LoadFromFile(path) ");
return(scores);
}
scores = objectSerialization.loadObjectFromXML <DocumentSelectResult>(path, logger);
String filename = Path.GetFileNameWithoutExtension(path);
scores.name = scores.name.or(filename);
scores.description = scores.description.or("Precompiled scores loaded from [" + path + "]");
return(scores);
}
public IDocumentSetClass GetClass(String className, ILogBuilder logger)
{
IDocumentSetClass output = null;
foreach (IDocumentSetClass cl in items)
{
if (cl.name == className)
{
logger.log("Class found by name");
output = cl;
break;
}
if (cl.treeLetterAcronim == className)
{
logger.log("Class found by TLA");
output = cl;
break;
}
}
if (output == null)
{
logger.log("No class found under [" + className + "] name nor tree letter accronim");
}
return(output);
}
///// <summary>
///// Vrsi selektovanje profila na osnovu podesavanja i SampleTaker algoritma
///// </summary>
///// <param name="runStamp"></param>
///// <param name="groups"></param>
///// <param name="sampler"></param>
///// <param name="customWhere"></param>
///// <returns></returns>
//public List<webSiteProfile> selectProfiles(this IGrou String runStamp = null, List<String> groups = null,
// sampleSettings sampler = null, String customWhere = "", Boolean onlyCurrentTest = false)
////where T : IEnumerable<webSiteProfile>, new()
//{
// List<webSiteProfile> output = new List<webSiteProfile>();
// String wSql = makeSqlWhere(runStamp, groups, customWhere, onlyCurrentTest);
// // source
// //IRelationEnabledCollection rl = this as IRelationEnabledCollection;
// output = this.selectItems<webSiteProfile>(wSql, -1, selectItemsMode.sqlWhere, selectItemsResultType.managed);
// return output;
//}
///// <summary>
///// 2017c: Selects the group.
///// </summary>
///// <param name="groupSet">The group set.</param>
///// <param name="group">The group.</param>
///// <param name="loger">The loger.</param>
///// <param name="source">The source.</param>
///// <param name="runStamp">The run stamp.</param>
///// <param name="customWhere">The custom where.</param>
///// <param name="onlyCurrentTest">if set to <c>true</c> [only current test].</param>
///// <returns></returns>
//public static IEnumerable<webSiteProfile> selectGroup(this sampleGroupSet groupSet, sampleGroupItem group, ILogBuilder loger,webSiteProfileCollection source, String runStamp = null, String customWhere = "", Boolean onlyCurrentTest = false)
//{
// //List<IReal> wbp = new List<IProfile>();
// //String whereQ = groupSet.makeSqlWhere(group, loger, runStamp, customWhere, onlyCurrentTest);
// ////.selectItems(whereQ); // webSiteProfiles.selectItems<webSiteProfile>(sql, -1, selectItemsMode.sqlWhere, selectItemsResultType.managed);
// // return wbp;
//}
/// <summary>
/// Pravi WHERE dodatak SQL upita -
/// </summary>
/// <param name="runStamp">Ako nije definisan napravice trenuitni</param>
/// <param name="groups">Grupe - ako nije definisan ucitace sve</param>
/// <param name="customWhere">Proizvoljan WHERE upit</param>
/// <param name="onlyCurrentTest">Da li proverava RunStamp</param>
/// <returns></returns>
public static string makeSqlWhere(this sampleGroupSet groupSet, sampleGroupItem group, ILogBuilder loger, string runStamp = null, string customWhere = "", bool onlyCurrentTest = false)
{
loger.log("Creating sample filter query for: " + group.groupTitle + " (" + group.groupTag + ")");
string output = "";
string whereRunStamp = "";
string whereGroups = "";
if (onlyCurrentTest)
{
if (string.IsNullOrEmpty(runStamp))
{
}
whereRunStamp = "((lastRunStamp <> \"" + runStamp + "\") OR (lastRunStamp IS NULL))";
}
whereGroups = groupSet.db_groupTagFieldName + " LIKE '%" + group.groupTag + "%'";
if (!string.IsNullOrEmpty(whereRunStamp))
{
output += whereRunStamp + "";
}
if (!string.IsNullOrEmpty(whereGroups))
{
if (!string.IsNullOrEmpty(output))
{
output += " OR " + whereGroups;
}
else
{
output = whereGroups;
}
}
if (!string.IsNullOrEmpty(customWhere))
{
if (!string.IsNullOrEmpty(output))
{
output += " OR " + customWhere;
}
else
{
output = customWhere;
}
}
loger.log("Query to filter from " + group.count + " entries: " + output);
return(output);
}
public static DataTable SaveXML(this DataTable source, folderNode folder, string filenamePrefix = "", bool clearMeta = true, bool checkContent = true, ILogBuilder logger = null)
{
if (source.TableName.isNullOrEmpty())
{
source.TableName = filenamePrefix + source.GetHashCode().ToString() + ".xml";
}
string path = folder.pathFor(filenamePrefix + source.GetHashCode().ToString() + ".xml", getWritableFileMode.autoRenameExistingOnOtherDate);
if (clearMeta)
{
source = source.CleanMeta();
}
if (checkContent)
{
foreach (DataRow row in source.Rows)
{
foreach (DataColumn dc in source.Columns)
{
var t = row[dc]?.GetType();
if (t.IsClass)
{
if (logger != null)
{
logger.log(" === unallowed content detected in the table to save " + path);
}
row[dc] = dc.DataType.GetDefaultValue();
}
}
}
}
bool notOkToSave = false;
foreach (DataColumn dc in source.Columns)
{
bool okType = dc.DataType.checkIfDataTypeIsAllowed();
if (!okType)
{
notOkToSave = true;
break;
}
}
if (!notOkToSave)
{
objectSerialization.saveObjectToXML(source, path);
}
else
{
if (logger != null)
{
logger.log("Can't save XML for table: " + source.TableName + " to " + path);
}
}
return(source);
}
public void LoadCompleteInformation(classificationReportCollectionSettings setup, folderNode folder, ILogBuilder logger)
{
DeployPath(folder.path);
rootFolder = new folderStructure(rootPath, "Root", "Root directory with experiment reports");
var resultFiles = rootFolder.findFiles(setup.FILENAME_CLASSIFICATION_RESULTS, SearchOption.AllDirectories);
String basePath = rootPath;
foreach (String rFile in resultFiles)
{
String dirPath = Path.GetDirectoryName(rFile);
String dirSubPath = dirPath.Substring(basePath.Length);
classificationReportExpanded rep = classificationReportExpanded.LoadSimpleReport(rFile, logger);
folderNode reportFolder = rootFolder.Add(dirSubPath, rep.Name, "Experiment [" + rep.Name + "] data");
rep.folder = reportFolder;
String subCollectionDirPath = reportFolder.parent.path.Substring(rootFolder.path.Length);
classificationReportCollection hostCollection = this;
if (!subCollectionDirPath.isNullOrEmpty())
{
if (reportFolder.parent != rootFolder)
{
hostCollection = AddOrGetChild(subCollectionDirPath);
}
}
classificationReportExpanded rep_existing = hostCollection.FirstOrDefault(x => x.filepath == rep.filepath);
if (rep_existing != null)
{
if (rep.filecreation > rep_existing.filecreation)
{
hostCollection.Remove(rep_existing);
logger.log("Replacing older report [" + rep_existing.filepath + "] with new version");
}
}
hostCollection.Add(rep);
reportCompleteList.Add(rep);
//if (LoadExtraInformation != null)
//{
// LoadExtraInformation.Invoke(rep, hostCollection, setup, logger);
//}
}
logger.log("Reports (" + Count + ") directly imported from [" + rootPath + "] - Total reports, including [" + Children.Count + "] sub collections is [" + resultFiles.Count + "]");
}
public void LogConstruct(ILogBuilder logger)
{
if (logger != null)
{
logger.log("Primary Terms [ " + PrimaryTerms.Count + " ] -- Secondary Terms [" + SecondaryTerms.Count + " ]");
logger.log("Primary Chunks [ " + primaryChunks.Count + " ] -- Sec.Chunks [" + secondaryChunks.Count + " ]");
logger.log("Reserve Terms [ " + RelevantTerms.Count + " ] -- Trash bin [" + TrashBin.Count + " ]");
}
}
public void SaveKnowledgeInstancesForClasses(kFoldValidationCase validationCase, ILogBuilder logger)
{
logger.log("Saving knowledge objects for [" + validationCase.name + "] classes");
foreach (IWebFVExtractorKnowledge knowledge in registry[validationCase].Values)
{
knowledge.OnBeforeSave();
}
logger.log("Knowledge saved for [" + validationCase.name + "] classes");
}
/// <summary>
/// Prepares resource file for querying and loads the grammar tag converter specification
/// </summary>
/// <param name="resourceFilePath">The text resource file path</param>
/// <param name="grammSpecFilename">The Excel file with the grammar tag converter</param>
/// <param name="output">The output.</param>
protected void Setup(string resourceFilePath, string grammSpecFilename, ILogBuilder output = null)
{
if (resourceFilePath.isNullOrEmpty())
{
imbACE.Services.terminal.aceTerminalInput.askYesNo("Resource file path is empty (textResourceResolverBase.Setup)!");
throw new ArgumentNullException(nameof(resourceFilePath));
return;
}
if (grammSpecFilename.isNullOrEmpty())
{
imbACE.Services.terminal.aceTerminalInput.askYesNo("Grammar conversion specification file path is empty (textResourceResolverBase.Setup)!");
throw new ArgumentNullException(nameof(grammSpecFilename));
return;
}
resourceFileOperater = new fileTextOperater(resourceFilePath, true);
grammTagConverter = new resourceConverterForGramaticTags();
if (grammSpecFilename.EndsWith(".xlsx"))
{
if (output != null)
{
output.log("Grammar conversion specification loading from Excel file");
}
grammTagConverter.LoadSpecificationExcelFile(grammSpecFilename, output);
}
else if (grammSpecFilename.EndsWith(".csv"))
{
string filebase = Path.GetFileNameWithoutExtension(grammSpecFilename);
string filepath = Path.GetDirectoryName(grammSpecFilename);
if (output != null)
{
output.log("Grammar conversion specification loading from CSV files");
}
string gramSpecFileFormat = filepath + Path.DirectorySeparatorChar + filebase + "_format.csv";
string gramSpecFileTranslation = filepath + Path.DirectorySeparatorChar + filebase + "_translation.csv";
grammTagConverter.LoadSpecificationCSV(gramSpecFileFormat, gramSpecFileTranslation, output);
}
else
{
if (output != null)
{
output.log("Grammar conversion file format not recognized from the filepath! [" + grammSpecFilename + "]");
}
else
{
throw new ArgumentOutOfRangeException("File format not recognized for " + nameof(textResourceResolverBase) + " Setup call.", nameof(grammSpecFilename));
}
}
}
//private Object resourceLock = new Object();
public void SaveCaseKnowledgeInstances(ILogBuilder logger)
{
logger.log("Saving case knowledge objects for [" + validationCollection.extractor.name + "]");
foreach (IWebFVExtractorKnowledge knowledge in registryForCases.Values)
{
knowledge.OnBeforeSave();
reusableKnowledgeSaved.Add(knowledge.name, true);
}
logger.log("Knowledge case saved for [" + validationCollection.extractor.name + "]");
}
/// <summary>
/// Builds a <see cref="settingsPropertyEntry"/> from contained data
/// </summary>
/// <param name="column">The column.</param>
/// <param name="skipExisting">if set to <c>true</c> [skip existing].</param>
/// <param name="log">The log.</param>
/// <returns></returns>
public settingsPropertyEntry BuildPCE(DataColumn column, Boolean skipExisting = true, ILogBuilder log = null)
{
settingsPropertyEntry pce = new settingsPropertyEntry(column);
PropertyCollection pc = pce.exportPropertyCollection();
foreach (var pair in definitions.keyValuePairs)
{
if (skipExisting)
{
if (pc.ContainsKey(pair.resolvedKey))
{
if (!pc[pair.resolvedKey].toStringSafe().isNullOrEmpty())
{
if (log != null)
{
log.log(" Deploy [" + pair.key + "] = false, because destination is not empty (skipExisting=" + skipExisting.ToString() + ")");
}
continue;
}
}
}
switch (pair.resolvedKey)
{
default:
if (pair.resolvedKey is imbAttributeName attName)
{
pce.deploy(attName, pair.value);
if (log != null)
{
log.log("Set[" + pair.key + "] = " + pair.value.toStringSafe());
}
}
else if (pair.resolvedKey is templateFieldDataTable tfdt)
{
pce.deploy(tfdt, pair.value);
if (log != null)
{
log.log("Set[" + pair.key + "] = " + pair.value.toStringSafe());
}
}
else
{
if (log != null)
{
log.log(column.Table.GetTitle() + "." + column.ColumnName + " <- entry not recognized [" + pair.key + "]");
}
}
break;
}
}
return(pce);
}
public override List <pipelineTaskSubjectContentToken> process(IEnumerable <pipelineTaskSubjectContentToken> _input, ILogBuilder logger)
{
settings.checkReady();
List <pipelineTaskSubjectContentToken> output = new List <pipelineTaskSubjectContentToken>();
List <pipelineTaskSubjectContentToken> next = new List <pipelineTaskSubjectContentToken>();
next = _input.ToList();
next.Sort((x, y) => String.CompareOrdinal(x.currentForm, y.currentForm));
while (currentIteration > 0)
{
List <pipelineTaskSubjectContentToken> MCNext = new List <pipelineTaskSubjectContentToken>();
foreach (pipelineTaskSubjectContentToken sub in next)
{
MCNext.AddRange(processIteration(sub), true);
}
if (settings.keepAllInOutput)
{
output.AddRange(MCNext, true);
}
else
{
output = MCNext;
}
logger.log("[" + currentIteration + "] chunk construction in[" + next.Count + "] new[" + MCNext.Count + "] out[" + output.Count + "]");
if (next.Count == output.Count)
{
logger.log("Aborting the process since last iteation produced no changes");
break;
}
next = MCNext.ToList();
if (MCNext.Count == 0)
{
break;
}
currentIteration--;
}
return(output);
//return base.process(_input, logger);
}
public htmlContentPage tokenizeContent(ILogBuilder pRecordLog, HtmlDocument htmlDoc, basicLanguage language, node page)
{
var starttime = DateTime.Now;
htmlContentPage contentPage = new htmlContentPage();
contentPage.acceptSourcePage(page);
string domain = page.domain;
object[] resources = new object[] { language, page, flags, sentenceFlags, tokenFlags, preprocessFlags };
var ctb = contentTreeBuilder.getInstance(htmlDoc.CreateNavigator(), domain, page);
contentPage.treeBuilder = ctb;
var blocks = ctb.tree.breakToBlocks();
int b = 0;
for (int bi = 0; bi < blocks.Count; bi++)
{
imbTreeNodeBlock bl = blocks[bi];
b++;
makeBlock(bl, contentPage, language, resources);
// pRecordLog.close();
}
contentPage.recountItems();
contentPage.primaryFlaging(resources);
contentPage.secondaryFlaging(resources);
// <---------------
// pRecordLog.log("SKIP: complete exploration of all tokens is turned off.");
// contentPage.saveCache();
pRecordLog.log("Basic semantic analysis done. Closing the process.");
var time = DateTime.Now.Subtract(starttime);
pRecordLog.log("Tree-building and tokenization [" + page.url + "] done in: " + time.TotalMilliseconds.getSeconds(4) + "s");
return(contentPage);
}
/// <summary>
/// Gets the lexicon items.
/// </summary>
/// <param name="termForm">The term form.</param>
/// <param name="loger">The loger.</param>
/// <returns></returns>
private lexiconResponse _getLexiconItems(List <string> termForms, ILogBuilder loger = null, bool callWithAutoDiscovery = true)
{
lexiconResponse output = new lexiconResponse();
//lock (getLexiconItemsLock)
//{
output.setType(lexiconResponse.responseType.lexicon);
ILexiconItem tis_i = null;
try
{
//var tis_is = lexiconContext.TermInstances.Where(x => x.Equals(termForm)); // x=>x.name == termForm
var tis_is = lexiconContext.TermInstances.Where(x => termForms.Contains(x.name)).FirstOrDefault(); //
if (tis_is != null)
{
output.Add(tis_is);
}
}
catch (Exception ex)
{
if (loger != null)
{
loger.log("Lexicon query failed: " + termForms.toCsvInLine(",") + " (lemmas)");
}
}
if (tis_i == null)
{
try
{
var til_is = lexiconContext.TermLemmas.Where(x => termForms.Contains(x.name)).FirstOrDefault(); // var til_is = lexiconContext.TermLemmas.Where(x => x.Equals(termForm));
if (til_is != null)
{
output.Add(til_is);
}
}
catch (Exception ex)
{
if (loger != null)
{
loger.log("Lexicon query failed: " + termForms.toCsvInLine(",") + " (instances)");
}
}
}
//}
return(output);
}
/// <summary>
/// Loads the object from XML.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="filepath">The filepath.</param>
/// <returns></returns>
public static T loadObjectFromXML <T>(String filepath, ILogBuilder logger = null) where T : class
{
if (!File.Exists(filepath))
{
return(default(T));
}
FileInfo fi = new FileInfo(filepath);
if (fi.Length == 0)
{
if (logger != null)
{
logger.log("Loading XML object from [" + filepath + "] aborted because file has 0 bytes. Default instance is created.");
}
return(default(T));
}
Type t = typeof(T);
String xmlString = File.ReadAllText(filepath);
if (!(xmlString.Contains(t.Name) || xmlString.StartsWith("<?xml")))
{
T altOutput = imbTypeExtensions.getInstance(t) as T;
if (logger != null)
{
logger.log("XML structure failed");
}
return(altOutput);
}
//if (!xmlString.Contains(t.Name))
//{
// if (logger != null) logger.log("Loading XML object from [" + filepath + "] aborted because the file seems not to be serialization of type [" + t.Name + "]");
// return default(T);
//}
XmlSerializer deserializer = new XmlSerializer(typeof(T));
//TextReader reader = new StreamReader(filepath);
StringReader stringReader = new StringReader(xmlString);
object obj = deserializer.Deserialize(stringReader);
T output = (T)obj;
stringReader.Close();
// reader.Close();
return(output);
}
/// <summary>
/// Adds the link.
/// </summary>
/// <param name="from">From.</param>
/// <param name="to">To.</param>
/// <param name="type">The type.</param>
/// <param name="description">The description.</param>
/// <returns></returns>
public diagramLink AddLink(diagramNode from, diagramNode to, diagramLinkTypeEnum type, string description = "", string __hash = "", ILogBuilder logger = null)
{
diagramLink output = new diagramLink();
output.from = from;
output.to = to;
output.type = type;
output.name = getUID(output);
output.description = description;
links.Add(output);
output.parent = this;
if (!__hash.isNullOrEmpty())
{
if (linkByHash.ContainsKey(__hash))
{
if (logger != null)
{
logger.log("AddLink() failed :: " + __hash);
}
}
else
{
linkByHash.Add(__hash, output);
}
}
return(output);
}
/// <summary>
/// Creates new node in the model
/// </summary>
/// <param name="__description">The description.</param>
/// <param name="type">The type.</param>
/// <param name="__name">The name.</param>
/// <returns></returns>
public diagramNode AddNode(string __description, diagramNodeShapeEnum type, string __name = "", string __hash = "", ILogBuilder logger = null)
{
diagramNode output = new diagramNode();
output.description = __description;
output.shapeType = type;
if (!isNodeNameAcceptable(__name))
{
__name = __name + getUID(output, true, false);
}
output.name = __name;
output.parent = this;
nodes.Add(output.name, output);
if (!__hash.isNullOrEmpty())
{
if (linkByHash.ContainsKey(__hash))
{
if (logger != null)
{
logger.log("AddNode() failed :: " + __hash);
}
}
else
{
nodeByHash.Add(__hash, output);
}
}
return(output);
}
/// <summary>
/// Filters the space model features.
/// </summary>
/// <param name="spaceModel">The space model.</param>
/// <param name="selectedFeatures">The selected features.</param>
/// <param name="log">The log.</param>
/// <returns></returns>
public static Int32 FilterSpaceModelFeatures(this SpaceModel spaceModel, WeightDictionary selectedFeatures, ILogBuilder log)
{
Int32 i = 0;
Int32 s = spaceModel.documents.Count() / 5;
Int32 c_filter_out = 0;
List <String> keys = selectedFeatures.GetKeys();
List <String> termsToRemove = spaceModel.terms.GetTokensOtherThan(keys);
for (int i2 = 0; i2 < spaceModel.documents.Count; i2++)
{
c_filter_out += spaceModel.documents[i2].FilterSelectedFeatures(termsToRemove, false);
if (i > s)
{
Double r = i2.GetRatio(spaceModel.documents.Count());
log.log("Filter SelectedFeatures [" + r.ToString("P2") + "]");
i = 0;
}
i++;
}
spaceModel.terms_known_label.FilterTokens(termsToRemove, false);
spaceModel.terms_unknown_label.FilterTokens(termsToRemove, false);
return(c_filter_out);
}
/// <summary>
/// Processes the specified cloud.
/// </summary>
/// <param name="cloud">The cloud.</param>
/// <param name="logger">The logger.</param>
/// <returns></returns>
public lemmaSemanticWeaverResult Process(lemmaSemanticCloud cloud, ILogBuilder logger)
{
Int32 c = cloud.links.Count;
lemmaSemanticWeaverResult weaverResult = new lemmaSemanticWeaverResult(cloud);
wordSimilarityResultSet output = new wordSimilarityResultSet();
List <String> words = cloud.nodes.Select(x => x.name).ToList();
if (useSimilarity)
{
output = similarWords.GetResult(words);
foreach (var pair in output)
{
var link = cloud.GetLink(pair.wordA, pair.wordB);
if (link == null)
{
cloud.AddLink(pair.wordA, pair.wordB, pair.score, LINK_OF_SIMILARWORDS);
}
}
weaverResult.linkRatioAfterWS = cloud.GetLinkPerNodeRatio();
weaverResult.similarWords = output;
}
/* NOT WORKING ---- TEMPORARLY DISABLED
* if (useDictionary)
* {
* Stack<String> wordsToTest = new Stack<string>();
* words.ForEach(x => wordsToTest.Push(x));
*
* while (wordsToTest.Any())
* {
* String word = wordsToTest.Pop();
* apertiumDictionaryResult result = apertium.queryForSynonyms(word, apertiumDictQueryScope.exact);
*
* var synonims = result.GetNativeWords();
*
* var synonimNodes = wordsToTest.Where(x => synonims.Contains(x));
*
* foreach (String syn in synonimNodes)
* {
* wordsToTest.RemoveAll(x=>x==syn);
* cloud.AddLink(word, syn, 1, LINK_OF_DICTIONARYSYNONIMS);
*
* weaverResult.appertiumNotes.Add(word + " -> " + syn);
* }
* weaverResult.linkRatioAfterDS = cloud.GetLinkPerNodeRatio();
* }
* }*/
if (c != cloud.links.Count)
{
if (logger != null)
{
logger.log("Weaver created [" + (cloud.links.Count - c) + "] new links in the cloud [" + cloud.className + "]");
}
}
return(weaverResult);
}
