/// <summary> /// Loads the specified folder. /// </summary> /// <param name="folder">The folder - to be scanned for experiment data</param> /// <param name="logger">The logger.</param> public void Load(folderNode _folder, ILogBuilder logger) { folder = _folder; try { var experimentFiles = folder.findFiles("experimentSetup.xml", SearchOption.AllDirectories, false); foreach (var path in experimentFiles) { try { var exp = new secondaryReportOnExperiment(path, logger, folder); items.Add(exp); allItems.AddRange(exp.items); allTopItems.Add(exp.topPerformer); // logger.log("Experiment report processed [" + path + "] : [" + allItems.Count + "]"); } catch (Exception ex) { logger.log(ex.LogException("secondaryReport.Load", "SECREP_LOAD=>" + path)); } } } catch (Exception ex) { logger.log(ex.LogException("secondaryReport.Load", "SECREP_LOAD")); } }
public void Save(ILogBuilder loger = null) { try { setContentLines(contentLines.ToList()); if (loger != null) { if (!info.Exists) { loger.log("File [" + path + "] not saved since the content is empty."); } else { loger.log("File [" + path + "] saved. Size: " + imbStringFormats.getKByteCountFormated(getByteSize()) + " - lines: " + contentLines.Count()); } } Accept(); } catch (Exception ex) { if (loger != null) { loger.log("File [" + path + "] not saved due exception: " + ex.Message); } } }
/// <summary> /// Applies the index of the manual page --- loads external, manually authored truth index /// </summary> /// <param name="manualPageIndex">Index of the manual page.</param> /// <param name="loger">The loger.</param> /// <param name="callDomainIndexRecheck">if set to <c>true</c> [call domain index recheck].</param> public void ApplyManualPageIndex(string manualPageIndex = "pageIndex_revision.xlsx", ILogBuilder loger = null, bool callDomainIndexRecheck = true) { string path = folder.pathFor(manualPageIndex); if (File.Exists(path)) { DataTable manualIndex = path.deserializeDataTable(dataTableExportEnum.excel, folder); int loaded = pageIndexTable.Load(manualIndex, loger, objectTableUpdatePolicy.overwrite); if (loger != null) { loger.log("Page index entries updated: " + loaded); } if (callDomainIndexRecheck) { Recheck(loger); } isFullTrustMode = true; } else { loger.log("Excel file not found on: " + path); } }
/// <summary> /// Loads the semantic clouds of the specified FVE /// </summary> /// <param name="fve">The fve.</param> /// <param name="logger">The logger.</param> public void LoadSemanticClouds(semanticFVExtractor fve, ILogBuilder logger) { DirectoryInfo di = folder; var dirs = di.GetDirectories(fve.name); if (!dirs.Any()) { logger.log("Failed to find subfolder for FVE [" + fve.name + "]"); } else { DirectoryInfo dir = dirs.First(); folderNode fveFolder = dir; var allCloudFiles = fveFolder.findFiles("*Cloud.xml", SearchOption.AllDirectories); Int32 cl = 0; foreach (String cloudFile in allCloudFiles) { if (cloudFile.Contains("General") || cloudFile.Contains("SharedKnowledge")) { } else { semanticClouds.Add(fve.name, objectSerialization.loadObjectFromXML <lemmaSemanticCloud>(cloudFile, logger)); cl++; } } logger.log("Semantic clouds loaded [" + cl + "] for " + fve.name); } }
/// <summary> /// Loads the bin. /// </summary> /// <param name="folder">The folder.</param> /// <param name="logger">The logger.</param> /// <param name="filenamePrefix">The filename prefix.</param> public Boolean LoadBin(folderNode folder, ILogBuilder logger, String filenamePrefix = "lexicResource") { var files = folder.findFiles(filenamePrefix + "_*.bin", SearchOption.TopDirectoryOnly); Int32 c = 0; foreach (var pair in files) { String filename = Path.GetFileNameWithoutExtension(pair); String letter = filename.Replace(filenamePrefix + "_", ""); ConcurrentDictionary <String, lexicInflection> dict = Accord.IO.Serializer.Load <ConcurrentDictionary <String, lexicInflection> >(pair); if (dict.Any()) { c++; logger.log("File [" + filename + "] loaded --> index [" + letter + "]"); if (items.ContainsKey(letter)) { items[letter].Clear(); items[letter].AddRange(dict); } else { items.TryAdd(letter, dict); } } } if (c > 0) { logger.log("[" + c + "] lexic files loaded from [" + folder.path + "]"); return(true); } return(false); }
/// <summary> /// Loads the table from the specified filepath /// </summary> /// <param name="filepath">The filepath.</param> public void Load(string filepath, ILogBuilder loger) { sourceFile = new fileunit(filepath, true); int cl = sourceFile.contentLines.Count; loger.log("Lexicon terms coding twins definitions: " + cl); int lind = 0; int lmax = cl / 20; lind = lmax; LoadCount = 0; foreach (string ln in sourceFile.contentLines) { lind--; SetEntryFromString(ln); LoadCount++; if (lind <= 0) { lind = lmax; loger.log("Coding twins loaded: " + LoadCount); } } loger.log("Coding twins completly loaded: " + LoadCount); }
/// <summary> /// Performing post processing of FV knowledge /// </summary> /// <param name="validationCase">The validation case.</param> /// <param name="tools">The tools.</param> /// <param name="logger">The logger.</param> public override void DoFVPostProcessing(kFoldValidationCase validationCase, classifierTools tools, ILogBuilder logger) { List <lemmaSemanticCloud> clouds = new List <lemmaSemanticCloud>(); foreach (var docClass in validationCase.context.classes.GetClasses()) { var knowledge = validationCase.knowledgeLibrary.GetKnowledgeInstance <semanticFVExtractorKnowledge>(docClass, validationCase, logger); knowledge.semanticCloudFiltered = knowledge.semanticCloud.Clone(); clouds.Add(knowledge.semanticCloudFiltered); knowledge.semanticCloud.className = docClass.name; knowledge.semanticCloudFiltered.className = docClass.name + "flt"; if (settings.semanticCloudFilter.isActive) { knowledge.semanticCloudFiltered.description = "Semantic cloud filtered with cloud matrix"; } else { knowledge.semanticCloudFiltered.description = "Semantic cloud filter is off - this is initial cloud"; } } if (settings.semanticCloudFilter.isActive) { logger.log(validationCase.name + ": Cloud matrix creation starts..."); cloudMatrix matrix = new cloudMatrix(validationCase.name, "Cloud overlap matrix of [" + clouds.Count + "] for fold [" + validationCase.name + "] of experiment [" + validationCase.context.setup.name + "]"); matrix.build(clouds, logger); matrix.TransformClouds(settings.semanticCloudFilter, logger); if (tools.operation.doMakeGraphForClassClouds) { foreach (var cloud in clouds) { if (tools.operation.doUseSimpleGraphs) { cloud.GetSimpleGraph(true).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite)); } else { var converter = lemmaSemanticCloud.GetDGMLConverter(); converter.ConvertToDMGL(cloud).Save(validationCase.caseFolder.pathFor("class_" + cloud.className + "_reducedCloud", getWritableFileMode.overwrite)); } } } //logger.log(validationCase.name + ": Cloud matrix report creation ..."); // matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_initial"); // matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_initial"); // matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_norm_reduced"); // matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_overlap_abs_reduced"); // matrix.BuildTable(settings.semanticCloudFilter, cloudMatrixDataTableType.stateAfterReduction | cloudMatrixDataTableType.maxCloudFrequency | cloudMatrixDataTableType.normalizedValues).GetReportAndSave(validationCase.folder, appManager.AppInfo, "matrix_CF_norm_reduced"); // logger.log(validationCase.name + ": Cloud matrix report done."); } else { logger.log(validationCase.name + ": Cloud matrix is not active"); } }
/// <summary> /// Preloads the lexicon. /// </summary> /// <param name="loger">The loger.</param> /// <param name="context">The context.</param> public void preloadLexicon(ILogBuilder loger, semanticLexiconContext context) { loger.log("Preloading Semantic Lexicon"); if (isLexiconPreloaded) { loger.log("Semantic Lexicon is already loaded"); return; } double ratio = 0; int i = 0; int repIndex = lemmaCount / 20; foreach (ITermLemma lemma in context.TermLemmas) { Add(lemma); i++; repIndex--; if (repIndex == 0) { repIndex = lemmaCount / 20; ratio = (double)i / (double)lemmaCount; loger.log("Loaded [" + i + "] lemmas (" + ratio.ToString("P2") + ")"); } } doAllowLexiconQuery = false; isLexiconPreloaded = true; loger.log("Semantic Lexicon preload done"); }
/// <summary> /// Initializes loading of plugins /// </summary> /// <param name="output">The output.</param> protected void loadPlugins(ILogBuilder output) { var assemblies = AppDomain.CurrentDomain.GetAssemblies(); Type pluginType = typeof(T); foreach (Assembly ass in assemblies) { try { if (ass.FullName.StartsWith("imb") || AllowNonImbAssemblies) { var types = ass.GetTypes(); foreach (Type type in types) { if (type.IsInterface || type.IsAbstract) { continue; } else { if (type.GetInterface(pluginType.FullName) != null) { registerPlugin(type, type.Namespace, output); } } } } } catch (Exception ex) { output.log("Assembly [" + ass.FullName + "] type harvest failed"); output.log("Exception: " + ex.Message); } } }
/// <summary> /// Saves the bin. /// </summary> /// <param name="folder">The folder.</param> /// <param name="logger">The logger.</param> /// <param name="skipExisting">if set to <c>true</c> [skip existing].</param> /// <param name="filenamePrefix">The filename prefix.</param> public Boolean SaveBin(folderNode folder, ILogBuilder logger, Boolean skipExisting = true, String filenamePrefix = "lexicResource") { Int32 c = 0; foreach (var pair in items) { String pbin = folder.pathFor(filenamePrefix + "_" + pair.Key + ".bin", imbSCI.Data.enums.getWritableFileMode.none, "Binary serialized lexic entries starting with [" + pair.Key + "]"); if (skipExisting && File.Exists(pbin)) { logger.log("File [" + pbin + "] exists. Skipping binary serialization"); } else { IFormatter formatter = new BinaryFormatter(); Stream stream = new FileStream(pair.Key, FileMode.Create, FileAccess.Write, FileShare.None); formatter.Serialize(stream, pair.Value); stream.Close(); //ConcurrentDictionary<String, lexicInflection> dict = pair.Value; //dict.Save(pbin); c++; } } if (c > 0) { logger.log("[" + c + "] lexic files serialized to [" + folder.path + "]"); return(true); } return(false); }
/// <summary> /// Features the vector construction. /// </summary> /// <param name="context">The context.</param> /// <param name="log">The log.</param> public void FeatureVectorConstruction(OperationContext context, ILogBuilder log) { // deploying feature vector space constructor featureSpaceConstructor.Deploy(constructorSettings, context.vectorSpace); featureSpaceConstructor.Deploy(constructorSettings, context.SelectedFeatures.GetKeys()); Int32 i = 0; Int32 s = 100; foreach (IVector vector in context.vectorSpace.documents) { var lab = designateSpaceLabel(context, vector); FeatureVector fv = featureSpaceConstructor.ConstructFeatureVector(vector); context.featureVectorByName.Add(vector.name, fv); context.featureSpace.documents.Add(fv); context.featureSpace.labelToDocumentAssociations.Add(fv, lab, 1); if (i % s == 0) { Double r = i.GetRatio(context.spaceModel.documents.Count); log.log("Building feature vectors [" + r.ToString("P2") + "] : [" + i + "/" + context.vectorSpace.documents.Count + "]"); } i++; } log.log("Feature vector construction [" + context.featureSpace.documents.Count + "] done"); //if (context.reportOptions.HasFlag(PlanesReportOptions.report_featureVectors)) //{ // var dt = context.featureSpace.MakeTable(featureSpaceConstructor, "FeatureSpace", "Feature space"); // notes.SaveDataTable(dt, notes.folder_feature); //} }
/// <summary> /// Transforms to fv dictionary. /// </summary> /// <param name="context">The context.</param> /// <param name="TermWeightModel">The term weight model.</param> /// <param name="function">The function.</param> /// <returns></returns> public static FeatureVectorSetDictionary TransformToFVDictionaryAsCategorySimilarity(this DocumentSelectResult context, FeatureWeightModel TermWeightModel, IVectorSimilarityFunction function, ILogBuilder log) { log.log("... Category Similarity ..."); List <string> selectedTerms = context.selectedFeatures.GetKeys(); //.entries.Select(x => x.name)?.ToList(); Dictionary <String, WeightDictionary> categoryDictionarties = new Dictionary <string, WeightDictionary>(); foreach (SpaceLabel label in context.spaceModel.labels) { Relationship <SpaceLabel, SpaceCategoryModel> categoryModel = context.spaceModel.LabelToCategoryLinks.GetAllRelationships(label).FirstOrDefault(); var c = TermWeightModel.GetWeights(selectedTerms, categoryModel.NodeB, context.spaceModel, label); categoryDictionarties.Add(label.name, c); } FeatureVectorSetDictionary dict = new FeatureVectorSetDictionary(); String domainNameLast = ""; Double total = context.Count; Int32 i = 0; Int32 p = (context.Count / 20); foreach (var entry in context.items) { i++; WeightDictionary documentWeights = TermWeightModel.GetWeights(selectedTerms, entry.spaceDocument, context.spaceModel); FeatureVector fv = new FeatureVector(entry.AssignedID); fv.dimensions = new double[context.spaceModel.labels.Count]; Int32 c = 0; Parallel.ForEach(context.spaceModel.labels, (label) => { var docToClassSimilarity = function.ComputeSimilarity(categoryDictionarties[label.name], documentWeights); fv.dimensions[context.spaceModel.labels.IndexOf(label)] = docToClassSimilarity; }); Int32 r = i % p; if (r == 0) { log.Append(" [" + i.GetRatio(context.Count).ToString("P2") + "] "); } dict.GetOrAdd(entry.DomainID).Add(fv, -1); } foreach (KeyValuePair <string, FeatureVectorWithLabelIDSet> pair in dict) { pair.Value.CloseDeploy(); } log.log("... Preparation done..."); return(dict); }
/// <summary> /// Loads from file. /// </summary> /// <param name="path">The path.</param> /// <param name="logger">The logger.</param> /// <returns></returns> public static DocumentSelectResult LoadFromFile(String path, ILogBuilder logger) { DocumentSelectResult scores = null; if (path.Trim().isNullOrEmpty()) { logger.log(" _ NO PATH SPECIFIED _ for DocumentSelectResult.LoadFromFile()"); return(scores); } if (!File.Exists(path)) { logger.log(" _ FILE NOT FOUND _ for DocumentSelectResult.LoadFromFile(path) "); return(scores); } scores = objectSerialization.loadObjectFromXML <DocumentSelectResult>(path, logger); String filename = Path.GetFileNameWithoutExtension(path); scores.name = scores.name.or(filename); scores.description = scores.description.or("Precompiled scores loaded from [" + path + "]"); return(scores); }
public IDocumentSetClass GetClass(String className, ILogBuilder logger) { IDocumentSetClass output = null; foreach (IDocumentSetClass cl in items) { if (cl.name == className) { logger.log("Class found by name"); output = cl; break; } if (cl.treeLetterAcronim == className) { logger.log("Class found by TLA"); output = cl; break; } } if (output == null) { logger.log("No class found under [" + className + "] name nor tree letter accronim"); } return(output); }
///// <summary> ///// Vrsi selektovanje profila na osnovu podesavanja i SampleTaker algoritma ///// </summary> ///// <param name="runStamp"></param> ///// <param name="groups"></param> ///// <param name="sampler"></param> ///// <param name="customWhere"></param> ///// <returns></returns> //public List<webSiteProfile> selectProfiles(this IGrou String runStamp = null, List<String> groups = null, // sampleSettings sampler = null, String customWhere = "", Boolean onlyCurrentTest = false) ////where T : IEnumerable<webSiteProfile>, new() //{ // List<webSiteProfile> output = new List<webSiteProfile>(); // String wSql = makeSqlWhere(runStamp, groups, customWhere, onlyCurrentTest); // // source // //IRelationEnabledCollection rl = this as IRelationEnabledCollection; // output = this.selectItems<webSiteProfile>(wSql, -1, selectItemsMode.sqlWhere, selectItemsResultType.managed); // return output; //} ///// <summary> ///// 2017c: Selects the group. ///// </summary> ///// <param name="groupSet">The group set.</param> ///// <param name="group">The group.</param> ///// <param name="loger">The loger.</param> ///// <param name="source">The source.</param> ///// <param name="runStamp">The run stamp.</param> ///// <param name="customWhere">The custom where.</param> ///// <param name="onlyCurrentTest">if set to <c>true</c> [only current test].</param> ///// <returns></returns> //public static IEnumerable<webSiteProfile> selectGroup(this sampleGroupSet groupSet, sampleGroupItem group, ILogBuilder loger,webSiteProfileCollection source, String runStamp = null, String customWhere = "", Boolean onlyCurrentTest = false) //{ // //List<IReal> wbp = new List<IProfile>(); // //String whereQ = groupSet.makeSqlWhere(group, loger, runStamp, customWhere, onlyCurrentTest); // ////.selectItems(whereQ); // webSiteProfiles.selectItems<webSiteProfile>(sql, -1, selectItemsMode.sqlWhere, selectItemsResultType.managed); // // return wbp; //} /// <summary> /// Pravi WHERE dodatak SQL upita - /// </summary> /// <param name="runStamp">Ako nije definisan napravice trenuitni</param> /// <param name="groups">Grupe - ako nije definisan ucitace sve</param> /// <param name="customWhere">Proizvoljan WHERE upit</param> /// <param name="onlyCurrentTest">Da li proverava RunStamp</param> /// <returns></returns> public static string makeSqlWhere(this sampleGroupSet groupSet, sampleGroupItem group, ILogBuilder loger, string runStamp = null, string customWhere = "", bool onlyCurrentTest = false) { loger.log("Creating sample filter query for: " + group.groupTitle + " (" + group.groupTag + ")"); string output = ""; string whereRunStamp = ""; string whereGroups = ""; if (onlyCurrentTest) { if (string.IsNullOrEmpty(runStamp)) { } whereRunStamp = "((lastRunStamp <> \"" + runStamp + "\") OR (lastRunStamp IS NULL))"; } whereGroups = groupSet.db_groupTagFieldName + " LIKE '%" + group.groupTag + "%'"; if (!string.IsNullOrEmpty(whereRunStamp)) { output += whereRunStamp + ""; } if (!string.IsNullOrEmpty(whereGroups)) { if (!string.IsNullOrEmpty(output)) { output += " OR " + whereGroups; } else { output = whereGroups; } } if (!string.IsNullOrEmpty(customWhere)) { if (!string.IsNullOrEmpty(output)) { output += " OR " + customWhere; } else { output = customWhere; } } loger.log("Query to filter from " + group.count + " entries: " + output); return(output); }
public static DataTable SaveXML(this DataTable source, folderNode folder, string filenamePrefix = "", bool clearMeta = true, bool checkContent = true, ILogBuilder logger = null) { if (source.TableName.isNullOrEmpty()) { source.TableName = filenamePrefix + source.GetHashCode().ToString() + ".xml"; } string path = folder.pathFor(filenamePrefix + source.GetHashCode().ToString() + ".xml", getWritableFileMode.autoRenameExistingOnOtherDate); if (clearMeta) { source = source.CleanMeta(); } if (checkContent) { foreach (DataRow row in source.Rows) { foreach (DataColumn dc in source.Columns) { var t = row[dc]?.GetType(); if (t.IsClass) { if (logger != null) { logger.log(" === unallowed content detected in the table to save " + path); } row[dc] = dc.DataType.GetDefaultValue(); } } } } bool notOkToSave = false; foreach (DataColumn dc in source.Columns) { bool okType = dc.DataType.checkIfDataTypeIsAllowed(); if (!okType) { notOkToSave = true; break; } } if (!notOkToSave) { objectSerialization.saveObjectToXML(source, path); } else { if (logger != null) { logger.log("Can't save XML for table: " + source.TableName + " to " + path); } } return(source); }
public void LoadCompleteInformation(classificationReportCollectionSettings setup, folderNode folder, ILogBuilder logger) { DeployPath(folder.path); rootFolder = new folderStructure(rootPath, "Root", "Root directory with experiment reports"); var resultFiles = rootFolder.findFiles(setup.FILENAME_CLASSIFICATION_RESULTS, SearchOption.AllDirectories); String basePath = rootPath; foreach (String rFile in resultFiles) { String dirPath = Path.GetDirectoryName(rFile); String dirSubPath = dirPath.Substring(basePath.Length); classificationReportExpanded rep = classificationReportExpanded.LoadSimpleReport(rFile, logger); folderNode reportFolder = rootFolder.Add(dirSubPath, rep.Name, "Experiment [" + rep.Name + "] data"); rep.folder = reportFolder; String subCollectionDirPath = reportFolder.parent.path.Substring(rootFolder.path.Length); classificationReportCollection hostCollection = this; if (!subCollectionDirPath.isNullOrEmpty()) { if (reportFolder.parent != rootFolder) { hostCollection = AddOrGetChild(subCollectionDirPath); } } classificationReportExpanded rep_existing = hostCollection.FirstOrDefault(x => x.filepath == rep.filepath); if (rep_existing != null) { if (rep.filecreation > rep_existing.filecreation) { hostCollection.Remove(rep_existing); logger.log("Replacing older report [" + rep_existing.filepath + "] with new version"); } } hostCollection.Add(rep); reportCompleteList.Add(rep); //if (LoadExtraInformation != null) //{ // LoadExtraInformation.Invoke(rep, hostCollection, setup, logger); //} } logger.log("Reports (" + Count + ") directly imported from [" + rootPath + "] - Total reports, including [" + Children.Count + "] sub collections is [" + resultFiles.Count + "]"); }
public void LogConstruct(ILogBuilder logger) { if (logger != null) { logger.log("Primary Terms [ " + PrimaryTerms.Count + " ] -- Secondary Terms [" + SecondaryTerms.Count + " ]"); logger.log("Primary Chunks [ " + primaryChunks.Count + " ] -- Sec.Chunks [" + secondaryChunks.Count + " ]"); logger.log("Reserve Terms [ " + RelevantTerms.Count + " ] -- Trash bin [" + TrashBin.Count + " ]"); } }
public void SaveKnowledgeInstancesForClasses(kFoldValidationCase validationCase, ILogBuilder logger) { logger.log("Saving knowledge objects for [" + validationCase.name + "] classes"); foreach (IWebFVExtractorKnowledge knowledge in registry[validationCase].Values) { knowledge.OnBeforeSave(); } logger.log("Knowledge saved for [" + validationCase.name + "] classes"); }
/// <summary> /// Prepares resource file for querying and loads the grammar tag converter specification /// </summary> /// <param name="resourceFilePath">The text resource file path</param> /// <param name="grammSpecFilename">The Excel file with the grammar tag converter</param> /// <param name="output">The output.</param> protected void Setup(string resourceFilePath, string grammSpecFilename, ILogBuilder output = null) { if (resourceFilePath.isNullOrEmpty()) { imbACE.Services.terminal.aceTerminalInput.askYesNo("Resource file path is empty (textResourceResolverBase.Setup)!"); throw new ArgumentNullException(nameof(resourceFilePath)); return; } if (grammSpecFilename.isNullOrEmpty()) { imbACE.Services.terminal.aceTerminalInput.askYesNo("Grammar conversion specification file path is empty (textResourceResolverBase.Setup)!"); throw new ArgumentNullException(nameof(grammSpecFilename)); return; } resourceFileOperater = new fileTextOperater(resourceFilePath, true); grammTagConverter = new resourceConverterForGramaticTags(); if (grammSpecFilename.EndsWith(".xlsx")) { if (output != null) { output.log("Grammar conversion specification loading from Excel file"); } grammTagConverter.LoadSpecificationExcelFile(grammSpecFilename, output); } else if (grammSpecFilename.EndsWith(".csv")) { string filebase = Path.GetFileNameWithoutExtension(grammSpecFilename); string filepath = Path.GetDirectoryName(grammSpecFilename); if (output != null) { output.log("Grammar conversion specification loading from CSV files"); } string gramSpecFileFormat = filepath + Path.DirectorySeparatorChar + filebase + "_format.csv"; string gramSpecFileTranslation = filepath + Path.DirectorySeparatorChar + filebase + "_translation.csv"; grammTagConverter.LoadSpecificationCSV(gramSpecFileFormat, gramSpecFileTranslation, output); } else { if (output != null) { output.log("Grammar conversion file format not recognized from the filepath! [" + grammSpecFilename + "]"); } else { throw new ArgumentOutOfRangeException("File format not recognized for " + nameof(textResourceResolverBase) + " Setup call.", nameof(grammSpecFilename)); } } }
//private Object resourceLock = new Object(); public void SaveCaseKnowledgeInstances(ILogBuilder logger) { logger.log("Saving case knowledge objects for [" + validationCollection.extractor.name + "]"); foreach (IWebFVExtractorKnowledge knowledge in registryForCases.Values) { knowledge.OnBeforeSave(); reusableKnowledgeSaved.Add(knowledge.name, true); } logger.log("Knowledge case saved for [" + validationCollection.extractor.name + "]"); }
/// <summary> /// Builds a <see cref="settingsPropertyEntry"/> from contained data /// </summary> /// <param name="column">The column.</param> /// <param name="skipExisting">if set to <c>true</c> [skip existing].</param> /// <param name="log">The log.</param> /// <returns></returns> public settingsPropertyEntry BuildPCE(DataColumn column, Boolean skipExisting = true, ILogBuilder log = null) { settingsPropertyEntry pce = new settingsPropertyEntry(column); PropertyCollection pc = pce.exportPropertyCollection(); foreach (var pair in definitions.keyValuePairs) { if (skipExisting) { if (pc.ContainsKey(pair.resolvedKey)) { if (!pc[pair.resolvedKey].toStringSafe().isNullOrEmpty()) { if (log != null) { log.log(" Deploy [" + pair.key + "] = false, because destination is not empty (skipExisting=" + skipExisting.ToString() + ")"); } continue; } } } switch (pair.resolvedKey) { default: if (pair.resolvedKey is imbAttributeName attName) { pce.deploy(attName, pair.value); if (log != null) { log.log("Set[" + pair.key + "] = " + pair.value.toStringSafe()); } } else if (pair.resolvedKey is templateFieldDataTable tfdt) { pce.deploy(tfdt, pair.value); if (log != null) { log.log("Set[" + pair.key + "] = " + pair.value.toStringSafe()); } } else { if (log != null) { log.log(column.Table.GetTitle() + "." + column.ColumnName + " <- entry not recognized [" + pair.key + "]"); } } break; } } return(pce); }
public override List <pipelineTaskSubjectContentToken> process(IEnumerable <pipelineTaskSubjectContentToken> _input, ILogBuilder logger) { settings.checkReady(); List <pipelineTaskSubjectContentToken> output = new List <pipelineTaskSubjectContentToken>(); List <pipelineTaskSubjectContentToken> next = new List <pipelineTaskSubjectContentToken>(); next = _input.ToList(); next.Sort((x, y) => String.CompareOrdinal(x.currentForm, y.currentForm)); while (currentIteration > 0) { List <pipelineTaskSubjectContentToken> MCNext = new List <pipelineTaskSubjectContentToken>(); foreach (pipelineTaskSubjectContentToken sub in next) { MCNext.AddRange(processIteration(sub), true); } if (settings.keepAllInOutput) { output.AddRange(MCNext, true); } else { output = MCNext; } logger.log("[" + currentIteration + "] chunk construction in[" + next.Count + "] new[" + MCNext.Count + "] out[" + output.Count + "]"); if (next.Count == output.Count) { logger.log("Aborting the process since last iteation produced no changes"); break; } next = MCNext.ToList(); if (MCNext.Count == 0) { break; } currentIteration--; } return(output); //return base.process(_input, logger); }
public htmlContentPage tokenizeContent(ILogBuilder pRecordLog, HtmlDocument htmlDoc, basicLanguage language, node page) { var starttime = DateTime.Now; htmlContentPage contentPage = new htmlContentPage(); contentPage.acceptSourcePage(page); string domain = page.domain; object[] resources = new object[] { language, page, flags, sentenceFlags, tokenFlags, preprocessFlags }; var ctb = contentTreeBuilder.getInstance(htmlDoc.CreateNavigator(), domain, page); contentPage.treeBuilder = ctb; var blocks = ctb.tree.breakToBlocks(); int b = 0; for (int bi = 0; bi < blocks.Count; bi++) { imbTreeNodeBlock bl = blocks[bi]; b++; makeBlock(bl, contentPage, language, resources); // pRecordLog.close(); } contentPage.recountItems(); contentPage.primaryFlaging(resources); contentPage.secondaryFlaging(resources); // <--------------- // pRecordLog.log("SKIP: complete exploration of all tokens is turned off."); // contentPage.saveCache(); pRecordLog.log("Basic semantic analysis done. Closing the process."); var time = DateTime.Now.Subtract(starttime); pRecordLog.log("Tree-building and tokenization [" + page.url + "] done in: " + time.TotalMilliseconds.getSeconds(4) + "s"); return(contentPage); }
/// <summary> /// Gets the lexicon items. /// </summary> /// <param name="termForm">The term form.</param> /// <param name="loger">The loger.</param> /// <returns></returns> private lexiconResponse _getLexiconItems(List <string> termForms, ILogBuilder loger = null, bool callWithAutoDiscovery = true) { lexiconResponse output = new lexiconResponse(); //lock (getLexiconItemsLock) //{ output.setType(lexiconResponse.responseType.lexicon); ILexiconItem tis_i = null; try { //var tis_is = lexiconContext.TermInstances.Where(x => x.Equals(termForm)); // x=>x.name == termForm var tis_is = lexiconContext.TermInstances.Where(x => termForms.Contains(x.name)).FirstOrDefault(); // if (tis_is != null) { output.Add(tis_is); } } catch (Exception ex) { if (loger != null) { loger.log("Lexicon query failed: " + termForms.toCsvInLine(",") + " (lemmas)"); } } if (tis_i == null) { try { var til_is = lexiconContext.TermLemmas.Where(x => termForms.Contains(x.name)).FirstOrDefault(); // var til_is = lexiconContext.TermLemmas.Where(x => x.Equals(termForm)); if (til_is != null) { output.Add(til_is); } } catch (Exception ex) { if (loger != null) { loger.log("Lexicon query failed: " + termForms.toCsvInLine(",") + " (instances)"); } } } //} return(output); }
/// <summary> /// Loads the object from XML. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="filepath">The filepath.</param> /// <returns></returns> public static T loadObjectFromXML <T>(String filepath, ILogBuilder logger = null) where T : class { if (!File.Exists(filepath)) { return(default(T)); } FileInfo fi = new FileInfo(filepath); if (fi.Length == 0) { if (logger != null) { logger.log("Loading XML object from [" + filepath + "] aborted because file has 0 bytes. Default instance is created."); } return(default(T)); } Type t = typeof(T); String xmlString = File.ReadAllText(filepath); if (!(xmlString.Contains(t.Name) || xmlString.StartsWith("<?xml"))) { T altOutput = imbTypeExtensions.getInstance(t) as T; if (logger != null) { logger.log("XML structure failed"); } return(altOutput); } //if (!xmlString.Contains(t.Name)) //{ // if (logger != null) logger.log("Loading XML object from [" + filepath + "] aborted because the file seems not to be serialization of type [" + t.Name + "]"); // return default(T); //} XmlSerializer deserializer = new XmlSerializer(typeof(T)); //TextReader reader = new StreamReader(filepath); StringReader stringReader = new StringReader(xmlString); object obj = deserializer.Deserialize(stringReader); T output = (T)obj; stringReader.Close(); // reader.Close(); return(output); }
/// <summary> /// Adds the link. /// </summary> /// <param name="from">From.</param> /// <param name="to">To.</param> /// <param name="type">The type.</param> /// <param name="description">The description.</param> /// <returns></returns> public diagramLink AddLink(diagramNode from, diagramNode to, diagramLinkTypeEnum type, string description = "", string __hash = "", ILogBuilder logger = null) { diagramLink output = new diagramLink(); output.from = from; output.to = to; output.type = type; output.name = getUID(output); output.description = description; links.Add(output); output.parent = this; if (!__hash.isNullOrEmpty()) { if (linkByHash.ContainsKey(__hash)) { if (logger != null) { logger.log("AddLink() failed :: " + __hash); } } else { linkByHash.Add(__hash, output); } } return(output); }
/// <summary> /// Creates new node in the model /// </summary> /// <param name="__description">The description.</param> /// <param name="type">The type.</param> /// <param name="__name">The name.</param> /// <returns></returns> public diagramNode AddNode(string __description, diagramNodeShapeEnum type, string __name = "", string __hash = "", ILogBuilder logger = null) { diagramNode output = new diagramNode(); output.description = __description; output.shapeType = type; if (!isNodeNameAcceptable(__name)) { __name = __name + getUID(output, true, false); } output.name = __name; output.parent = this; nodes.Add(output.name, output); if (!__hash.isNullOrEmpty()) { if (linkByHash.ContainsKey(__hash)) { if (logger != null) { logger.log("AddNode() failed :: " + __hash); } } else { nodeByHash.Add(__hash, output); } } return(output); }
/// <summary> /// Filters the space model features. /// </summary> /// <param name="spaceModel">The space model.</param> /// <param name="selectedFeatures">The selected features.</param> /// <param name="log">The log.</param> /// <returns></returns> public static Int32 FilterSpaceModelFeatures(this SpaceModel spaceModel, WeightDictionary selectedFeatures, ILogBuilder log) { Int32 i = 0; Int32 s = spaceModel.documents.Count() / 5; Int32 c_filter_out = 0; List <String> keys = selectedFeatures.GetKeys(); List <String> termsToRemove = spaceModel.terms.GetTokensOtherThan(keys); for (int i2 = 0; i2 < spaceModel.documents.Count; i2++) { c_filter_out += spaceModel.documents[i2].FilterSelectedFeatures(termsToRemove, false); if (i > s) { Double r = i2.GetRatio(spaceModel.documents.Count()); log.log("Filter SelectedFeatures [" + r.ToString("P2") + "]"); i = 0; } i++; } spaceModel.terms_known_label.FilterTokens(termsToRemove, false); spaceModel.terms_unknown_label.FilterTokens(termsToRemove, false); return(c_filter_out); }
/// <summary> /// Processes the specified cloud. /// </summary> /// <param name="cloud">The cloud.</param> /// <param name="logger">The logger.</param> /// <returns></returns> public lemmaSemanticWeaverResult Process(lemmaSemanticCloud cloud, ILogBuilder logger) { Int32 c = cloud.links.Count; lemmaSemanticWeaverResult weaverResult = new lemmaSemanticWeaverResult(cloud); wordSimilarityResultSet output = new wordSimilarityResultSet(); List <String> words = cloud.nodes.Select(x => x.name).ToList(); if (useSimilarity) { output = similarWords.GetResult(words); foreach (var pair in output) { var link = cloud.GetLink(pair.wordA, pair.wordB); if (link == null) { cloud.AddLink(pair.wordA, pair.wordB, pair.score, LINK_OF_SIMILARWORDS); } } weaverResult.linkRatioAfterWS = cloud.GetLinkPerNodeRatio(); weaverResult.similarWords = output; } /* NOT WORKING ---- TEMPORARLY DISABLED * if (useDictionary) * { * Stack<String> wordsToTest = new Stack<string>(); * words.ForEach(x => wordsToTest.Push(x)); * * while (wordsToTest.Any()) * { * String word = wordsToTest.Pop(); * apertiumDictionaryResult result = apertium.queryForSynonyms(word, apertiumDictQueryScope.exact); * * var synonims = result.GetNativeWords(); * * var synonimNodes = wordsToTest.Where(x => synonims.Contains(x)); * * foreach (String syn in synonimNodes) * { * wordsToTest.RemoveAll(x=>x==syn); * cloud.AddLink(word, syn, 1, LINK_OF_DICTIONARYSYNONIMS); * * weaverResult.appertiumNotes.Add(word + " -> " + syn); * } * weaverResult.linkRatioAfterDS = cloud.GetLinkPerNodeRatio(); * } * }*/ if (c != cloud.links.Count) { if (logger != null) { logger.log("Weaver created [" + (cloud.links.Count - c) + "] new links in the cloud [" + cloud.className + "]"); } } return(weaverResult); }