/* * public static WeightDictionary GetChildrenWT(WeightDictionary output, SpaceDocumentModel model, FeatureWeightModel weightModel, SpaceModel space, List<string> FV) * { * if (output == null) * { * output = new WeightDictionary(model.name, ""); * } * if (model.Children.Any()) * { * foreach (var child in model.Children) * { * GetChildrenWT(output, child, weightModel, space, FV); * * } * * } * else * { * var wd = weightModel.GetWeights(FV, model, space); * output.Merge(wd.index.Values, model.weight); * } * return output; * * }*/ public static T BlendToVector <T>(this SpaceDocumentModel model, FeatureWeightModel weightModel, SpaceModel space, List <string> FV) where T : VectorDocument, new() { T output = new T(); output.name = model.name; var leafs = model.GetLeafs(); foreach (var leaf in leafs) { var wd = weightModel.GetWeights(FV, model, space); output.terms.Merge(wd); } //output.terms = //output.Merge(wd.index.Values, model.weight); //WeightDictionary wd = new WeightDictionary(model.name, ""); // GetChildrenWT(wd, model, weightModel, space, FV); return(output); }
public void SetEntry(String _domainID, WebSiteDocument _webDocument, SpaceDocumentModel _spaceDocument, TextDocument _textDocument) { type = DocumentSelectEntryType.unknown; DomainID = _domainID; webDocument = _webDocument; spaceDocument = _spaceDocument; textDocument = _textDocument; if (textDocument != null) { AssignedID = textDocument.name; type |= DocumentSelectEntryType.textDocument; } if (spaceDocument != null) { AssignedID = spaceDocument.name; type |= DocumentSelectEntryType.spaceDocument; } if (webDocument != null) { AssignedID = _webDocument.AssignedID; type |= DocumentSelectEntryType.webDocument; } }
/// <summary> /// Gets the weights. /// </summary> /// <param name="termWhiteList">The term white list.</param> /// <param name="document">The document.</param> /// <param name="space">The space.</param> /// <param name="label">The label.</param> /// <returns></returns> public WeightDictionary GetWeights(List <String> termWhiteList, SpaceDocumentModel document, SpaceModel space, SpaceLabel label = null) { WeightDictionary output = new WeightDictionary(); output.name = GetSignature() + "_" + document.name; output.description = "Feature weight table constructed by [" + GetSignature() + "] for features [" + termWhiteList.Count + "] in document [" + document.name + "]"; output.nDimensions = nDimensions; if (KERNELOPTION_USE_WHITELISTTERMS) { foreach (String term in termWhiteList) { if (document.terms.Contains(term)) { throw new NotImplementedException(); //output.entries.Add(entry); } } } else { List <String> terms = document.terms.GetTokens(); for (int i = 0; i < document.terms.Count; i++) { String term = terms[i]; WeightDictionaryEntry entry = new WeightDictionaryEntry(term, 0); if (DoUseLocalFunction) { entry = LocalFunction.GetElementFactorEntry(term, document); } foreach (FeatureWeightFactor gf in GlobalFactors) { entry = entry * (gf.GlobalFunction.GetElementFactorEntry(term, space, label) * gf.weight); } if (document.weight != 1) { entry = entry * document.weight; } output.Merge(entry); //output.AddEntry(term, entry.dimensions, false); } } return(output); }
/// <summary> /// Gets the local element factor /// </summary> /// <param name="term">The term.</param> /// <param name="document">The document.</param> /// <returns></returns> public double GetElementFactor(String term, SpaceDocumentModel document) { if (DoUseLocalFunction && LocalFunction != null) { if (LocalFunction.IsEnabled) { Double TF = LocalFunction.GetElementFactor(term, document); return(TF); } } return(1); }
/// <summary> /// Nests the space document model. /// </summary> /// <param name="distionary">The distionary.</param> /// <param name="key">The key.</param> /// <param name="log">The log.</param> /// <returns></returns> public static SpaceDocumentModel NestSpaceDocumentModel(this Dictionary <String, SpaceDocumentModel> distionary, String key, ILogBuilder log) { SpaceDocumentModel output = new SpaceDocumentModel(); output.name = key; foreach (var p in distionary) { output.Children.Add(p.Value); output.terms.MergeDictionary(p.Value.terms); } return(output); }
public WeightDictionaryEntry GetCompositeEntry(String term, SpaceDocumentModel document, SpaceModel space) { WeightDictionaryEntry output = new WeightDictionaryEntry(term, 0); List <Double> dimensions = new List <double>(); dimensions.Add(LocalFunction.GetElementFactor(term, document)); foreach (var gf in GlobalFactors) { dimensions.Add(gf.GlobalFunction.GetElementFactor(term, space) * gf.weight); } output.dimensions = dimensions.ToArray(); return(output); }
/// <summary> /// Nests the complete space document model. /// </summary> /// <param name="nest">The nest.</param> /// <param name="log">The log.</param> /// <returns></returns> public static SpaceDocumentModel NestCompleteSpaceDocumentModel(this Dictionary <string, Dictionary <string, Dictionary <string, SpaceDocumentModel> > > nest, String name, ILogBuilder log) { SpaceDocumentModel output = new SpaceDocumentModel(); Dictionary <String, SpaceDocumentModel> rootTmp = new Dictionary <string, SpaceDocumentModel>(); foreach (var pairCategory in nest) { Dictionary <String, SpaceDocumentModel> catTmp = new Dictionary <string, SpaceDocumentModel>(); foreach (var pairWebsite in pairCategory.Value) { catTmp.Add(pairWebsite.Key, pairWebsite.Value.NestSpaceDocumentModel(pairWebsite.Key, log)); } rootTmp.Add(pairCategory.Key, catTmp.NestSpaceDocumentModel(pairCategory.Key, log)); } output = rootTmp.NestSpaceDocumentModel(name, log); return(output); }
/// <summary> /// Prepares the factor by processing the context /// </summary> /// <param name="context">The context.</param> /// <param name="log">The log.</param> public override void Prepare(DocumentSelectResult context, ILogBuilder log) { statsByAssignedID.Clear(); foreach (DocumentSelectResultEntry docEntry in context.items) { instanceCountCollection <string> ft = new instanceCountCollection <string>(); if (docEntry.type.HasFlag(DocumentSelectEntryType.spaceDocument)) { SpaceDocumentModel document = docEntry.spaceDocument; foreach (var term in document.terms.GetTokens()) { ft.AddInstance(term, document.terms.GetTokenFrequency(term)); } } else if (docEntry.type.HasFlag(DocumentSelectEntryType.textDocument)) { String content = docEntry.textDocument.content; // document.ToString(); List <String> tkns = content.getTokens(true, true, true, false, 4); foreach (String tkn in tkns) { String stem = tkn; if (useStems) { stem = context.stemmingContext.Stem(tkn); } ft.AddInstance(stem); } } statsByAssignedID.Add(docEntry.AssignedID, ft); assignedIDs.Add(docEntry.AssignedID); } }
/// <summary> /// Spaces the model population. /// </summary> /// <param name="context">The context.</param> /// <param name="log">The log.</param> public void SpaceModelPopulation(OperationContext context, ILogBuilder log) { log.log("Space model population"); context.stemmContext = new StemmingContext(stemmer); // modelling the documents foreach (var pair in context.textDocuments) { var doc = pair.Value; SpaceDocumentModel model = spaceConstructor.ConstructDocument(doc.content, doc.name, context.spaceModel, context.stemmContext, tokenizer); foreach (String label in doc.labels) { SpaceLabel sLabel = null; sLabel = context.spaceLabels[label]; context.spaceModel.LabelToDocumentLinks.Add(sLabel, model, 1); } context.spaceModel.documents.Add(model); if (doc.labels.Contains(SpaceLabel.UNKNOWN)) { context.spaceModel.terms_unknown_label.MergeDictionary(model.terms); } else { context.spaceModel.terms.MergeDictionary(model.terms); } } log.log("Space model -- documents created [" + context.spaceModel.documents.Count + "]"); }
// public abstract WeightDictionaryEntry GetElementFactor(string term, SpaceDocumentModel document); public abstract double GetElementFactor(string term, SpaceDocumentModel document);
public DocumentSelectResultEntry(SpaceDocumentModel document) { spaceDocument = document; AssignedID = document.name; type = DocumentSelectEntryType.spaceDocument; }
/// <summary> /// Spaces the model population. /// </summary> /// <param name="context">The context.</param> /// <param name="log">The log.</param> public void SpaceModelPopulation(OperationContext context, ILogBuilder log) { log.log("Space model population"); context.stemmContext = new StemmingContext(stemmer); context.tokenizer = tokenizer; context.entityMetrics = new Dictionary <String, ContentMetrics>(); foreach (KeyValuePair <String, TextDocumentSet> pair in context.renderSiteByDomain) { SpaceLabel spaceLabel = context.spaceLabelsDomains[pair.Key]; SpaceDocumentModel modelOfSite = new SpaceDocumentModel(); modelOfSite.name = pair.Key; modelOfSite.labels.Add(spaceLabel.name); foreach (TextDocumentLayerCollection textLayer in pair.Value) { SpaceDocumentModel modelOfPage = new SpaceDocumentModel(textLayer.name); ContentMetrics metrics = null; if (DoKeepContentMetrics) { metrics = new ContentMetrics(textLayer.name); } foreach (var renderLayer in textLayer) { SpaceDocumentModel modelOfLayer = new SpaceDocumentModel(modelOfPage.name + renderLayer.name); modelOfLayer = spaceConstructor.ConstructDocument(renderLayer.content, modelOfPage.name + renderLayer.name, context.spaceModel, context.stemmContext, tokenizer, spaceLabel.name != SpaceLabel.UNKNOWN, metrics); modelOfLayer.weight = renderLayer.layerWeight; modelOfLayer.documentScope = DocumentBlenderFunctionOptions.layerLevel; modelOfPage.Children.Add(modelOfLayer); } modelOfPage.documentScope = DocumentBlenderFunctionOptions.pageLevel; if (DoKeepContentMetrics) { context.entityMetrics.Add(metrics.Name, metrics); } // modelOfPage.Flatten(false); modelOfSite.Children.Add(modelOfPage); } modelOfSite.documentScope = DocumentBlenderFunctionOptions.siteLevel; context.spaceModel.documents.Add(modelOfSite); foreach (String label in modelOfSite.labels) { SpaceLabel sLabel = null; sLabel = context.spaceLabels[label]; context.spaceModel.LabelToDocumentLinks.Add(sLabel, modelOfSite, 1); } modelOfSite.Flatten(false); /* * if (modelOfSite.labels.Contains(SpaceLabel.UNKNOWN)) * { * context.spaceModel.terms_unknown_label.MergeDictionary(modelOfSite.terms); * } * else * { * context.spaceModel.terms_known_label.MergeDictionary(modelOfSite.terms); * }*/ modelOfSite.PropagateLabels(); // modelOfSite.SetLabel(spaceLabel, context.spaceModel); //context.spaceModel.LabelToDocumentLinks.Add(spaceLabel, modelOfSite, 1.0); } log.log("Space model -- documents created [" + context.spaceModel.documents.Count + "]"); }
public override WeightDictionaryEntry GetElementFactorEntry(string term, SpaceDocumentModel document) { return(new WeightDictionaryEntry(term, GetElementFactor(term, document))); }
public override double GetElementFactor(string term, SpaceDocumentModel document) { if (!IsEnabled) { return(1); } TokenDictionary docDict = document.GetTerms(true, true); Double TF = docDict.GetTokenFrequency(term); switch (computation) { case TFComputation.modifiedTF: if (!index.ContainsKey(term)) { return(0); } Double Tt = index[term]; // training_terms.GetTokenFrequency(term); Double length_d = docDict.Count; //.GetTokenCount(); Double mTF_above = TF * Math.Log(SqrTc / Tt); Double mTF_below_2nd = (length_d * length_d) / SqrTc; Double mTF_below = Math.Log(docDict.GetSumSquareFrequencies() * mTF_below_2nd); return(mTF_above / mTF_below); break; } Double divisor = GetDivisor(docDict); //if (TFN_index.ContainsKey(document)) //{ // divisor = TFN_index[document]; //} //else //{ // divisor //} switch (computation) { default: case TFComputation.normal: return(TF / divisor); break; case TFComputation.squareRooted: return(Math.Sqrt(TF / divisor)); break; case TFComputation.glasgow: return(Math.Log(TF + 1) / divisor); break; } }
/// <summary> /// Executes the plane method, invoking contained functions according to the settings /// </summary> /// <param name="inputContext">The input context - related to this plane.</param> /// <param name="generalContext">General execution context, attached to the <see cref="T:imbNLP.Toolkit.Planes.PlanesMethodDesign" /></param> /// <param name="logger">The logger.</param> /// <returns> /// Retur /// </returns> public IPlaneContext ExecutePlaneMethod(IPlaneContext inputContext, ExperimentModelExecutionContext generalContext, ILogBuilder logger) { notes.logStartPhase("[2] Corpus Plane - execution", ""); ICorpusPlaneContext context = (ICorpusPlaneContext)inputContext; VectorPlaneContext outputContext = new VectorPlaneContext(); outputContext.provider.StoreAndReceive(context); context.stemmContext = new StemmingContext(stemmer); Dictionary <String, SpaceDocumentModel> documentVsModel = new Dictionary <string, SpaceDocumentModel>(); // modelling the documents foreach (TextDocument doc in context.corpus_documents) { SpaceDocumentModel model = spaceConstructor.ConstructDocument(doc.content, doc.name, context.space, context.stemmContext, tokenizer); List <SpaceLabel> labels = spaceConstructor.GetLabels(doc.labels, context.space); Boolean isUnknownLabel = true; foreach (SpaceLabel label in labels) { if (label.name != SpaceLabel.UNKNOWN) { isUnknownLabel = false; } context.space.LabelToDocumentLinks.Add(label, model, 1); } context.space.documents.Add(model); if (!isUnknownLabel) { context.space.terms.MergeDictionary(model.terms); } documentVsModel.Add(doc.name, model); } if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_fold_textrender)) { foreach (TextDocument doc in context.corpus_documents) { String prefix = doc.labels.FirstOrDefault(); if (prefix.isNullOrEmpty()) { prefix = SpaceLabel.UNKNOWN; } String fn = prefix + "_" + doc.name; String pth = notes.folder_entity.pathFor(fn.getFilename("txt"), imbSCI.Data.enums.getWritableFileMode.overwrite, "Textual representation of website [" + doc.name + "], produced by rendering and blending settings", true); doc.content.saveStringToFile(pth, imbSCI.Data.enums.getWritableFileMode.overwrite); } } if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_fold_stats)) { foreach (WebSiteDocumentsSet ds in context.dataset) { DataTable dt = ds.MakeTable(documentVsModel); notes.SaveDataTable(dt, notes.folder_entity); } var dt_vsm = context.space.LabelToDocumentLinks.MakeTable("LabelToDocument", "Relationships between labels and documents in the primary Vector Space Model"); notes.SaveDataTable(dt_vsm, notes.folder_corpus); } if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_corpusDictionary)) { notes.SaveDataTable(context.space.terms.MakeTable("corpus_stats", "Training set dictionary, after stemming", generalContext.DictionaryReportLimit), notes.folder_corpus); } #region SELECTING THE FEATURES // forming corpus global weight context.SelectedFeatures = new WeightDictionary(); List <KeyValuePair <string, double> > filter_result = filter.SelectFeatures(context.space); List <string> FV = new List <string>(); FV.AddRange(filter_result.Select(x => x.Key)); if (filter_result.Any()) { foreach (var pair in filter_result) { context.SelectedFeatures.AddEntry(pair.Key, pair.Value); } if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_selectedFeatures)) { notes.SaveDataTable(context.SelectedFeatures.MakeTable("selected_features", "Features selected for BoW construction", new List <string>() { filter.function.shortName }, generalContext.DictionaryReportLimit), notes.folder_corpus); } } else { logger.log("-- Feature selection function returned zero set. All features [" + context.space.terms.Count + "] are therefore accepted as selected."); var tkns = context.space.terms.GetTokens(); foreach (var tkn in tkns) { context.SelectedFeatures.AddEntry(tkn, 1); } } #endregion notes.log("Selected features [" + context.SelectedFeatures.entries.Count + "] by [" + filter.functionSettings.functionName + "]"); //context.space = //weightModel.Deploy(); outputContext.vectorSpace = new Vectors.VectorSpace(); foreach (SpaceLabel label in context.space.labels) { var docs = context.space.LabelToDocumentLinks.GetAllLinked(label); if (label.name != SpaceLabel.UNKNOWN) { SpaceCategoryModel categoryModel = new SpaceCategoryModel(label, docs); context.space.LabelToCategoryLinks.Add(label, categoryModel, 1); context.space.categories.Add(categoryModel); notes.log("Class [" + categoryModel.name + "] BoW model created - terms[" + categoryModel.terms.Count + "] "); } } outputContext.LabelToDocumentLinks = context.space.LabelToDocumentLinks; // preparing the model weightModel.PrepareTheModel(context.space); // logger.log(":: Creating VectorSpace instances for documents"); // building document VSM foreach (SpaceDocumentModel docModel in context.space.documents) { var wd = weightModel.GetWeights(FV, docModel, context.space); VectorDocument docVec = new VectorDocument(docModel.name); docVec.terms = wd; if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_documentBoWModels)) { DataTable dt = wd.MakeTable("docVec_" + docModel.name, "Document vector model", null, 10000); notes.SaveDataTable(dt, notes.folder_vector); } outputContext.vectorSpace.documents.Add(docVec); } // logger.log(":: Creating VectorSpace instances for categories"); // building category VSM foreach (SpaceCategoryModel catModel in context.space.categories) { var wd = weightModel.GetWeights(FV, catModel, context.space); VectorLabel catVec = new VectorLabel(catModel.name); catVec.terms = wd; if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_documentBoWModels)) { DataTable dt = wd.MakeTable("catVec_" + catModel.name, "Document vector model", null, 10000); notes.SaveDataTable(dt, notes.folder_vector); } outputContext.vectorSpace.labels.Add(catVec); } if (generalContext.reportOptions.HasFlag(PlanesReportOptions.report_documentBoWModels)) { foreach (SpaceCategoryModel catModel in context.space.categories) { var dt = catModel.terms.MakeTable("cat_" + catModel.name, "Vector Space BoW weighted model, representing a category"); notes.SaveDataTable(dt, notes.folder_vector); } } notes.logEndPhase(); return(outputContext); }
public static histogramModel GetHistogram(this SpaceDocumentModel dictionary, Int32 binCount = 50) { histogramModel model = dictionary.terms.GetRankedTokenFrequency().GetHistogramModel(dictionary.name, x => x.Value, binCount); //new histogramModel(binCount, dictionary.name); return(model); }
public Double GetWeight(String term, SpaceDocumentModel document, SpaceModel space, SpaceLabel label = null) { return(GetElementFactor(term, document) * GetElementFactor(term, space, label)); }
///// <summary> ///// Transforms to fv dictionary. ///// </summary> ///// <param name="context">The context.</param> ///// <param name="TermWeightModel">The term weight model.</param> ///// <param name="function">The function.</param> ///// <returns></returns> //public static FeatureVectorSetDictionary TransformToFVDictionaryAsPageInCategorySimilarity(this DocumentSelectResult context, FeatureWeightModel TermWeightModel, IVectorSimilarityFunction function, ILogBuilder log) //{ // log.log("... Page Similarity ..."); // List<string> selectedTerms = context.selectedFeatures.GetKeys(); // var ByDomain = context.GetByDomain(log); // Dictionary<string, List<string>> assignIDByLabel = context.featureSpace.labelToDocumentAssociations.GetAllRelationShipByName(true); // var ByCategory = context.GetByAssignIDCategory(assignIDByLabel,log); // Dictionary<String, List<DocumentSelectResultEntry>> EntryByLabel = new Dictionary<string, List<DocumentSelectResultEntry>>(); // Dictionary<String, WeightDictionary> documentDictionarties = new Dictionary<string, WeightDictionary>(); // foreach (var entry in context.items) // { // WeightDictionary documentWeights = TermWeightModel.GetWeights(selectedTerms, entry.spaceDocument, context.spaceModel); // documentDictionarties.Add(entry.AssignedID, documentWeights); // } // FeatureVectorSetDictionary dict = new FeatureVectorSetDictionary(); // Double total = context.Count; // Int32 i = 0; // Int32 p = (context.Count / 10); // //List<List<Double>> matrix = new List<List<double>>(); // //foreach (var entry in context.items) // //{ // // matrix.Add(new List<double>()); // //} // //for (int x = 0; x < context.items.Count; x++) // //{ // // for (int y = 0; y < context.items.Count; x++) // // { // // } // //} // ConcurrentDictionary<String, Double> computedPairs = new ConcurrentDictionary<string, double>(); // foreach (var domainPair in ByCategory) // { // List<DocumentSelectResultEntry> relatives = ByCategory[domainPair.Key].ToList(); // foreach (var entry in relatives) // { // i++; // FeatureVector fv = new FeatureVector(entry.AssignedID); // // List<Double> d = new List<>(); // fv.dimensions = new double[relatives.Count - 1]; // // List<String> keys = documentDictionarties.Keys.ToList(); // Int32 hostInd = relatives.IndexOf(entry); // Int32 c = 0; // //foreach (var pair in documentDictionarties) // //{ // Parallel.ForEach(relatives, (pair) => // { // Int32 ind = relatives.IndexOf(pair); // keys.IndexOf(pair.AssignedID); // if (ind >= hostInd) // { // ind = ind - 1; // } // if (pair.AssignedID != entry.AssignedID) // { // Double docToClassSimilarity = 0; // if (computedPairs.ContainsKey(entry.AssignedID + pair.AssignedID)) // { // docToClassSimilarity = computedPairs[entry.AssignedID + pair.AssignedID]; // } // else if (computedPairs.ContainsKey(pair.AssignedID + entry.AssignedID)) // { // docToClassSimilarity = computedPairs[pair.AssignedID + entry.AssignedID]; // } // else // { // var vecA = documentDictionarties[pair.AssignedID]; // var vecB = documentDictionarties[entry.AssignedID]; // docToClassSimilarity = function.ComputeSimilarity(vecA, vecB); // if (docToClassSimilarity > 0) // { // } // if (!computedPairs.ContainsKey(entry.AssignedID + pair.AssignedID)) // { // computedPairs.GetOrAdd(entry.AssignedID + pair.AssignedID, docToClassSimilarity); // //computedPairs.AddOrUpdate(entry.AssignedID + pair.Key, docToClassSimilarity); // } // else if (!computedPairs.ContainsKey(pair.AssignedID + entry.AssignedID)) // { // computedPairs.GetOrAdd(pair.AssignedID + entry.AssignedID, docToClassSimilarity); // } // } // fv.dimensions[ind] = docToClassSimilarity; // } // }); // Int32 r = i % p; // if (r == 0) // { // log.Append(" [" + i.GetRatio(context.Count).ToString("P2") + "] "); // } // dict.GetOrAdd(entry.DomainID).Add(fv, -1); // } // } // //foreach (KeyValuePair<string, FeatureVectorWithLabelIDSet> pair in dict) // //{ // // pair.Value.CloseDeploy(); // //} // log.log("... Preparation finished ..."); // return dict; //} /// <summary> /// Transforms to fv dictionary. /// </summary> /// <param name="context">The context.</param> /// <param name="TermWeightModel">The term weight model.</param> /// <param name="function">The function.</param> /// <returns></returns> public static FeatureVectorSetDictionary TransformToFVDictionaryAsSiteSimilarity(this DocumentSelectResult context, FeatureWeightModel TermWeightModel, IVectorSimilarityFunction function, ILogBuilder log) { log.log("... Site Similarity ..."); List <string> selectedTerms = context.selectedFeatures.GetKeys(); //.entries.Select(x => x.name)?.ToList(); Dictionary <String, WeightDictionary> categoryDictionarties = new Dictionary <string, WeightDictionary>(); Dictionary <String, WeightDictionary> documentDictionarties = new Dictionary <string, WeightDictionary>(); var byDomain = context.GetByDomain(log); FeatureVectorSetDictionary dict = new FeatureVectorSetDictionary(); Double total = context.Count; Int32 i = 0; Int32 p = (context.Count / 10); foreach (var pair in byDomain) { i++; SpaceDocumentModel siteModel = new SpaceDocumentModel(); foreach (var ent in pair.Value) { WeightDictionary documentWeights = TermWeightModel.GetWeights(selectedTerms, ent.spaceDocument, context.spaceModel); documentDictionarties.Add(ent.AssignedID, documentWeights); siteModel.Children.Add(ent.spaceDocument); //siteModel.terms.MergeDictionary(ent.spaceDocument.terms); } siteModel.Flatten(false); categoryDictionarties.Add(pair.Key, TermWeightModel.GetWeights(selectedTerms, siteModel, context.spaceModel)); foreach (var ent in pair.Value) { FeatureVector fv = new FeatureVector(ent.AssignedID); fv.dimensions = new double[context.spaceModel.labels.Count]; // documentDictionarties[ent.AssignedID].entries var docToClassSimilarity = function.ComputeSimilarity(categoryDictionarties[pair.Key], documentDictionarties[ent.AssignedID]); fv.dimensions[0] = docToClassSimilarity; dict.GetOrAdd(pair.Key).Add(fv, -1); } Int32 r = i % p; if (r == 0) { log.Append(" [" + i.GetRatio(context.Count).ToString("P2") + "] "); } } foreach (KeyValuePair <string, FeatureVectorWithLabelIDSet> pair in dict) { pair.Value.CloseDeploy(); } log.log("... Preparation finished ..."); return(dict); }
public abstract WeightDictionaryEntry GetElementFactorEntry(string term, SpaceDocumentModel document);
/// <summary> /// Prepares the context. /// </summary> /// <param name="context">The context.</param> /// <param name="log">The log.</param> /// <returns></returns> public static DocumentSelectResult PrepareContext(this OperationContext context, DocumentRankingMethod ranking, folderNode folder, ILogBuilder log) { DocumentSelectResult selectContext = new DocumentSelectResult(); selectContext.stemmingContext = context.stemmContext; selectContext.spaceModel = context.spaceModel; selectContext.folder = folder; if (ranking != null) { selectContext.name = ranking.model.GetSignature(); selectContext.query = ranking.query; builderForText builder = new builderForText(); ranking.Describe(builder); builder.AppendLine("Selected features [" + selectContext.selectedFeatures.description + "]."); selectContext.description = builder.GetContent().Replace(Environment.NewLine, ""); } selectContext.selectedFeatures = context.SelectedFeatures; foreach (KeyValuePair <string, WebSiteDocuments> pair in context.webSiteByDomain) { selectContext.domainNameToGraph.Add(pair.Key, pair.Value?.extensions?.graph); foreach (WebSiteDocument doc in pair.Value.documents) { DocumentSelectResultEntry entry = new DocumentSelectResultEntry(); TextDocument text = null; string err = ""; //if (context.textDocuments.ContainsKey(doc.AssignedID)) //{ // text = context.textDocuments[doc.AssignedID]; //} //else //{ // err += "Failed to find text document for [" + doc.AssignedID + "]"; //} SpaceDocumentModel spaceDocument = context.spaceModel.documents.FirstOrDefault(x => x.name == doc.AssignedID); if (spaceDocument == null) { err += "Failed to find space model document for [" + doc.AssignedID + "]"; } string dn = pair.Value.domain; entry.SetEntry(dn, doc, spaceDocument, text); if (!entry.HasTextOrSpaceModel) { log.log(err); } selectContext.items.Add(entry); //entry.SetEntry( context.context.webDocumentByAssignedID[pair.Key], webDocIDToDomain[aID], webDocumentRegistry[aID], spaceDocumentRegistry[aID], textDocumentRegistry[aID]); } } // PREPARATION OF MODEL if (ranking != null) { ranking.model.Prepare(selectContext, log); } return(selectContext); }
public void Prepare(SpaceModel spaceModel, ILogBuilder log, bool excludeUnknown = true) { var labels = spaceModel.LabelToDocumentLinks.GetAllDistinctNames(); if (excludeUnknown) { labels.Remove(SpaceLabel.UNKNOWN); } datasetStatsModel = new SpaceDocumentStatsModel("Stats", log); datasetStatsModel.documentScope = Entity.DocumentBlenderFunctionOptions.datasetLevel; flatDataSetStatsModel = new SpaceDocumentStatsModel("FlatStats", log); flatDataSetStatsModel.documentScope = Entity.DocumentBlenderFunctionOptions.datasetLevel; foreach (string label in labels) { var documents = spaceModel.documents.Where(x => x.labels.Contains(label)); //var documents = spaceModel.LabelToDocumentLinks.GetAllLinkedB(label); SpaceDocumentModel labelDocModel = new SpaceDocumentModel(label); labelDocModel.documentScope = Entity.DocumentBlenderFunctionOptions.categoryLevel; labelDocModel.Children.AddRange(documents); labelDocModel.Flatten(false); // var categoryModel = new SpaceDocumentStatsModel(labelDocModel, log); SpaceDocumentStatsModel categoryModel = new SpaceDocumentStatsModel(labelDocModel.name, log); categoryModel.LearnFrom(labelDocModel, log, true); datasetStatsModel.Children.Add(categoryModel); datasetStatsModel.terms.MergeDictionary(labelDocModel.terms); datasetStatsModel.termsChildCount.CountTokens(labelDocModel.terms.GetTokens()); if (log != null) { log.log("Statistics for category [" + label + "]"); } var allChildren = labelDocModel.GetAllChildren(); for (int i = 0; i < allChildren.Count; i++) { totalCounts.DirectCount(allChildren[i].documentScope); } } if (settings.RequiredScopes.HasFlag(CWPAnalysisScopeEnum.flatSiteLevel)) { if (log != null) { log.log("Creating flat report"); } foreach (var document in spaceModel.documents) { if (!document.labels.Contains(SpaceLabel.UNKNOWN)) { flatDataSetStatsModel.LearnFrom(document, log, true); } } } }