///// <summary> ///// Normalization divisor for document ///// </summary> ///// <value> ///// The index of the TFN. ///// </value> //protected Dictionary<SpaceDocumentModel, Double> TFN_index { get; set; } = new Dictionary<SpaceDocumentModel, Double>(); public override void PrepareTheModel(SpaceModel _space, ILogBuilder log) { var space = _space; TokenDictionary training_terms = space.GetTerms(true, true); List <SpaceLabel> labels = space.labels.ToList(); shortName = GetFunctionName(computation); if (!IsEnabled) { return; } switch (computation) { case TFComputation.modifiedTF: SqrTc = Math.Sqrt(training_terms.GetSumFrequency()); break; default: //foreach (SpaceDocumentModel document in space.documents) //{ // TFN_index.Add(document, GetDivisor(document)); //} break; } index = training_terms.ToFrequencyDictionary(); }
/// <summary> /// Makes ranked table with term frequencies /// </summary> /// <param name="terms">The terms.</param> /// <param name="name">The name.</param> /// <param name="description">The description.</param> /// <param name="limit">The limit.</param> /// <returns></returns> public static DataTable MakeTable(this TokenDictionary terms, string name, string description, Int32 limit = 1000) { DataTable table = new DataTable(); table.SetTitle(name); table.SetDescription(description); table.SetAdditionalInfoEntry("Dictinct terms", terms.Count, "Total distinct terms in the dictionary"); table.SetAdditionalInfoEntry("Max frequency", terms.GetMaxFrequency(), "Highest frequency"); table.SetAdditionalInfoEntry("Total tokens", terms.GetSumFrequency(), "Total number of tokens extracted from the corpus/document, i.e. sum of all frequencies"); DataColumn column_rank = table.Add("Rank", "Rank by frequency", "R", typeof(Int32), imbSCI.Core.enums.dataPointImportance.normal).SetWidth(20); DataColumn column_id = table.Add("ID", "Token ID", "id", typeof(Int32), imbSCI.Core.enums.dataPointImportance.normal).SetWidth(20); DataColumn column_token = table.Add("Token", "Token", "t", typeof(String), imbSCI.Core.enums.dataPointImportance.normal).SetWidth(50); DataColumn column_freq = table.Add("Frequency", "Absolute number of token occurrences in the corpus/document", "TF", typeof(Int32), imbSCI.Core.enums.dataPointImportance.normal).SetWidth(30); // var tokens = terms.GetTokens(); var list = terms.GetRankedTokenFrequency(limit); Int32 c = 1; foreach (var pair in list) { var dr = table.NewRow(); dr[column_rank] = c; dr[column_id] = terms.GetTokenID(pair.Key); dr[column_token] = pair.Key; dr[column_freq] = pair.Value; c++; table.Rows.Add(dr); } if (terms.Count > limit) { table.AddExtra("Table contains only top [" + limit + "] entries, out of [" + terms.Count + "] enumerated in the dictionary"); } return(table); }