private void PrintNormalizedTagsStats(TextDocument[] documents) { var tags = documents.SelectMany(x => x.Tags).Distinct().ToArray(); var stats = tags.Select(tag => new TagStatistics { Name = tag, Count = documents.Count(p => p.Tags.Contains(tag)) }).ToArray(); Logger.Log("Normalized tags stats"); foreach (var tag in stats.OrderByDescending(x => x.Count)) { Logger.Log("Tag = {0}, count = {1}", tag.Name, tag.Count); } PrintDelimeter(); }
public FeatureSelectionResult Select(TextDocument[] documents, FeatureSelectionParams featureSelectionParams) { var words = documents.SelectMany(p => p.Words).Distinct().ToArray(); var documentsCount = documents.Where(d => d.Tags.Contains(featureSelectionParams.TargetTag)).GetDocumentsCountByWord(); var featuredWords = words .Where(documentsCount.ContainsKey) .Select(w => new WeightedWord { Word = w, Metric = documentsCount[w] }) .OrderByDescending(w => w.Metric) .Take(words.Length * featureSelectionParams.UpperBoundPercent / 100) .Skip(words.Length * featureSelectionParams.LowerBoundPercent / 100) .ToArray(); return new FeatureSelectionResult { FeaturedWords = featuredWords }; }
public FeatureSelectionResult Select(TextDocument[] documents, FeatureSelectionParams featureSelectionParams) { cache = new Dictionary<string, double>(); var words = documents.SelectMany(p => p.Words).Distinct().ToArray(); var documentsCount = documents.GetDocumentsCountByWord(); words = words.Where(w => documentsCount.ContainsKey(w) && documentsCount[w] >= featureSelectionParams.MinDocCount && documentsCount[w] <= featureSelectionParams.MaxDocCount).ToArray(); var featuredWords = words .Select(w => new WeightedWord { Word = w, Metric = CalculateChiSquared(w, documents, featureSelectionParams.TargetTag) }) .OrderByDescending(w => w.Metric) .Take(words.Length * featureSelectionParams.UpperBoundPercent / 100) .Skip(words.Length * featureSelectionParams.LowerBoundPercent / 100) .ToArray(); return new FeatureSelectionResult { FeaturedWords = featuredWords }; }