static List <WeightedItem <T> > GetCumulativeWeightedList <T>(IEnumerable <WeightedItem <T> > items)
{
var result = new List <WeightedItem <T> >();
var relevantItems = items
.Where(i => i.Weight > 0)
.ToArray();
if (relevantItems.Length > 0)
{
var total = Sum(relevantItems.Select(i => i.Weight));
uint threshold = 0;
foreach (var i in relevantItems.Take(relevantItems.Length - 1))
{
threshold += Scale(i.Weight, total);
result.Add(WeightedItem.Create(threshold, i.Item));
}
var lastItem = relevantItems[relevantItems.Length - 1].Item;
result.Add(WeightedItem.Create(uint.MaxValue, lastItem));
}
return(result);
}
public static SerializableTrigramStatistics Count(IEnumerable <string> wordlist)
{
var count = GetTrigramCount(wordlist);
var prefixWeights = count
.GroupBy(kv => Tuple.Create(kv.Key.Item1, kv.Key.Item2), kv => kv.Value)
.Select(g => WeightedItem.Create(Sum(g), g.Key))
.OrderBy(i => i.Item.Item1).ThenBy(i => i.Item.Item2);
return(new SerializableTrigramStatistics
{
PrefixWeights = GetCumulativeWeightedList(prefixWeights),
TrigramWeights = count
.GroupBy(
kv => Tuple.Create(kv.Key.Item1, kv.Key.Item2),
kv => Tuple.Create(kv.Key.Item3, kv.Value))
.ToDictionary(
g => g.Key,
g => GetCumulativeWeightedList(
g.Select(t => WeightedItem.Create(t.Item2, t.Item1)).OrderBy(i => i.Item))),
});
}
