public void Smooth(int ngramSize, TSeq[] sequences, Func <TSeq, IEnumerable <TItem> > itemsSelector, Direction dir, ConditionalFrequencyDistribution <Ngram <TItem>, TItem> cfd)
{
_cfd = cfd;
_dir = dir;
if (ngramSize > 1)
{
_lowerOrderModel = new NgramModel <TSeq, TItem>(ngramSize - 1, sequences, itemsSelector, dir, new WittenBellSmoother <TSeq, TItem>());
}
}
public void NoSamples()
{
var words = new string[0];
NgramModel <string, char> model = NgramModel <string, char> .Train(2, words, w => w, new MaxLikelihoodSmoother <string, char>());
Assert.That(model.GetProbability('a', new Ngram <char>("l")), Is.EqualTo(0));
Assert.That(model.GetProbability('l', new Ngram <char>("l")), Is.EqualTo(0));
Assert.That(model.GetProbability('e', new Ngram <char>("l")), Is.EqualTo(0));
Assert.That(model.GetProbability('t', new Ngram <char>("l")), Is.EqualTo(0));
}
public void Ngrams()
{
var words = new[] { "#call#", "#stall#", "#hello#", "#the#", "#a#", "#test#", "#income#", "#unproduce#" };
NgramModel <string, char>[] models = NgramModel <string, char> .TrainAll(10, words, w => w, () => new MaxLikelihoodSmoother <string, char>()).ToArray();
Assert.That(models[0].Ngrams.Count, Is.EqualTo(16));
Assert.That(models[1].Ngrams.Count, Is.EqualTo(36));
Assert.That(models[7].Ngrams.Count, Is.EqualTo(5));
Assert.That(models[8].Ngrams.Count, Is.EqualTo(3));
Assert.That(models[9].Ngrams.Count, Is.EqualTo(2));
}
public void GetProbabilityRightToLeft()
{
var words = new[] { "#call#", "#stall#", "#hello#", "#the#", "#a#", "#test#", "#income#", "#unproduce#" };
NgramModel <string, char> model = NgramModel <string, char> .Train(2, words, w => w, Direction.RightToLeft, new MaxLikelihoodSmoother <string, char>());
Assert.That(model.GetProbability('a', new Ngram <char>("l")), Is.EqualTo(0.333).Within(0.001));
Assert.That(model.GetProbability('l', new Ngram <char>("l")), Is.EqualTo(0.5));
Assert.That(model.GetProbability('e', new Ngram <char>("l")), Is.EqualTo(0.166).Within(0.001));
Assert.That(model.GetProbability('t', new Ngram <char>("l")), Is.EqualTo(0.0));
Assert.That(model.GetProbability('c', new Ngram <char>("a")), Is.EqualTo(0.333).Within(0.001));
Assert.That(model.GetProbability('t', new Ngram <char>("a")), Is.EqualTo(0.333).Within(0.001));
Assert.That(model.GetProbability('#', new Ngram <char>("a")), Is.EqualTo(0.333).Within(0.001));
Assert.That(model.GetProbability('l', new Ngram <char>("a")), Is.EqualTo(0.0));
}
public static IEnumerable <NgramModel <TSeq, TItem> > TrainAll(int maxNgramSize, IEnumerable <TSeq> sequences, Func <TSeq, IEnumerable <TItem> > itemsSelector, Direction dir,
Func <INgramModelSmoother <TSeq, TItem> > smootherFactory)
{
TSeq[] seqArray = sequences.ToArray();
var model = new NgramModel <TSeq, TItem>(maxNgramSize, seqArray, itemsSelector, dir, smootherFactory());
var models = new NgramModel <TSeq, TItem> [maxNgramSize];
for (int i = maxNgramSize - 1; i >= 0; i--)
{
models[i] = model;
if (i > 0)
{
model = model.Smoother.LowerOrderModel ?? new NgramModel <TSeq, TItem>(i, seqArray, itemsSelector, dir, smootherFactory());
}
}
return(models);
}
public void Smooth(int ngramSize, TSeq[] sequences, Func <TSeq, IEnumerable <TItem> > itemsSelector, Direction dir, ConditionalFrequencyDistribution <Ngram <TItem>, TItem> cfd)
{
_cfd = cfd;
_dir = dir;
int totalN1 = 0, totalN2 = 0, totalN3 = 0, totalN4 = 0;
_bigNs.Clear();
foreach (Ngram <TItem> cond in cfd.Conditions)
{
int n1 = 0, n2 = 0, n3 = 0, n4 = 0;
int nGreater = 0;
FrequencyDistribution <TItem> freqDist = cfd[cond];
foreach (TItem item in freqDist.ObservedSamples)
{
if (freqDist[item] == 1)
{
n1++;
}
else if (freqDist[item] == 2)
{
n2++;
}
else if (freqDist[item] > 2)
{
if (freqDist[item] == 3)
{
n3++;
}
else if (freqDist[item] == 4)
{
n4++;
}
nGreater++;
}
}
totalN1 += n1;
totalN2 += n2;
totalN3 += n3;
totalN4 += n4;
_bigNs[cond] = Tuple.Create(n1, n2, nGreater);
}
_discount1 = 0;
_discount2 = 0;
_discount3 = 0;
double y = 0;
if (totalN1 > 0)
{
y = (double)totalN1 / (totalN1 + (2 * totalN2));
_discount1 = 1 - (2 * y * ((double)totalN2 / totalN1));
}
if (totalN2 > 0)
{
_discount2 = 2 - (3 * y * ((double)totalN3 / totalN2));
}
if (totalN3 > 0)
{
_discount3 = 3 - (4 * y * ((double)totalN4 / totalN3));
}
if (ngramSize > 1)
{
_lowerOrderModel = new NgramModel <TSeq, TItem>(ngramSize - 1, sequences, itemsSelector, dir, new ModifiedKneserNeySmoother <TSeq, TItem>());
}
}