public static Edu.Stanford.Nlp.Stats.Distribution <E> AbsolutelyDiscountedDistribution <E>(ICounter <E> counter, int numberOfKeys, double discount)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
double total = counter.TotalCount();
double reservedMass = 0.0;
foreach (E key in counter.KeySet())
{
double count = counter.GetCount(key);
if (count > discount)
{
double newCount = (count - discount) / total;
norm.counter.SetCount(key, newCount);
// a positive count left over
// System.out.println("seen: " + newCount);
reservedMass += discount;
}
else
{
// count <= discount
reservedMass += count;
}
}
// if the count <= discount, don't put key in counter, and we treat it as unseen!!
norm.numberOfKeys = numberOfKeys;
norm.reservedMass = reservedMass / total;
// System.out.println("UNSEEN: " + reservedMass / total / (numberOfKeys - counter.size()));
return(norm);
}
// ----------------------------------------------------------------------------
/// <summary>
/// Returns a Distribution that uses prior as a Dirichlet prior
/// weighted by weight.
/// </summary>
/// <remarks>
/// Returns a Distribution that uses prior as a Dirichlet prior
/// weighted by weight. Essentially adds "pseudo-counts" for each Object
/// in prior equal to that Object's mass in prior times weight,
/// then normalizes.
/// <p>
/// WARNING: If unseen item is encountered in c, total may not be 1.
/// NOTE: This will not work if prior is a DynamicDistribution
/// to fix this, you could add a CounterView to Distribution and use that
/// in the linearCombination call below
/// </remarks>
/// <param name="weight">multiplier of prior to get "pseudo-count"</param>
/// <returns>new Distribution</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> DistributionWithDirichletPrior <E>(ICounter <E> c, Edu.Stanford.Nlp.Stats.Distribution <E> prior, double weight)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
double totalWeight = c.TotalCount() + weight;
if (prior is Distribution.DynamicDistribution)
{
throw new NotSupportedException("Cannot make normalized counter with Dynamic prior.");
}
norm.counter = Counters.LinearCombination(c, 1 / totalWeight, prior.counter, weight / totalWeight);
norm.numberOfKeys = prior.numberOfKeys;
norm.reservedMass = prior.reservedMass * weight / totalWeight;
//System.out.println("totalCount: " + norm.totalCount());
return(norm);
}
//--- end JM added
/// <param name="s">a Collection of keys.</param>
public static Edu.Stanford.Nlp.Stats.Distribution <E> GetUniformDistribution <E>(ICollection <E> s)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
norm.numberOfKeys = s.Count;
norm.reservedMass = 0;
double total = s.Count;
double count = 1.0 / total;
foreach (E key in s)
{
norm.counter.SetCount(key, count);
}
return(norm);
}
/// <param name="s">a Collection of keys.</param>
public static Edu.Stanford.Nlp.Stats.Distribution <E> GetPerturbedUniformDistribution <E>(ICollection <E> s, Random r)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
norm.numberOfKeys = s.Count;
norm.reservedMass = 0;
double total = s.Count;
double prob = 1.0 / total;
double stdev = prob / 1000.0;
foreach (E key in s)
{
norm.counter.SetCount(key, prob + (r.NextGaussian() * stdev));
}
return(norm);
}
/// <summary>
/// Creates a smoothed Distribution using Lidstone's law, ie adds lambda (typically
/// between 0 and 1) to every item, including unseen ones, and divides by the total count.
/// </summary>
/// <returns>a new Lidstone smoothed Distribution</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> LaplaceSmoothedDistribution <E>(ICounter <E> counter, int numberOfKeys, double lambda)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
double total = counter.TotalCount();
double newTotal = total + (lambda * numberOfKeys);
double reservedMass = ((double)numberOfKeys - counter.Size()) * lambda / newTotal;
norm.numberOfKeys = numberOfKeys;
norm.reservedMass = reservedMass;
foreach (E key in counter.KeySet())
{
double count = counter.GetCount(key);
norm.counter.SetCount(key, (count + lambda) / newTotal);
}
return(norm);
}
/// <summary>Creates a Good-Turing smoothed Distribution from the given counter.</summary>
/// <returns>a new Good-Turing smoothed Distribution.</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> GoodTuringSmoothedCounter <E>(ICounter <E> counter, int numberOfKeys)
{
// gather count-counts
int[] countCounts = GetCountCounts(counter);
// if count-counts are unreliable, we shouldn't be using G-T
// revert to laplace
for (int i = 1; i <= 10; i++)
{
if (countCounts[i] < 3)
{
return(LaplaceSmoothedDistribution(counter, numberOfKeys, 0.5));
}
}
double observedMass = counter.TotalCount();
double reservedMass = countCounts[1] / observedMass;
// calculate and cache adjusted frequencies
// also adjusting total mass of observed items
double[] adjustedFreq = new double[10];
for (int freq = 1; freq < 10; freq++)
{
adjustedFreq[freq] = (double)(freq + 1) * (double)countCounts[freq + 1] / countCounts[freq];
observedMass -= (freq - adjustedFreq[freq]) * countCounts[freq];
}
double normFactor = (1.0 - reservedMass) / observedMass;
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
// fill in the new Distribution, renormalizing as we go
foreach (E key in counter.KeySet())
{
int origFreq = (int)Math.Round(counter.GetCount(key));
if (origFreq < 10)
{
norm.counter.SetCount(key, adjustedFreq[origFreq] * normFactor);
}
else
{
norm.counter.SetCount(key, origFreq * normFactor);
}
}
norm.numberOfKeys = numberOfKeys;
norm.reservedMass = reservedMass;
return(norm);
}
//---- end cdm added
//--- JM added for Distributions
/// <summary>Assuming that c has a total count < 1, returns a new Distribution using the counts in c as probabilities.</summary>
/// <remarks>
/// Assuming that c has a total count < 1, returns a new Distribution using the counts in c as probabilities.
/// If c has a total count > 1, returns a normalized distribution with no remaining mass.
/// </remarks>
public static Edu.Stanford.Nlp.Stats.Distribution <E> GetDistributionFromPartiallySpecifiedCounter <E>(ICounter <E> c, int numKeys)
{
Edu.Stanford.Nlp.Stats.Distribution <E> d;
double total = c.TotalCount();
if (total >= 1.0)
{
d = GetDistribution(c);
d.numberOfKeys = numKeys;
}
else
{
d = new Edu.Stanford.Nlp.Stats.Distribution <E>();
d.numberOfKeys = numKeys;
d.counter = c;
d.reservedMass = 1.0 - total;
}
return(d);
}
public static Edu.Stanford.Nlp.Stats.Distribution <E> GetDistributionWithReservedMass <E>(ICounter <E> counter, double reservedMass)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
norm.numberOfKeys = counter.Size();
norm.reservedMass = reservedMass;
double total = counter.TotalCount() * (1 + reservedMass);
if (total == 0.0)
{
total = 1.0;
}
foreach (E key in counter.KeySet())
{
double count = counter.GetCount(key) / total;
// if (Double.isNaN(count) || count < 0.0 || count> 1.0 ) throw new RuntimeException("count=" + counter.getCount(key) + " total=" + total);
norm.counter.SetCount(key, count);
}
return(norm);
}
/// <summary>
/// Creates a smoothed Distribution with Laplace smoothing, but assumes an explicit
/// count of "UNKNOWN" items.
/// </summary>
/// <remarks>
/// Creates a smoothed Distribution with Laplace smoothing, but assumes an explicit
/// count of "UNKNOWN" items. Thus anything not in the original counter will have
/// probability zero.
/// </remarks>
/// <param name="counter">the counter to normalize</param>
/// <param name="lambda">the value to add to each count</param>
/// <param name="Unk">the UNKNOWN symbol</param>
/// <returns>a new Laplace-smoothed distribution</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> LaplaceWithExplicitUnknown <E>(ICounter <E> counter, double lambda, E Unk)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
double total = counter.TotalCount() + (lambda * (counter.Size() - 1));
norm.numberOfKeys = counter.Size();
norm.reservedMass = 0.0;
foreach (E key in counter.KeySet())
{
if (key.Equals(Unk))
{
norm.counter.SetCount(key, counter.GetCount(key) / total);
}
else
{
norm.counter.SetCount(key, (counter.GetCount(key) + lambda) / total);
}
}
return(norm);
}
public static Edu.Stanford.Nlp.Stats.Distribution <E> GetPerturbedDistribution <E>(ICounter <E> wordCounter, Random r)
{
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Edu.Stanford.Nlp.Stats.Distribution <E>();
norm.counter = new ClassicCounter <E>();
norm.numberOfKeys = wordCounter.Size();
norm.reservedMass = 0;
double totalCount = wordCounter.TotalCount();
double stdev = 1.0 / norm.numberOfKeys / 1000.0;
// tiny relative to average value
foreach (E key in wordCounter.KeySet())
{
double prob = wordCounter.GetCount(key) / totalCount;
double perturbedProb = prob + (r.NextGaussian() * stdev);
if (perturbedProb < 0.0)
{
perturbedProb = 0.0;
}
norm.counter.SetCount(key, perturbedProb);
}
return(norm);
}
// ----------------------------------------------------------------------------
/// <summary>
/// Creates a Distribution from the given counter using Gale & Sampsons'
/// "simple Good-Turing" smoothing.
/// </summary>
/// <returns>a new simple Good-Turing smoothed Distribution.</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> SimpleGoodTuring <E>(ICounter <E> counter, int numberOfKeys)
{
// check arguments
ValidateCounter(counter);
int numUnseen = numberOfKeys - counter.Size();
if (numUnseen < 1)
{
throw new ArgumentException(string.Format("ERROR: numberOfKeys %d must be > size of counter %d!", numberOfKeys, counter.Size()));
}
// do smoothing
int[][] cc = CountCounts2IntArrays(CollectCountCounts(counter));
int[] r = cc[0];
// counts
int[] n = cc[1];
// counts of counts
Edu.Stanford.Nlp.Stats.SimpleGoodTuring sgt = new Edu.Stanford.Nlp.Stats.SimpleGoodTuring(r, n);
// collate results
ICounter <int> probsByCount = new ClassicCounter <int>();
double[] probs = sgt.GetProbabilities();
for (int i = 0; i < probs.Length; i++)
{
probsByCount.SetCount(r[i], probs[i]);
}
// make smoothed distribution
Edu.Stanford.Nlp.Stats.Distribution <E> dist = new Edu.Stanford.Nlp.Stats.Distribution <E>();
dist.counter = new ClassicCounter <E>();
foreach (KeyValuePair <E, double> entry in counter.EntrySet())
{
E item = entry.Key;
int count = (int)Math.Round(entry.Value);
dist.counter.SetCount(item, probsByCount.GetCount(count));
}
dist.numberOfKeys = numberOfKeys;
dist.reservedMass = sgt.GetProbabilityForUnseen();
return(dist);
}
/// <summary>
/// Like normalizedCounterWithDirichletPrior except probabilities are
/// computed dynamically from the counter and prior instead of all at once up front.
/// </summary>
/// <remarks>
/// Like normalizedCounterWithDirichletPrior except probabilities are
/// computed dynamically from the counter and prior instead of all at once up front.
/// The main advantage of this is if you are making many distributions from relatively
/// sparse counters using the same relatively dense prior, the prior is only represented
/// once, for major memory savings.
/// </remarks>
/// <param name="weight">multiplier of prior to get "pseudo-count"</param>
/// <returns>new Distribution</returns>
public static Edu.Stanford.Nlp.Stats.Distribution <E> DynamicCounterWithDirichletPrior <E>(ICounter <E> c, Edu.Stanford.Nlp.Stats.Distribution <E> prior, double weight)
{
double totalWeight = c.TotalCount() + weight;
Edu.Stanford.Nlp.Stats.Distribution <E> norm = new Distribution.DynamicDistribution <E>(prior, weight / totalWeight);
norm.counter = new ClassicCounter <E>();
// this might be done more efficiently with entrySet but there isn't a way to get
// the entrySet from a Counter now. In most cases c will be small(-ish) anyway
foreach (E key in c.KeySet())
{
double count = c.GetCount(key) / totalWeight;
prior.AddToKeySet(key);
norm.counter.SetCount(key, count);
}
norm.numberOfKeys = prior.numberOfKeys;
return(norm);
}