/// <summary>for testing purposes only</summary> public static void Main(string[] args) { object[] a1 = new object[] { "a", "b" }; object[] a2 = new object[] { "a", "b" }; System.Console.Out.WriteLine(Arrays.Equals(a1, a2)); GeneralizedCounter <string> gc = new GeneralizedCounter <string>(3); gc.IncrementCount(Arrays.AsList(new string[] { "a", "j", "x" }), 3.0); gc.IncrementCount(Arrays.AsList(new string[] { "a", "l", "x" }), 3.0); gc.IncrementCount(Arrays.AsList(new string[] { "b", "k", "y" }), 3.0); gc.IncrementCount(Arrays.AsList(new string[] { "b", "k", "z" }), 3.0); System.Console.Out.WriteLine("incremented counts."); System.Console.Out.WriteLine(gc.DumpKeys()); System.Console.Out.WriteLine("string representation of generalized counter:"); System.Console.Out.WriteLine(gc.ToString()); gc.PrintKeySet(); System.Console.Out.WriteLine("entry set:\n" + gc.EntrySet()); ArrayPrintDouble(gc.GetCounts(Arrays.AsList(new string[] { "a", "j", "x" }))); ArrayPrintDouble(gc.GetCounts(Arrays.AsList(new string[] { "a", "j", "z" }))); ArrayPrintDouble(gc.GetCounts(Arrays.AsList(new string[] { "b", "k", "w" }))); ArrayPrintDouble(gc.GetCounts(Arrays.AsList(new string[] { "b", "k", "z" }))); GeneralizedCounter <string> gc1 = gc.Conditionalize(Arrays.AsList(new string[] { "a" })); gc1.IncrementCount(Arrays.AsList(new string[] { "j", "x" })); gc1.IncrementCount2D("j", "z"); GeneralizedCounter <string> gc2 = gc1.Conditionalize(Arrays.AsList(new string[] { "j" })); gc2.IncrementCount1D("x"); System.Console.Out.WriteLine("Pretty-printing gc after incrementing gc1:"); gc.PrettyPrint(); System.Console.Out.WriteLine("Total: " + gc.TotalCount()); gc1.PrintKeySet(); System.Console.Out.WriteLine("another entry set:\n" + gc1.EntrySet()); ClassicCounter <IList <string> > c = gc.CounterView(); System.Console.Out.WriteLine("string representation of counter view:"); System.Console.Out.WriteLine(c.ToString()); double d1 = c.GetCount(Arrays.AsList(new string[] { "a", "j", "x" })); double d2 = c.GetCount(Arrays.AsList(new string[] { "a", "j", "w" })); System.Console.Out.WriteLine(d1 + " " + d2); ClassicCounter <IList <string> > c1 = gc1.CounterView(); System.Console.Out.WriteLine("Count of {j,x} -- should be 3.0\t" + c1.GetCount(Arrays.AsList(new string[] { "j", "x" }))); System.Console.Out.WriteLine(c.KeySet() + " size " + c.KeySet().Count); System.Console.Out.WriteLine(c1.KeySet() + " size " + c1.KeySet().Count); System.Console.Out.WriteLine(c1.Equals(c)); System.Console.Out.WriteLine(c.Equals(c1)); System.Console.Out.WriteLine(c.Equals(c)); System.Console.Out.WriteLine("### testing equality of regular Counter..."); ClassicCounter <string> z1 = new ClassicCounter <string>(); ClassicCounter <string> z2 = new ClassicCounter <string>(); z1.IncrementCount("a1"); z1.IncrementCount("a2"); z2.IncrementCount("b"); System.Console.Out.WriteLine(z1.Equals(z2)); System.Console.Out.WriteLine(z1.ToString()); System.Console.Out.WriteLine(z1.KeySet().ToString()); }
public virtual double NumOccurances(E @object) { if (@object == null) { throw new Exception("You cannot ask for the number of occurances of null."); } return(sampled.GetCount(@object)); }
public virtual void TestLogNormalize() { ClassicCounter <string> c = new ClassicCounter <string>(); c.IncrementCount("a", Math.Log(4.0)); c.IncrementCount("b", Math.Log(2.0)); c.IncrementCount("c", Math.Log(1.0)); c.IncrementCount("d", Math.Log(1.0)); Counters.LogNormalizeInPlace(c); NUnit.Framework.Assert.AreEqual(c.GetCount("a"), -0.693, Tolerance); NUnit.Framework.Assert.AreEqual(c.GetCount("b"), -1.386, Tolerance); NUnit.Framework.Assert.AreEqual(c.GetCount("c"), -2.079, Tolerance); NUnit.Framework.Assert.AreEqual(c.GetCount("d"), -2.079, Tolerance); NUnit.Framework.Assert.AreEqual(Counters.LogSum(c), 0.0, Tolerance); }
private double Percentage(OUT key, ClassicCounter <OUT> guessed, ClassicCounter <OUT> guessedCorrect) { double thisGuessed = guessed.GetCount(key); double thisGuessedCorrect = guessedCorrect.GetCount(key); return((thisGuessed == 0.0) ? 0.0 : thisGuessedCorrect / thisGuessed); }
public virtual void TestSerializeStringCounter() { ICounter <string> counts = new ClassicCounter <string>(); for (int @base = -10; @base < 10; ++@base) { if (@base == 0) { continue; } for (int exponent = -100; exponent < 100; ++exponent) { double number = Math.Pow(Math.Pi * @base, exponent); counts.SetCount(double.ToString(number), number); } } File tmp = File.CreateTempFile("counts", ".tab.gz"); tmp.DeleteOnExit(); Counters.SerializeStringCounter(counts, tmp.GetPath()); ICounter <string> reread = Counters.DeserializeStringCounter(tmp.GetPath()); foreach (KeyValuePair <string, double> entry in reread.EntrySet()) { double old = counts.GetCount(entry.Key); NUnit.Framework.Assert.AreEqual(old, entry.Value, Math.Abs(old) / 1e5); } }
public virtual double GetCount(K1 o1, K2 o2) { ClassicCounter <K2> c = GetCounter(o1); if (c.TotalCount() == 0.0 && !c.KeySet().Contains(o2)) { return(DefaultReturnValue()); } return(c.GetCount(o2)); }
public virtual ClassicCounter <Pair <K1, K2> > Flatten() { ClassicCounter <Pair <K1, K2> > result = new ClassicCounter <Pair <K1, K2> >(); result.SetDefaultReturnValue(defaultValue); foreach (K1 key1 in FirstKeySet()) { ClassicCounter <K2> inner = GetCounter(key1); foreach (K2 key2 in inner.KeySet()) { result.SetCount(new Pair <K1, K2>(key1, key2), inner.GetCount(key2)); } } return(result); }
/// <summary> /// A simple String representation of this TwoDimensionalCounter, which has the /// String representation of each key pair on a separate line, followed by the /// count for that pair. /// </summary> /// <remarks> /// A simple String representation of this TwoDimensionalCounter, which has the /// String representation of each key pair on a separate line, followed by the /// count for that pair. The items are tab separated, so the result is a /// tab-separated value (TSV) file. Iff none of the keys contain spaces, it /// will also be possible to treat this as whitespace separated fields. /// </remarks> public override string ToString() { StringBuilder buff = new StringBuilder(); foreach (K1 key1 in map.Keys) { ClassicCounter <K2> c = GetCounter(key1); foreach (K2 key2 in c.KeySet()) { double score = c.GetCount(key2); buff.Append(key1).Append('\t').Append(key2).Append('\t').Append(score).Append('\n'); } } return(buff.ToString()); }
/// <summary>Produces a new ConditionalCounter.</summary> /// <returns>a new ConditionalCounter, where order of indices is reversed</returns> public static Edu.Stanford.Nlp.Stats.TwoDimensionalCounter <K2, K1> ReverseIndexOrder <K1, K2>(Edu.Stanford.Nlp.Stats.TwoDimensionalCounter <K1, K2> cc) { // they typing on the outerMF is violated a bit, but it'll work.... Edu.Stanford.Nlp.Stats.TwoDimensionalCounter <K2, K1> result = new Edu.Stanford.Nlp.Stats.TwoDimensionalCounter((MapFactory)cc.outerMF, (MapFactory)cc.innerMF); foreach (K1 key1 in cc.FirstKeySet()) { ClassicCounter <K2> c = cc.GetCounter(key1); foreach (K2 key2 in c.KeySet()) { double count = c.GetCount(key2); result.SetCount(key2, key1, count); } } return(result); }
public static Multinomial <F> DrawSample <F>(Random random, ICounter <F> parameters) { ICounter <F> multParameters = new ClassicCounter <F>(); double sum = 0.0; foreach (F o in parameters.KeySet()) { double parameter = Gamma.DrawSample(random, parameters.GetCount(o)); sum += parameter; multParameters.SetCount(o, parameter); } foreach (F o_1 in multParameters.KeySet()) { multParameters.SetCount(o_1, multParameters.GetCount(o_1) / sum); } return(new Multinomial <F>(multParameters)); }
public virtual void Clean() { foreach (K1 key1 in Generics.NewHashSet(map.Keys)) { ClassicCounter <K2> c = map[key1]; foreach (K2 key2 in Generics.NewHashSet(c.KeySet())) { if (SloppyMath.IsCloseTo(0.0, c.GetCount(key2))) { c.Remove(key2); } } if (c.KeySet().IsEmpty()) { Sharpen.Collections.Remove(map, key1); } } }
private void DisplayHelper(ICollection <OUT> keys, PrintWriter pw, ClassicCounter <OUT> guessed, ClassicCounter <OUT> guessedCorrect, ClassicCounter <OUT> gold, ClassicCounter <OUT> goldCorrect) { IDictionary <OUT, string> pads = GetPads(keys); foreach (OUT key in keys) { double thisGuessed = guessed.GetCount(key); double thisGuessedCorrect = guessedCorrect.GetCount(key); double precision = (thisGuessed == 0.0) ? 0.0 : thisGuessedCorrect / thisGuessed; lastPrecision.SetCount(key, precision); double thisGold = gold.GetCount(key); double thisGoldCorrect = goldCorrect.GetCount(key); double recall = (thisGold == 0.0) ? 0.0 : thisGoldCorrect / thisGold; lastRecall.SetCount(key, recall); double f1 = F1(precision, recall); lastF1.SetCount(key, f1); string pad = pads[key]; pw.Println(key + pad + "\t" + "P: " + FormatNumber(precision) + "\ton " + FormatCount(thisGuessed) + " objects\tR: " + FormatNumber(recall) + "\ton " + FormatCount(thisGold) + " objects\tF1: " + FormatNumber(f1)); } }
// ---------------------------------------------------------------------------- /// <summary> /// Creates a Distribution from the given counter using Gale & Sampsons' /// "simple Good-Turing" smoothing. /// </summary> /// <returns>a new simple Good-Turing smoothed Distribution.</returns> public static Edu.Stanford.Nlp.Stats.Distribution <E> SimpleGoodTuring <E>(ICounter <E> counter, int numberOfKeys) { // check arguments ValidateCounter(counter); int numUnseen = numberOfKeys - counter.Size(); if (numUnseen < 1) { throw new ArgumentException(string.Format("ERROR: numberOfKeys %d must be > size of counter %d!", numberOfKeys, counter.Size())); } // do smoothing int[][] cc = CountCounts2IntArrays(CollectCountCounts(counter)); int[] r = cc[0]; // counts int[] n = cc[1]; // counts of counts Edu.Stanford.Nlp.Stats.SimpleGoodTuring sgt = new Edu.Stanford.Nlp.Stats.SimpleGoodTuring(r, n); // collate results ICounter <int> probsByCount = new ClassicCounter <int>(); double[] probs = sgt.GetProbabilities(); for (int i = 0; i < probs.Length; i++) { probsByCount.SetCount(r[i], probs[i]); } // make smoothed distribution Edu.Stanford.Nlp.Stats.Distribution <E> dist = new Edu.Stanford.Nlp.Stats.Distribution <E>(); dist.counter = new ClassicCounter <E>(); foreach (KeyValuePair <E, double> entry in counter.EntrySet()) { E item = entry.Key; int count = (int)Math.Round(entry.Value); dist.counter.SetCount(item, probsByCount.GetCount(count)); } dist.numberOfKeys = numberOfKeys; dist.reservedMass = sgt.GetProbabilityForUnseen(); return(dist); }
public virtual void TestClassicCounterHistoricalMain() { c.SetCount("p", 0); c.SetCount("q", 2); ClassicCounter <string> small_c = new ClassicCounter <string>(c); ICounter <string> c7 = c.GetFactory().Create(); c7.AddAll(c); NUnit.Framework.Assert.AreEqual(c.TotalCount(), 2.0); c.IncrementCount("p"); NUnit.Framework.Assert.AreEqual(c.TotalCount(), 3.0); c.IncrementCount("p", 2.0); NUnit.Framework.Assert.AreEqual(Counters.Min(c), 2.0); NUnit.Framework.Assert.AreEqual(Counters.Argmin(c), "q"); // Now p is p=3.0, q=2.0 c.SetCount("w", -5.0); c.SetCount("x", -4.5); IList <string> biggestKeys = new List <string>(c.KeySet()); NUnit.Framework.Assert.AreEqual(biggestKeys.Count, 4); biggestKeys.Sort(Counters.ToComparator(c, false, true)); NUnit.Framework.Assert.AreEqual("w", biggestKeys[0]); NUnit.Framework.Assert.AreEqual("x", biggestKeys[1]); NUnit.Framework.Assert.AreEqual("p", biggestKeys[2]); NUnit.Framework.Assert.AreEqual("q", biggestKeys[3]); NUnit.Framework.Assert.AreEqual(Counters.Min(c), -5.0, Tolerance); NUnit.Framework.Assert.AreEqual(Counters.Argmin(c), "w"); NUnit.Framework.Assert.AreEqual(Counters.Max(c), 3.0, Tolerance); NUnit.Framework.Assert.AreEqual(Counters.Argmax(c), "p"); if (integral) { NUnit.Framework.Assert.AreEqual(Counters.Mean(c), -1.0); } else { NUnit.Framework.Assert.AreEqual(Counters.Mean(c), -1.125, Tolerance); } if (!integral) { // only do this for floating point counters. Too much bother to rewrite c.SetCount("x", -2.5); ClassicCounter <string> c2 = new ClassicCounter <string>(c); NUnit.Framework.Assert.AreEqual(3.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(2.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(-5.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(-2.5, c2.GetCount("x")); ICounter <string> c3 = c.GetFactory().Create(); foreach (string str in c2.KeySet()) { c3.IncrementCount(str); } NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("p")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("q")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("w")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("x")); Counters.AddInPlace(c2, c3, 10.0); NUnit.Framework.Assert.AreEqual(13.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(12.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(5.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(7.5, c2.GetCount("x")); c3.AddAll(c); NUnit.Framework.Assert.AreEqual(4.0, c3.GetCount("p")); NUnit.Framework.Assert.AreEqual(3.0, c3.GetCount("q")); NUnit.Framework.Assert.AreEqual(-4.0, c3.GetCount("w")); NUnit.Framework.Assert.AreEqual(-1.5, c3.GetCount("x")); Counters.SubtractInPlace(c3, c); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("p")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("q")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("w")); NUnit.Framework.Assert.AreEqual(1.0, c3.GetCount("x")); foreach (string str_1 in c.KeySet()) { c3.IncrementCount(str_1); } NUnit.Framework.Assert.AreEqual(2.0, c3.GetCount("p")); NUnit.Framework.Assert.AreEqual(2.0, c3.GetCount("q")); NUnit.Framework.Assert.AreEqual(2.0, c3.GetCount("w")); NUnit.Framework.Assert.AreEqual(2.0, c3.GetCount("x")); Counters.DivideInPlace(c2, c3); NUnit.Framework.Assert.AreEqual(6.5, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(6.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(2.5, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(3.75, c2.GetCount("x")); Counters.DivideInPlace(c2, 0.5); NUnit.Framework.Assert.AreEqual(13.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(12.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(5.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(7.5, c2.GetCount("x")); Counters.MultiplyInPlace(c2, 2.0); NUnit.Framework.Assert.AreEqual(26.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(24.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(10.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(15.0, c2.GetCount("x")); Counters.DivideInPlace(c2, 2.0); NUnit.Framework.Assert.AreEqual(13.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(12.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(5.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(7.5, c2.GetCount("x")); foreach (string str_2 in c2.KeySet()) { c2.IncrementCount(str_2); } NUnit.Framework.Assert.AreEqual(14.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(13.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(6.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(8.5, c2.GetCount("x")); foreach (string str_3 in c.KeySet()) { c2.IncrementCount(str_3); } NUnit.Framework.Assert.AreEqual(15.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(14.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(7.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(9.5, c2.GetCount("x")); c2.AddAll(small_c); NUnit.Framework.Assert.AreEqual(15.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(16.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(7.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(9.5, c2.GetCount("x")); NUnit.Framework.Assert.AreEqual(new HashSet <string>(Arrays.AsList("p", "q")), Counters.KeysAbove(c2, 14)); NUnit.Framework.Assert.AreEqual(new HashSet <string>(Arrays.AsList("q")), Counters.KeysAt(c2, 16)); NUnit.Framework.Assert.AreEqual(new HashSet <string>(Arrays.AsList("x", "w")), Counters.KeysBelow(c2, 9.5)); Counters.AddInPlace(c2, small_c, -6); NUnit.Framework.Assert.AreEqual(15.0, c2.GetCount("p")); NUnit.Framework.Assert.AreEqual(4.0, c2.GetCount("q")); NUnit.Framework.Assert.AreEqual(7.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(9.5, c2.GetCount("x")); Counters.SubtractInPlace(c2, small_c); Counters.SubtractInPlace(c2, small_c); Counters.RetainNonZeros(c2); NUnit.Framework.Assert.AreEqual(15.0, c2.GetCount("p")); NUnit.Framework.Assert.IsFalse(c2.ContainsKey("q")); NUnit.Framework.Assert.AreEqual(7.0, c2.GetCount("w")); NUnit.Framework.Assert.AreEqual(9.5, c2.GetCount("x")); } // serialize to Stream if (c is ISerializable) { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream @out = new ObjectOutputStream(new BufferedOutputStream(baos)); @out.WriteObject(c); @out.Close(); // reconstitute byte[] bytes = baos.ToByteArray(); ObjectInputStream @in = new ObjectInputStream(new BufferedInputStream(new ByteArrayInputStream(bytes))); c = IOUtils.ReadObjectFromObjectStream(@in); @in.Close(); if (!this.integral) { NUnit.Framework.Assert.AreEqual(-2.5, c.TotalCount()); NUnit.Framework.Assert.AreEqual(-5.0, Counters.Min(c)); NUnit.Framework.Assert.AreEqual("w", Counters.Argmin(c)); } c.Clear(); if (!this.integral) { NUnit.Framework.Assert.AreEqual(0.0, c.TotalCount()); } } catch (IOException ioe) { Fail("IOException: " + ioe); } catch (TypeLoadException cce) { Fail("ClassNotFoundException: " + cce); } } }