public static void RunTests(bool interactive, char kc) { { using (MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64, 10)) { int a; gz.ReadLine(out a); gz.ReadLine(out a); for (int i = 0; i < 100; i++) { gz.ReadLine(out a); } int i3 = 333 + 333; } int i333 = 333 + 333; } { List<int> nums = new List<int>(); using (MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64, 10)) { for(;;) { nums.Clear(); if (!gz.ReadLineAppend(nums)) { break; } int i3 = 333 + 333; } } } { Console.WriteLine("Testing FixedArrayComboList with int key and 0-length value."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("ffooobj", 4, 0); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); int itercount = 200; int maxtimes = 5; for (int times = 0; times < maxtimes; times++) { for (int i = 0; i < itercount; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); facl.Add(keybuf, valuebuf); } } facl.SortBlocks(); int n = 0; FixedArrayComboListEnumerator[] enums = facl.GetEnumerators(); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int x = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); //Console.Write(" {0} ", x); n++; } } if (n != itercount * maxtimes) { throw new Exception("Whoops"); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing FixedArrayComboList with int key and 0-length value in reducer."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("ffooobj", 4, 0); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.AddBlock(@"127.0.0.1|arraycombo_logs1.txt|slaveid=1"); facl.Open(); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); int itercount = 200; int maxtimes = 5; for (int times = 0; times < maxtimes; times++) { for (int i = 0; i < itercount; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); facl.Add(keybuf, valuebuf); // 0-length value. } } facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { output.Add(key, values[0].Value); if(" + maxtimes.ToString() + @" != values.Length) { throw new OverflowException(); } } "); int n = 0; foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (ikey < 0 || ikey > itercount) { throw new Exception("Bad key!"); } } } if (n != itercount) // only itercount, not (itercount * maxtimes), because it was reduced from 5 to 1. { throw new Exception("Whoops"); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing FixedArrayComboList before-load: random data stress test and validation."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("fstv", 4, 4); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); Random rnd = new Random(); List<int> input = new List<int>(5000); Dictionary<int, bool> hi = new Dictionary<int, bool>(5000); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); { for (int i = 0; i < 5000; i++) { int x; do { if (0 == (i % 300)) { x = rnd.Next(30, 300); } else { x = rnd.Next(); } } while (hi.ContainsKey(x)); hi[x] = true; input.Add(x); } } StringBuilder blinput = new StringBuilder(@"int[] all = new int[] { ", 10000); for (int i = 0; i < input.Count; i++) { if (0 != i) { blinput.Append(", "); } blinput.Append(input[i]); } blinput.Append(" };"); facl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { " + blinput.ToString() + @" List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < all.Length; i++) { int k = 1 + all[i] % 8; if(all[i] < 500) { k += all[i]; } for(int j = 0; j != k; j++) { keybuf.Clear(); Entry.ToBytesAppend(all[i], keybuf); valuebuf.Clear(); Entry.ToBytesAppend(all[j], valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } } }"); facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(128); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { output.Add(key, values[0].Value); } "); int n = 0; foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (4 != en.Current.GetValueLength()) { throw new Exception("Value length incorrect"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (!hi.ContainsKey(ikey)) { throw new Exception("Unexpected key"); } int k = 1 + ikey % 8; if (ikey < 500) { k += ikey; } } } if (input.Count != n) { throw new Exception("Test failed: expected " + input.Count.ToString() + " key/values, got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("FixedArrayComboList test with dymamically created inline C# reducer"); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("faclxobj2", 4, 4); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); Random rnd = new Random(); Dictionary<int, int> kv = new Dictionary<int, int>(); int numkeys = 50; for (int i = 0; i != numkeys; i++) { int x; do { x = rnd.Next(); } while (kv.ContainsKey(x)); kv[x] = rnd.Next(); } if (kv.Count != numkeys) { throw new Exception("kv.Count != numkeys"); } foreach (KeyValuePair<int, int> pair in kv) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(pair.Key, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(pair.Value, valuebuf); facl.Add(keybuf, valuebuf); } facl.SortBlocks(); StringBuilder dictcode = new StringBuilder(128); dictcode.Append("kv = new Dictionary<int, int>();"); foreach (KeyValuePair<int, int> pair in kv) { dictcode.Append("kv[" + pair.Key.ToString() + "] = " + pair.Value.ToString() + ";"); } FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); Dictionary<int, int> kv; bool finalized = false; public void ReduceInitialize() { " + dictcode.ToString() + @" } public void ReduceFinalize() { if(finalized) { throw new Exception(" + "\"Double finalized!\"" + @"); } finalized = true; if(0 != kv.Count) { throw new Exception(" + "\"ReduceFinalize: \" + " + @"kv.Count.ToString()); } } public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { if(finalized) { throw new Exception(" + "\"Reduce after finalized!\"" + @"); } for(int i = 0; i != values.Length; i++) { output.Add(key, values[i].Value); // ... keybuf.Clear(); key.AppendTo(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); values[i].Value.AppendTo(valuebuf); int ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf); if(!kv.ContainsKey(ikey)) { throw new Exception(" + "\"Reduce doesn't contain key: \" + " + @"ikey.ToString()); } if(kv[ikey] != ivalue) { throw new Exception(" + "\"Reduce value mismatch for key: \" + " + @"ikey.ToString()); } kv.Remove(ikey); } } "); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); en.Current.CopyValueAppend(valuebuf); int ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf); if (!kv.ContainsKey(ikey)) { throw new Exception("Enumeration doesn't contain key: " + ikey.ToString()); } if (kv[ikey] != ivalue) { throw new Exception("Enumeration value mismatch for key: " + ikey.ToString()); } kv.Remove(ikey); } } if (0 != kv.Count) { throw new Exception("Enumeration failure: " + kv.Count.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Confirming correct zblock distribution with FixedArrayComboList:"); if (interactive) { Console.WriteLine("Press a key..."); Console.ReadKey(); } MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("faclzfoo", 4, 4); facl.AddBlock(@"127.0.0.1|intcombo_logs0.txt|slaveid=0"); facl.Open(); System.Collections.Generic.Queue<int> wheel = new Queue<int>(); System.Collections.Generic.List<int> al = new List<int>(); Random rnd = new Random(System.DateTime.Now.Millisecond); System.Collections.Hashtable ht = new System.Collections.Hashtable(); List<byte> keybuf = new List<byte>(); //fill wheel for (int i = 1; i != 200; i++) { int x = rnd.Next(5656, 76688989); object o = ht[x]; int y = i; if (null != o) { y += (int)o; } ht[x] = y; for (int j = 0; j != i; j++) { al.Add(x); } } foreach (int o in al) { wheel.Enqueue(o); } int added = 0; while (0 != wheel.Count) { int x = rnd.Next(0, wheel.Count - 1); for (int foo = 0; foo != x; foo++) { wheel.Enqueue(wheel.Dequeue()); } int key = (int)wheel.Dequeue(); keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(key, keybuf); facl.Add(keybuf, keybuf); added++; } if (interactive) { Console.WriteLine("Press a key to sort..."); Console.ReadKey(); } facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumerators(); System.Collections.Generic.List<int> outlist = new List<int>(); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int key = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); outlist.Add(key); } Console.WriteLine(); } int lastkey = 0; foreach (int key in outlist) { if (key != lastkey) { if (0 != lastkey) { if (0 != (int)ht[lastkey]) { throw new Exception("Order error"); } } } lastkey = key; object blah = ht[key]; ht[key] = (int)blah - 1; } al.Sort(); outlist.Sort(); if (al.Count != outlist.Count) { throw new Exception("foo"); } for (int i = 0; i != al.Count; i++) { if ((int)al[i] != (int)outlist[i]) { throw new Exception("bar"); } } Console.WriteLine("Done"); if (interactive) { Console.WriteLine("Press a key to close..."); Console.ReadKey(); } facl.Close(); } { Console.WriteLine("Testing GzipFastLoader..."); MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64); int i = 0; for (; !gz.EOF; i++) { int a, b, c, d; if (!gz.ReadLine(out a, out b, out c, out d)) { throw new Exception("Whoops, it was supposed to hit gz.EOF first"); } if (0 == (i % 4)) { if (0 != a || 0 != b || 0 != c || 0 != d) { throw new Exception("Line " + (i + 1).ToString() + " should give me 0 but it doesn't!!"); } } else { if (i != a) { throw new Exception("i != a"); } switch (i) { case 1: if (0 != b) throw new Exception(i.ToString()); break; case 2: if (0 != c) throw new Exception(i.ToString()); break; case 3: if (0 != d) throw new Exception(i.ToString()); break; default: if (0 == d) throw new Exception(i.ToString()); break; } } } if (10 != i) { throw new Exception("Bad number of lines!"); } Console.WriteLine("Done"); } #if OBSOLETE { Console.WriteLine("Testing ArrayComboList with simple before-load (primitive mapper)."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("foomap", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < 20; i++) { keybuf.Clear(); Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); Entry.ToBytesAppend64(42, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } }"); acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumerators(); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 8) { throw new Exception("Value length mismatch"); } n++; } } if (20 != n) { throw new Exception("Test failed: expected 20 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList before-load against regular add."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("foomap2", 4); acl.atype = true; // Must have exactly 1 block(slave) for this test to work!! acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); { for (int i = 0; i < 20; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend64(42, valuebuf); acl.Add(keybuf, valuebuf); } } acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < 20; i++) { keybuf.Clear(); Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); Entry.ToBytesAppend64(42, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } }"); { for (int i = 0; i < 20; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend64(42, valuebuf); acl.Add(keybuf, valuebuf); } } acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { if(3 != values.Length) { throw new Exception(3.ToString() + '!'.ToString() + values.Length.ToString()); } for(int i = 0; i != values.Length; i++) { if(8 != values[i].Value.Length) { throw new Exception(3.ToString() + '!'.ToString() + values[i].Value.Length.ToString()); } valuebuf.Clear(); values[i].Value.AppendTo(valuebuf); long ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToLong(valuebuf); if(42 != ivalue) { throw new Exception(42.ToString() + '!'.ToString() + ivalue.ToString()); } } valuebuf.Clear(); Entry.ToBytesAppend64(42 * 3, valuebuf); output.Add(key, ByteSlice.Create(valuebuf)); } "); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 8) { throw new Exception("Value length mismatch"); } n++; } } if (20 != n) { throw new Exception("Test failed: expected 20 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList with GzipFastLoader in before-load."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("gzmap", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader(" + "\"intlines.txt.gz\"" + @", 64); int i = 0; for (; !gz.EOF; i++) { int a, b, c, d; gz.ReadLine(out a, out b, out c, out d); keybuf.Clear(); Entry.ToBytesAppend(a, keybuf); valuebuf.Clear(); Entry.ToBytesAppend(b, valuebuf); Entry.ToBytesAppend(c, valuebuf); Entry.ToBytesAppend(d, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } if (10 != i) { throw new Exception('!'.ToString()); } }"); acl.SortBlocks(); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); ArrayComboListEnumerator[] enums = acl.GetEnumerators(); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 12) { throw new Exception("Value length mismatch"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int a = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); en.Current.CopyValueAppend(valuebuf); int b = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 0); int c = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 4); int d = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 8); //if(0 != a) { switch (a) { case 0: break; case 1: if (0 != b) throw new Exception(a.ToString()); break; case 2: if (0 != c) throw new Exception(a.ToString()); break; case 3: if (0 != d) throw new Exception(a.ToString()); break; default: if (0 == d) throw new Exception(a.ToString()); break; } } } } if (10 != n) { throw new Exception("Test failed: expected 10 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList before-load: random data stress test and validation."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("stv", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); Random rnd = new Random(); List<int> input = new List<int>(5000); Dictionary<int, bool> hi = new Dictionary<int, bool>(5000); List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); { for (int i = 0; i < 5000; i++) { int x; do { if (0 == (i % 300)) { x = rnd.Next(30, 300); } else { x = rnd.Next(); } } while (hi.ContainsKey(x)); hi[x] = true; input.Add(x); } } StringBuilder blinput = new StringBuilder(@"int[] all = new int[] { ", 10000); for (int i = 0; i < input.Count; i++) { if (0 != i) { blinput.Append(", "); } blinput.Append(input[i]); } blinput.Append(" };"); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { " + blinput.ToString() + @" List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < all.Length; i++) { int k = 1 + all[i] % 8; if(all[i] < 500) { k += all[i]; } for(int j = 0; j != k; j++) { keybuf.Clear(); Entry.ToBytesAppend(all[i], keybuf); valuebuf.Clear(); Entry.ToBytesAppend(all[j], valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } } }"); acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(128); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { valuebuf.Clear(); for(int i = 0; i < values.Length; i++) { for(int j = 0; j < values[i].Value.Length; j++) { valuebuf.Add(values[i].Value[j]); } } output.Add(key, ByteSlice.Create(valuebuf)); } "); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (0 != (en.Current.GetValueLength() % 4)) { throw new Exception("Uneven value length"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (!hi.ContainsKey(ikey)) { throw new Exception("Unexpected key"); } int k = 1 + ikey % 8; if (ikey < 500) { k += ikey; } if (k * 4 != en.Current.GetValueLength()) { throw new Exception("Value length incorrect"); } /* valuebuf.Clear(); en.Current.CopyValueAppend(valuebuf); // Note: not in same order as added. for (int vv = 0; vv < en.Current.GetValueLength(); vv += 4) { int x = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, vv); } * */ } } if (input.Count != n) { throw new Exception("Test failed: expected " + input.Count.ToString() + " key/values, got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } #endif }
public static void RunTests(bool interactive, char kc) { { using (MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64, 10)) { int a; gz.ReadLine(out a); gz.ReadLine(out a); for (int i = 0; i < 100; i++) { gz.ReadLine(out a); } int i3 = 333 + 333; } int i333 = 333 + 333; } { List <int> nums = new List <int>(); using (MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64, 10)) { for (;;) { nums.Clear(); if (!gz.ReadLineAppend(nums)) { break; } int i3 = 333 + 333; } } } { Console.WriteLine("Testing FixedArrayComboList with int key and 0-length value."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("ffooobj", 4, 0); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); int itercount = 200; int maxtimes = 5; for (int times = 0; times < maxtimes; times++) { for (int i = 0; i < itercount; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); facl.Add(keybuf, valuebuf); } } facl.SortBlocks(); int n = 0; FixedArrayComboListEnumerator[] enums = facl.GetEnumerators(); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int x = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); //Console.Write(" {0} ", x); n++; } } if (n != itercount * maxtimes) { throw new Exception("Whoops"); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing FixedArrayComboList with int key and 0-length value in reducer."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("ffooobj", 4, 0); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.AddBlock(@"127.0.0.1|arraycombo_logs1.txt|slaveid=1"); facl.Open(); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); int itercount = 200; int maxtimes = 5; for (int times = 0; times < maxtimes; times++) { for (int i = 0; i < itercount; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); facl.Add(keybuf, valuebuf); // 0-length value. } } facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { output.Add(key, values[0].Value); if(" + maxtimes.ToString() + @" != values.Length) { throw new OverflowException(); } } "); int n = 0; foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (ikey < 0 || ikey > itercount) { throw new Exception("Bad key!"); } } } if (n != itercount) // only itercount, not (itercount * maxtimes), because it was reduced from 5 to 1. { throw new Exception("Whoops"); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing FixedArrayComboList before-load: random data stress test and validation."); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("fstv", 4, 4); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); Random rnd = new Random(); List <int> input = new List <int>(5000); Dictionary <int, bool> hi = new Dictionary <int, bool>(5000); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); { for (int i = 0; i < 5000; i++) { int x; do { if (0 == (i % 300)) { x = rnd.Next(30, 300); } else { x = rnd.Next(); } }while (hi.ContainsKey(x)); hi[x] = true; input.Add(x); } } StringBuilder blinput = new StringBuilder(@"int[] all = new int[] { ", 10000); for (int i = 0; i < input.Count; i++) { if (0 != i) { blinput.Append(", "); } blinput.Append(input[i]); } blinput.Append(" };"); facl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { " + blinput.ToString() + @" List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < all.Length; i++) { int k = 1 + all[i] % 8; if(all[i] < 500) { k += all[i]; } for(int j = 0; j != k; j++) { keybuf.Clear(); Entry.ToBytesAppend(all[i], keybuf); valuebuf.Clear(); Entry.ToBytesAppend(all[j], valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } } }"); facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(128); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { output.Add(key, values[0].Value); } "); int n = 0; foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (4 != en.Current.GetValueLength()) { throw new Exception("Value length incorrect"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (!hi.ContainsKey(ikey)) { throw new Exception("Unexpected key"); } int k = 1 + ikey % 8; if (ikey < 500) { k += ikey; } } } if (input.Count != n) { throw new Exception("Test failed: expected " + input.Count.ToString() + " key/values, got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("FixedArrayComboList test with dymamically created inline C# reducer"); MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("faclxobj2", 4, 4); facl.AddBlock(@"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); facl.Open(); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); Random rnd = new Random(); Dictionary <int, int> kv = new Dictionary <int, int>(); int numkeys = 50; for (int i = 0; i != numkeys; i++) { int x; do { x = rnd.Next(); }while (kv.ContainsKey(x)); kv[x] = rnd.Next(); } if (kv.Count != numkeys) { throw new Exception("kv.Count != numkeys"); } foreach (KeyValuePair <int, int> pair in kv) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(pair.Key, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(pair.Value, valuebuf); facl.Add(keybuf, valuebuf); } facl.SortBlocks(); StringBuilder dictcode = new StringBuilder(128); dictcode.Append("kv = new Dictionary<int, int>();"); foreach (KeyValuePair <int, int> pair in kv) { dictcode.Append("kv[" + pair.Key.ToString() + "] = " + pair.Value.ToString() + ";"); } FixedArrayComboListEnumerator[] enums = facl.GetEnumeratorsWithCode(@" List<byte> keybuf = new List<byte>(); List<byte> valuebuf = new List<byte>(); Dictionary<int, int> kv; bool finalized = false; public void ReduceInitialize() { " + dictcode.ToString() + @" } public void ReduceFinalize() { if(finalized) { throw new Exception(" + "\"Double finalized!\"" + @"); } finalized = true; if(0 != kv.Count) { throw new Exception(" + "\"ReduceFinalize: \" + " + @"kv.Count.ToString()); } } public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { if(finalized) { throw new Exception(" + "\"Reduce after finalized!\"" + @"); } for(int i = 0; i != values.Length; i++) { output.Add(key, values[i].Value); // ... keybuf.Clear(); key.AppendTo(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); values[i].Value.AppendTo(valuebuf); int ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf); if(!kv.ContainsKey(ikey)) { throw new Exception(" + "\"Reduce doesn't contain key: \" + " + @"ikey.ToString()); } if(kv[ikey] != ivalue) { throw new Exception(" + "\"Reduce value mismatch for key: \" + " + @"ikey.ToString()); } kv.Remove(ikey); } } "); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); en.Current.CopyValueAppend(valuebuf); int ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf); if (!kv.ContainsKey(ikey)) { throw new Exception("Enumeration doesn't contain key: " + ikey.ToString()); } if (kv[ikey] != ivalue) { throw new Exception("Enumeration value mismatch for key: " + ikey.ToString()); } kv.Remove(ikey); } } if (0 != kv.Count) { throw new Exception("Enumeration failure: " + kv.Count.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } facl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Confirming correct zblock distribution with FixedArrayComboList:"); if (interactive) { Console.WriteLine("Press a key..."); Console.ReadKey(); } MySpace.DataMining.DistributedObjects5.FixedArrayComboList facl = new MySpace.DataMining.DistributedObjects5.FixedArrayComboList("faclzfoo", 4, 4); facl.AddBlock(@"127.0.0.1|intcombo_logs0.txt|slaveid=0"); facl.Open(); System.Collections.Generic.Queue <int> wheel = new Queue <int>(); System.Collections.Generic.List <int> al = new List <int>(); Random rnd = new Random(System.DateTime.Now.Millisecond); System.Collections.Hashtable ht = new System.Collections.Hashtable(); List <byte> keybuf = new List <byte>(); //fill wheel for (int i = 1; i != 200; i++) { int x = rnd.Next(5656, 76688989); object o = ht[x]; int y = i; if (null != o) { y += (int)o; } ht[x] = y; for (int j = 0; j != i; j++) { al.Add(x); } } foreach (int o in al) { wheel.Enqueue(o); } int added = 0; while (0 != wheel.Count) { int x = rnd.Next(0, wheel.Count - 1); for (int foo = 0; foo != x; foo++) { wheel.Enqueue(wheel.Dequeue()); } int key = (int)wheel.Dequeue(); keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(key, keybuf); facl.Add(keybuf, keybuf); added++; } if (interactive) { Console.WriteLine("Press a key to sort..."); Console.ReadKey(); } facl.SortBlocks(); FixedArrayComboListEnumerator[] enums = facl.GetEnumerators(); System.Collections.Generic.List <int> outlist = new List <int>(); foreach (FixedArrayComboListEnumerator en in enums) { while (en.MoveNext()) { keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int key = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); outlist.Add(key); } Console.WriteLine(); } int lastkey = 0; foreach (int key in outlist) { if (key != lastkey) { if (0 != lastkey) { if (0 != (int)ht[lastkey]) { throw new Exception("Order error"); } } } lastkey = key; object blah = ht[key]; ht[key] = (int)blah - 1; } al.Sort(); outlist.Sort(); if (al.Count != outlist.Count) { throw new Exception("foo"); } for (int i = 0; i != al.Count; i++) { if ((int)al[i] != (int)outlist[i]) { throw new Exception("bar"); } } Console.WriteLine("Done"); if (interactive) { Console.WriteLine("Press a key to close..."); Console.ReadKey(); } facl.Close(); } { Console.WriteLine("Testing GzipFastLoader..."); MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader("intlines.txt.gz", 64); int i = 0; for (; !gz.EOF; i++) { int a, b, c, d; if (!gz.ReadLine(out a, out b, out c, out d)) { throw new Exception("Whoops, it was supposed to hit gz.EOF first"); } if (0 == (i % 4)) { if (0 != a || 0 != b || 0 != c || 0 != d) { throw new Exception("Line " + (i + 1).ToString() + " should give me 0 but it doesn't!!"); } } else { if (i != a) { throw new Exception("i != a"); } switch (i) { case 1: if (0 != b) { throw new Exception(i.ToString()); } break; case 2: if (0 != c) { throw new Exception(i.ToString()); } break; case 3: if (0 != d) { throw new Exception(i.ToString()); } break; default: if (0 == d) { throw new Exception(i.ToString()); } break; } } } if (10 != i) { throw new Exception("Bad number of lines!"); } Console.WriteLine("Done"); } #if OBSOLETE { Console.WriteLine("Testing ArrayComboList with simple before-load (primitive mapper)."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("foomap", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < 20; i++) { keybuf.Clear(); Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); Entry.ToBytesAppend64(42, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } }"); acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumerators(); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 8) { throw new Exception("Value length mismatch"); } n++; } } if (20 != n) { throw new Exception("Test failed: expected 20 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList before-load against regular add."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("foomap2", 4); acl.atype = true; // Must have exactly 1 block(slave) for this test to work!! acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); { for (int i = 0; i < 20; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend64(42, valuebuf); acl.Add(keybuf, valuebuf); } } acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < 20; i++) { keybuf.Clear(); Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); Entry.ToBytesAppend64(42, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } }"); { for (int i = 0; i < 20; i++) { keybuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend(i, keybuf); valuebuf.Clear(); MySpace.DataMining.DistributedObjects.Entry.ToBytesAppend64(42, valuebuf); acl.Add(keybuf, valuebuf); } } acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { if(3 != values.Length) { throw new Exception(3.ToString() + '!'.ToString() + values.Length.ToString()); } for(int i = 0; i != values.Length; i++) { if(8 != values[i].Value.Length) { throw new Exception(3.ToString() + '!'.ToString() + values[i].Value.Length.ToString()); } valuebuf.Clear(); values[i].Value.AppendTo(valuebuf); long ivalue = MySpace.DataMining.DistributedObjects.Entry.BytesToLong(valuebuf); if(42 != ivalue) { throw new Exception(42.ToString() + '!'.ToString() + ivalue.ToString()); } } valuebuf.Clear(); Entry.ToBytesAppend64(42 * 3, valuebuf); output.Add(key, ByteSlice.Create(valuebuf)); } "); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 8) { throw new Exception("Value length mismatch"); } n++; } } if (20 != n) { throw new Exception("Test failed: expected 20 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList with GzipFastLoader in before-load."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("gzmap", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); MySpace.DataMining.DistributedObjects.GzipFastLoader gz = new MySpace.DataMining.DistributedObjects.GzipFastLoader(" + "\"intlines.txt.gz\"" + @", 64); int i = 0; for (; !gz.EOF; i++) { int a, b, c, d; gz.ReadLine(out a, out b, out c, out d); keybuf.Clear(); Entry.ToBytesAppend(a, keybuf); valuebuf.Clear(); Entry.ToBytesAppend(b, valuebuf); Entry.ToBytesAppend(c, valuebuf); Entry.ToBytesAppend(d, valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } if (10 != i) { throw new Exception('!'.ToString()); } }"); acl.SortBlocks(); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); ArrayComboListEnumerator[] enums = acl.GetEnumerators(); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (en.Current.GetValueLength() != 12) { throw new Exception("Value length mismatch"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int a = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); valuebuf.Clear(); en.Current.CopyValueAppend(valuebuf); int b = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 0); int c = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 4); int d = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, 8); //if(0 != a) { switch (a) { case 0: break; case 1: if (0 != b) { throw new Exception(a.ToString()); } break; case 2: if (0 != c) { throw new Exception(a.ToString()); } break; case 3: if (0 != d) { throw new Exception(a.ToString()); } break; default: if (0 == d) { throw new Exception(a.ToString()); } break; } } } } if (10 != n) { throw new Exception("Test failed: expected 10 key/values, only got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } { Console.WriteLine("Testing ArrayComboList before-load: random data stress test and validation."); MySpace.DataMining.DistributedObjects5.ArrayComboList acl = new MySpace.DataMining.DistributedObjects5.ArrayComboList("stv", 4); acl.atype = true; acl.AddBlock("64", "128", @"127.0.0.1|arraycombo_logs0.txt|slaveid=0"); acl.Open(); Random rnd = new Random(); List <int> input = new List <int>(5000); Dictionary <int, bool> hi = new Dictionary <int, bool>(5000); List <byte> keybuf = new List <byte>(); List <byte> valuebuf = new List <byte>(); { for (int i = 0; i < 5000; i++) { int x; do { if (0 == (i % 300)) { x = rnd.Next(30, 300); } else { x = rnd.Next(); } }while (hi.ContainsKey(x)); hi[x] = true; input.Add(x); } } StringBuilder blinput = new StringBuilder(@"int[] all = new int[] { ", 10000); for (int i = 0; i < input.Count; i++) { if (0 != i) { blinput.Append(", "); } blinput.Append(input[i]); } blinput.Append(" };"); acl.BeforeLoad(@" public virtual void OnBeforeLoad(MySpace.DataMining.DistributedObjects.LoadOutput output) { " + blinput.ToString() + @" List<byte> keybuf = new List<byte>(128); List<byte> valuebuf = new List<byte>(128); for (int i = 0; i < all.Length; i++) { int k = 1 + all[i] % 8; if(all[i] < 500) { k += all[i]; } for(int j = 0; j != k; j++) { keybuf.Clear(); Entry.ToBytesAppend(all[i], keybuf); valuebuf.Clear(); Entry.ToBytesAppend(all[j], valuebuf); output.Add(ByteSlice.Create(keybuf), ByteSlice.Create(valuebuf)); } } }"); acl.SortBlocks(); ArrayComboListEnumerator[] enums = acl.GetEnumeratorsWithCode(@" List<byte> valuebuf = new List<byte>(128); public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output) { valuebuf.Clear(); for(int i = 0; i < values.Length; i++) { for(int j = 0; j < values[i].Value.Length; j++) { valuebuf.Add(values[i].Value[j]); } } output.Add(key, ByteSlice.Create(valuebuf)); } "); int n = 0; foreach (ArrayComboListEnumerator en in enums) { while (en.MoveNext()) { if (en.Current.GetKeyLength() != 4) { throw new Exception("Key length mismatch"); } if (0 != (en.Current.GetValueLength() % 4)) { throw new Exception("Uneven value length"); } n++; keybuf.Clear(); en.Current.CopyKeyAppend(keybuf); int ikey = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(keybuf); if (!hi.ContainsKey(ikey)) { throw new Exception("Unexpected key"); } int k = 1 + ikey % 8; if (ikey < 500) { k += ikey; } if (k * 4 != en.Current.GetValueLength()) { throw new Exception("Value length incorrect"); } /* * valuebuf.Clear(); * en.Current.CopyValueAppend(valuebuf); // Note: not in same order as added. * for (int vv = 0; vv < en.Current.GetValueLength(); vv += 4) * { * int x = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(valuebuf, vv); * } * */ } } if (input.Count != n) { throw new Exception("Test failed: expected " + input.Count.ToString() + " key/values, got " + n.ToString()); } if (interactive) { Console.WriteLine("Press a key to close"); Console.ReadKey(); } acl.Close(); Console.WriteLine("Done"); } #endif }