public void TestHashTableSimpleSmallMMP() { int n = 100000; int l = 7; int seed = 1; //Generate 10 elements with different value for l IEnumerable <Tuple <ulong, int> > S = Generator.CreateStream(n, 63, seed); //Maps all keys from S to [0,127] BigInteger a = new BigInteger(Generator.GenerateBits(89, seed)); BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1)); HashTableChaining <int> table_MMP = new HashTableChaining <int>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l); foreach (Tuple <ulong, int> elem in S) { table_MMP.increment(elem.Item1, (Number)elem.Item2); } foreach (Tuple <ulong, int> elem in S) { Assert.Equal(elem.Item2, table_MMP.get(elem.Item1)); } output.WriteLine("Streamsize: " + n); output.WriteLine("Our hashtable size: " + table_MMP.Count); output.WriteLine(table_MMP.ToString()); }
public void TestHashTableSimpleSmallMS() { int n = 100000; int seed = 1; //Generate 10 elements with different value for l IEnumerable <Tuple <ulong, int> > S = Generator.CreateStream(n, 63, 1); //Maps all keys from S to [0,127] ulong a = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed))); int l = 7; HashTableChaining <int> table_MS = new HashTableChaining <int>(HashFunction.MultiplyShift(a, l), 1UL << l); foreach (Tuple <ulong, int> elem in S) { table_MS.increment(elem.Item1, (Number)elem.Item2); } foreach (Tuple <ulong, int> elem in S) { Assert.Equal(elem.Item2, table_MS.get(elem.Item1)); } output.WriteLine("Streamsize: " + n); output.WriteLine("Our hashtable size: " + table_MS.Count); output.WriteLine(table_MS.ToString()); }
public void TestEstimationOfS_Both() { int n = 10000000; int l_min = 0; // Should be single hashtable node int l_max = 20; // array boundary, as we shift 1 << 30 <=> 2^31 https://docs.microsoft.com/en-us/dotnet/api/system.int32.maxvalue?view=netcore-3.1 int l = l_min; int seed = 2; ulong a_64odd = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed))); BigInteger a = new BigInteger(Generator.GenerateBits(89, seed)); BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1)); Stopwatch sw = new Stopwatch(); while (l <= l_max) { sw.Restart(); HashTableChaining <long> tableMS = new HashTableChaining <long>(HashFunction.MultiplyShift(a_64odd, l), 1UL << l); HashTableChaining <long> tableMMP = new HashTableChaining <long>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l); IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed); foreach (Tuple <ulong, long> elem in S) { tableMS.increment(elem.Item1, (NumberLong)elem.Item2); tableMMP.increment(elem.Item1, (NumberLong)elem.Item2); } BigInteger Real_S_MS = Generator.RealCount <long>(tableMS); BigInteger Real_S_MMP = Generator.RealCount <long>(tableMMP); sw.Stop(); //output.WriteLine("MS table: \n " + tableMS); //output.WriteLine("MMP table: \n " + tableMMP); output.WriteLine("Iteration: {0,2}, time in ms: {1}", l, sw.ElapsedMilliseconds); output.WriteLine("\tMS table sum: " + Real_S_MS); output.WriteLine("\tMMP table sum: " + Real_S_MMP); l++; } }
public void Test100BCSResultsReduced() //takes too long... { int seed = 1; int n = 100000; //100000 int l = 7; int t_min = 29; int t_max = 31; IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed); HashTableChaining <long> tableMS = new HashTableChaining <long>( HashFunction.MultiplyShift(BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed))), l), 1UL << l ); foreach (Tuple <ulong, long> elem in S) { tableMS.increment(elem.Item1, (NumberLong)elem.Item2); } BigInteger Real_S = Generator.RealCount <long>(tableMS); string path = Directory.GetCurrentDirectory(); DirectoryInfo di = new DirectoryInfo(path); while (di.Name != "XUnit_RAD") { di = di.Parent; } if (!Directory.Exists(Path.Combine(di.FullName, "TestResult"))) { Directory.CreateDirectory(Path.Combine(di.FullName, "TestResult")); } string testfilePath = Path.Combine(di.FullName, "TestResult", "Test100BCSResultsReduced.csv"); if (File.Exists(testfilePath)) { File.Delete(testfilePath); } File.Create(testfilePath).Close(); for (int t = t_min; t <= t_max; t++) { BigInteger[] unsortedResults = new BigInteger[100]; for (int i = 0; i < 100; i++) { BasicCountSketch bcs = new BasicCountSketch(t); foreach (Tuple <ulong, long> elem in S) { bcs.Process(elem); } unsortedResults[i] = bcs.Estimate2ndMoment(); } BigInteger[] manyResults = unsortedResults.OrderBy(val => val).ToArray(); BigInteger meansquare = unsortedResults.Aggregate(BigInteger.Zero, (acc, x) => acc + (BigInteger)Math.Pow((long)x - (long)Real_S, 2)) / 100; BigInteger[] medianResults = new BigInteger[9]; for (int i = 0; i < 9; i++) { medianResults[i] = unsortedResults.Skip(i * 11).Take(11).OrderBy(val => val).ElementAt(5); } medianResults = medianResults.OrderBy(val => val).ToArray(); StringBuilder sb = new StringBuilder(); sb.AppendLine("Real S;" + Real_S + ";Expectation;" + Real_S + "\nVariance;" + Math.Round(2 * Math.Pow((long)Real_S, 2) / (1 << t), 2) + ";Value of t;" + t + "\nValue of m;" + (1 << t) + ";;"); sb.Append("MeanSquareErrors"); BigInteger average = 0; sb.Append(";" + meansquare); average += meansquare; sb.Append(";" + average / 10); sb.AppendLine(); sb.AppendLine("#Estimate;1st run;Average run;Real S"); for (int i = 0; i < 100; i++) { sb.Append(i + 1); average = 0; sb.Append(";" + manyResults[i]); average += manyResults[i]; sb.Append(";" + average / 10); sb.Append(";" + Real_S); sb.AppendLine(); } sb.AppendLine("#Estimate;1st run;Average run;Real S"); for (int i = 0; i < 9; i++) { sb.Append(i + 1); average = 0; sb.Append(";" + medianResults[i]); average += medianResults[i]; sb.Append(";" + average / 10); sb.Append(";" + Real_S); sb.AppendLine(); } sb.AppendLine(";;;"); File.AppendAllText(testfilePath, sb.ToString()); } }
public void Test100BCSResultsParallel() { int seed = 1; int n = 100000; //100000 int l = 17; //2^17 ~ 131000 int t_min = 1; int t_max = 17; //fails due to space on my machine at 29 IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed); BigInteger[] a_s = new BigInteger[] { new BigInteger(Generator.GenerateBits(89, 1)), new BigInteger(Generator.GenerateBits(89, 2)), new BigInteger(Generator.GenerateBits(89, 3)), new BigInteger(Generator.GenerateBits(89, 4)) }; HashTableChaining <long> table4MMP = new HashTableChaining <long>( HashFunction.CountSketchHashfunctions(HashFunction.kIndependentMultiplyModPrime(a_s), t_max).Item1, 1UL << t_max ); Stopwatch sw = new Stopwatch(); sw.Restart(); foreach (Tuple <ulong, long> elem in S) { table4MMP.increment(elem.Item1, (NumberLong)elem.Item2); } BigInteger Real_S = Generator.RealCount <long>(table4MMP); sw.Stop(); long Real_S_time = sw.ElapsedMilliseconds; string path = Directory.GetCurrentDirectory(); DirectoryInfo di = new DirectoryInfo(path); while (di.Name != "XUnit_RAD") { di = di.Parent; } if (!Directory.Exists(Path.Combine(di.FullName, "TestResult"))) { Directory.CreateDirectory(Path.Combine(di.FullName, "TestResult")); } string testfilePath = Path.Combine(di.FullName, "TestResult", "Test100BCSResultsParallel.csv"); if (File.Exists(testfilePath)) { File.Delete(testfilePath); } File.Create(testfilePath).Close(); for (int t = t_min; t <= t_max; t++) { BigInteger[][] unsortedResults = new BigInteger[10][]; for (int run = 0; run < 10; run++) { Task <BigInteger>[] resultTasks = new Task <BigInteger> [100]; sw.Restart(); for (int i = 0; i < 100; i++) { resultTasks[i] = Task <BigInteger> .Factory.StartNew((object obj) => { IEnumerable <Tuple <ulong, long> > S = (IEnumerable <Tuple <ulong, long> >)obj; BasicCountSketch bcs = new BasicCountSketch(t); foreach (Tuple <ulong, long> elem in S) { bcs.Process(elem); } return(bcs.Estimate2ndMoment()); }, S); } sw.Stop(); Task <BigInteger> .WaitAll(resultTasks); BigInteger[] results = resultTasks.Select(task => task.Result).ToArray(); unsortedResults[run] = results; } long t_iteration_time = sw.ElapsedMilliseconds; BigInteger[][] manyResults = new BigInteger[10][]; BigInteger[][] manyMedianResults = new BigInteger[10][]; BigInteger[] meansquare = new BigInteger[10]; for (int run = 0; run < 10; run++) { manyResults[run] = unsortedResults[run].OrderBy(val => val).ToArray(); meansquare[run] = unsortedResults[run].Aggregate(BigInteger.Zero, (acc, x) => acc + (BigInteger)Math.Pow((long)x - (long)Real_S, 2)) / 100; BigInteger[] medianResults = new BigInteger[9]; for (int i = 0; i < 9; i++) { medianResults[i] = unsortedResults[run].Skip(i * 11).Take(11).OrderBy(val => val).ElementAt(5); } manyMedianResults[run] = medianResults.OrderBy(val => val).ToArray(); } StringBuilder sb = new StringBuilder(); sb.AppendLine("Real S;" + Real_S + ";Expectation;" + Real_S + ";Variance;" + Math.Round(2 * Math.Pow((long)Real_S, 2) / (1 << t), 2) + ";Value of t;" + t + ";Value of m;" + (1 << t) + ";Time taken;" + t_iteration_time + ";"); sb.Append("MeanSquareErrors"); BigInteger average = 0; for (int run = 0; run < 10; run++) { sb.Append(";" + meansquare[run]); average += meansquare[run]; } sb.Append(";" + average / 10); sb.Append(";Real S time;" + Real_S_time); sb.AppendLine(); sb.AppendLine("#Estimate;1st run;2nd run;3rd run;4th run;5th run;6th run;7th run;8th run;9th run;10th run;Average run;Real S"); for (int i = 0; i < 100; i++) { sb.Append(i + 1); average = 0; for (int run = 0; run < 10; run++) { sb.Append(";" + manyResults[run][i]); average += manyResults[run][i]; } sb.Append(";" + average / 10); sb.Append(";" + Real_S); sb.AppendLine(); } sb.AppendLine("#Estimate;1st run;2nd run;3rd run;4th run;5th run;6th run;7th run;8th run;9th run;10th run;Average run;Real S"); for (int i = 0; i < 9; i++) { sb.Append(i + 1); average = 0; for (int run = 0; run < 10; run++) { sb.Append(";" + manyMedianResults[run][i]); average += manyMedianResults[run][i]; } sb.Append(";" + average / 10); sb.Append(";" + Real_S); sb.AppendLine(); } sb.AppendLine(";;;;;;;;;;;;"); File.AppendAllText(testfilePath, sb.ToString()); } }
public void TestEstimationOfS_Comparison() { int n = 100000; int l_min = 1; int l_max = 29; // array boundary, as we shift 1 << 31 <=> 2^31 https://docs.microsoft.com/en-us/dotnet/api/system.int32.maxvalue?view=netcore-3.1 int seed = 2; ulong a_64odd = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed))); BigInteger a = new BigInteger(Generator.GenerateBits(89, seed)); BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1)); //same S for all tests IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l_max, seed); Stopwatch sw = new Stopwatch(); for (int l = l_min; l <= l_max; l++) { if (l == 17) { n /= 10; } //output.WriteLine("Iteration: {0,2}", l); HashTableChaining <long> tableMMP = new HashTableChaining <long>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l); sw.Restart(); foreach (Tuple <ulong, long> elem in S) { tableMMP.increment(elem.Item1, (NumberLong)elem.Item2); } sw.Stop(); long tmp1 = sw.ElapsedMilliseconds; sw.Restart(); BigInteger Real_S_MMP = Generator.RealCount <long>(tableMMP); sw.Stop(); long tmp2 = sw.ElapsedMilliseconds; //output.WriteLine("\tMultiplyShift:"); //output.WriteLine("\t\tTime to increment: " + tmp1); //output.WriteLine("\t\tTime for summation: " + tmp2); //output.WriteLine("\t\tSum: " + Real_S_MS); BigInteger[] a_s = new BigInteger[] { new BigInteger(Generator.GenerateBits(89, 1)), new BigInteger(Generator.GenerateBits(89, 2)), new BigInteger(Generator.GenerateBits(89, 3)), new BigInteger(Generator.GenerateBits(89, 4)) }; HashTableChaining <long> table4MMP = new HashTableChaining <long>(HashFunction.CountSketchHashfunctions(HashFunction.kIndependentMultiplyModPrime(a_s), l).Item1, 1UL << l); sw.Restart(); foreach (Tuple <ulong, long> elem in S) { table4MMP.increment(elem.Item1, (NumberLong)elem.Item2); } sw.Stop(); long tmp3 = sw.ElapsedMilliseconds; sw.Restart(); BigInteger Real_S_4MMP = Generator.RealCount <long>(table4MMP); sw.Stop(); long tmp4 = sw.ElapsedMilliseconds; //output.WriteLine("\tMultiplyModPrime:"); //output.WriteLine("\t\tTime to increment: " + tmp3); //output.WriteLine("\t\tTime for summation: " + tmp4); //output.WriteLine("\t\tSum: " + Real_S_MMP); // //output.WriteLine("\tComparison:"); //output.WriteLine("\t\tMS vs. MMP increment: " + Math.Round(((double)tmp1) / tmp3, 2) + "x"); //output.WriteLine("\t\tMS vs. MMP summation: " + Math.Round(((double)tmp2) / tmp4, 2) + "x"); //output.WriteLine("\t\tSame sum: " + (Real_S_MS == Real_S_MMP)); double diff1 = tmp3 == 0 ? 1.0 : Math.Round(((double)tmp1) / tmp3, 2); double diff2 = tmp4 == 0 ? 1.0 : Math.Round(((double)tmp2) / tmp4, 2); output.WriteLine(l + " & " + n + " & " + tmp1 + " & " + tmp3 + " & " + diff1 + "x & " + tmp2 + " & " + tmp4 + " & " + diff2 + @"x \\\hline"); } }