public void TestHashTableSimpleSmallMMP()
{
int n = 100000;
int l = 7;
int seed = 1;
//Generate 10 elements with different value for l
IEnumerable <Tuple <ulong, int> > S = Generator.CreateStream(n, 63, seed);
//Maps all keys from S to [0,127]
BigInteger a = new BigInteger(Generator.GenerateBits(89, seed));
BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1));
HashTableChaining <int> table_MMP = new HashTableChaining <int>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l);
foreach (Tuple <ulong, int> elem in S)
{
table_MMP.increment(elem.Item1, (Number)elem.Item2);
}
foreach (Tuple <ulong, int> elem in S)
{
Assert.Equal(elem.Item2, table_MMP.get(elem.Item1));
}
output.WriteLine("Streamsize: " + n);
output.WriteLine("Our hashtable size: " + table_MMP.Count);
output.WriteLine(table_MMP.ToString());
}
public void TestHashTableSimpleSmallMS()
{
int n = 100000;
int seed = 1;
//Generate 10 elements with different value for l
IEnumerable <Tuple <ulong, int> > S = Generator.CreateStream(n, 63, 1);
//Maps all keys from S to [0,127]
ulong a = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed)));
int l = 7;
HashTableChaining <int> table_MS = new HashTableChaining <int>(HashFunction.MultiplyShift(a, l), 1UL << l);
foreach (Tuple <ulong, int> elem in S)
{
table_MS.increment(elem.Item1, (Number)elem.Item2);
}
foreach (Tuple <ulong, int> elem in S)
{
Assert.Equal(elem.Item2, table_MS.get(elem.Item1));
}
output.WriteLine("Streamsize: " + n);
output.WriteLine("Our hashtable size: " + table_MS.Count);
output.WriteLine(table_MS.ToString());
}
public void TestEstimationOfS_Both()
{
int n = 10000000;
int l_min = 0; // Should be single hashtable node
int l_max = 20; // array boundary, as we shift 1 << 30 <=> 2^31 https://docs.microsoft.com/en-us/dotnet/api/system.int32.maxvalue?view=netcore-3.1
int l = l_min;
int seed = 2;
ulong a_64odd = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed)));
BigInteger a = new BigInteger(Generator.GenerateBits(89, seed));
BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1));
Stopwatch sw = new Stopwatch();
while (l <= l_max)
{
sw.Restart();
HashTableChaining <long> tableMS = new HashTableChaining <long>(HashFunction.MultiplyShift(a_64odd, l), 1UL << l);
HashTableChaining <long> tableMMP = new HashTableChaining <long>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l);
IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed);
foreach (Tuple <ulong, long> elem in S)
{
tableMS.increment(elem.Item1, (NumberLong)elem.Item2);
tableMMP.increment(elem.Item1, (NumberLong)elem.Item2);
}
BigInteger Real_S_MS = Generator.RealCount <long>(tableMS);
BigInteger Real_S_MMP = Generator.RealCount <long>(tableMMP);
sw.Stop();
//output.WriteLine("MS table: \n " + tableMS);
//output.WriteLine("MMP table: \n " + tableMMP);
output.WriteLine("Iteration: {0,2}, time in ms: {1}", l, sw.ElapsedMilliseconds);
output.WriteLine("\tMS table sum: " + Real_S_MS);
output.WriteLine("\tMMP table sum: " + Real_S_MMP);
l++;
}
}
public void Test100BCSResultsReduced() //takes too long...
{
int seed = 1;
int n = 100000; //100000
int l = 7;
int t_min = 29;
int t_max = 31;
IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed);
HashTableChaining <long> tableMS = new HashTableChaining <long>(
HashFunction.MultiplyShift(BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed))), l),
1UL << l
);
foreach (Tuple <ulong, long> elem in S)
{
tableMS.increment(elem.Item1, (NumberLong)elem.Item2);
}
BigInteger Real_S = Generator.RealCount <long>(tableMS);
string path = Directory.GetCurrentDirectory();
DirectoryInfo di = new DirectoryInfo(path);
while (di.Name != "XUnit_RAD")
{
di = di.Parent;
}
if (!Directory.Exists(Path.Combine(di.FullName, "TestResult")))
{
Directory.CreateDirectory(Path.Combine(di.FullName, "TestResult"));
}
string testfilePath = Path.Combine(di.FullName, "TestResult", "Test100BCSResultsReduced.csv");
if (File.Exists(testfilePath))
{
File.Delete(testfilePath);
}
File.Create(testfilePath).Close();
for (int t = t_min; t <= t_max; t++)
{
BigInteger[] unsortedResults = new BigInteger[100];
for (int i = 0; i < 100; i++)
{
BasicCountSketch bcs = new BasicCountSketch(t);
foreach (Tuple <ulong, long> elem in S)
{
bcs.Process(elem);
}
unsortedResults[i] = bcs.Estimate2ndMoment();
}
BigInteger[] manyResults = unsortedResults.OrderBy(val => val).ToArray();
BigInteger meansquare = unsortedResults.Aggregate(BigInteger.Zero, (acc, x) => acc + (BigInteger)Math.Pow((long)x - (long)Real_S, 2)) / 100;
BigInteger[] medianResults = new BigInteger[9];
for (int i = 0; i < 9; i++)
{
medianResults[i] = unsortedResults.Skip(i * 11).Take(11).OrderBy(val => val).ElementAt(5);
}
medianResults = medianResults.OrderBy(val => val).ToArray();
StringBuilder sb = new StringBuilder();
sb.AppendLine("Real S;" + Real_S + ";Expectation;" + Real_S + "\nVariance;" + Math.Round(2 * Math.Pow((long)Real_S, 2) / (1 << t), 2) + ";Value of t;" + t + "\nValue of m;" + (1 << t) + ";;");
sb.Append("MeanSquareErrors");
BigInteger average = 0;
sb.Append(";" + meansquare);
average += meansquare;
sb.Append(";" + average / 10);
sb.AppendLine();
sb.AppendLine("#Estimate;1st run;Average run;Real S");
for (int i = 0; i < 100; i++)
{
sb.Append(i + 1);
average = 0;
sb.Append(";" + manyResults[i]);
average += manyResults[i];
sb.Append(";" + average / 10);
sb.Append(";" + Real_S);
sb.AppendLine();
}
sb.AppendLine("#Estimate;1st run;Average run;Real S");
for (int i = 0; i < 9; i++)
{
sb.Append(i + 1);
average = 0;
sb.Append(";" + medianResults[i]);
average += medianResults[i];
sb.Append(";" + average / 10);
sb.Append(";" + Real_S);
sb.AppendLine();
}
sb.AppendLine(";;;");
File.AppendAllText(testfilePath, sb.ToString());
}
}
public void Test100BCSResultsParallel()
{
int seed = 1;
int n = 100000; //100000
int l = 17; //2^17 ~ 131000
int t_min = 1;
int t_max = 17; //fails due to space on my machine at 29
IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l, seed);
BigInteger[] a_s = new BigInteger[]
{
new BigInteger(Generator.GenerateBits(89, 1)),
new BigInteger(Generator.GenerateBits(89, 2)),
new BigInteger(Generator.GenerateBits(89, 3)),
new BigInteger(Generator.GenerateBits(89, 4))
};
HashTableChaining <long> table4MMP = new HashTableChaining <long>(
HashFunction.CountSketchHashfunctions(HashFunction.kIndependentMultiplyModPrime(a_s), t_max).Item1,
1UL << t_max
);
Stopwatch sw = new Stopwatch();
sw.Restart();
foreach (Tuple <ulong, long> elem in S)
{
table4MMP.increment(elem.Item1, (NumberLong)elem.Item2);
}
BigInteger Real_S = Generator.RealCount <long>(table4MMP);
sw.Stop();
long Real_S_time = sw.ElapsedMilliseconds;
string path = Directory.GetCurrentDirectory();
DirectoryInfo di = new DirectoryInfo(path);
while (di.Name != "XUnit_RAD")
{
di = di.Parent;
}
if (!Directory.Exists(Path.Combine(di.FullName, "TestResult")))
{
Directory.CreateDirectory(Path.Combine(di.FullName, "TestResult"));
}
string testfilePath = Path.Combine(di.FullName, "TestResult", "Test100BCSResultsParallel.csv");
if (File.Exists(testfilePath))
{
File.Delete(testfilePath);
}
File.Create(testfilePath).Close();
for (int t = t_min; t <= t_max; t++)
{
BigInteger[][] unsortedResults = new BigInteger[10][];
for (int run = 0; run < 10; run++)
{
Task <BigInteger>[] resultTasks = new Task <BigInteger> [100];
sw.Restart();
for (int i = 0; i < 100; i++)
{
resultTasks[i] = Task <BigInteger> .Factory.StartNew((object obj) =>
{
IEnumerable <Tuple <ulong, long> > S = (IEnumerable <Tuple <ulong, long> >)obj;
BasicCountSketch bcs = new BasicCountSketch(t);
foreach (Tuple <ulong, long> elem in S)
{
bcs.Process(elem);
}
return(bcs.Estimate2ndMoment());
}, S);
}
sw.Stop();
Task <BigInteger> .WaitAll(resultTasks);
BigInteger[] results = resultTasks.Select(task => task.Result).ToArray();
unsortedResults[run] = results;
}
long t_iteration_time = sw.ElapsedMilliseconds;
BigInteger[][] manyResults = new BigInteger[10][];
BigInteger[][] manyMedianResults = new BigInteger[10][];
BigInteger[] meansquare = new BigInteger[10];
for (int run = 0; run < 10; run++)
{
manyResults[run] = unsortedResults[run].OrderBy(val => val).ToArray();
meansquare[run] = unsortedResults[run].Aggregate(BigInteger.Zero, (acc, x) => acc + (BigInteger)Math.Pow((long)x - (long)Real_S, 2)) / 100;
BigInteger[] medianResults = new BigInteger[9];
for (int i = 0; i < 9; i++)
{
medianResults[i] = unsortedResults[run].Skip(i * 11).Take(11).OrderBy(val => val).ElementAt(5);
}
manyMedianResults[run] = medianResults.OrderBy(val => val).ToArray();
}
StringBuilder sb = new StringBuilder();
sb.AppendLine("Real S;" + Real_S + ";Expectation;" + Real_S + ";Variance;" + Math.Round(2 * Math.Pow((long)Real_S, 2) / (1 << t), 2) + ";Value of t;" + t + ";Value of m;" + (1 << t) + ";Time taken;" + t_iteration_time + ";");
sb.Append("MeanSquareErrors");
BigInteger average = 0;
for (int run = 0; run < 10; run++)
{
sb.Append(";" + meansquare[run]);
average += meansquare[run];
}
sb.Append(";" + average / 10);
sb.Append(";Real S time;" + Real_S_time);
sb.AppendLine();
sb.AppendLine("#Estimate;1st run;2nd run;3rd run;4th run;5th run;6th run;7th run;8th run;9th run;10th run;Average run;Real S");
for (int i = 0; i < 100; i++)
{
sb.Append(i + 1);
average = 0;
for (int run = 0; run < 10; run++)
{
sb.Append(";" + manyResults[run][i]);
average += manyResults[run][i];
}
sb.Append(";" + average / 10);
sb.Append(";" + Real_S);
sb.AppendLine();
}
sb.AppendLine("#Estimate;1st run;2nd run;3rd run;4th run;5th run;6th run;7th run;8th run;9th run;10th run;Average run;Real S");
for (int i = 0; i < 9; i++)
{
sb.Append(i + 1);
average = 0;
for (int run = 0; run < 10; run++)
{
sb.Append(";" + manyMedianResults[run][i]);
average += manyMedianResults[run][i];
}
sb.Append(";" + average / 10);
sb.Append(";" + Real_S);
sb.AppendLine();
}
sb.AppendLine(";;;;;;;;;;;;");
File.AppendAllText(testfilePath, sb.ToString());
}
}
public void TestEstimationOfS_Comparison()
{
int n = 100000;
int l_min = 1;
int l_max = 29; // array boundary, as we shift 1 << 31 <=> 2^31 https://docs.microsoft.com/en-us/dotnet/api/system.int32.maxvalue?view=netcore-3.1
int seed = 2;
ulong a_64odd = BitConverter.ToUInt64(Generator.MakeOdd(Generator.GenerateBits(64, seed)));
BigInteger a = new BigInteger(Generator.GenerateBits(89, seed));
BigInteger b = new BigInteger(Generator.GenerateBits(89, seed + 1));
//same S for all tests
IEnumerable <Tuple <ulong, long> > S = Generator.CreateStreamLong(n, l_max, seed);
Stopwatch sw = new Stopwatch();
for (int l = l_min; l <= l_max; l++)
{
if (l == 17)
{
n /= 10;
}
//output.WriteLine("Iteration: {0,2}", l);
HashTableChaining <long> tableMMP = new HashTableChaining <long>(HashFunction.MultiplyModPrime(a, b, l), 1UL << l);
sw.Restart();
foreach (Tuple <ulong, long> elem in S)
{
tableMMP.increment(elem.Item1, (NumberLong)elem.Item2);
}
sw.Stop();
long tmp1 = sw.ElapsedMilliseconds;
sw.Restart();
BigInteger Real_S_MMP = Generator.RealCount <long>(tableMMP);
sw.Stop();
long tmp2 = sw.ElapsedMilliseconds;
//output.WriteLine("\tMultiplyShift:");
//output.WriteLine("\t\tTime to increment: " + tmp1);
//output.WriteLine("\t\tTime for summation: " + tmp2);
//output.WriteLine("\t\tSum: " + Real_S_MS);
BigInteger[] a_s = new BigInteger[]
{
new BigInteger(Generator.GenerateBits(89, 1)),
new BigInteger(Generator.GenerateBits(89, 2)),
new BigInteger(Generator.GenerateBits(89, 3)),
new BigInteger(Generator.GenerateBits(89, 4))
};
HashTableChaining <long> table4MMP = new HashTableChaining <long>(HashFunction.CountSketchHashfunctions(HashFunction.kIndependentMultiplyModPrime(a_s), l).Item1, 1UL << l);
sw.Restart();
foreach (Tuple <ulong, long> elem in S)
{
table4MMP.increment(elem.Item1, (NumberLong)elem.Item2);
}
sw.Stop();
long tmp3 = sw.ElapsedMilliseconds;
sw.Restart();
BigInteger Real_S_4MMP = Generator.RealCount <long>(table4MMP);
sw.Stop();
long tmp4 = sw.ElapsedMilliseconds;
//output.WriteLine("\tMultiplyModPrime:");
//output.WriteLine("\t\tTime to increment: " + tmp3);
//output.WriteLine("\t\tTime for summation: " + tmp4);
//output.WriteLine("\t\tSum: " + Real_S_MMP);
//
//output.WriteLine("\tComparison:");
//output.WriteLine("\t\tMS vs. MMP increment: " + Math.Round(((double)tmp1) / tmp3, 2) + "x");
//output.WriteLine("\t\tMS vs. MMP summation: " + Math.Round(((double)tmp2) / tmp4, 2) + "x");
//output.WriteLine("\t\tSame sum: " + (Real_S_MS == Real_S_MMP));
double diff1 = tmp3 == 0 ? 1.0 : Math.Round(((double)tmp1) / tmp3, 2);
double diff2 = tmp4 == 0 ? 1.0 : Math.Round(((double)tmp2) / tmp4, 2);
output.WriteLine(l + " & " + n + " & " + tmp1 + " & " + tmp3 + " & " + diff1 + "x & " + tmp2 + " & " + tmp4 + " & " + diff2 + @"x \\\hline");
}
}
