public async Task MNATest()
{
var sm = new MutableSeries <int, int>();
var count = 1_0;
var sum = 0;
using (Benchmark.Run("MNA"))
{
var _ = Task.Run(() =>
{
for (int i = 0; i < count; i++)
{
sm.TryAppend(i, i);
}
sm.MarkReadOnly();
});
var c = sm.GetAsyncCursor();
while (await c.MoveNextAsync())
{
sum += c.CurrentValue;
}
}
Assert.IsTrue(sum > 0);
Benchmark.Dump();
}
public unsafe void CouldSerializeSortedMap2()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new MutableSeries <int, int>();
for (var i = 0; i < 10000; i++)
{
sm.Add(i, i);
}
var len = BinarySerializer.SizeOf(sm, out var temp);
var len2 = BinarySerializer.Write(sm, buffer.Span, temp);
Assert.AreEqual(len, len2);
Console.WriteLine($"Useful: {sm.RowCount * 8}");
Console.WriteLine($"Total: {len}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((len * 1.0) / (sm.RowCount * 8.0))}");
Series <int, int> sm2 = null;
var len3 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(len, len3);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public unsafe void CouldSerializeRegularSortedMapWithZstd()
{
var rng = new Random();
var dest = (Memory <byte>) new byte[1000000];
var buffer = dest;
var handle = buffer.Pin();
var ptr = (IntPtr)handle.Pointer;
var sm = new MutableSeries <DateTime, decimal>();
for (var i = 0; i < 1000; i++)
{
sm.Add(DateTime.Today.AddSeconds(i), (decimal)Math.Round(i + rng.NextDouble(), 2));
}
var sizeOf = BinarySerializer.SizeOf(sm, out var tmp);
var written = BinarySerializer.Write(sm, dest.Span, tmp);
Assert.AreEqual(sizeOf, written);
Console.WriteLine($"Useful: {sm.RowCount * 24}");
Console.WriteLine($"Total: {written}");
// NB interesting that with converting double to decimal savings go from 65% to 85%,
// even calculated from (8+8) base size not decimal's 16 size
Console.WriteLine($"Savings: {1.0 - ((written * 1.0) / (sm.RowCount * 24.0))}");
Series <DateTime, decimal> sm2 = null;
var len2 = BinarySerializer.Read(buffer.Span, out sm2);
Assert.AreEqual(written, len2);
Assert.IsTrue(sm2.Keys.SequenceEqual(sm.Keys));
Assert.IsTrue(sm2.Values.SequenceEqual(sm.Values));
}
public void NewSeries()
{
var s = new Series <int, int>(Array.Empty <int>(), Array.Empty <int>());
Assert.AreEqual(s.Mutability, Mutability.ReadOnly);
Assert.AreEqual(s.KeySorting, KeySorting.Strong);
Assert.AreEqual(0, s.RowCount.Value);
s.Dispose();
Assert.IsTrue(s.IsDisposed);
var aps = new AppendSeries <int, int>();
Assert.AreEqual(aps.Mutability, Mutability.AppendOnly);
Assert.AreEqual(aps.KeySorting, KeySorting.Strong);
Assert.AreEqual(0, aps.RowCount.Value);
aps.Dispose();
Assert.IsTrue(aps.IsDisposed);
var mus = new MutableSeries <int, int>();
Assert.AreEqual(mus.Mutability, Mutability.Mutable);
Assert.AreEqual(mus.KeySorting, KeySorting.Strong);
Assert.AreEqual(0, mus.RowCount.Value);
mus.MarkAppendOnly();
mus.MarkAppendOnly(); // ignored, does not throw
Assert.AreEqual(Mutability.AppendOnly, mus.Mutability);
mus.MarkReadOnly();
mus.MarkReadOnly(); // ignored, does not throw
Assert.AreEqual(Mutability.ReadOnly, mus.Mutability);
Assert.Throws <InvalidOperationException>(() => { mus.MarkAppendOnly(); });
mus.Dispose();
Assert.IsTrue(mus.IsDisposed);
}
public async Task AddSpeed()
{
const int count = 5_000_000;
for (int r = 0; r < 10; r++)
{
var sl = new SortedList <int, int>();
var sm = new MutableSeries <int, int>();
var c = sm.GetCursor();
using (Benchmark.Run("SL", count))
{
for (int i = 0; i < count; i++)
{
if (i != 2)
{
sl.Add(i, i);
}
}
}
using (Benchmark.Run("Series", count))
{
for (int i = 0; i < count; i++)
{
if (i != 2)
{
sm.Append(i, i);
}
}
}
}
Benchmark.Dump();
}
public void EnumerateScmSpeed()
{
const int count = 10_000_000;
var sl = new SortedList <int, int>();
var sm = new MutableSeries <int, int>();
for (int i = 0; i < count; i++)
{
if (i % 1000 != 0)
{
sl.Add(i, i);
sm.Add(i, i);
//scm.Add(i, i);
}
}
sm.MarkReadOnly();
// scm.Complete();
for (int r = 0; r < 20; r++)
{
//var sum1 = 0L;
//using (Benchmark.Run("SL", count))
//{
// using (var c = sl.GetEnumerator())
// {
// while (c.MoveNext())
// {
// sum1 += c.Current.Value;
// }
// }
//}
//Assert.True(sum1 > 0);
var sum2 = 0L;
using (Benchmark.Run("SM Current.Value", count))
{
using (var c = sm.GetEnumerator())
{
while (c.MoveNext())
{
sum2 += c.Current.Value;
}
}
}
//Assert.AreEqual(sum1, sum2);
var sum3 = 0L;
using (Benchmark.Run("SM CurrentValue", count))
{
using (var c = sm.GetEnumerator())
{
while (c.MoveNext())
{
sum3 += c.CurrentValue;
}
}
}
//Assert.AreEqual(sum1, sum3);
//var sum4 = 0L;
//using (Benchmark.Run("SCM Current.Value", count))
//{
// using (var c = scm.GetEnumerator())
// {
// while (c.MoveNext())
// {
// sum4 += c.Current.Value;
// }
// }
//}
//Assert.AreEqual(sum1, sum4);
//var sum5 = 0L;
//using (Benchmark.Run("SCM CurrentValue", count))
//{
// using (var c = scm.GetEnumerator())
// {
// while (c.MoveNext())
// {
// sum5 += c.CurrentValue;
// }
// }
//}
//Assert.AreEqual(sum1, sum5);
}
Benchmark.Dump();
}
public void TGVSpeed()
{
for (int size = 0; size < 3; size++)
{
var count = (int)(1024 * Math.Pow(2, size));
const int mult = 1000;
var sl = new SortedList <DateTime, int>();
var sm = new MutableSeries <DateTime, int>();
var start = DateTime.Today.ToUniversalTime();
for (int i = 0; i < count; i++)
{
if (i != 2) // make irregular
{
sl.Add(start.AddTicks(i), i);
sm.Add(start.AddTicks(i), i);
}
}
Assert.IsFalse(sm.IsCompleted);
sm.MarkReadOnly();
Assert.IsTrue(sm.IsCompleted);
for (int r = 0; r < 20; r++)
{
//var sum1 = 0L;
//using (Benchmark.Run("SL", count * mult, true))
//{
// for (int j = 0; j < mult; j++)
// {
// for (int i = 0; i < count; i++)
// {
// if (sl.TryGetValue(start.AddTicks(i), out var v))
// {
// sum1 += v;
// }
// }
// }
//}
//Assert.True(sum1 > 0);
var sum2 = 0L;
using (Benchmark.Run("SM", count * mult, true))
{
for (int j = 0; j < mult; j++)
{
for (int i = 0; i < count; i++)
{
if (sm.TryGetValue(start.AddTicks(i), out var v))
{
sum2 += v;
}
}
}
}
//Assert.True(sum2 > 0);
//Assert.AreEqual(sum1, sum2);
//var sum3 = 0L;
//using (Benchmark.Run("SCM", count * mult, true))
//{
// for (int j = 0; j < mult; j++)
// {
// for (int i = 0; i < count; i++)
// {
// if (scm.TryGetValue(start.AddTicks(i), out var v))
// {
// sum3 += v;
// }
// }
// }
//}
//Assert.True(sum3 > 0);
//Assert.AreEqual(sum2, sum3);
}
Benchmark.Dump($"Size = {Math.Pow(2, size)}k elements");
}
}
public void SortedMapNotifierTest()
{
var rounds = 100_000;
for (int r = 0; r < rounds; r++)
{
var count = 1_000_000;
var cnt1 = 0;
var cnt2 = 0;
var cnt3 = 0;
var cnt4 = 0;
var sm1 = new MutableSeries <int, int>();
// sm1._isSynchronized = false;
var addTask = Task.Run(async() =>
{
// await Task.Delay(5000);
try
{
// sm1.TryAddLast(0, 0);
for (int i = 0; i < count; i++)
{
if (i != 2)
{
sm1.TryAppend(i, i);
Thread.SpinWait(5);
//if (i % 250000 == 0)
//{
// GC.Collect(0, GCCollectionMode.Forced, false);
//}
}
}
sm1.MarkReadOnly();
//Console.WriteLine("cnt1: " + cnt1);
//Console.WriteLine("cnt2: " + cnt2);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
});
// addTask.Wait();
using (Benchmark.Run("SM.Updated", count * 5))
{
var t1 = Task.Run(async() =>
{
Thread.CurrentThread.Name = "MNA1";
try
{
using (var cursor1 = sm1.GetAsyncCursor())
{
// Console.WriteLine("MNA1 started");
while (await cursor1.MoveNextAsync())
{
AsyncCursor.LogFinished();
if (cnt1 == 2)
{
cnt1++;
// Console.WriteLine("MNA1 moving");
}
if (cursor1.CurrentKey != cnt1)
{
ThrowHelper.ThrowInvalidOperationException("Wrong cursor enumeration");
}
cnt1++;
//if (c % 250000 == 0)
//{
// GC.Collect(0, GCCollectionMode.Forced, false);
// Console.WriteLine(c);
//}
}
if (cnt1 != count)
{
ThrowHelper.ThrowInvalidOperationException($"1 Cannot move to count: c={cnt1}, count={count}");
}
if (AsyncCursor.SyncCount == 0)
{
Console.WriteLine("SyncCount == 0");
}
Thread.MemoryBarrier();
}
}
catch (Exception e)
{
Console.WriteLine("MNA1 ex: " + e);
}
});
var t2 = Task.Run(async() =>
{
Thread.CurrentThread.Name = "MNA2";
try
{
using (var cursor2 = sm1.GetAsyncCursor())
{
// Console.WriteLine("MNA2 started");
while (await cursor2.MoveNextAsync())
{
AsyncCursor.LogFinished();
if (cnt2 == 2)
{
cnt2++;
// Console.WriteLine("MNA2 moving");
}
if (cursor2.CurrentKey != cnt2)
{
ThrowHelper.ThrowInvalidOperationException("Wrong cursor enumeration");
}
cnt2++;
}
if (cnt2 != count)
{
ThrowHelper.ThrowInvalidOperationException($"2 Cannot move to count: c={cnt2}, count={count}");
}
if (AsyncCursor.SyncCount == 0)
{
Console.WriteLine("SyncCount == 0");
}
}
}
catch (Exception e)
{
Console.WriteLine("MNA2 ex: " + e);
}
});
var t3 = Task.Run(async() =>
{
Thread.CurrentThread.Name = "MNA3";
try
{
using (var cursor3 = sm1.GetAsyncCursor())
{
// Console.WriteLine("MNA2 started");
while (await cursor3.MoveNextAsync())
{
AsyncCursor.LogFinished();
if (cnt3 == 2)
{
cnt3++;
// Console.WriteLine("MNA2 moving");
}
if (cursor3.CurrentKey != cnt3)
{
ThrowHelper.ThrowInvalidOperationException("Wrong cursor enumeration");
}
cnt3++;
}
if (cnt3 != count)
{
ThrowHelper.ThrowInvalidOperationException($"3 Cannot move to count: c={cnt3}, count={count}");
}
if (AsyncCursor.SyncCount == 0)
{
Console.WriteLine("SyncCount == 0");
}
}
}
catch (Exception e)
{
Console.WriteLine("MNA3 ex: " + e);
}
});
var t4 = Task.Run(async() =>
{
Thread.CurrentThread.Name = "MNA4";
try
{
using (var cursor4 = sm1.GetAsyncCursor())
{
// Console.WriteLine("MNA2 started");
while (await cursor4.MoveNextAsync())
{
AsyncCursor.LogFinished();
if (cnt4 == 2)
{
cnt4++;
// Console.WriteLine("MNA2 moving");
}
if (cursor4.CurrentKey != cnt4)
{
ThrowHelper.ThrowInvalidOperationException("Wrong cursor enumeration");
}
cnt4++;
}
if (cnt4 != count)
{
ThrowHelper.ThrowInvalidOperationException($"4 Cannot move to count: c={cnt4}, count={count}");
}
if (AsyncCursor.SyncCount == 0)
{
Console.WriteLine("SyncCount == 0");
}
}
}
catch (Exception e)
{
Console.WriteLine("MNA3 ex: " + e);
}
});
var finished = false;
while (!finished)
{
finished = Task.WhenAll(addTask, t1, t2, t3, t4).Wait(2000);
//Console.WriteLine("cnt1: " + cnt1);
//Console.WriteLine("cnt2: " + cnt2);
}
Console.WriteLine($"{r}: Sync: {AsyncCursor.SyncCount}, Async: {AsyncCursor.AsyncCount}, Await: {AsyncCursor.AwaitCount}, Skipped: {AsyncCursor.SkippedCount}, Missed: {AsyncCursor.MissedCount}, Finished: {AsyncCursor.FinishedCount}");
AsyncCursor.ResetCounters();
}
}
Benchmark.Dump();
}
public void CouldMoveNextBench()
{
var count = 1000_000;
var rounds = 20;
var mult = 1_00;
var bcImm = CreateIntBaseContainer(count, count);
bcImm.Flags = new Flags((byte)Mutability.ReadOnly | (byte)KeySorting.Strong);
var bcMut = CreateIntBaseContainer(count, count);
bcMut.Flags = new Flags((byte)Mutability.Mutable | (byte)KeySorting.Strong);
BlockCursor <int, object, BaseContainer <int> >[] useMut = new BlockCursor <int, object, BaseContainer <int> > [count];
var rng = new Random(42);
var sm = new MutableSeries <int, int>();
var sl = new SortedList <int, int>(count);
for (int i = 0; i < count; i++)
{
var x = rng.NextDouble();
// if ((i & 1) == 0) // this is perfectly predicted on i7-8700, give ~300 MOPS
if (x > 0.50) // this is not predicted at all, performance drops to ~130MOPS, but if we always use the Sync implementation the perf is the same ~300 MOPS, always NoSync ~360 MOPS
{
useMut[i] = new BlockCursor <int, object, BaseContainer <int> >(bcMut);
}
else
{
useMut[i] = new BlockCursor <int, object, BaseContainer <int> >(bcImm);
}
sm.Add(i, i);
sl.Add(i, i);
}
var series = new Series <int, int>(Enumerable.Range(0, count).ToArray(),
Enumerable.Range(0, count).ToArray());
for (int r = 0; r < rounds; r++)
{
//MoveNextBenchBranch(useMut, count, mult);
//MoveNextBenchMut(bcMut, count, mult);
//MoveNextBenchImm(bcImm, count, mult);
//MoveNextBenchSL(sl, count, mult);
MoveNextBenchSM(sm, count, mult);
// MoveNextBenchSCM(scm, count, mult);
MoveNextBenchSeries(series, count, mult);
}
Benchmark.Dump();
bcImm.Dispose();
bcMut.Dispose();
}
internal DataBlockSource(MutableSeries <TKey, DataBlock> blockSeries)
{
_blockSeries = blockSeries;
}
public DataBlockSource()
{
_blockSeries = new MutableSeries <TKey, DataBlock>();
}
