public Dictionary <string, IReadOnlyOrderedMap <DateTime, double> > GetImplementation()
{
var implemetations = new Dictionary <string, IReadOnlyOrderedMap <DateTime, double> >();
var scm_irregular_small = new SortedChunkedMap <DateTime, double>();
var scm_irregular_big = new SortedChunkedMap <DateTime, double>();
var scm_regular_small = new SortedChunkedMap <DateTime, double>();
var scm_regular_big = new SortedChunkedMap <DateTime, double>();
scm_irregular_small.Add(DateTime.Today.AddDays(-2), -2.0);
scm_irregular_big.Add(DateTime.Today.AddDays(-2), -2.0);
for (int i = 0; i < _big; i++)
{
if (i < _small)
{
scm_irregular_small.Add(DateTime.Today.AddDays(i), i);
scm_regular_small.Add(DateTime.Today.AddDays(i), i);
}
scm_irregular_big.Add(DateTime.Today.AddDays(i), i);
scm_regular_big.Add(DateTime.Today.AddDays(i), i);
}
implemetations.Add("scm_irregular_small", scm_irregular_small);
implemetations.Add("scm_regular_small", scm_regular_small);
implemetations.Add("scm_irregular_big", scm_irregular_big);
implemetations.Add("scm_regular_big", scm_regular_big);
return(implemetations);
}
public void CouldUseBaseSeriesAddOperator()
{
var count = 1000000;
var sm1 = new SortedChunkedMap <int, double>();
var sm2 = new SortedChunkedMap <int, double>();
var result = new SortedMap <int, double>();
double expected = 0;
sm1.Add(0, 0);
sm2.Add(0, 0);
for (int i = 2; i < count; i++)
{
sm1.Add(i, i);
if (i % 1 == 0)
{
sm2.Add(i, 2 * i);
expected += i + 2 * i;
result.Add(i, i + 2 * i);
}
}
for (int round = 0; round < 20; round++)
{
double actual = 0;
using (Benchmark.Run("ZipDiscrete", (int)sm1.Count + (int)sm2.Count))
{
var calculated = sm1 + sm2;
using (var c = calculated.GetEnumerator())
{
while (c.MoveNext())
{
actual += c.CurrentValue;
}
}
//foreach (var cursorSeries in calculated)
//{
// actual += cursorSeries.Value;
//}
}
Assert.AreEqual(expected, actual);
//double actual1 = 0;
//using (Benchmark.Run("LINQ", (int)sm1.Count + (int)sm2.Count))
//{
// var linq = sm1.Zip(sm2, (l, r) => l.Value + r.Value);
// foreach (var l in linq)
// {
// actual1 += l;
// }
//}
//Assert.AreEqual(expected, actual);
}
Benchmark.Dump();
}
public void OuterMapCachesInnerMaps(SortedChunkedMap <int, int> scm)
{
scm.RemoveMany(int.MinValue, Lookup.GE);
for (int i = 0; i < 10000; i++)
{
scm.Add(i, i);
}
var outer = scm.outerMap;
var oc = outer.GetCursor();
var first = outer.First;
scm.Add(10001, 10001);
Assert.True(oc.MoveFirst());
Assert.ReferenceEquals(first.Value, oc.CurrentValue);
var last = outer.Last;
Assert.True(oc.MoveLast());
Assert.ReferenceEquals(last.Value, oc.CurrentValue);
// TODO other cases
scm.RemoveMany(int.MinValue, Lookup.GE);
}
public void CouldAddMoreThanChunkSize()
{
var scm = new SortedChunkedMap <int, int>(1024);
var cnt = 0;
for (int i = 0; i < 10000; i = i + 2)
{
scm.Add(i, i);
cnt++;
}
for (int i = 1; i < 10000; i = i + 2)
{
scm.Add(i, i);
cnt++;
}
Assert.AreEqual(cnt, scm.Count);
Assert.IsTrue(scm.outerMap.First.Value.Count > 1024);
for (int i = 0; i < 10000; i++)
{
Assert.AreEqual(i, scm[i]);
}
}
public void AddExistingThrowsAndKeepsVersion()
{
var sm = new SortedChunkedMap <long, long>();
sm.Add(1, 1);
Assert.AreEqual(1, sm.Version);
Assert.Throws <ArgumentException>(() => sm.Add(1, 1));
Assert.AreEqual(1, sm.Version);
}
public void CouldCreateSCM()
{
var scm_irregular_small = new SortedChunkedMap <DateTime, double>();
scm_irregular_small.Add(DateTime.Today.AddDays(-2), -2.0);
for (int i = 0; i < _small; i++)
{
scm_irregular_small.Add(DateTime.Today.AddDays(i), i);
}
}
public void CouldRemoeAllFromBothEndsOnEmpty()
{
var map = new SortedChunkedMap <long, long>();
map.Add(1, 1);
Assert.True(map.RemoveMany(map.First.Key, Lookup.GE));
Assert.AreEqual(0, map.Count);
Assert.True(map.IsEmpty);
map.Add(1, 1);
Assert.True(map.RemoveMany(map.Last.Key, Lookup.LE));
Assert.AreEqual(0, map.Count);
Assert.True(map.IsEmpty);
}
public void RemoveFirstLastIncrementsVersion()
{
var map = new SortedChunkedMap <long, long>();
map.Add(1, 1);
Assert.AreEqual(1, map.Version);
map.Add(2, 2);
Assert.AreEqual(2, map.Version);
KeyValuePair <long, long> tmp;
map.RemoveFirst(out tmp);
Assert.AreEqual(3, map.Version);
map.RemoveLast(out tmp);
Assert.AreEqual(4, map.Version);
}
public void CouldRandomlyAddRemoveLast()
{
var map = new SortedChunkedMap <int, int>(50);
var list = new List <int>();
var count = 10000;
for (int i = 0; i < count; i++)
{
list.Add(i);
}
Shuffle(list);
for (int i = 0; i < count; i++)
{
map.Add(list[i], list[i]);
Assert.AreEqual(i + 1, map.Version);
}
for (int i = 0; i < count; i++)
{
KeyValuePair <int, int> tmp;
Assert.True(map.RemoveLast(out tmp));
Assert.AreEqual(i + count + 1, map.Version);
}
Assert.True(map.IsEmpty);
}
public async void CouldReadReadOnlyChildWhileAddingToParent() {
// TODO if we change the first element to -1 and add from 0, some weird assertion related to regular keys fails
var total = 1000;
var scm = new SortedChunkedMap<int, int>(50);
scm.IsSynchronized = true;
scm.AddLast(1, 1);
var addTask = Task.Run(() => {
for (int i = 2; i < total + 2; i++) {
scm.Add(i, i);
//scm[i] = i;
Thread.Sleep(5);
}
scm.IsMutable = false; // this will trigger a false return of MoveNextAsync()
});
var reader = scm.ReadOnly();
Console.WriteLine("Writer IsMutable: {0}", scm.IsMutable);
Console.WriteLine("Reader IsMutable: {0}", reader.IsMutable);
var cnt = 0;
using (var c = reader.GetCursor()) {
var couldMove = await c.MoveNext(CancellationToken.None);
while (couldMove) {
if (cnt % 100 == 0) Console.WriteLine("{0} - {1}", c.CurrentKey, c.CurrentValue);
cnt++;
couldMove = await c.MoveNext(CancellationToken.None);
}
}
addTask.Wait();
Assert.AreEqual(cnt, total + 1);
Console.WriteLine("Writer IsMutable: {0}", scm.IsMutable);
Console.WriteLine("Reader IsMutable: {0}", reader.IsMutable);
//(scm as IPersistentOrderedMap<int, int>).Dispose();
}
public void DemoAndStratIssue()
{
var series = new SortedChunkedMap <DateTime, double> [2];
var scm1 = new SortedChunkedMap <DateTime, double>();
var scm2 = new SortedChunkedMap <DateTime, double>();
var today = DateTime.UtcNow.Date;
for (int i = 0; i < 10000; i = i + 2)
{
scm1.Add(today.AddMilliseconds(i), i);
}
for (int i = 1; i < 10000; i = i + 2)
{
scm2.Add(today.AddMilliseconds(i), i);
}
series[0] = scm1;
series[1] = scm2;
// Zip on repeated used to throw
var sm = series.Select(x => x.Repeat()).ToArray().Zip((k, vArr) => {
if (Math.Abs(vArr.Sum(x => Math.Sign(x))) == vArr.Length)
{
return(vArr.Average());
}
else
{
return(0.0);
}
}).ToSortedMap();
Console.WriteLine(sm.Count);
}
public void CouldMoveAsyncOnEmptySCM() {
var sm = new SortedChunkedMap<DateTime, double>();
var c = sm.GetCursor();
var moveTask = c.MoveNext(CancellationToken.None);
sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0);
var result = moveTask.Result;
Assert.IsTrue(result);
}
public void CouldUseCacheExtensionMethod()
{
var count = 100;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap <DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async() => {
await Task.Delay(50);
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
var cached = sm.Cache();
double sum = 0.0;
var sumTask = Task.Run(async() => {
var c = cached.GetCursor();
while (c.MoveNext())
{
sum += c.CurrentValue;
}
Assert.AreEqual(0, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
//await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None))
{
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop)
{
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldCalculateSMAInRealTime()
{
var totalCount = 10000;
var sm = new SortedChunkedMap <int, double>();
var tcs = new TaskCompletionSource <bool>();
Task.Run(async() => {
for (int i = 0; i < 20; i++)
{
sm.Add(i, i);
if (i == 0)
{
tcs.SetResult(true);
}
}
await Task.Delay(100);
for (int i = 20; i < totalCount; i++)
{
//await Task.Delay(1); // 15 msec
sm.Add(i, i);
}
sm.Complete();
});
var sma = sm.SMA(10, true);
var c = sma.GetCursor();
//Thread.Sleep(200);
Assert.IsTrue(tcs.Task.Result);
Assert.IsTrue(c.MoveNext(CancellationToken.None).Result);
Assert.IsTrue(c.MoveNext());
var ii = 2;
while (c.MoveNext(CancellationToken.None).Result) //
//Console.WriteLine("Key: {0}, value: {1}", c.CurrentKey, c.CurrentValue);
{
ii++;
}
Assert.AreEqual(totalCount, ii);
Console.WriteLine("finished");
}
public void CouldMoveAsyncOnEmptySCM()
{
var sm = new SortedChunkedMap <DateTime, double>();
var c = sm.GetCursor();
var moveTask = c.MoveNext(CancellationToken.None);
sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0);
var result = moveTask.Result;
Assert.IsTrue(result);
}
public void CouldRemoveMany()
{
var scm = new SortedChunkedMap <int, int>();
scm.Add(1, 1);
var removed = scm.RemoveMany(0, Lookup.GE);
Assert.IsTrue(removed);
Assert.IsTrue(scm.IsEmpty);
Assert.IsTrue(scm.Count == 0);
}
public async void CouldReadReadOnlyChildWhileAddingToParent()
{
// TODO if we change the first element to -1 and add from 0, some weird assertion related to regular keys fails
var total = 150;
var scm = new SortedChunkedMap <int, int>(50);
scm.IsSynchronized = true;
scm.AddLast(1, 1);
var cc = 1;
var addTask = Task.Run(() => {
for (int i = 2; i < total + 2; i++)
{
cc++;
if (cc == 50)
{
Console.WriteLine("Next bucket");
}
scm.Add(i, i);
//scm[i] = i;
Thread.Sleep(5);
}
scm.Complete(); // this will trigger a false return of MoveNextAsync()
});
//Thread.Sleep(5000000);
var reader = scm.ReadOnly();
Console.WriteLine("Writer IsReadOnly: {0}", scm.IsReadOnly);
Console.WriteLine("Reader IsReadOnly: {0}", reader.IsReadOnly);
var cnt = 0;
using (var c = reader.GetCursor()) {
var couldMove = await c.MoveNext(CancellationToken.None);
while (couldMove)
{
if (cnt % 100 == 0)
{
Console.WriteLine("{0} - {1}", c.CurrentKey, c.CurrentValue);
}
cnt++;
couldMove = await c.MoveNext(CancellationToken.None);
}
}
addTask.Wait();
Thread.Sleep(200);
Assert.AreEqual(cnt, total + 1);
Console.WriteLine("Writer IsReadOnly: {0}", scm.IsReadOnly);
Console.WriteLine("Reader IsReadOnly: {0}", reader.IsReadOnly);
//(scm as IPersistentSeries<int, int>).Dispose();
}
public void CouldMoveAtOnNonEmpty()
{
var scm = new SortedChunkedMap <int, int>();
scm.Add(1, 1);
var c = scm.GetCursor();
Assert.IsFalse(c.MoveAt(1, Lookup.GT));
Assert.IsTrue(c.MoveAt(1, Lookup.GE));
Assert.IsTrue(c.MoveAt(1, Lookup.EQ));
Assert.IsTrue(c.MoveAt(1, Lookup.LE));
Assert.IsFalse(c.MoveAt(1, Lookup.LT));
}
public void CouldUseDoExtensionMethod()
{
var count = 100;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap <DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async() =>
{
await Task.Delay(50);
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
// signal data completion
sm.Complete();
});
var cached = sm; //.Cache();
double sum = 0.0;
var doTask = cached.Do((k, v) =>
{
Console.WriteLine($"{k} - {v}");
sum += v;
});
addTask.Wait();
doTask.Wait();
//Thread.Sleep(100);
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldRemoveFirst() {
var scm = new SortedChunkedMap<int, int>(50);
for (int i = 0; i < 100; i++) {
scm.Add(i, i);
}
scm.Remove(50);
Assert.AreEqual(50, scm.outerMap.Last.Key);
KeyValuePair<int, int> kvp;
scm.RemoveFirst(out kvp);
Assert.AreEqual(0, kvp.Value);
Assert.AreEqual(1, scm.First.Value);
Assert.AreEqual(0, scm.outerMap.First.Key);
}
public void CouldRemoveFirst()
{
var scm = new SortedChunkedMap <int, int>();
for (int i = 0; i < 10000; i++)
{
scm.Add(i, i);
}
KeyValuePair <int, int> result;
var removed = scm.RemoveFirst(out result);
Assert.IsTrue(removed);
Assert.IsTrue(!scm.IsEmpty);
Assert.IsTrue(scm.Count == 9999);
Assert.IsTrue(scm.First.Key == 1);
}
public void CouldAddWitDefaultMathProviderViaOperator()
{
var sm = new SortedChunkedMap <DateTime, double>(4000);
var count = 10000;
OptimizationSettings.AlwaysBatch = true;
for (int i = 0; i < count; i++)
{
if (i % 99 != 0)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
}
var sw = new Stopwatch();
sw.Start();
var sum = 0.0;
for (int rounds = 0; rounds < 1000; rounds++)
{
var sm2 = sm + 3.1415926;
var bmvc = sm2.GetCursor();
while (bmvc.MoveNext())
{
sum += bmvc.CurrentValue;
}
//foreach (var kvp in sm2)
//{
// sum += kvp.Value;
//}
}
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000 * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
Console.WriteLine(sum);
var c = 0;
//foreach (var kvp in sm2) {
// Assert.AreEqual(c + 1, kvp.Value);
// c++;
//}
OptimizationSettings.AlwaysBatch = false;
}
public void MoveNextAsyncBenchmark()
{
// this benchmark shows that simple async enumeration gives 13+ mops,
// this means than we should use parallel enumeration on joins.
// the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p))))
// but they are optimized for single thread: when movenext is called on the outer cursor,
// the whole chain enumerates synchronously.
// During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible
// but not too big, while on real-time stream there is no alternative at all.
// Join algos should be paralell and task-based by default
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap <DateTime, double>();
//sm.IsSynchronized = true;
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sw.Start();
double sum = 0.0;
var c = sm.GetCursor();
//c.MoveNext();
while (c.CurrentValue < count - 1.0)
{
Task.Run(async() => await c.MoveNext(CancellationToken.None)).Wait();
sum += c.CurrentValue;
}
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldRemoveFirst()
{
var scm = new SortedChunkedMap <int, int>(50);
for (int i = 0; i < 100; i++)
{
scm.Add(i, i);
}
scm.Remove(50);
Assert.AreEqual(50, scm.outerMap.Last.Key);
KeyValuePair <int, int> kvp;
scm.RemoveFirst(out kvp);
Assert.AreEqual(0, kvp.Value);
Assert.AreEqual(1, scm.First.Value);
Assert.AreEqual(0, scm.outerMap.First.Key);
}
public void SmaDeviationTest()
{
var count = 8193;
var data = new SortedChunkedMap <int, double>();
for (int i = 0; i < count; i++)
{
data.Add(i, i);
}
var dc = data.GetCursor();
dc.MoveFirst();
var dc2 = dc.Clone();
Assert.IsFalse(dc.MovePrevious());
Assert.IsFalse(dc2.MovePrevious());
//Assert.AreEqual(8192, dc.CurrentKey);
var sma = data.SMA(2, true);
var sma2 = data.Window(2, 1, true).Map(w => w.Values.Average());
var ii = 0;
foreach (var kvp in sma2)
{
//Assert.AreEqual(kvp.Value, ii);
ii++;
}
Assert.AreEqual(count, ii);
//var smaSm = sma.ToSortedMap();
//Assert.AreEqual(count, smaSm.Count());
//var deviation = (data/sma - 1);
//var deviationSm = deviation;
//var smaDirection = deviation.Map(Math.Sign);
//Assert.AreEqual(count, smaDirection.Count());
//Assert.AreEqual(count, deviation.Count());
}
public void AsyncCoursorOnEmptyMapWaitsForValues()
{
var scm = new SortedChunkedMap <int, double>();
var scmSma = scm.SMA(20, true);
var c = scmSma.GetCursor();
var task = c.MoveNext(CancellationToken.None);
var moved = task.Wait(100);
// timeout
Assert.IsFalse(moved);
scm.Add(1, 1.0);
Assert.IsTrue(task.Result);
Console.WriteLine("moved " + task.Result);
task = c.MoveNext(CancellationToken.None);
moved = task.Wait(100);
// timeout
Assert.IsFalse(moved);
scm.Complete();
Assert.IsFalse(task.Result);
Console.WriteLine("moved " + task.Result);
}
public void CouldAddWithYeppMathProvider()
{
var sm = new SortedChunkedMap <DateTime, double>();
var count = 1000;
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var sw = new Stopwatch();
sw.Start();
var sum = 0.0;
for (int rounds = 0; rounds < 10000; rounds++)
{
var bmvc = new BatchMapValuesCursor <DateTime, double, double>(sm.GetCursor,
(v) => v + 3.1415926, YeppMathProviderSample); //
while (bmvc.MoveNext())
{
sum += bmvc.CurrentValue;
}
}
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 10000 * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
Console.WriteLine(sum);
var c = 0;
//foreach (var kvp in sm2)
//{
// Assert.AreEqual(c + 1, kvp.Value);
// c++;
//}
}
public void CouldCreateSCM() {
var scm_irregular_small = new SortedChunkedMap<DateTime, double>();
scm_irregular_small.Add(DateTime.Today.AddDays(-2), -2.0);
for (int i = 0; i < _small; i++) {
scm_irregular_small.Add(DateTime.Today.AddDays(i), i);
}
}
public void CouldMoveNextBench()
{
var count = 1000_000;
var rounds = 20;
var mult = 1_00;
var bcImm = CreateIntBaseContainer(count, count);
bcImm._flags = new Flags((byte)Mutability.ReadOnly | (byte)KeySorting.Strong);
var bcMut = CreateIntBaseContainer(count, count);
bcMut._flags = new Flags((byte)Mutability.Mutable | (byte)KeySorting.Strong);
BlockCursor <int, object, BaseContainer <int> >[] useMut = new BlockCursor <int, object, BaseContainer <int> > [count];
var rng = new Random(42);
var sm = new SortedMap <int, int>(count);
var scm = new SortedChunkedMap <int, int>();
var sl = new SortedList <int, int>(count);
for (int i = 0; i < count; i++)
{
var x = rng.NextDouble();
// if ((i & 1) == 0) // this is perfectly predicted on i7-8700, give ~300 MOPS
if (x > 0.50) // this is not predicted at all, performance drops to ~130MOPS, but if we always use the Sync implementation the perf is the same ~300 MOPS, always NoSync ~360 MOPS
{
useMut[i] = new BlockCursor <int, object, BaseContainer <int> >(bcMut);
}
else
{
useMut[i] = new BlockCursor <int, object, BaseContainer <int> >(bcImm);
}
sm.Add(i, i);
scm.Add(i, i);
sl.Add(i, i);
}
var series = new Spreads.Collections.Experimental.Series <int, int>(Enumerable.Range(0, count).ToArray(),
Enumerable.Range(0, count).ToArray());
for (int r = 0; r < rounds; r++)
{
//MoveNextBenchBranch(useMut, count, mult);
//MoveNextBenchMut(bcMut, count, mult);
//MoveNextBenchImm(bcImm, count, mult);
//MoveNextBenchSL(sl, count, mult);
MoveNextBenchSM(sm, count, mult);
// MoveNextBenchSCM(scm, count, mult);
MoveNextBenchSeries(series, count, mult);
}
Benchmark.Dump();
bcImm.Dispose();
bcMut.Dispose();
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_spinwait()
{
OptimizationSettings.ArrayPool = new DoubleArrayPool();
for (int r = 0; r < 10; r++)
{
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap <DateTime, double>();
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async() => {
await Task.Delay(50);
for (int i = 5; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
double sum = 0.0;
var sumTask = Task.Run(() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum += c.CurrentValue;
}
Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
sw.Start();
while (true)
{
// spinwait
while (!c.MoveNext())
{
}
;
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop)
{
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
}
public void TGVSpeed()
{
for (int size = 0; size < 3; size++)
{
var count = (int)(1024 * Math.Pow(2, size));
const int mult = 1000;
var sl = new SortedList <DateTime, int>();
var sm = new SortedMap <DateTime, int>(count);
var scm = new SortedChunkedMap <DateTime, int>();
var start = DateTime.Today.ToUniversalTime();
for (int i = 0; i < count; i++)
{
if (i != 2) // make irregular
{
sl.Add(start.AddTicks(i), i);
sm.Add(start.AddTicks(i), i);
scm.Add(start.AddTicks(i), i);
}
}
Assert.IsFalse(sm.IsCompleted);
Assert.IsFalse(sm.IsCompleted);
Assert.IsTrue(sm.IsSynchronized);
sm.Complete();
Assert.IsTrue(sm.IsCompleted);
Assert.IsTrue(sm.IsCompleted);
Assert.IsFalse(sm.IsSynchronized);
scm.Complete();
for (int r = 0; r < 20; r++)
{
//var sum1 = 0L;
//using (Benchmark.Run("SL", count * mult, true))
//{
// for (int j = 0; j < mult; j++)
// {
// for (int i = 0; i < count; i++)
// {
// if (sl.TryGetValue(start.AddTicks(i), out var v))
// {
// sum1 += v;
// }
// }
// }
//}
//Assert.True(sum1 > 0);
var sum2 = 0L;
using (Benchmark.Run("SM", count * mult, true))
{
for (int j = 0; j < mult; j++)
{
for (int i = 0; i < count; i++)
{
if (sm.TryGetValueUnchecked(start.AddTicks(i), out var v))
{
sum2 += v;
}
}
}
}
//Assert.True(sum2 > 0);
//Assert.AreEqual(sum1, sum2);
//var sum3 = 0L;
//using (Benchmark.Run("SCM", count * mult, true))
//{
// for (int j = 0; j < mult; j++)
// {
// for (int i = 0; i < count; i++)
// {
// if (scm.TryGetValue(start.AddTicks(i), out var v))
// {
// sum3 += v;
// }
// }
// }
//}
//Assert.True(sum3 > 0);
//Assert.AreEqual(sum2, sum3);
}
Benchmark.Dump($"Size = {Math.Pow(2, size)}k elements");
}
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor()
{
var count = 1000; //00000;
var sw = new Stopwatch();
//var mre = new ManualResetEventSlim(true);
sw.Start();
//var sm = new SortedMap<DateTime, double>();
var sm = new SortedChunkedMap<DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async () => {
await Task.Delay(50);
//try {
for (int i = 5; i < count; i++) {
//mre.Wait();
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
NOP(500);
}
//} catch(Exception e)
//{
// Console.WriteLine(e.Message);
//}
});
double sum = 0.0;
int cnt = 0;
var sumTask = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
if ((int)c.CurrentValue != cnt) {
Console.WriteLine("Wrong sequence");
} else {
//Console.WriteLine("Sync move");
}
sum += c.CurrentValue;
cnt++;
}
//Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(10);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
if ((int) c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
Assert.Fail("Wrong sequence");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
//mre.Set();
}
Console.WriteLine("Finished 1");
});
double sum2 = 0.0;
var sumTask2 = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum2 += c.CurrentValue;
}
Assert.AreEqual(10, sum2);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum2 += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
//mre.Set();
}
Console.WriteLine("Finished 2");
});
double sum3 = 0.0;
var sumTask3 = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum3 += c.CurrentValue;
}
Assert.AreEqual(10, sum3);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(100);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum3 += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
//mre.Set();
}
Console.WriteLine("Finished 3");
});
sumTask.Wait();
sumTask2.Wait();
sumTask3.Wait();
addTask.Wait();
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
if(expectedSum != sum) Console.WriteLine("Sum 1 is wrong");
if (expectedSum != sum2) Console.WriteLine("Sum 2 is wrong");
if (expectedSum != sum3) Console.WriteLine("Sum 3 is wrong");
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
}
public void CouldAddWithYeppMathProvider() {
var sm = new SortedChunkedMap<DateTime, double>();
var count = 1000;
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var sw = new Stopwatch();
sw.Start();
var sum = 0.0;
for (int rounds = 0; rounds < 10000; rounds++) {
var bmvc = new BatchMapValuesCursor<DateTime, double, double>(sm.GetCursor,
(v) => v + 3.1415926, YeppMathProviderSample); //
while (bmvc.MoveNext()) {
sum += bmvc.CurrentValue;
}
}
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 10000 * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
Console.WriteLine(sum);
var c = 0;
//foreach (var kvp in sm2)
//{
// Assert.AreEqual(c + 1, kvp.Value);
// c++;
//}
}
public void MoveNextAsyncBenchmark() {
// this benchmark shows that simple async enumeration gives 13+ mops,
// this means than we should use parallel enumeration on joins.
// the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p))))
// but they are optimized for single thread: when movenext is called on the outer cursor,
// the whole chain enumerates synchronously.
// During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible
// but not too big, while on real-time stream there is no alternative at all.
// Join algos should be paralell and task-based by default
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap<DateTime, double>();
//sm.IsSynchronized = true;
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sw.Start();
double sum = 0.0;
var c = sm.GetCursor();
//c.MoveNext();
while (c.CurrentValue < count - 1.0) {
Task.Run(async () => await c.MoveNext(CancellationToken.None)).Wait();
sum += c.CurrentValue;
}
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void BugFromStrategies2() {
var sw = new Stopwatch();
var sm1 = new SortedChunkedMap<int, int>();
var sm2 = new SortedChunkedMap<int, int>();
sm1.Add(0, 0);
sm2.Add(-100500, 0);
for (int i = 2; i < 100; i++) {
sm1.Add(i, i);
if (i % 10 == 0) {
sm2.Add(i, i);
}
}
// assertion failure
var repeated = sm2.Map(x => x + 0).Repeat(); //.ToSortedMap();
var cursor1 = repeated.GetCursor();
var result = repeated.Zip(sm1, (k, p, d) => p); //.Lag(1u); // .Fill(0)
var cursor = result.GetCursor();
//Assert.IsTrue(cursor.MoveNext());
var clone = cursor.Clone();
//Assert.IsTrue(clone.MoveNext());
//Assert.IsTrue(clone.MoveNext());
var sm = result.ToSortedMap();
//Console.WriteLine(result.Count());
}
public void CouldAddWitDefaultMathProviderViaOperator() {
var sm = new SortedChunkedMap<DateTime, double>(4000);
var count = 10000;
OptimizationSettings.AlwaysBatch = true;
for (int i = 0; i < count; i++) {
if(i % 99 != 0)
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var sw = new Stopwatch();
sw.Start();
var sum = 0.0;
for (int rounds = 0; rounds < 1000; rounds++)
{
var sm2 = sm + 3.1415926;
var bmvc = sm2.GetCursor();
while (bmvc.MoveNext()) {
sum += bmvc.CurrentValue;
}
//foreach (var kvp in sm2)
//{
// sum += kvp.Value;
//}
}
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000 * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
Console.WriteLine(sum);
var c = 0;
//foreach (var kvp in sm2) {
// Assert.AreEqual(c + 1, kvp.Value);
// c++;
//}
}
public void CouldRemoveFirst() {
var scm = new SortedChunkedMap<int, int>();
for (int i = 0; i < 10000; i++) {
scm.Add(i, i);
}
KeyValuePair<int, int> result;
var removed = scm.RemoveFirst(out result);
Assert.IsTrue(removed);
Assert.IsTrue(!scm.IsEmpty);
Assert.IsTrue(scm.Count == 9999);
Assert.IsTrue(scm.First.Key == 1);
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor()
{
var count = 1000; //00000;
var sw = new Stopwatch();
//var mre = new ManualResetEventSlim(true);
sw.Start();
//var sm = new SortedMap<DateTime, double>();
var sm = new SortedChunkedMap <DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async() => {
await Task.Delay(50);
//try {
for (int i = 5; i < count; i++)
{
//mre.Wait();
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
NOP(500);
}
//} catch(Exception e)
//{
// Console.WriteLine(e.Message);
//}
});
double sum = 0.0;
int cnt = 0;
var sumTask = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
if ((int)c.CurrentValue != cnt)
{
Console.WriteLine("Wrong sequence");
}
else
{
//Console.WriteLine("Sync move");
}
sum += c.CurrentValue;
cnt++;
}
//Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(10);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
if ((int)c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
Assert.Fail("Wrong sequence");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
//mre.Set();
}
Console.WriteLine("Finished 1");
});
double sum2 = 0.0;
var sumTask2 = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum2 += c.CurrentValue;
}
Assert.AreEqual(10, sum2);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum2 += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
//mre.Set();
}
Console.WriteLine("Finished 2");
});
double sum3 = 0.0;
var sumTask3 = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum3 += c.CurrentValue;
}
Assert.AreEqual(10, sum3);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(100);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum3 += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
//mre.Set();
}
Console.WriteLine("Finished 3");
});
sumTask.Wait();
sumTask2.Wait();
sumTask3.Wait();
addTask.Wait();
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
if (expectedSum != sum)
{
Console.WriteLine("Sum 1 is wrong");
}
if (expectedSum != sum2)
{
Console.WriteLine("Sum 2 is wrong");
}
if (expectedSum != sum3)
{
Console.WriteLine("Sum 3 is wrong");
}
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
}
public void CouldRemoveMany() {
var scm = new SortedChunkedMap<int, int>();
scm.Add(1, 1);
var removed = scm.RemoveMany(0, Lookup.GE);
Assert.IsTrue(removed);
Assert.IsTrue(scm.IsEmpty);
Assert.IsTrue(scm.Count == 0);
}
public void CouldUseCacheExtensionMethod() {
var count = 100;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap<DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async () => {
await Task.Delay(50);
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
var cached = sm.Cache();
double sum = 0.0;
var sumTask = Task.Run(async () => {
var c = cached.GetCursor();
while (c.MoveNext()) {
sum += c.CurrentValue;
}
Assert.AreEqual(0, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
//await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None)) {
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop) {
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldUseDoExtensionMethod() {
var count = 100;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap<DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async () => {
await Task.Delay(50);
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
// signal data completion
sm.isMutable = false;
});
var cached = sm.Cache();
double sum = 0.0;
cached.Do((k, v) =>
{
sum += v;
});
addTask.Wait();
Thread.Sleep(100);
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void EnumerateScmSpeed()
{
const int count = 10_000_000;
var sl = new SortedList <int, int>();
var sm = new SortedMap <int, int>();
sm.IsSynchronized = false;
var scm = new SortedChunkedMap <int, int>();
scm.IsSynchronized = false;
for (int i = 0; i < count; i++)
{
if (i % 1000 != 0)
{
sl.Add(i, i);
sm.Add(i, i);
scm.Add(i, i);
}
}
//var ism = new ImmutableSortedMap<int, int>(sm);
long sum;
for (int r = 0; r < 20; r++)
{
sum = 0L;
using (Benchmark.Run("SL", count))
{
using (var c = sl.GetEnumerator())
{
while (c.MoveNext())
{
sum += c.Current.Value;
}
}
}
Assert.True(sum > 0);
sum = 0L;
using (Benchmark.Run("SM", count))
{
using (var c = sm.GetEnumerator())
{
while (c.MoveNext())
{
sum += c.Current.Value;
}
}
}
Assert.True(sum > 0);
//sum = 0L;
//using (Benchmark.Run("ISM", count))
//{
// foreach (var item in ism)
// {
// sum += item.Value;
// }
//}
//Assert.True(sum > 0);
sum = 0L;
using (Benchmark.Run("SCM", count))
{
using (var c = scm.GetEnumerator())
{
while (c.MoveNext())
{
sum += c.Current.Value;
}
}
}
Assert.True(sum > 0);
}
Benchmark.Dump();
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_spinwait()
{
OptimizationSettings.ArrayPool = new DoubleArrayPool();
for (int r = 0; r < 10; r++) {
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap<DateTime, double>();
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async () => {
await Task.Delay(50);
for (int i = 5; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
double sum = 0.0;
var sumTask = Task.Run(() => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum += c.CurrentValue;
}
Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
sw.Start();
while (true) {
// spinwait
while (!c.MoveNext()) {
}
;
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop) {
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
}
public Dictionary<string, IReadOnlyOrderedMap<DateTime, double>> GetImplementation() {
var implemetations = new Dictionary<string, IReadOnlyOrderedMap<DateTime, double>>();
var sm_irregular_small = new SortedMap<DateTime, double>();
var sm_irregular_big = new SortedMap<DateTime, double>();
var sm_regular_small = new SortedMap<DateTime, double>();
var sm_regular_big = new SortedMap<DateTime, double>();
var scm_irregular_small = new SortedChunkedMap<DateTime, double>();
var scm_irregular_big = new SortedChunkedMap<DateTime, double>();
var scm_regular_small = new SortedChunkedMap<DateTime, double>();
var scm_regular_big = new SortedChunkedMap<DateTime, double>();
sm_irregular_small.Add(DateTime.Today.AddDays(-2), -2.0);
sm_irregular_big.Add(DateTime.Today.AddDays(-2), -2.0);
scm_irregular_small.Add(DateTime.Today.AddDays(-2), -2.0);
scm_irregular_big.Add(DateTime.Today.AddDays(-2), -2.0);
for (int i = 0; i < _big; i++)
{
if (i < _small)
{
sm_irregular_small.Add(DateTime.Today.AddDays(i), i);
sm_regular_small.Add(DateTime.Today.AddDays(i), i);
scm_irregular_small.Add(DateTime.Today.AddDays(i), i);
scm_regular_small.Add(DateTime.Today.AddDays(i), i);
}
sm_irregular_big.Add(DateTime.Today.AddDays(i), i);
sm_regular_big.Add(DateTime.Today.AddDays(i), i);
scm_irregular_big.Add(DateTime.Today.AddDays(i), i);
scm_regular_big.Add(DateTime.Today.AddDays(i), i);
}
// SM regular
implemetations.Add("sm_irregular_small", sm_irregular_small);
implemetations.Add("sm_regular_small", sm_regular_small);
implemetations.Add("scm_irregular_small", scm_irregular_small);
implemetations.Add("scm_regular_small", scm_regular_small);
implemetations.Add("sm_irregular_big", sm_irregular_big);
implemetations.Add("sm_regular_big", sm_regular_big);
implemetations.Add("scm_irregular_big", scm_irregular_big);
implemetations.Add("scm_regular_big", scm_regular_big);
return implemetations;
}