private static void MoveNextBenchSCM(SortedChunkedMap <int, int> sm, int count, int mult) { // warm up for (int _ = 0; _ < 1; _++) { var cSM = sm.GetCursor(); for (int i = 0; i < count; i++) { cSM.MoveNext(); } } using (Benchmark.Run("SCM", count * mult)) { long sum = 1L; for (int _ = 0; _ < mult; _++) { var cSM = sm.GetCursor(); for (int i = 0; i < count; i++) { cSM.MoveNext(); sum = cSM.CurrentValue; } } Assert.IsTrue(sum != 0); } }
public void CouldMoveAsyncOnEmptySCM() { var sm = new SortedChunkedMap<DateTime, double>(); var c = sm.GetCursor(); var moveTask = c.MoveNext(CancellationToken.None); sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0); var result = moveTask.Result; Assert.IsTrue(result); }
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_StartEmpty() { var count = 1000000; var sw = new Stopwatch(); sw.Start(); var sm = new SortedChunkedMap <DateTime, double>(); sm.IsSynchronized = true; var addTask = Task.Run(async() => { await Task.Delay(50); for (int i = 0; i < count; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); //await Task.Delay(1); } }); double sum = 0.0; var sumTask = Task.Run(async() => { var c = sm.GetCursor(); while (c.MoveNext()) { sum += c.CurrentValue; } Assert.AreEqual(0, sum); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); //await Task.Delay(50); while (await c.MoveNext(CancellationToken.None)) { sum += c.CurrentValue; //Console.WriteLine("Current key: {0}", c.CurrentKey); if (c.CurrentKey == stop) { break; } } }); sumTask.Wait(); addTask.Wait(); sw.Stop(); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } Assert.AreEqual(expectedSum, sum); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); }
public void CouldMoveAtOnEmpty() { var scm = new SortedChunkedMap <int, int>(); var c = scm.GetCursor(); Assert.IsFalse(c.MoveAt(1, Lookup.GT)); Assert.IsFalse(c.MoveAt(1, Lookup.GE)); Assert.IsFalse(c.MoveAt(1, Lookup.EQ)); Assert.IsFalse(c.MoveAt(1, Lookup.LE)); Assert.IsFalse(c.MoveAt(1, Lookup.LT)); }
public void CouldMoveAsyncOnEmptySCM() { var sm = new SortedChunkedMap <DateTime, double>(); var c = sm.GetCursor(); var moveTask = c.MoveNext(CancellationToken.None); sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0); var result = moveTask.Result; Assert.IsTrue(result); }
public void CouldMoveAtGE() { var scm = new SortedChunkedMap<int, int>(50); for (int i = 0; i < 100; i++) { scm[i] = i; } var cursor = scm.GetCursor(); cursor.MoveAt(-100, Lookup.GE); Assert.AreEqual(0, cursor.CurrentKey); Assert.AreEqual(0, cursor.CurrentValue); var shouldBeFalse = cursor.MoveAt(-100, Lookup.LE); Assert.IsFalse(shouldBeFalse); }
public void MoveNextAsyncBenchmark() { // this benchmark shows that simple async enumeration gives 13+ mops, // this means than we should use parallel enumeration on joins. // the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p)))) // but they are optimized for single thread: when movenext is called on the outer cursor, // the whole chain enumerates synchronously. // During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible // but not too big, while on real-time stream there is no alternative at all. // Join algos should be paralell and task-based by default var count = 100000; var sw = new Stopwatch(); var sm = new SortedChunkedMap <DateTime, double>(); //sm.IsSynchronized = true; for (int i = 0; i < count; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); } sw.Start(); double sum = 0.0; var c = sm.GetCursor(); //c.MoveNext(); while (c.CurrentValue < count - 1.0) { Task.Run(async() => await c.MoveNext(CancellationToken.None)).Wait(); sum += c.CurrentValue; } sw.Stop(); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } Assert.AreEqual(expectedSum, sum); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); }
public void CouldMoveAtGE() { var scm = new SortedChunkedMap <int, int>(50); for (int i = 0; i < 100; i++) { scm[i] = i; } var cursor = scm.GetCursor(); cursor.MoveAt(-100, Lookup.GE); Assert.AreEqual(0, cursor.CurrentKey); Assert.AreEqual(0, cursor.CurrentValue); var shouldBeFalse = cursor.MoveAt(-100, Lookup.LE); Assert.IsFalse(shouldBeFalse); }
public void SmaDeviationTest() { var count = 8193; var data = new SortedChunkedMap <int, double>(); for (int i = 0; i < count; i++) { data.Add(i, i); } var dc = data.GetCursor(); dc.MoveFirst(); var dc2 = dc.Clone(); Assert.IsFalse(dc.MovePrevious()); Assert.IsFalse(dc2.MovePrevious()); //Assert.AreEqual(8192, dc.CurrentKey); var sma = data.SMA(2, true); var sma2 = data.Window(2, 1, true).Map(w => w.Values.Average()); var ii = 0; foreach (var kvp in sma2) { //Assert.AreEqual(kvp.Value, ii); ii++; } Assert.AreEqual(count, ii); //var smaSm = sma.ToSortedMap(); //Assert.AreEqual(count, smaSm.Count()); //var deviation = (data/sma - 1); //var deviationSm = deviation; //var smaDirection = deviation.Map(Math.Sign); //Assert.AreEqual(count, smaDirection.Count()); //Assert.AreEqual(count, deviation.Count()); }
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor() { var count = 1000; //00000; var sw = new Stopwatch(); //var mre = new ManualResetEventSlim(true); sw.Start(); //var sm = new SortedMap<DateTime, double>(); var sm = new SortedChunkedMap<DateTime, double>(); //sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0); sm.IsSynchronized = true; for (int i = 0; i < 5; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); } var addTask = Task.Run(async () => { await Task.Delay(50); //try { for (int i = 5; i < count; i++) { //mre.Wait(); sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); //await Task.Delay(1); NOP(500); } //} catch(Exception e) //{ // Console.WriteLine(e.Message); //} }); double sum = 0.0; int cnt = 0; var sumTask = Task.Run(async () => { var c = sm.GetCursor(); while (c.MoveNext()) { if ((int)c.CurrentValue != cnt) { Console.WriteLine("Wrong sequence"); } else { //Console.WriteLine("Sync move"); } sum += c.CurrentValue; cnt++; } //Assert.AreEqual(10, sum); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(10); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum += c.CurrentValue; if (c.CurrentKey == stop) { break; } if ((int) c.CurrentValue != cnt) { //Console.WriteLine("Wrong sequence"); Assert.Fail("Wrong sequence"); } else { //Console.WriteLine("Async move"); } cnt++; //mre.Set(); } Console.WriteLine("Finished 1"); }); double sum2 = 0.0; var sumTask2 = Task.Run(async () => { var c = sm.GetCursor(); while (c.MoveNext()) { sum2 += c.CurrentValue; } Assert.AreEqual(10, sum2); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(50); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum2 += c.CurrentValue; if (c.CurrentKey == stop) { break; } //mre.Set(); } Console.WriteLine("Finished 2"); }); double sum3 = 0.0; var sumTask3 = Task.Run(async () => { var c = sm.GetCursor(); while (c.MoveNext()) { sum3 += c.CurrentValue; } Assert.AreEqual(10, sum3); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(100); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum3 += c.CurrentValue; if (c.CurrentKey == stop) { break; } //mre.Set(); } Console.WriteLine("Finished 3"); }); sumTask.Wait(); sumTask2.Wait(); sumTask3.Wait(); addTask.Wait(); sw.Stop(); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } if(expectedSum != sum) Console.WriteLine("Sum 1 is wrong"); if (expectedSum != sum2) Console.WriteLine("Sum 2 is wrong"); if (expectedSum != sum3) Console.WriteLine("Sum 3 is wrong"); Assert.AreEqual(expectedSum, sum, "Sum 1"); Assert.AreEqual(expectedSum, sum2, "Sum 2"); Assert.AreEqual(expectedSum, sum3, "Sum 3"); }
public void MoveNextAsyncBenchmark() { // this benchmark shows that simple async enumeration gives 13+ mops, // this means than we should use parallel enumeration on joins. // the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p)))) // but they are optimized for single thread: when movenext is called on the outer cursor, // the whole chain enumerates synchronously. // During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible // but not too big, while on real-time stream there is no alternative at all. // Join algos should be paralell and task-based by default var count = 100000; var sw = new Stopwatch(); var sm = new SortedChunkedMap<DateTime, double>(); //sm.IsSynchronized = true; for (int i = 0; i < count; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); } sw.Start(); double sum = 0.0; var c = sm.GetCursor(); //c.MoveNext(); while (c.CurrentValue < count - 1.0) { Task.Run(async () => await c.MoveNext(CancellationToken.None)).Wait(); sum += c.CurrentValue; } sw.Stop(); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } Assert.AreEqual(expectedSum, sum); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); }
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_StartEmpty() { var count = 1000000; var sw = new Stopwatch(); sw.Start(); var sm = new SortedChunkedMap<DateTime, double>(); sm.IsSynchronized = true; var addTask = Task.Run(async () => { await Task.Delay(50); for (int i = 0; i < count; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); //await Task.Delay(1); } }); double sum = 0.0; var sumTask = Task.Run(async () => { var c = sm.GetCursor(); while (c.MoveNext()) { sum += c.CurrentValue; } Assert.AreEqual(0, sum); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); //await Task.Delay(50); while (await c.MoveNext(CancellationToken.None)) { sum += c.CurrentValue; //Console.WriteLine("Current key: {0}", c.CurrentKey); if (c.CurrentKey == stop) { break; } } }); sumTask.Wait(); addTask.Wait(); sw.Stop(); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } Assert.AreEqual(expectedSum, sum); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); }
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor() { var count = 1000; //00000; var sw = new Stopwatch(); //var mre = new ManualResetEventSlim(true); sw.Start(); //var sm = new SortedMap<DateTime, double>(); var sm = new SortedChunkedMap <DateTime, double>(); //sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0); sm.IsSynchronized = true; for (int i = 0; i < 5; i++) { sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); } var addTask = Task.Run(async() => { await Task.Delay(50); //try { for (int i = 5; i < count; i++) { //mre.Wait(); sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i); //await Task.Delay(1); NOP(500); } //} catch(Exception e) //{ // Console.WriteLine(e.Message); //} }); double sum = 0.0; int cnt = 0; var sumTask = Task.Run(async() => { var c = sm.GetCursor(); while (c.MoveNext()) { if ((int)c.CurrentValue != cnt) { Console.WriteLine("Wrong sequence"); } else { //Console.WriteLine("Sync move"); } sum += c.CurrentValue; cnt++; } //Assert.AreEqual(10, sum); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(10); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum += c.CurrentValue; if (c.CurrentKey == stop) { break; } if ((int)c.CurrentValue != cnt) { //Console.WriteLine("Wrong sequence"); Assert.Fail("Wrong sequence"); } else { //Console.WriteLine("Async move"); } cnt++; //mre.Set(); } Console.WriteLine("Finished 1"); }); double sum2 = 0.0; var sumTask2 = Task.Run(async() => { var c = sm.GetCursor(); while (c.MoveNext()) { sum2 += c.CurrentValue; } Assert.AreEqual(10, sum2); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(50); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum2 += c.CurrentValue; if (c.CurrentKey == stop) { break; } //mre.Set(); } Console.WriteLine("Finished 2"); }); double sum3 = 0.0; var sumTask3 = Task.Run(async() => { var c = sm.GetCursor(); while (c.MoveNext()) { sum3 += c.CurrentValue; } Assert.AreEqual(10, sum3); var stop = DateTime.UtcNow.Date.AddSeconds(count - 1); await Task.Delay(100); while (await c.MoveNext(CancellationToken.None)) { //mre.Reset(); sum3 += c.CurrentValue; if (c.CurrentKey == stop) { break; } //mre.Set(); } Console.WriteLine("Finished 3"); }); sumTask.Wait(); sumTask2.Wait(); sumTask3.Wait(); addTask.Wait(); sw.Stop(); Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50); Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)); double expectedSum = 0.0; for (int i = 0; i < count; i++) { expectedSum += i; } if (expectedSum != sum) { Console.WriteLine("Sum 1 is wrong"); } if (expectedSum != sum2) { Console.WriteLine("Sum 2 is wrong"); } if (expectedSum != sum3) { Console.WriteLine("Sum 3 is wrong"); } Assert.AreEqual(expectedSum, sum, "Sum 1"); Assert.AreEqual(expectedSum, sum2, "Sum 2"); Assert.AreEqual(expectedSum, sum3, "Sum 3"); }