private static void MoveNextBenchSCM(SortedChunkedMap <int, int> sm, int count, int mult)
{
// warm up
for (int _ = 0; _ < 1; _++)
{
var cSM = sm.GetCursor();
for (int i = 0; i < count; i++)
{
cSM.MoveNext();
}
}
using (Benchmark.Run("SCM", count * mult))
{
long sum = 1L;
for (int _ = 0; _ < mult; _++)
{
var cSM = sm.GetCursor();
for (int i = 0; i < count; i++)
{
cSM.MoveNext();
sum = cSM.CurrentValue;
}
}
Assert.IsTrue(sum != 0);
}
}
public void CouldMoveAsyncOnEmptySCM() {
var sm = new SortedChunkedMap<DateTime, double>();
var c = sm.GetCursor();
var moveTask = c.MoveNext(CancellationToken.None);
sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0);
var result = moveTask.Result;
Assert.IsTrue(result);
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_StartEmpty()
{
var count = 1000000;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap <DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async() => {
await Task.Delay(50);
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
double sum = 0.0;
var sumTask = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum += c.CurrentValue;
}
Assert.AreEqual(0, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
//await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None))
{
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop)
{
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldMoveAtOnEmpty()
{
var scm = new SortedChunkedMap <int, int>();
var c = scm.GetCursor();
Assert.IsFalse(c.MoveAt(1, Lookup.GT));
Assert.IsFalse(c.MoveAt(1, Lookup.GE));
Assert.IsFalse(c.MoveAt(1, Lookup.EQ));
Assert.IsFalse(c.MoveAt(1, Lookup.LE));
Assert.IsFalse(c.MoveAt(1, Lookup.LT));
}
public void CouldMoveAsyncOnEmptySCM()
{
var sm = new SortedChunkedMap <DateTime, double>();
var c = sm.GetCursor();
var moveTask = c.MoveNext(CancellationToken.None);
sm.Add(DateTime.UtcNow.Date.AddSeconds(0), 0);
var result = moveTask.Result;
Assert.IsTrue(result);
}
public void CouldMoveAtGE() {
var scm = new SortedChunkedMap<int, int>(50);
for (int i = 0; i < 100; i++) {
scm[i] = i;
}
var cursor = scm.GetCursor();
cursor.MoveAt(-100, Lookup.GE);
Assert.AreEqual(0, cursor.CurrentKey);
Assert.AreEqual(0, cursor.CurrentValue);
var shouldBeFalse = cursor.MoveAt(-100, Lookup.LE);
Assert.IsFalse(shouldBeFalse);
}
public void MoveNextAsyncBenchmark()
{
// this benchmark shows that simple async enumeration gives 13+ mops,
// this means than we should use parallel enumeration on joins.
// the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p))))
// but they are optimized for single thread: when movenext is called on the outer cursor,
// the whole chain enumerates synchronously.
// During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible
// but not too big, while on real-time stream there is no alternative at all.
// Join algos should be paralell and task-based by default
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap <DateTime, double>();
//sm.IsSynchronized = true;
for (int i = 0; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sw.Start();
double sum = 0.0;
var c = sm.GetCursor();
//c.MoveNext();
while (c.CurrentValue < count - 1.0)
{
Task.Run(async() => await c.MoveNext(CancellationToken.None)).Wait();
sum += c.CurrentValue;
}
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldMoveAtGE()
{
var scm = new SortedChunkedMap <int, int>(50);
for (int i = 0; i < 100; i++)
{
scm[i] = i;
}
var cursor = scm.GetCursor();
cursor.MoveAt(-100, Lookup.GE);
Assert.AreEqual(0, cursor.CurrentKey);
Assert.AreEqual(0, cursor.CurrentValue);
var shouldBeFalse = cursor.MoveAt(-100, Lookup.LE);
Assert.IsFalse(shouldBeFalse);
}
public void SmaDeviationTest()
{
var count = 8193;
var data = new SortedChunkedMap <int, double>();
for (int i = 0; i < count; i++)
{
data.Add(i, i);
}
var dc = data.GetCursor();
dc.MoveFirst();
var dc2 = dc.Clone();
Assert.IsFalse(dc.MovePrevious());
Assert.IsFalse(dc2.MovePrevious());
//Assert.AreEqual(8192, dc.CurrentKey);
var sma = data.SMA(2, true);
var sma2 = data.Window(2, 1, true).Map(w => w.Values.Average());
var ii = 0;
foreach (var kvp in sma2)
{
//Assert.AreEqual(kvp.Value, ii);
ii++;
}
Assert.AreEqual(count, ii);
//var smaSm = sma.ToSortedMap();
//Assert.AreEqual(count, smaSm.Count());
//var deviation = (data/sma - 1);
//var deviationSm = deviation;
//var smaDirection = deviation.Map(Math.Sign);
//Assert.AreEqual(count, smaDirection.Count());
//Assert.AreEqual(count, deviation.Count());
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor()
{
var count = 1000; //00000;
var sw = new Stopwatch();
//var mre = new ManualResetEventSlim(true);
sw.Start();
//var sm = new SortedMap<DateTime, double>();
var sm = new SortedChunkedMap<DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async () => {
await Task.Delay(50);
//try {
for (int i = 5; i < count; i++) {
//mre.Wait();
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
NOP(500);
}
//} catch(Exception e)
//{
// Console.WriteLine(e.Message);
//}
});
double sum = 0.0;
int cnt = 0;
var sumTask = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
if ((int)c.CurrentValue != cnt) {
Console.WriteLine("Wrong sequence");
} else {
//Console.WriteLine("Sync move");
}
sum += c.CurrentValue;
cnt++;
}
//Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(10);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
if ((int) c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
Assert.Fail("Wrong sequence");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
//mre.Set();
}
Console.WriteLine("Finished 1");
});
double sum2 = 0.0;
var sumTask2 = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum2 += c.CurrentValue;
}
Assert.AreEqual(10, sum2);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum2 += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
//mre.Set();
}
Console.WriteLine("Finished 2");
});
double sum3 = 0.0;
var sumTask3 = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum3 += c.CurrentValue;
}
Assert.AreEqual(10, sum3);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(100);
while (await c.MoveNext(CancellationToken.None)) {
//mre.Reset();
sum3 += c.CurrentValue;
if (c.CurrentKey == stop) {
break;
}
//mre.Set();
}
Console.WriteLine("Finished 3");
});
sumTask.Wait();
sumTask2.Wait();
sumTask3.Wait();
addTask.Wait();
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
if(expectedSum != sum) Console.WriteLine("Sum 1 is wrong");
if (expectedSum != sum2) Console.WriteLine("Sum 2 is wrong");
if (expectedSum != sum3) Console.WriteLine("Sum 3 is wrong");
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
}
public void MoveNextAsyncBenchmark() {
// this benchmark shows that simple async enumeration gives 13+ mops,
// this means than we should use parallel enumeration on joins.
// the idea is that a chain of calculations could be somewhat heavy, e.g. rp(ma(log(zipLag(c p -> c/p))))
// but they are optimized for single thread: when movenext is called on the outer cursor,
// the whole chain enumerates synchronously.
// During joins, we must make these evaluations parallel. The net overhead of tasks over existing data is visible
// but not too big, while on real-time stream there is no alternative at all.
// Join algos should be paralell and task-based by default
var count = 100000;
var sw = new Stopwatch();
var sm = new SortedChunkedMap<DateTime, double>();
//sm.IsSynchronized = true;
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sw.Start();
double sum = 0.0;
var c = sm.GetCursor();
//c.MoveNext();
while (c.CurrentValue < count - 1.0) {
Task.Run(async () => await c.MoveNext(CancellationToken.None)).Wait();
sum += c.CurrentValue;
}
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_StartEmpty() {
var count = 1000000;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedChunkedMap<DateTime, double>();
sm.IsSynchronized = true;
var addTask = Task.Run(async () => {
await Task.Delay(50);
for (int i = 0; i < count; i++) {
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
}
});
double sum = 0.0;
var sumTask = Task.Run(async () => {
var c = sm.GetCursor();
while (c.MoveNext()) {
sum += c.CurrentValue;
}
Assert.AreEqual(0, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
//await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None)) {
sum += c.CurrentValue;
//Console.WriteLine("Current key: {0}", c.CurrentKey);
if (c.CurrentKey == stop) {
break;
}
}
});
sumTask.Wait();
addTask.Wait();
sw.Stop();
double expectedSum = 0.0;
for (int i = 0; i < count; i++) {
expectedSum += i;
}
Assert.AreEqual(expectedSum, sum);
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor()
{
var count = 1000; //00000;
var sw = new Stopwatch();
//var mre = new ManualResetEventSlim(true);
sw.Start();
//var sm = new SortedMap<DateTime, double>();
var sm = new SortedChunkedMap <DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
sm.IsSynchronized = true;
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var addTask = Task.Run(async() => {
await Task.Delay(50);
//try {
for (int i = 5; i < count; i++)
{
//mre.Wait();
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
//await Task.Delay(1);
NOP(500);
}
//} catch(Exception e)
//{
// Console.WriteLine(e.Message);
//}
});
double sum = 0.0;
int cnt = 0;
var sumTask = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
if ((int)c.CurrentValue != cnt)
{
Console.WriteLine("Wrong sequence");
}
else
{
//Console.WriteLine("Sync move");
}
sum += c.CurrentValue;
cnt++;
}
//Assert.AreEqual(10, sum);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(10);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
if ((int)c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
Assert.Fail("Wrong sequence");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
//mre.Set();
}
Console.WriteLine("Finished 1");
});
double sum2 = 0.0;
var sumTask2 = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum2 += c.CurrentValue;
}
Assert.AreEqual(10, sum2);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(50);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum2 += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
//mre.Set();
}
Console.WriteLine("Finished 2");
});
double sum3 = 0.0;
var sumTask3 = Task.Run(async() => {
var c = sm.GetCursor();
while (c.MoveNext())
{
sum3 += c.CurrentValue;
}
Assert.AreEqual(10, sum3);
var stop = DateTime.UtcNow.Date.AddSeconds(count - 1);
await Task.Delay(100);
while (await c.MoveNext(CancellationToken.None))
{
//mre.Reset();
sum3 += c.CurrentValue;
if (c.CurrentKey == stop)
{
break;
}
//mre.Set();
}
Console.WriteLine("Finished 3");
});
sumTask.Wait();
sumTask2.Wait();
sumTask3.Wait();
addTask.Wait();
sw.Stop();
Console.WriteLine("Elapsed msec: {0}", sw.ElapsedMilliseconds - 50);
Console.WriteLine("Ops: {0}", Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2));
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
if (expectedSum != sum)
{
Console.WriteLine("Sum 1 is wrong");
}
if (expectedSum != sum2)
{
Console.WriteLine("Sum 2 is wrong");
}
if (expectedSum != sum3)
{
Console.WriteLine("Sum 3 is wrong");
}
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
}
