public void Range_queries()
{
// should be optimized as a between query (GeLe operator)
var query = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 10 && o.Quantity >= 2);
Assert.AreEqual(1, query.Elements.Count);
Assert.AreEqual(1, query.Elements[0].Elements.Count);
Assert.AreEqual(QueryOperator.GeLe, query.Elements[0].Elements[0].Operator);
Assert.IsTrue(query.IsValid);
Console.WriteLine(query.ToString());
// should be optimized ad GtLe
query = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 10 && o.Quantity > 2);
Assert.AreEqual(1, query.Elements.Count);
Assert.AreEqual(1, query.Elements[0].Elements.Count);
Assert.AreEqual(QueryOperator.GtLe, query.Elements[0].Elements[0].Operator);
Assert.IsTrue(query.IsValid);
Console.WriteLine(query.ToString());
// should be optimized ad GeLt
query = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity < 10 && o.Quantity >= 2);
Assert.AreEqual(1, query.Elements.Count);
Assert.AreEqual(1, query.Elements[0].Elements.Count);
Assert.AreEqual(QueryOperator.GeLt, query.Elements[0].Elements[0].Operator);
Assert.IsTrue(query.IsValid);
Console.WriteLine(query.ToString());
// should be optimized ad GtLt
query = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity < 10 && o.Quantity > 2);
Assert.AreEqual(1, query.Elements.Count);
Assert.AreEqual(1, query.Elements[0].Elements.Count);
Assert.AreEqual(QueryOperator.GtLt, query.Elements[0].Elements[0].Operator);
Assert.IsTrue(query.IsValid);
Console.WriteLine(query.ToString());
}
public void Speed_test_sql_vs_linq()
{
var schema = TypedSchemaFactory.FromType <Order>();
// warm-up
var categories = new string[] { "geek", "games" };
var query1 = ExpressionTreeHelper.PredicateToQuery <Order>(
o => o.IsDelivered && categories.Contains(o.Category) || o.Amount > 100 && o.Amount < 200,
schema.CollectionName);
var query2 = new Parser().ParseSql($"select * from {schema.CollectionName} where isdelivered = true and category in (geek, games) or amount > 100 and amount < 200").ToQuery(schema);
Assert.AreEqual(query1.ToString(), query2.ToString());
const int iterations = 1000;
{
var clock = new Stopwatch();
clock.Start();
for (int i = 0; i < iterations; i++)
{
query1 = ExpressionTreeHelper.PredicateToQuery <Order>(
o => o.IsDelivered && categories.Contains(o.Category) || o.Amount > 100 && o.Amount < 200,
schema.CollectionName);
}
clock.Stop();
Console.WriteLine($"{iterations} iterations with linq took {clock.ElapsedMilliseconds} ms");
}
{
var clock = new Stopwatch();
clock.Start();
for (int i = 0; i < iterations; i++)
{
query2 = new Parser().ParseSql($"select * from {schema.CollectionName} where isdelivered = true and category in (geek, games) or amount > 100 and amount < 200").ToQuery(schema);
}
clock.Stop();
Console.WriteLine($"{iterations} iterations with sql took {clock.ElapsedMilliseconds} ms");
}
}
public void Query_performance()
{
var schema = TypedSchemaFactory.FromType <Order>();
var queries = WhereClausesForOrders()
.Select(w => ExpressionTreeHelper.PredicateToQuery(w, schema.CollectionName)).ToList();
var count = queries.Count;
var objects = Order.GenerateTestData(100_000);
var packed = objects.Select(o => PackedObject.Pack(o, schema)).ToList();
var ds = new DataStore(schema, new NullEvictionPolicy(), new FullTextConfig());
ds.InternalPutMany(packed, true);
var watch = new Stopwatch();
for (var i = 0; i < count; i++)
{
var qm = new QueryManager(ds);
const int iterations = 100;
// warm up
var returned = qm.ProcessQuery(queries[i]).Count;
// run
watch.Restart();
for (var j = 0; j < iterations; j++)
{
returned = qm.ProcessQuery(queries[i]).Count;
}
watch.Stop();
Console.WriteLine($"{queries[i]} returned {returned} took {watch.ElapsedMilliseconds / iterations} ms");
Console.WriteLine("execution plan:");
Console.WriteLine(qm.ExecutionPlan);
Console.WriteLine();
}
}
public void Test_functions_vs_queries()
{
var schema = TypedSchemaFactory.FromType <Order>();
var queries = WhereClausesForOrders()
.Select(w => ExpressionTreeHelper.PredicateToQuery(w, schema.CollectionName)).ToList();
var predicates = WhereClausesForOrders().Select(w => w.Compile()).ToList();
var count = queries.Count;
Assert.AreEqual(count, predicates.Count);
var objects = Order.GenerateTestData(1000);
var packed = objects.Select(o => PackedObject.Pack(o, schema)).ToList();
var ds = new DataStore(schema, new NullEvictionPolicy(), new FullTextConfig());
ds.InternalPutMany(packed, true);
for (var i = 0; i < count; i++)
{
var fromObjects = objects.Where(predicates[i]).ToList();
var qm = new QueryManager(ds);
var fromDataSource = qm.ProcessQuery(queries[i]);
Console.WriteLine($"{queries[i]} returned {fromDataSource.Count}");
Console.WriteLine("execution plan:");
Console.WriteLine(qm.ExecutionPlan);
Console.WriteLine();
Assert.AreEqual(fromObjects.Count, fromDataSource.Count);
}
}
public void Atomic_query_subset()
{
var products1 = new [] { 101, 102, 103 };
var products2 = new [] { 101, 102 };
var q1 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered && products1.Contains(o.ProductId));
var q2 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered && products2.Contains(o.ProductId));
Assert.IsTrue(q1.IsSubsetOf(q1));
Assert.IsTrue(q2.IsSubsetOf(q1));
Assert.IsFalse(q1.IsSubsetOf(q2));
var q3 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 10);
var q4 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity < 10);
var q5 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 9);
Assert.IsTrue(q5.IsSubsetOf(q3));
Assert.IsTrue(q4.IsSubsetOf(q3));
Assert.IsFalse(q3.IsSubsetOf(q5));
// should be optimized as a between query (GeLe operator)
var q6 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 10 && o.Quantity >= 2);
Assert.AreEqual(1, q6.Elements.Count);
Assert.AreEqual(1, q6.Elements[0].Elements.Count);
Assert.AreEqual(QueryOperator.GeLe, q6.Elements[0].Elements[0].Operator);
var q7 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 4 && o.Quantity >= 2);
Assert.IsTrue(q7.IsSubsetOf(q6));
var q8 = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.Quantity <= 11 && o.Quantity >= 2);
Assert.IsFalse(q8.IsSubsetOf(q6));
}
public void Distinct_operator()
{
var objects = Order.GenerateTestData(100_000);
var schema = TypedSchemaFactory.FromType <Order>();
var packed = objects.Select(o => PackedObject.Pack(o, schema)).ToList();
var ds = new DataStore(schema, new NullEvictionPolicy(), new FullTextConfig());
ds.InternalPutMany(packed, true);
// empty query
{
// result from linq2object to be compared with query
var raw = objects.Select(o => new { o.Category, o.ClientId }).Distinct().ToList();
var q = new OrQuery(schema.CollectionName)
{
Distinct = true
};
q.SelectClause.Add(new SelectItem {
Name = "Category", Alias = "Category"
});
q.SelectClause.Add(new SelectItem {
Name = "ClientId", Alias = "ClientId"
});
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Assert.AreEqual(raw.Count, result.Count);
}
// atomic query
{
// result from linq2object to be compared with query
var raw = objects.Where(o => o.IsDelivered).Select(o => new { o.Category, o.ClientId }).Distinct().ToList();
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered);
q.Distinct = true;
q.SelectClause.Add(new SelectItem {
Name = "Category", Alias = "Category"
});
q.SelectClause.Add(new SelectItem {
Name = "ClientId", Alias = "ClientId"
});
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Assert.AreEqual(raw.Count, result.Count);
}
// simple and query
{
// result from linq2object to be compared with query
var raw = objects.Where(o => o.IsDelivered && o.Amount < 100).Select(o => new { o.Category, o.ClientId }).Distinct().ToList();
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered && o.Amount < 100);
q.Distinct = true;
q.SelectClause.Add(new SelectItem {
Name = "Category", Alias = "Category"
});
q.SelectClause.Add(new SelectItem {
Name = "ClientId", Alias = "ClientId"
});
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Assert.AreEqual(raw.Count, result.Count);
}
// complex or query
{
// result from linq2object to be compared with query
var raw = objects.Where(o => o.IsDelivered && o.Amount < 100 || o.Category == "sf" && o.Quantity > 1).Select(o => new { o.Category, o.ClientId }).Distinct().ToList();
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered && o.Amount < 100 || o.Category == "sf" && o.Quantity > 1);
q.Distinct = true;
q.SelectClause.Add(new SelectItem {
Name = "Category", Alias = "Category"
});
q.SelectClause.Add(new SelectItem {
Name = "ClientId", Alias = "ClientId"
});
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Assert.AreEqual(raw.Count, result.Count);
q.Take = 3;
result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Assert.AreEqual(3, result.Count);
}
}
public void Order_by()
{
var objects = Order.GenerateTestData(100_000);
var schema = TypedSchemaFactory.FromType <Order>();
var packed = objects.Select(o => PackedObject.Pack(o, schema)).ToList();
var ds = new DataStore(schema, new NullEvictionPolicy(), new FullTextConfig());
ds.InternalPutMany(packed, true);
// empty query
{
var q = new OrQuery(schema.CollectionName)
{
OrderByProperty = "Amount"
};
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(objects.Count, result.Count);
// check sorted ascending
for (int i = 0; i < result.Count - 1; i++)
{
Assert.LessOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
q.OrderByIsDescending = true;
result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(objects.Count, result.Count);
// check sorted descending
for (int i = 0; i < result.Count - 1; i++)
{
Assert.GreaterOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
}
// atomic query
{
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered == false);
// result from linq2object to be compared with ordered query
var raw = objects.Where(o => o.IsDelivered == false).ToList();
q.OrderByProperty = "Amount";
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
// check sorted ascending
for (int i = 0; i < result.Count - 1; i++)
{
Assert.LessOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
q.OrderByIsDescending = true;
result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
// check sorted descending
for (int i = 0; i < result.Count - 1; i++)
{
Assert.GreaterOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
}
// simple AND query
{
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered == false && o.Category == "geek");
var raw = objects.Where(o => o.IsDelivered == false && o.Category == "geek").ToList();
q.OrderByProperty = "Amount";
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
for (int i = 0; i < result.Count - 1; i++)
{
Assert.LessOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
q.OrderByIsDescending = true;
result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
for (int i = 0; i < result.Count - 1; i++)
{
Assert.GreaterOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
}
// complex OR query
{
var q = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered == false && o.Category == "geek" || o.Category == "sf");
var raw = objects.Where(o => o.IsDelivered == false && o.Category == "geek" || o.Category == "sf").ToList();
q.OrderByProperty = "Amount";
var qm = new QueryManager(ds);
var result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
for (int i = 0; i < result.Count - 1; i++)
{
Assert.LessOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
q.OrderByIsDescending = true;
result = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).ToList();
Console.WriteLine(qm.ExecutionPlan.ToString());
Assert.AreEqual(raw.Count, result.Count);
for (int i = 0; i < result.Count - 1; i++)
{
Assert.GreaterOrEqual((int)result[i].Amount * 10000, (int)result[i + 1].Amount * 10000);
}
// check that TAKE operator is applied after ORDER BY
q.Take = 1;
var max = qm.ProcessQuery(q).Select(PackedObject.Unpack <Order>).Single();
Assert.AreEqual(max.Amount, result[0].Amount);
}
}
public void Check_execution_plans()
{
var schema = TypedSchemaFactory.FromType <AllKindsOfProperties>();
var objects = GenerateAllKinds(100_000);
var packed = objects.Select(o => PackedObject.Pack(o, schema)).ToList();
var ds = new DataStore(schema, new NullEvictionPolicy(), new FullTextConfig());
ds.InternalPutMany(packed, true);
var qm = new QueryManager(ds);
// first time for warm-up
qm.ProcessQuery(ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Tags.Contains("food")));
qm.ProcessQuery(ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Tags.Contains("food")));
Console.WriteLine(qm.ExecutionPlan);
Assert.AreEqual(1, qm.ExecutionPlan.QueryPlans.Count);
Assert.IsTrue(qm.ExecutionPlan.QueryPlans[0].SimpleQueryStrategy);
Assert.IsFalse(qm.ExecutionPlan.QueryPlans[0].FullScan);
// first time for warm-up
qm.ProcessQuery(ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a =>
a.Tags.Contains("food") && a.Tags.Contains("space")));
var result =
qm.ProcessQuery(ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a =>
a.Tags.Contains("food") && a.Tags.Contains("space")));
Assert.AreEqual(0, result.Count);
Assert.AreEqual(1, qm.ExecutionPlan.QueryPlans.Count);
Assert.IsFalse(qm.ExecutionPlan.QueryPlans[0].SimpleQueryStrategy);
Assert.IsFalse(qm.ExecutionPlan.QueryPlans[0].FullScan,
"this query should be solved by an index not a full-scan");
Console.WriteLine(qm.ExecutionPlan);
// first time for warm-up
qm.ProcessQuery(
ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Quantity > 1 && a.Quantity < 2));
result = qm.ProcessQuery(
ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Quantity > 1 && a.Quantity < 2));
Console.WriteLine(qm.ExecutionPlan);
Assert.AreEqual(0, result.Count);
Assert.AreEqual(1, qm.ExecutionPlan.QueryPlans.Count);
Assert.IsTrue(qm.ExecutionPlan.QueryPlans[0].SimpleQueryStrategy,
"this query should have been optimized as a range query and executed as a simple query");
Assert.IsTrue(qm.ExecutionPlan.QueryPlans[0].FullScan,
"this query should be executed as full-scan as the index is not ordered");
// first time for warm-up
qm.ProcessQuery(
ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Quantity >= 1 || a.Quantity <= 2));
result = qm.ProcessQuery(
ExpressionTreeHelper.PredicateToQuery <AllKindsOfProperties>(a => a.Quantity >= 1 || a.Quantity <= 2));
Console.WriteLine(qm.ExecutionPlan);
Assert.AreEqual(100_000, result.Count);
Assert.AreEqual(2, qm.ExecutionPlan.QueryPlans.Count, "this query should have been decomposed in two queries");
Assert.IsTrue(qm.ExecutionPlan.QueryPlans[0].SimpleQueryStrategy);
// empty query. Should return everything
result = qm.ProcessQuery(OrQuery.Empty <AllKindsOfProperties>());
Assert.AreEqual(100_000, result.Count);
Console.WriteLine(qm.ExecutionPlan);
}
public void Split_transaction_request()
{
var schema = TypedSchemaFactory.FromType <Order>();
var requests = new List <DataRequest>();
// simple put requests
for (int i = 0; i < 100; i++)
{
var order1 = new Order {
Id = Guid.NewGuid(), Category = "geek", ProductId = 123, Quantity = 1
};
var putRequest = new PutRequest("orders");
var packed1 = PackedObject.Pack(order1, schema, "orders");
putRequest.Items.Add(packed1);
requests.Add(putRequest);
}
// conditional put requests
for (int i = 0; i < 100; i++)
{
var order1 = new Order {
Id = Guid.NewGuid(), Category = "geek", ProductId = 123, Quantity = 1
};
var putRequest = new PutRequest("orders");
var packed1 = PackedObject.Pack(order1, schema, "orders");
putRequest.Items.Add(packed1);
putRequest.Predicate = ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered, "orders");
requests.Add(putRequest);
}
// simple delete requests
for (int i = 0; i < 100; i++)
{
var order1 = new Order {
Id = Guid.NewGuid(), Category = "geek", ProductId = 123, Quantity = 1
};
var packed1 = PackedObject.Pack(order1, schema, "orders1");
var deleteRequest = new RemoveRequest("orders1", packed1.PrimaryKey);
requests.Add(deleteRequest);
}
// delete many request
var deleteMany = new RemoveManyRequest(ExpressionTreeHelper.PredicateToQuery <Order>(o => o.IsDelivered, "orders"));
requests.Add(deleteMany);
var transactionRequest = new TransactionRequest(requests)
{
TransactionId = Guid.NewGuid()
};
var split = transactionRequest.SplitByServer(k => k.GetHashCode() % 5, 5);
var total = split.Values.Sum(r => r.ChildRequests.Count);
// 300 uniformly distributed + 5 (delete many) cloned on each server
Assert.AreEqual(305, total);
Assert.IsTrue(split.Values.All(s => s.TransactionId == transactionRequest.TransactionId));
var tr0 = split[0];
Assert.IsTrue(tr0.ConditionalRequests.All(r => r.HasCondition));
Assert.IsTrue(tr0.ConditionalRequests.Any());
var deleteManyCount = tr0.ChildRequests.Count(r => r is RemoveManyRequest);
Assert.AreEqual(1, deleteManyCount);
Assert.AreEqual(2, tr0.AllCollections.Length);// orders and orders1
}