public void ParsingAndQueryTest_ReturnsAndQuery()
{
//Arrange
string query = "kiwi banana";
AndQuery expected = new AndQuery(
new List <IQueryComponent> {
new TermLiteral("kiwi"), new TermLiteral("banana")
}
);
//Act
IQueryComponent actual = parser.ParseQuery(query);
//Assert
actual.Should().BeOfType(typeof(AndQuery));
((AndQuery)actual).Components.Should().HaveSameCount(expected.Components);
}
/// <summary>
/// Parse a stack selector.
/// </summary>
public IQuery descendents_selector()
{
bool isRoot = tokenizer.Token == QueryTokens.Slash;
if (isRoot)
{
tokenizer.Advance();
}
var query = compound_selector();
if (isRoot)
{
var rootQuery = new RootQuery();
query = new AndQuery(query, rootQuery);
}
if (tokenizer.AtEnd)
{
// Exhausted tokens, return single filter.
return(query);
}
do
{
if (tokenizer.Token == QueryTokens.Slash)
{
query = new ParentQuery(query);
}
else if (tokenizer.Token == QueryTokens.GreaterThan)
{
query = new AncestorQuery(query);
}
else
{
throw new ApplicationException("Unexpected token: " + tokenizer.TokenString);
}
tokenizer.Advance(); // Chew up the separator.
var otherQuery = compound_selector();
query = new AndQuery(otherQuery, query);
} while (!tokenizer.AtEnd);
return(query);
}
public void RealLife()
{
var analyzer = new SimpleAnalyzer();
var store = new InMemoryStore();
var searchIndex = new SearchIndex(analyzer, store);
var doc1 = new Document();
doc1.AddField(new StringField("name", "name1", FieldFlags.Stored));
doc1.AddField(new StringField("content", "hello world", FieldFlags.Analyzed));
searchIndex.AddDocument(doc1);
var doc2 = new Document();
doc2.AddField(new StringField("name", "name2", FieldFlags.Stored));
doc2.AddField(new StringField("content", "hi pretty world", FieldFlags.Analyzed));
searchIndex.AddDocument(doc2);
var doc3 = new Document();
doc3.AddField(new StringField("name", "name3", FieldFlags.Stored));
doc3.AddField(new StringField("content", "hell worm", FieldFlags.Analyzed));
searchIndex.AddDocument(doc3);
var query = new AndQuery();
query.Subqueries.Add(new TermQuery(new Term("content", "hell*")));
query.Subqueries.Add(new TermQuery(new Term("content", "wor*")));
var docs = searchIndex.Search(query);
Assert.AreEqual(2, docs.Count);
var name1 = searchIndex.GetFieldValue(docs[0], "name");
var name2 = searchIndex.GetFieldValue(docs[1], "name");
CollectionAssert.AreEquivalent(new[] { "name1", "name3" }, new[] { name1, name2 });
searchIndex.RemoveDocument(new Term("name", "name1"));
docs = searchIndex.Search(query);
Assert.AreEqual(1, docs.Count);
Assert.AreEqual("name3", searchIndex.GetFieldValue(docs[0], "name"));
}
public void result_is_true_when_inputs_are_true()
{
var mockGameObject = new Mock <GameObject>();
var child1 = new Mock <IQuery>();
child1
.Setup(m => m.Match(mockGameObject.Object))
.Returns(true);
var child2 = new Mock <IQuery>();
child2
.Setup(m => m.Match(mockGameObject.Object))
.Returns(true);
var testObject = new AndQuery(child1.Object, child2.Object);
Assert.True(testObject.Match(mockGameObject.Object));
}
public void AndQueryRun()
{
var postings1 = new long[] { 1, 2, 3, 4 };
var postings2 = new long[] { 3, 4, 5, 6 };
var runner = new Mock <IQueryRunner>(MockBehavior.Strict);
var query1 = new Mock <IQuery>();
query1.Setup(x => x.Run(runner.Object)).Returns(postings1);
var query2 = new Mock <IQuery>();
query2.Setup(x => x.Run(runner.Object)).Returns(postings2);
var andQuery = new AndQuery();
andQuery.Subqueries.Add(query1.Object);
andQuery.Subqueries.Add(query2.Object);
CollectionAssert.AreEquivalent(new[] { 3, 4 }, andQuery.Run(runner.Object));
}
private void VisitAndExpression(BinaryExpression binaryExpression, AndQuery andExpression)
{
if (IsLeafExpression(binaryExpression.Left))
{
andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Left));
}
else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso)
{
VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression);
}
else if (binaryExpression.Left.NodeType == ExpressionType.Extension)
{
if (binaryExpression.Left is SubQueryExpression subQuery)
{
var leaf = new AtomicQuery();
andExpression.Elements.Add(leaf);
VisitContainsExpression(subQuery, leaf);
}
}
else
{
throw new NotSupportedException("Query too complex");
}
if (IsLeafExpression(binaryExpression.Right))
{
andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Right));
}
else if (binaryExpression.Right.NodeType == ExpressionType.Extension)
{
if (binaryExpression.Right is SubQueryExpression subQuery)
{
var leaf = new AtomicQuery();
andExpression.Elements.Add(leaf);
VisitContainsExpression(subQuery, leaf);
}
}
else
{
throw new NotSupportedException("Query too complex");
}
}
/// <summary>
/// Parse a compound selector.
/// </summary>
public IQuery compound_selector()
{
var nameSelectorAllowed = true;
var query = selector(ref nameSelectorAllowed);
if (tokenizer.AtEnd || IsDescendentsSeparator(tokenizer.Token))
{
// There is only a single selector, no need to create a compound selector.
return(query);
}
do
{
var otherQuery = selector(ref nameSelectorAllowed);
query = new AndQuery(otherQuery, query);
} while (!tokenizer.AtEnd &&
!IsDescendentsSeparator(tokenizer.Token));
return(query);
}
private void VisitBinaryExpression(BinaryExpression binaryExpression, OrQuery rootExpression)
{
// manage AND expressions
if (binaryExpression.NodeType == ExpressionType.AndAlso)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
VisitAndExpression(binaryExpression, andExpression);
}
// manage OR expressions
else if (binaryExpression.NodeType == ExpressionType.OrElse)
{
VisitOrExpression(binaryExpression, rootExpression);
}
// manage simple expressions like a > 10
else if (IsLeafExpression(binaryExpression))
{
AndQuery andExpression;
if (!rootExpression.MultipleWhereClauses)
{
andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
}
else // if multiple where clauses consider them as expressions linked by AND
{
andExpression = rootExpression.Elements[0];
}
andExpression.Elements.Add(VisitLeafExpression(binaryExpression));
}
else
{
throw new NotSupportedException("Query too complex");
}
}
//TODO add unit test for timestamp synchronization (coverage)
/// <summary>
/// Optimistic synchronization using a timestamp property
/// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp"
/// It also updates this property withe DateTime.Now
/// If you use this you should never modify the timestamp manually when updating the object
/// </summary>
/// <param name="newValue"></param>
public void UpdateWithTimestampSynchronization <T>(T newValue)
{
var prop = newValue.GetType().GetProperty("Timestamp");
if (prop == null)
{
throw new CacheException($"No Timestamp property found on type {typeof(T).Name}");
}
if (!prop.CanWrite)
{
throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable");
}
var oldTimestamp = prop.GetValue(newValue);
var kv = KeyInfo.ValueToKeyValue(oldTimestamp,
new KeyInfo(KeyDataType.IntKey, KeyType.ScalarIndex, "Timestamp"));
var q = new AtomicQuery(kv);
var andQuery = new AndQuery();
andQuery.Elements.Add(q);
var orq = new OrQuery(typeof(T));
orq.Elements.Add(andQuery);
var now = DateTime.Now;
var newTimestamp = now.AddTicks(1); // add one to be sure its different
prop.SetValue(newValue, newTimestamp);
var packed = Pack(newValue);
_itemsToPut.Add(packed);
_conditions.Add(orq);
}
public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index)
{
if (whereClause.Predicate is BinaryExpression expression)
{
VisitBinaryExpression(expression, RootExpression);
}
else
{
if (whereClause.Predicate is SubQueryExpression subQuery)
{
AndQuery andExpression;
if (!RootExpression.MultipleWhereClauses)
{
andExpression = new AndQuery();
RootExpression.Elements.Add(andExpression);
}
else // multiple where clauses are joined by AND
{
andExpression = RootExpression.Elements[0];
}
var leaf = new AtomicQuery();
andExpression.Elements.Add(leaf);
VisitContainsExpression(subQuery, leaf);
}
else
{
throw new NotSupportedException("Incorrect query");
}
}
RootExpression.MultipleWhereClauses = true;
base.VisitWhereClause(whereClause, queryModel, index);
}
/// <summary>
/// Rank the indexes that can be used to resolve the query. Lower rank means more discriminant index (smaller result)
/// </summary>
/// <param name="andQuery"></param>
/// <returns></returns>
IList <IndexRanking> GetIndexesForQuery(AndQuery andQuery)
{
var result = new List <IndexRanking>();
foreach (var atomicQuery in andQuery.Elements)
{
var name = atomicQuery.PropertyName;
var index = _dataStore.TryGetIndex(name);
if (atomicQuery.Operator == QueryOperator.Eq || atomicQuery.Operator == QueryOperator.In || atomicQuery.Operator == QueryOperator.Contains)
{
// for primary or unique key we do not need more than one index
if (index.IndexType == IndexType.Primary || index.IndexType == IndexType.Unique)
{
// no need to count for the primary index. Waste of time as it wil always be the only index used
return(new List <IndexRanking> {
new IndexRanking(index, atomicQuery, -1)
});
}
var indexResultCount = index.GetCount(atomicQuery.Values, atomicQuery.Operator);
result.Add(new IndexRanking(index, atomicQuery, indexResultCount));
}
else if (atomicQuery.IsComparison) // in this case we can only use ordered indexes
{
if (index != null && index.IndexType == IndexType.Ordered)
{
var indexResultCount = index.GetCount(atomicQuery.Values, atomicQuery.Operator);
result.Add(new IndexRanking(index, atomicQuery, indexResultCount));
}
}
}
return(result);
}
private void InternalVisitWhereClause(Expression whereClause, bool not = false)
{
if (whereClause is BinaryExpression expression)
{
VisitBinaryExpression(expression, RootExpression);
}
else if (whereClause is MemberExpression memberExpression)
{
if (memberExpression.Type == typeof(bool))
{
// Generalize for more complex expressions
VisitMemberExpression(memberExpression, RootExpression, not);
}
}
else if (whereClause is MethodCallExpression call)
{
VisitMethodCall(call, RootExpression);
}
else
{
if (whereClause is SubQueryExpression subQuery)
{
var atomicQuery = VisitContainsExpression(subQuery, not);
var andQuery = new AndQuery();
andQuery.Elements.Add(atomicQuery);
RootExpression.Elements.Add(andQuery);
}
else
{
throw new NotSupportedException("Incorrect query");
}
}
RootExpression.MultipleWhereClauses = true;
}
/// <summary>
/// Optimistic synchronization using a timestamp property
/// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp"
/// It also updates this property withe DateTime.Now
/// If you use this you should never modify the timestamp manually
/// </summary>
/// <param name="newValue"></param>
public void UpdateWithTimestampSynchronization(T newValue)
{
var prop = newValue.GetType().GetProperty("Timestamp");
if (prop == null)
{
throw new CacheException($"No Timestamp property found on type {typeof(T).Name}");
}
if (!prop.CanWrite)
{
throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable");
}
var oldTimestamp = prop.GetValue(newValue);
var kv = new KeyValue(oldTimestamp,
new KeyInfo("Timestamp", 0, IndexType.Dictionary));
var q = new AtomicQuery(_collectionSchema.KeyByName(kv.KeyName), kv);
var andQuery = new AndQuery();
andQuery.Elements.Add(q);
var orq = new OrQuery(_collectionName);
orq.Elements.Add(andQuery);
var now = DateTime.Now;
var newTimestamp = now.AddTicks(1); // add one to be sure its different
prop.SetValue(newValue, newTimestamp);
_client.UpdateIf(Pack(newValue), orq);
}
IList <PackedObject> ProcessAndQuery(AndQuery query, ExecutionPlan executionPlan)
{
if (query.Elements.Count == 1)
{
return(ProcessSimpleQuery(query.Elements[0], executionPlan));
}
var queryExecutionPlan = new QueryExecutionPlan(query.ToString());
// this method can be called in parallel. The only common data is the global execution plan
lock (executionPlan)
{
executionPlan.QueryPlans.Add(queryExecutionPlan);
}
queryExecutionPlan.StartPlanning();
var indexesToUse = GetIndexesForQuery(query);
queryExecutionPlan.EndPlanning();
// this will contain all queries that have can not be resolved by indexes and need to be checked manually
var restOfTheQuery = query.Clone();
ISet <PackedObject> result = null;
var finalResult = new List <PackedObject>();
if (indexesToUse.Count == 1) // only one index can be used so do not bother with extra logic
{
queryExecutionPlan.StartIndexUse();
var plan = indexesToUse[0];
queryExecutionPlan.Trace($"single index: {plan.ResolvedQuery.PropertyName}");
result = plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator);
// this query was resolved by an index so no need to check it manually
restOfTheQuery.Elements.Remove(plan.ResolvedQuery);
queryExecutionPlan.EndIndexUse();
}
else if (indexesToUse.Count > 1)
{
queryExecutionPlan.StartIndexUse();
foreach (var plan in indexesToUse.OrderBy(p => p.Ranking).Take(2)) // no more than two indexes
{
if (result == null)
{
result = plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator);
queryExecutionPlan.Trace($"first index: {plan.ResolvedQuery.PropertyName} = {plan.Ranking}");
}
else
{
result.IntersectWith(plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator));
queryExecutionPlan.Trace($"then index: {plan.ResolvedQuery.PropertyName} = {plan.Ranking} => {result.Count}");
}
// do not work too hard if indexes found nothing
if (result.Count == 0)
{
break;
}
// this query was resolved by an index so no need to check it manually
restOfTheQuery.Elements.Remove(plan.ResolvedQuery);
}
queryExecutionPlan.EndIndexUse();
}
else // no index can be used so proceed to full-scan
{
queryExecutionPlan.FullScan = true;
queryExecutionPlan.StartScan();
var res = _dataStore.PrimaryIndex.GetAll().Where(o => restOfTheQuery.Match(o)).ToList();
queryExecutionPlan.EndScan();
return(res);
}
if (result != null)
{
if (restOfTheQuery.Elements.Count == 0) // empty query left; fully resolved by indexes
{
return(result.ToList());
}
queryExecutionPlan.StartScan();
foreach (var item in result)
{
if (restOfTheQuery.Match(item))
{
finalResult.Add(item);
}
}
queryExecutionPlan.EndScan();
}
return(finalResult);
}
//TODO add unit test for OR expression with Contains
/// <summary>
/// OR expression can be present only at root level
/// </summary>
/// <param name="binaryExpression"></param>
/// <param name="rootExpression"></param>
private void VisitOrExpression(BinaryExpression binaryExpression, OrQuery rootExpression)
{
// visit left part
if (IsLeafExpression(binaryExpression.Left))
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Left));
}
else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression);
}
else if (binaryExpression.Left.NodeType == ExpressionType.Extension)
{
if (binaryExpression.Left is SubQueryExpression subQuery)
{
AndQuery andExpression;
if (!rootExpression.MultipleWhereClauses)
{
andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
}
else // multiple where clauses are joined by AND
{
andExpression = rootExpression.Elements[0];
}
var leaf = new AtomicQuery();
andExpression.Elements.Add(leaf);
VisitContainsExpression(subQuery, leaf);
}
}
else if (binaryExpression.Left.NodeType == ExpressionType.OrElse)
{
VisitOrExpression((BinaryExpression)binaryExpression.Left, rootExpression);
}
else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression);
}
else
{
throw new NotSupportedException("Query too complex");
}
// visit right part
if (IsLeafExpression(binaryExpression.Right))
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Right));
}
else if (binaryExpression.Right.NodeType == ExpressionType.Extension)
{
if (binaryExpression.Right is SubQueryExpression subQuery)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
if (rootExpression.MultipleWhereClauses)
{
throw new NotSupportedException(
"Multiple where clauses can be used only with simple expressions");
}
var leaf = new AtomicQuery();
andExpression.Elements.Add(leaf);
VisitContainsExpression(subQuery, leaf);
}
}
else if (binaryExpression.Right.NodeType == ExpressionType.OrElse)
{
VisitOrExpression((BinaryExpression)binaryExpression.Right, rootExpression);
}
else if (binaryExpression.Right.NodeType == ExpressionType.AndAlso)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
VisitAndExpression((BinaryExpression)binaryExpression.Right, andExpression);
}
else
{
throw new NotSupportedException("Query too complex");
}
}
private static void ParseWhere(Node @where, OrQuery query, CollectionSchema schema)
{
var orNodes = where.Children.Where(c => c.Token == "or").ToList();
if (orNodes.Count != 1)
{
throw new NotSupportedException("Query too complex");
}
var orNode = orNodes.Single();
var andNodes = orNode.Children.Where(c => c.Token == "and").ToList();
foreach (var andNode in andNodes)
{
var andQuery = new AndQuery();
foreach (var node in andNode.Children)
{
var operands = node.Children.Select(c => c.Token).ToList();
var op = ParseOperator(node.Token, operands.LastOrDefault());
if (op == QueryOperator.In || op == QueryOperator.NotIn)
{
var metadata = schema.KeyByName(operands[0]);
if (metadata != null)
{
List <KeyValue> values = new List <KeyValue>();
foreach (var val in operands.Skip(1))
{
object value = JExtensions.SmartParse(val);
values.Add(new KeyValue(value, metadata));
}
andQuery.Elements.Add(new AtomicQuery(metadata, values, op));
}
else
{
throw new NotSupportedException($"Can not parse query after IN operator. {operands[0]} is a server-side value");
}
}
else if (op == QueryOperator.Contains || op == QueryOperator.NotContains) // for contains operator the collection property should be at the left side
{
var metadata = schema.KeyByName(operands[0]);
object value = JExtensions.SmartParse(operands[1]);
andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op));
}
else if (op == QueryOperator.StrStartsWith || op == QueryOperator.StrEndsWith || op == QueryOperator.StrContains) // for string operators the property should be at the left side
{
var metadata = schema.KeyByName(operands[0]);
object value = operands[1].Trim('%');
andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op));
}
else if (operands.Count == 2)// binary operators
{
var metadata = schema.KeyByName(operands[0]);
// by default property name first
if (metadata != null)
{
object value = JExtensions.SmartParse(operands[1]);
andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op));
}
else // try value first
{
metadata = schema.KeyByName(operands[1]);
if (metadata == null)
{
throw new NotSupportedException($"Can not parse query. Neither {operands[0]} nor {operands[1]} is a server-side value");
}
object value = JExtensions.SmartParse(operands[0]);;
andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), Reverse(op)));
}
}
}
query.Elements.Add(andQuery);
}
}
