public void ParsingAndQueryTest_ReturnsAndQuery() { //Arrange string query = "kiwi banana"; AndQuery expected = new AndQuery( new List <IQueryComponent> { new TermLiteral("kiwi"), new TermLiteral("banana") } ); //Act IQueryComponent actual = parser.ParseQuery(query); //Assert actual.Should().BeOfType(typeof(AndQuery)); ((AndQuery)actual).Components.Should().HaveSameCount(expected.Components); }
/// <summary> /// Parse a stack selector. /// </summary> public IQuery descendents_selector() { bool isRoot = tokenizer.Token == QueryTokens.Slash; if (isRoot) { tokenizer.Advance(); } var query = compound_selector(); if (isRoot) { var rootQuery = new RootQuery(); query = new AndQuery(query, rootQuery); } if (tokenizer.AtEnd) { // Exhausted tokens, return single filter. return(query); } do { if (tokenizer.Token == QueryTokens.Slash) { query = new ParentQuery(query); } else if (tokenizer.Token == QueryTokens.GreaterThan) { query = new AncestorQuery(query); } else { throw new ApplicationException("Unexpected token: " + tokenizer.TokenString); } tokenizer.Advance(); // Chew up the separator. var otherQuery = compound_selector(); query = new AndQuery(otherQuery, query); } while (!tokenizer.AtEnd); return(query); }
public void RealLife() { var analyzer = new SimpleAnalyzer(); var store = new InMemoryStore(); var searchIndex = new SearchIndex(analyzer, store); var doc1 = new Document(); doc1.AddField(new StringField("name", "name1", FieldFlags.Stored)); doc1.AddField(new StringField("content", "hello world", FieldFlags.Analyzed)); searchIndex.AddDocument(doc1); var doc2 = new Document(); doc2.AddField(new StringField("name", "name2", FieldFlags.Stored)); doc2.AddField(new StringField("content", "hi pretty world", FieldFlags.Analyzed)); searchIndex.AddDocument(doc2); var doc3 = new Document(); doc3.AddField(new StringField("name", "name3", FieldFlags.Stored)); doc3.AddField(new StringField("content", "hell worm", FieldFlags.Analyzed)); searchIndex.AddDocument(doc3); var query = new AndQuery(); query.Subqueries.Add(new TermQuery(new Term("content", "hell*"))); query.Subqueries.Add(new TermQuery(new Term("content", "wor*"))); var docs = searchIndex.Search(query); Assert.AreEqual(2, docs.Count); var name1 = searchIndex.GetFieldValue(docs[0], "name"); var name2 = searchIndex.GetFieldValue(docs[1], "name"); CollectionAssert.AreEquivalent(new[] { "name1", "name3" }, new[] { name1, name2 }); searchIndex.RemoveDocument(new Term("name", "name1")); docs = searchIndex.Search(query); Assert.AreEqual(1, docs.Count); Assert.AreEqual("name3", searchIndex.GetFieldValue(docs[0], "name")); }
public void result_is_true_when_inputs_are_true() { var mockGameObject = new Mock <GameObject>(); var child1 = new Mock <IQuery>(); child1 .Setup(m => m.Match(mockGameObject.Object)) .Returns(true); var child2 = new Mock <IQuery>(); child2 .Setup(m => m.Match(mockGameObject.Object)) .Returns(true); var testObject = new AndQuery(child1.Object, child2.Object); Assert.True(testObject.Match(mockGameObject.Object)); }
public void AndQueryRun() { var postings1 = new long[] { 1, 2, 3, 4 }; var postings2 = new long[] { 3, 4, 5, 6 }; var runner = new Mock <IQueryRunner>(MockBehavior.Strict); var query1 = new Mock <IQuery>(); query1.Setup(x => x.Run(runner.Object)).Returns(postings1); var query2 = new Mock <IQuery>(); query2.Setup(x => x.Run(runner.Object)).Returns(postings2); var andQuery = new AndQuery(); andQuery.Subqueries.Add(query1.Object); andQuery.Subqueries.Add(query2.Object); CollectionAssert.AreEquivalent(new[] { 3, 4 }, andQuery.Run(runner.Object)); }
private void VisitAndExpression(BinaryExpression binaryExpression, AndQuery andExpression) { if (IsLeafExpression(binaryExpression.Left)) { andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Left)); } else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso) { VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression); } else if (binaryExpression.Left.NodeType == ExpressionType.Extension) { if (binaryExpression.Left is SubQueryExpression subQuery) { var leaf = new AtomicQuery(); andExpression.Elements.Add(leaf); VisitContainsExpression(subQuery, leaf); } } else { throw new NotSupportedException("Query too complex"); } if (IsLeafExpression(binaryExpression.Right)) { andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Right)); } else if (binaryExpression.Right.NodeType == ExpressionType.Extension) { if (binaryExpression.Right is SubQueryExpression subQuery) { var leaf = new AtomicQuery(); andExpression.Elements.Add(leaf); VisitContainsExpression(subQuery, leaf); } } else { throw new NotSupportedException("Query too complex"); } }
/// <summary> /// Parse a compound selector. /// </summary> public IQuery compound_selector() { var nameSelectorAllowed = true; var query = selector(ref nameSelectorAllowed); if (tokenizer.AtEnd || IsDescendentsSeparator(tokenizer.Token)) { // There is only a single selector, no need to create a compound selector. return(query); } do { var otherQuery = selector(ref nameSelectorAllowed); query = new AndQuery(otherQuery, query); } while (!tokenizer.AtEnd && !IsDescendentsSeparator(tokenizer.Token)); return(query); }
private void VisitBinaryExpression(BinaryExpression binaryExpression, OrQuery rootExpression) { // manage AND expressions if (binaryExpression.NodeType == ExpressionType.AndAlso) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); VisitAndExpression(binaryExpression, andExpression); } // manage OR expressions else if (binaryExpression.NodeType == ExpressionType.OrElse) { VisitOrExpression(binaryExpression, rootExpression); } // manage simple expressions like a > 10 else if (IsLeafExpression(binaryExpression)) { AndQuery andExpression; if (!rootExpression.MultipleWhereClauses) { andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); } else // if multiple where clauses consider them as expressions linked by AND { andExpression = rootExpression.Elements[0]; } andExpression.Elements.Add(VisitLeafExpression(binaryExpression)); } else { throw new NotSupportedException("Query too complex"); } }
//TODO add unit test for timestamp synchronization (coverage) /// <summary> /// Optimistic synchronization using a timestamp property /// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp" /// It also updates this property withe DateTime.Now /// If you use this you should never modify the timestamp manually when updating the object /// </summary> /// <param name="newValue"></param> public void UpdateWithTimestampSynchronization <T>(T newValue) { var prop = newValue.GetType().GetProperty("Timestamp"); if (prop == null) { throw new CacheException($"No Timestamp property found on type {typeof(T).Name}"); } if (!prop.CanWrite) { throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable"); } var oldTimestamp = prop.GetValue(newValue); var kv = KeyInfo.ValueToKeyValue(oldTimestamp, new KeyInfo(KeyDataType.IntKey, KeyType.ScalarIndex, "Timestamp")); var q = new AtomicQuery(kv); var andQuery = new AndQuery(); andQuery.Elements.Add(q); var orq = new OrQuery(typeof(T)); orq.Elements.Add(andQuery); var now = DateTime.Now; var newTimestamp = now.AddTicks(1); // add one to be sure its different prop.SetValue(newValue, newTimestamp); var packed = Pack(newValue); _itemsToPut.Add(packed); _conditions.Add(orq); }
public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index) { if (whereClause.Predicate is BinaryExpression expression) { VisitBinaryExpression(expression, RootExpression); } else { if (whereClause.Predicate is SubQueryExpression subQuery) { AndQuery andExpression; if (!RootExpression.MultipleWhereClauses) { andExpression = new AndQuery(); RootExpression.Elements.Add(andExpression); } else // multiple where clauses are joined by AND { andExpression = RootExpression.Elements[0]; } var leaf = new AtomicQuery(); andExpression.Elements.Add(leaf); VisitContainsExpression(subQuery, leaf); } else { throw new NotSupportedException("Incorrect query"); } } RootExpression.MultipleWhereClauses = true; base.VisitWhereClause(whereClause, queryModel, index); }
/// <summary> /// Rank the indexes that can be used to resolve the query. Lower rank means more discriminant index (smaller result) /// </summary> /// <param name="andQuery"></param> /// <returns></returns> IList <IndexRanking> GetIndexesForQuery(AndQuery andQuery) { var result = new List <IndexRanking>(); foreach (var atomicQuery in andQuery.Elements) { var name = atomicQuery.PropertyName; var index = _dataStore.TryGetIndex(name); if (atomicQuery.Operator == QueryOperator.Eq || atomicQuery.Operator == QueryOperator.In || atomicQuery.Operator == QueryOperator.Contains) { // for primary or unique key we do not need more than one index if (index.IndexType == IndexType.Primary || index.IndexType == IndexType.Unique) { // no need to count for the primary index. Waste of time as it wil always be the only index used return(new List <IndexRanking> { new IndexRanking(index, atomicQuery, -1) }); } var indexResultCount = index.GetCount(atomicQuery.Values, atomicQuery.Operator); result.Add(new IndexRanking(index, atomicQuery, indexResultCount)); } else if (atomicQuery.IsComparison) // in this case we can only use ordered indexes { if (index != null && index.IndexType == IndexType.Ordered) { var indexResultCount = index.GetCount(atomicQuery.Values, atomicQuery.Operator); result.Add(new IndexRanking(index, atomicQuery, indexResultCount)); } } } return(result); }
private void InternalVisitWhereClause(Expression whereClause, bool not = false) { if (whereClause is BinaryExpression expression) { VisitBinaryExpression(expression, RootExpression); } else if (whereClause is MemberExpression memberExpression) { if (memberExpression.Type == typeof(bool)) { // Generalize for more complex expressions VisitMemberExpression(memberExpression, RootExpression, not); } } else if (whereClause is MethodCallExpression call) { VisitMethodCall(call, RootExpression); } else { if (whereClause is SubQueryExpression subQuery) { var atomicQuery = VisitContainsExpression(subQuery, not); var andQuery = new AndQuery(); andQuery.Elements.Add(atomicQuery); RootExpression.Elements.Add(andQuery); } else { throw new NotSupportedException("Incorrect query"); } } RootExpression.MultipleWhereClauses = true; }
/// <summary> /// Optimistic synchronization using a timestamp property /// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp" /// It also updates this property withe DateTime.Now /// If you use this you should never modify the timestamp manually /// </summary> /// <param name="newValue"></param> public void UpdateWithTimestampSynchronization(T newValue) { var prop = newValue.GetType().GetProperty("Timestamp"); if (prop == null) { throw new CacheException($"No Timestamp property found on type {typeof(T).Name}"); } if (!prop.CanWrite) { throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable"); } var oldTimestamp = prop.GetValue(newValue); var kv = new KeyValue(oldTimestamp, new KeyInfo("Timestamp", 0, IndexType.Dictionary)); var q = new AtomicQuery(_collectionSchema.KeyByName(kv.KeyName), kv); var andQuery = new AndQuery(); andQuery.Elements.Add(q); var orq = new OrQuery(_collectionName); orq.Elements.Add(andQuery); var now = DateTime.Now; var newTimestamp = now.AddTicks(1); // add one to be sure its different prop.SetValue(newValue, newTimestamp); _client.UpdateIf(Pack(newValue), orq); }
IList <PackedObject> ProcessAndQuery(AndQuery query, ExecutionPlan executionPlan) { if (query.Elements.Count == 1) { return(ProcessSimpleQuery(query.Elements[0], executionPlan)); } var queryExecutionPlan = new QueryExecutionPlan(query.ToString()); // this method can be called in parallel. The only common data is the global execution plan lock (executionPlan) { executionPlan.QueryPlans.Add(queryExecutionPlan); } queryExecutionPlan.StartPlanning(); var indexesToUse = GetIndexesForQuery(query); queryExecutionPlan.EndPlanning(); // this will contain all queries that have can not be resolved by indexes and need to be checked manually var restOfTheQuery = query.Clone(); ISet <PackedObject> result = null; var finalResult = new List <PackedObject>(); if (indexesToUse.Count == 1) // only one index can be used so do not bother with extra logic { queryExecutionPlan.StartIndexUse(); var plan = indexesToUse[0]; queryExecutionPlan.Trace($"single index: {plan.ResolvedQuery.PropertyName}"); result = plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator); // this query was resolved by an index so no need to check it manually restOfTheQuery.Elements.Remove(plan.ResolvedQuery); queryExecutionPlan.EndIndexUse(); } else if (indexesToUse.Count > 1) { queryExecutionPlan.StartIndexUse(); foreach (var plan in indexesToUse.OrderBy(p => p.Ranking).Take(2)) // no more than two indexes { if (result == null) { result = plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator); queryExecutionPlan.Trace($"first index: {plan.ResolvedQuery.PropertyName} = {plan.Ranking}"); } else { result.IntersectWith(plan.Index.GetMany(plan.ResolvedQuery.Values, plan.ResolvedQuery.Operator)); queryExecutionPlan.Trace($"then index: {plan.ResolvedQuery.PropertyName} = {plan.Ranking} => {result.Count}"); } // do not work too hard if indexes found nothing if (result.Count == 0) { break; } // this query was resolved by an index so no need to check it manually restOfTheQuery.Elements.Remove(plan.ResolvedQuery); } queryExecutionPlan.EndIndexUse(); } else // no index can be used so proceed to full-scan { queryExecutionPlan.FullScan = true; queryExecutionPlan.StartScan(); var res = _dataStore.PrimaryIndex.GetAll().Where(o => restOfTheQuery.Match(o)).ToList(); queryExecutionPlan.EndScan(); return(res); } if (result != null) { if (restOfTheQuery.Elements.Count == 0) // empty query left; fully resolved by indexes { return(result.ToList()); } queryExecutionPlan.StartScan(); foreach (var item in result) { if (restOfTheQuery.Match(item)) { finalResult.Add(item); } } queryExecutionPlan.EndScan(); } return(finalResult); }
//TODO add unit test for OR expression with Contains /// <summary> /// OR expression can be present only at root level /// </summary> /// <param name="binaryExpression"></param> /// <param name="rootExpression"></param> private void VisitOrExpression(BinaryExpression binaryExpression, OrQuery rootExpression) { // visit left part if (IsLeafExpression(binaryExpression.Left)) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Left)); } else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression); } else if (binaryExpression.Left.NodeType == ExpressionType.Extension) { if (binaryExpression.Left is SubQueryExpression subQuery) { AndQuery andExpression; if (!rootExpression.MultipleWhereClauses) { andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); } else // multiple where clauses are joined by AND { andExpression = rootExpression.Elements[0]; } var leaf = new AtomicQuery(); andExpression.Elements.Add(leaf); VisitContainsExpression(subQuery, leaf); } } else if (binaryExpression.Left.NodeType == ExpressionType.OrElse) { VisitOrExpression((BinaryExpression)binaryExpression.Left, rootExpression); } else if (binaryExpression.Left.NodeType == ExpressionType.AndAlso) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); VisitAndExpression((BinaryExpression)binaryExpression.Left, andExpression); } else { throw new NotSupportedException("Query too complex"); } // visit right part if (IsLeafExpression(binaryExpression.Right)) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); andExpression.Elements.Add(VisitLeafExpression((BinaryExpression)binaryExpression.Right)); } else if (binaryExpression.Right.NodeType == ExpressionType.Extension) { if (binaryExpression.Right is SubQueryExpression subQuery) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); if (rootExpression.MultipleWhereClauses) { throw new NotSupportedException( "Multiple where clauses can be used only with simple expressions"); } var leaf = new AtomicQuery(); andExpression.Elements.Add(leaf); VisitContainsExpression(subQuery, leaf); } } else if (binaryExpression.Right.NodeType == ExpressionType.OrElse) { VisitOrExpression((BinaryExpression)binaryExpression.Right, rootExpression); } else if (binaryExpression.Right.NodeType == ExpressionType.AndAlso) { var andExpression = new AndQuery(); rootExpression.Elements.Add(andExpression); VisitAndExpression((BinaryExpression)binaryExpression.Right, andExpression); } else { throw new NotSupportedException("Query too complex"); } }
private static void ParseWhere(Node @where, OrQuery query, CollectionSchema schema) { var orNodes = where.Children.Where(c => c.Token == "or").ToList(); if (orNodes.Count != 1) { throw new NotSupportedException("Query too complex"); } var orNode = orNodes.Single(); var andNodes = orNode.Children.Where(c => c.Token == "and").ToList(); foreach (var andNode in andNodes) { var andQuery = new AndQuery(); foreach (var node in andNode.Children) { var operands = node.Children.Select(c => c.Token).ToList(); var op = ParseOperator(node.Token, operands.LastOrDefault()); if (op == QueryOperator.In || op == QueryOperator.NotIn) { var metadata = schema.KeyByName(operands[0]); if (metadata != null) { List <KeyValue> values = new List <KeyValue>(); foreach (var val in operands.Skip(1)) { object value = JExtensions.SmartParse(val); values.Add(new KeyValue(value, metadata)); } andQuery.Elements.Add(new AtomicQuery(metadata, values, op)); } else { throw new NotSupportedException($"Can not parse query after IN operator. {operands[0]} is a server-side value"); } } else if (op == QueryOperator.Contains || op == QueryOperator.NotContains) // for contains operator the collection property should be at the left side { var metadata = schema.KeyByName(operands[0]); object value = JExtensions.SmartParse(operands[1]); andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op)); } else if (op == QueryOperator.StrStartsWith || op == QueryOperator.StrEndsWith || op == QueryOperator.StrContains) // for string operators the property should be at the left side { var metadata = schema.KeyByName(operands[0]); object value = operands[1].Trim('%'); andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op)); } else if (operands.Count == 2)// binary operators { var metadata = schema.KeyByName(operands[0]); // by default property name first if (metadata != null) { object value = JExtensions.SmartParse(operands[1]); andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), op)); } else // try value first { metadata = schema.KeyByName(operands[1]); if (metadata == null) { throw new NotSupportedException($"Can not parse query. Neither {operands[0]} nor {operands[1]} is a server-side value"); } object value = JExtensions.SmartParse(operands[0]);; andQuery.Elements.Add(new AtomicQuery(metadata, new KeyValue(value, metadata), Reverse(op))); } } } query.Elements.Add(andQuery); } }