public static DynamicQueryMapping Create(DocumentDatabase database, IndexQuery query, string entityName) { var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query.Query); if (query.SortedFields != null) { foreach (var sortedField in query.SortedFields) { if (sortedField.Field.StartsWith(Constants.RandomFieldName)) { continue; } fields.Add(Tuple.Create(sortedField.Field, sortedField.Field)); } } var dynamicQueryMapping = new DynamicQueryMapping { AggregationOperation = query.AggregationOperation.RemoveOptionals(), DynamicAggregation = query.AggregationOperation.HasFlag(AggregationOperation.Dynamic), ForEntityName = entityName, SortDescriptors = GetSortInfo(fieldName => { if (fields.Any(x => x.Item2 == fieldName || x.Item2 == (fieldName + "_Range")) == false) { fields.Add(Tuple.Create(fieldName, fieldName)); } }) }; dynamicQueryMapping.SetupFieldsToIndex(query, fields); dynamicQueryMapping.FindIndexName(database, dynamicQueryMapping, query); return(dynamicQueryMapping); }
public void TestParse() { var parser = new SimpleQueryParser(new QueryParserConfig()); var expression = parser.Parse("Hello World -ÄãºÃ°¡").Expression; Assert.IsInstanceOfType(expression, typeof(And)); var andExp = expression as And; var andExpList = andExp.ToList(); Assert.IsInstanceOfType(andExpList[0], typeof(Word)); Assert.IsInstanceOfType(andExpList[1], typeof(Word)); Assert.IsInstanceOfType(andExpList[2], typeof(Not)); Assert.AreEqual("hello", (andExpList[0] as Word).Value); Assert.AreEqual("world", (andExpList[1] as Word).Value); Assert.IsInstanceOfType((andExpList[2] as Not).Operand, typeof(And)); var notAndExp = (andExpList[2] as Not).Operand as And; var notWord1 = notAndExp.ToList()[0]; var notWord2 = notAndExp.ToList()[1]; Assert.IsInstanceOfType(notWord1, typeof(Word)); Assert.IsInstanceOfType(notWord2, typeof(Word)); Assert.AreEqual("ÄãºÃ", (notWord1 as Word).Value); Assert.AreEqual("°¡", (notWord2 as Word).Value); }
public Query CreateQuery(ILuceneQueryService builder, LuceneQueryContext context, string type, JObject query) { if (type != "simple_query_string") { return(null); } var queryString = query["query"]?.Value <string>(); var fields = query["fields"]?.Values <string>() ?? new string[0]; var defaultOperator = query["default_operator"]?.Value <string>().ToLowerInvariant() ?? "or"; var weights = fields.ToDictionary(field => field, field => 1.0f); var queryParser = new SimpleQueryParser(context.DefaultAnalyzer, weights); switch (defaultOperator) { case "and": queryParser.DefaultOperator = Occur.MUST; break; case "or": queryParser.DefaultOperator = Occur.SHOULD; break; } return(queryParser.Parse(queryString)); }
public Query CreateQuery(ILuceneQueryService builder, LuceneQueryContext context, string type, JObject query) { if (type != "simple_query_string") { return(null); } var queryString = query["query"]?.Value <string>(); var fields = query["fields"]?.Value <string>() ?? ""; var defaultOperator = query["default_operator"]?.Value <string>() ?? "and"; var queryParser = new SimpleQueryParser(context.DefaultAnalyzer, ""); return(queryParser.Parse(queryString)); }
public static DynamicQueryMapping Create(DocumentDatabase database, IndexQuery query, string entityName) { var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query); if (query.SortedFields != null) { foreach (var sortedField in query.SortedFields) { var field = sortedField.Field; if (field.StartsWith(Constants.RandomFieldName)) { continue; } if (field.StartsWith(Constants.CustomSortFieldName)) { continue; } if (field == Constants.TemporaryScoreValue) { continue; } if (field.EndsWith("_Range")) { field = field.Substring(0, field.Length - "_Range".Length); } fields.Add(Tuple.Create(SimpleQueryParser.TranslateField(field), field)); } } var dynamicQueryMapping = new DynamicQueryMapping { ForEntityName = entityName, HighlightedFields = query.HighlightedFields.EmptyIfNull().Select(x => x.Field).ToArray(), SortDescriptors = GetSortInfo(fieldName => { if (fields.Any(x => x.Item2 == fieldName || x.Item2 == (fieldName + "_Range")) == false) { fields.Add(Tuple.Create(fieldName, fieldName)); } }, query) }; dynamicQueryMapping.SetupFieldsToIndex(query, fields); dynamicQueryMapping.SetupSortDescriptors(dynamicQueryMapping.SortDescriptors); dynamicQueryMapping.FindIndexName(database, dynamicQueryMapping, query); return(dynamicQueryMapping); }
public Query CreateQuery(ILuceneQueryService builder, LuceneQueryContext context, string type, JObject query) { if (type != "simple_query_string") { return(null); } var queryString = query["query"]?.Value <string>(); var fields = query["fields"]?.Values <string>() ?? new string[0]; var defaultOperator = query["default_operator"]?.Value <string>().ToLowerInvariant() ?? "or"; var weight = 1.0f; var weights = fields.ToDictionary(field => field, field => weight); foreach (var field in fields) { var fieldWeightArray = field.Split('^', 2); if (fieldWeightArray.Length > 1 && Single.TryParse(fieldWeightArray[1], out weight)) { weights.Remove(field); weights.Add(fieldWeightArray[0], weight); } } var queryParser = new SimpleQueryParser(context.DefaultAnalyzer, weights); switch (defaultOperator) { case "and": queryParser.DefaultOperator = Occur.MUST; break; case "or": queryParser.DefaultOperator = Occur.SHOULD; break; } return(queryParser.Parse(queryString)); }
public static DynamicQueryMapping Create(DocumentDatabase database, IndexQuery query, string entityName) { var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query.Query); if (query.SortedFields != null) { foreach (var sortedField in query.SortedFields) { fields.Add(Tuple.Create(sortedField.Field, sortedField.Field)); } } var dynamicQueryMapping = new DynamicQueryMapping { AggregationOperation = query.AggregationOperation & ~AggregationOperation.Dynamic, DynamicAggregation = (query.AggregationOperation & AggregationOperation.Dynamic) == AggregationOperation.Dynamic, ForEntityName = entityName, SortDescriptors = GetSortInfo(fields) }; dynamicQueryMapping.SetupFieldsToIndex(query, fields); dynamicQueryMapping.FindIndexName(database, dynamicQueryMapping, query); return(dynamicQueryMapping); }
public DynamicQueryOptimizerResult SelectAppropriateIndex( string entityName, IndexQuery indexQuery, List <Explanation> explanations = null) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(new DynamicQueryOptimizerResult("", DynamicQueryMatchType.Failure)); } if (string.IsNullOrEmpty(indexQuery.Query) && // we optimize for empty queries to use Raven/DocumentsByEntityName (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested database.IndexDefinitionStorage.Contains("Raven/DocumentsByEntityName")) // and Raven/DocumentsByEntityName exists { if (string.IsNullOrEmpty(entityName) == false) { indexQuery.Query = "Tag:" + entityName; } return(new DynamicQueryOptimizerResult("Raven/DocumentsByEntityName", DynamicQueryMatchType.Complete)); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery).Select(x => x.Item2).ToArray(); var normalizedFieldsQueriedUpon = fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceInvalidCharactersForFields).ToArray(); var distinctSelectManyFields = new HashSet <string>(); foreach (var field in fieldsQueriedUpon) { var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < parts.Length; i++) { distinctSelectManyFields.Add(string.Join(",", parts.Take(i))); } } ExplainDelegate explain = (index, rejectionReason) => { }; if (explanations != null) { explain = (index, rejectionReason) => explanations.Add(new Explanation { Index = index, Reason = rejectionReason() }); } // there is no reason why we can't use indexes with transform results // we merely need to disable the transform results for this particular query indexQuery.SkipTransformResults = true; var results = database.IndexDefinitionStorage.IndexNames .Select(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); var currentBestState = DynamicQueryMatchType.Complete; if (abstractViewGenerator == null) // there is no matching view generator { explain(indexName, () => "There is no matching view generator. Maybe the index in the process of being deleted?"); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (entityName == null) { if (abstractViewGenerator.ForEntityNames.Count != 0) { explain(indexName, () => "Query is not specific for entity name, but the index filter by entity names."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } else { if (abstractViewGenerator.ForEntityNames.Count > 1) // we only allow indexes with a single entity name { explain(indexName, () => "Index contains more than a single entity name, may result in a different type being returned."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.ForEntityNames.Count == 0) { explain(indexName, () => "Query is specific for entity name, but the index searches across all of them, may result in a different type being returned."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.ForEntityNames.Contains(entityName) == false) // for the specified entity name { explain(indexName, () => string.Format("Index does not apply to entity name: {0}", entityName)); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { explain(indexName, () => "Can't choose a map/reduce index for dynamic queries."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.HasWhereClause) // without a where clause { explain(indexName, () => "Can't choose an index with a where clause, it might filter things that the query is looking for."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one // in order to handle that, we count the number of select many that would happen because of the query // and match it to the number of select many in the index if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count) { explain(indexName, () => "Can't choose an index with a different number of from clauses / SelectMany, will affect queries like Count()."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false) { explain(indexName, () => { var missingFields = normalizedFieldsQueriedUpon.Where(s => abstractViewGenerator.ContainsFieldOnMap(s) == false); return("The following fields are missing: " + string.Join(", ", missingFields)); }); currentBestState = DynamicQueryMatchType.Partial; } var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (indexQuery.HighlightedFields != null && indexQuery.HighlightedFields.Length > 0) { var nonHighlightableFields = indexQuery .HighlightedFields .Where(x => !indexDefinition.Stores.ContainsKey(x.Field) || indexDefinition.Stores[x.Field] != FieldStorage.Yes || !indexDefinition.Indexes.ContainsKey(x.Field) || indexDefinition.Indexes[x.Field] != FieldIndexing.Analyzed || !indexDefinition.TermVectors.ContainsKey(x.Field) || indexDefinition.TermVectors[x.Field] != FieldTermVector.WithPositionsAndOffsets) .Select(x => x.Field) .ToArray(); if (nonHighlightableFields.Any()) { explain(indexName, () => "The following fields could not be highlighted because they are not stored, analyzed and using term vectors with positions and offsets: " + string.Join(", ", nonHighlightableFields)); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { var normalizedFieldName = DynamicQueryMapping.ReplaceInvalidCharactersForFields(sortedField.Field); if (normalizedFieldName.StartsWith(Constants.RandomFieldName)) { return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(normalizedFieldName) == false) { explain(indexName, () => "Rejected because index does not contains field '" + normalizedFieldName + "' which we need to sort on"); currentBestState = DynamicQueryMatchType.Partial; continue; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == normalizedFieldName); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(normalizedFieldName, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: explain(indexName, () => "The specified sort type is different than the default for field: " + normalizedFieldName); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (value != dynamicSortInfo.FieldType) { explain(indexName, () => "The specified sort type (" + dynamicSortInfo.FieldType + ") is different than the one specified for field '" + normalizedFieldName + "' (" + value + ")"); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } } if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0) { // none of the fields have custom analyzers if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { explain(indexName, () => { var fields = normalizedFieldsQueriedUpon.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have a custom analyzer: " + string.Join(", ", fields)); }); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (indexDefinition.Indexes != null && indexDefinition.Indexes.Count > 0) { //If any of the fields we want to query on are set to something other than the default, don't use the index var anyFieldWithNonDefaultIndexing = normalizedFieldsQueriedUpon.Where(x => { FieldIndexing analyzedInfo; if (indexDefinition.Indexes.TryGetValue(x, out analyzedInfo)) { if (analyzedInfo != FieldIndexing.Default) { return(true); } } return(false); }); if (anyFieldWithNonDefaultIndexing.Any()) { explain(indexName, () => { var fields = anyFieldWithNonDefaultIndexing.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have aren't using default indexing: " + string.Join(", ", fields)); }); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (currentBestState != DynamicQueryMatchType.Complete && indexDefinition.Type != "Auto") { return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } return(new DynamicQueryOptimizerResult(indexName, currentBestState)); }) .Where(result => result.MatchType != DynamicQueryMatchType.Failure) .GroupBy(x => x.MatchType) .ToDictionary(x => x.Key, x => x.ToArray()); DynamicQueryOptimizerResult[] optimizerResults; if (results.TryGetValue(DynamicQueryMatchType.Complete, out optimizerResults) && optimizerResults.Length > 0) { DynamicQueryOptimizerResult[] prioritizedResults = null; database.TransactionalStorage.Batch(accessor => { prioritizedResults = optimizerResults.OrderByDescending(result => { var stats = accessor.Indexing.GetIndexStats(result.IndexName); if (stats == null) { return(Etag.Empty); } return(stats.LastIndexedEtag); }) .ThenByDescending(result => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator( result.IndexName); if (abstractViewGenerator == null) { return(-1); } return(abstractViewGenerator.CountOfFields); }) .ToArray(); }); for (int i = 1; i < prioritizedResults.Length; i++) { explain(prioritizedResults[i].IndexName, () => "Wasn't the widest / most unstable index matching this query"); } return(prioritizedResults[0]); } if (results.TryGetValue(DynamicQueryMatchType.Partial, out optimizerResults) && optimizerResults.Length > 0) { return(optimizerResults.OrderByDescending(x => { var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(x.IndexName); if (viewGenerator == null) { return -1; } return viewGenerator.CountOfFields; }).First()); } return(new DynamicQueryOptimizerResult("<invalid index>", DynamicQueryMatchType.Failure)); }
public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } if (string.IsNullOrEmpty(indexQuery.Query) && // we optimize for empty queries to use Raven/DocumentsByEntityName (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested database.IndexDefinitionStorage.Contains("Raven/DocumentsByEntityName")) // and Raven/DocumentsByEntityName exists { if (string.IsNullOrEmpty(entityName) == false) { indexQuery.Query = "Tag:" + entityName; } return("Raven/DocumentsByEntityName"); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2).ToArray(); var normalizedFieldsQueriedUpon = fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceIndavlidCharactersForFields).ToArray(); var distinctSelectManyFields = new HashSet <string>(); foreach (var field in fieldsQueriedUpon) { var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < parts.Length; i++) { distinctSelectManyFields.Add(string.Join(",", parts.Take(i))); } } return(database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is no matching view generator { return false; } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { return false; } if (abstractViewGenerator.TransformResultsDefinition != null) // we can't choose an index with transform results { return false; } if (abstractViewGenerator.HasWhereClause) // without a where clause { return false; } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one // in order to handle that, we count the number of select many that would happen because of the query // and match it to the number of select many in the index if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count) { return false; } if (entityName == null) { if (abstractViewGenerator.ForEntityNames.Count != 0) { return false; } } else { if (abstractViewGenerator.ForEntityNames.Count != 1 || // we only allow indexes with a single entity name abstractViewGenerator.ForEntityNames.Contains(entityName) == false) // for the specified entity name { return false; } } if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false) { return false; } var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return false; } if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { if (sortedField.Field.StartsWith(Constants.RandomFieldName)) { continue; } // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(sortedField.Field) == false) { return false; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: return false; } } if (value != dynamicSortInfo.FieldType) { return false; // different sort order, there is a problem here } } } if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0) { // none of the fields have custom analyzers if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { return false; } } return true; }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return -1; } return abstractViewGenerator.CountOfFields; }) .FirstOrDefault()); }
public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2); return(database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is no matching view generator { return false; } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { return false; } if (abstractViewGenerator.TransformResultsDefinition != null) // we can't choose an index with transform results { return false; } if (abstractViewGenerator.HasWhereClause) // without a where clause { return false; } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one if (abstractViewGenerator.CountOfSelectMany > 1) { return false; } if (abstractViewGenerator.ForEntityName != entityName) // for the specified entity name { return false; } return fieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap); }) .Where(indexName => { var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return false; } var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) { return false; } if (indexQuery.SortedFields != null) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(sortedField.Field) == false) { return false; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: return false; } } if (value != dynamicSortInfo.FieldType) { return false; // different sort order, there is a problem here } } } if (indexDefinition.Analyzers != null) { // none of the fields have custom analyzers if (fieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { return false; } } return true; }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return -1; } return abstractViewGenerator.CountOfFields; }) .FirstOrDefault()); }
public static DynamicQueryMapping Create(string entityName, IndexQueryServerSide query) { var result = new DynamicQueryMapping { ForCollection = entityName, }; IEnumerable <DynamicQueryMappingItem> dynamicMapFields; string[] numericFields; if (query.DynamicMapReduceFields == null) { // auto map query var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query); // TODO arek - not sure if we really need a Tuple<string, string> here if (query.SortedFields != null) { foreach (var sortedField in query.SortedFields) { var field = sortedField.Field; if (field == Constants.Indexing.Fields.IndexFieldScoreName) { continue; } if (field.StartsWith(Constants.Indexing.Fields.RandomFieldName) || field.StartsWith(Constants.Indexing.Fields.CustomSortFieldName)) { continue; } if (InvariantCompare.IsPrefix(field, Constants.Indexing.Fields.AlphaNumericFieldName, CompareOptions.None)) { field = SortFieldHelper.ExtractName(field); } if (InvariantCompare.IsSuffix(field, Constants.Indexing.Fields.RangeFieldSuffix, CompareOptions.None)) { field = field.Substring(0, field.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length); } fields.Add(Tuple.Create(SimpleQueryParser.TranslateField(field), field)); } } dynamicMapFields = fields.Select(x => new DynamicQueryMappingItem(x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix) ? x.Item1.Substring(0, x.Item1.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length) : x.Item1, FieldMapReduceOperation.None)); numericFields = fields.Where(x => x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix)).Select(x => x.Item1).Distinct().ToArray(); } else { // dynamic map-reduce query result.IsMapReduce = true; dynamicMapFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy == false).Select(x => new DynamicQueryMappingItem(x.Name, x.OperationType)); result.GroupByFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy).Select(x => x.Name).ToArray(); numericFields = null; } result.MapFields = dynamicMapFields .Where(x => x.Name != Constants.Indexing.Fields.DocumentIdFieldName) .OrderByDescending(x => x.Name.Length) .ToArray(); result.SortDescriptors = GetSortInfo(query.SortedFields, numericFields); result.HighlightedFields = query.HighlightedFields.EmptyIfNull().Select(x => x.Field).ToArray(); return(result); }
public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefor we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } return(database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return false; } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { return false; } if (abstractViewGenerator.TransformResultsDefinition != null) // we can't choose an index with transform results { return false; } if (abstractViewGenerator.ViewText.Contains("where")) // without a where clause { return false; } if (abstractViewGenerator.ForEntityName != entityName) // for the specified entity name { return false; } var items = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2); return items.All(abstractViewGenerator.ContainsFieldOnMap); }) .Where(indexName => { var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return false; } if (indexQuery.SortedFields != null) { foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { return false; } } } return true; }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return -1; } return abstractViewGenerator.CountOfFields; }) .FirstOrDefault()); }
public string SelectAppropriateIndex( string entityName, IndexQuery indexQuery, List <Explanation> explanations = null) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } if (string.IsNullOrEmpty(indexQuery.Query) && // we optimize for empty queries to use Raven/DocumentsByEntityName (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested database.IndexDefinitionStorage.Contains("Raven/DocumentsByEntityName")) // and Raven/DocumentsByEntityName exists { if (string.IsNullOrEmpty(entityName) == false) { indexQuery.Query = "Tag:" + entityName; } return("Raven/DocumentsByEntityName"); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery).Select(x => x.Item2).ToArray(); var normalizedFieldsQueriedUpon = fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceIndavlidCharactersForFields).ToArray(); var distinctSelectManyFields = new HashSet <string>(); foreach (var field in fieldsQueriedUpon) { var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < parts.Length; i++) { distinctSelectManyFields.Add(string.Join(",", parts.Take(i))); } } ExplainDelegate explain = (index, rejectionReason) => { }; if (explanations != null) { explain = (index, rejectionReason) => explanations.Add(new Explanation { Index = index, Reason = rejectionReason() }); } // there is no reason why we can't use indexes with transform results // we merely need to disable the transform results for this particular query indexQuery.SkipTransformResults = true; var results = database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is no matching view generator { explain(indexName, () => "There is no matching view generator. Maybe the index in the process of being deleted?"); return(false); } if (entityName == null) { if (abstractViewGenerator.ForEntityNames.Count != 0) { explain(indexName, () => "Query is not specific for entity name, but the index filter by entity names."); return(false); } } else { if (abstractViewGenerator.ForEntityNames.Count > 1) // we only allow indexes with a single entity name { explain(indexName, () => "Index contains more than a single entity name, may result in a different type being returned."); return(false); } if (abstractViewGenerator.ForEntityNames.Count == 0) { explain(indexName, () => "Query is specific for entity name, but the index searches across all of them, may result in a different type being returned."); return(false); } if (abstractViewGenerator.ForEntityNames.Contains(entityName) == false) // for the specified entity name { explain(indexName, () => string.Format("Index does not apply to entity name: {0}", entityName)); return(false); } } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { explain(indexName, () => "Can't choose a map/reduce index for dynamic queries."); return(false); } if (abstractViewGenerator.HasWhereClause) // without a where clause { explain(indexName, () => "Can't choose an index with a where clause, it might filter things that the query is looking for."); return(false); } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one // in order to handle that, we count the number of select many that would happen because of the query // and match it to the number of select many in the index if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count) { explain(indexName, () => "Can't choose an index with a different number of from clauses / SelectMany, will affect queries like Count()."); return(false); } if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false) { explain(indexName, () => { var missingFields = normalizedFieldsQueriedUpon.Where(s => abstractViewGenerator.ContainsFieldOnMap(s) == false); return("The following fields are missing: " + string.Join(", ", missingFields)); }); return(false); } var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return(false); } if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { if (sortedField.Field.StartsWith(Constants.RandomFieldName)) { continue; } // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(sortedField.Field) == false) { explain(indexName, () => "Rejected because index does not contains field '" + sortedField.Field + "' which we need to sort on"); return(false); } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: explain(indexName, () => "The specified sort type is different than the default for field: " + sortedField.Field); return(false); } } if (value != dynamicSortInfo.FieldType) { explain(indexName, () => "The specified sort type (" + dynamicSortInfo.FieldType + ") is different than the one specified for field '" + sortedField.Field + "' (" + value + ")"); return(false); // different sort order, there is a problem here } } } if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0) { // none of the fields have custom analyzers if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { explain(indexName, () => { var fields = normalizedFieldsQueriedUpon.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have a custom analyzer: " + string.Join(", ", fields)); }); return(false); } } if (indexDefinition.Indexes != null && indexDefinition.Indexes.Count > 0) { //If any of the fields we want to query on are set to something other than the default, don't use the index var anyFieldWithNonDefaultIndexing = normalizedFieldsQueriedUpon.Where(x => { FieldIndexing analysedInfo; if (indexDefinition.Indexes.TryGetValue(x, out analysedInfo)) { if (analysedInfo != FieldIndexing.Default) { return(true); } } return(false); }); if (anyFieldWithNonDefaultIndexing.Any()) { explain(indexName, () => { var fields = anyFieldWithNonDefaultIndexing.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have aren't using default indexing: " + string.Join(", ", fields)); }); return(false); } } return(true); }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there isn't a matching view generator { return(-1); } return(abstractViewGenerator.CountOfFields); }); string name = null; foreach (var indexName in results) { if (name == null) { name = indexName; } else { explain(indexName, () => "Wasn't the widest index matching this query."); } } explain(name ?? "Temporary index will be created", () => "Selected as best match"); return(name); }
public List <int> Search(string query) { var parser = new SimpleQueryParser(query); var parsedQuery = parser.Parse(); if (parsedQuery.Length == 1) { return(_index.Find(parsedQuery[0])); } var i = 0; var result = new List <int>(); while (i < parsedQuery.Length) { if (i == 0) { var invertedFirstArg = parsedQuery[i] == "NOT"; i = invertedFirstArg ? ++i : i; if (invertedFirstArg) { result = _dataSource.GetAllIds().Except(_index.Find(parsedQuery[i++])).ToList(); } else { result = _index.Find(parsedQuery[i++]); } } if (parsedQuery[i] == "AND") { i++; if (parsedQuery[i] == "NOT") { result = result.Except(_index.Find(parsedQuery[++i])).ToList(); } else { result = result.Intersect(_index.Find(parsedQuery[i])).ToList(); } } else if (parsedQuery[i] == "OR") { i++; if (parsedQuery[i] == "NOT") { result = result.Concat(_dataSource.GetAllIds().Except(_index.Find(parsedQuery[++i]))).ToList(); } else { result = result.Concat(_index.Find(parsedQuery[i])).ToList(); } } i++; } return(result); }