protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.HasExplicitFieldsToFetch) { return(base.RetrieveDocument(document, fieldsToFetch, score)); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return(base.RetrieveDocument(document, fieldsToFetch, score)); } var projection = RavenJObject.Parse(field.StringValue); if (fieldsToFetch.FetchAllStoredFields) { var fields = new HashSet <string>(document.GetFields().Select(x => x.Name)); fields.Remove(Constants.ReduceKeyFieldName); var documentFromFields = new RavenJObject(); AddFieldsToDocument(document, fields, documentFromFields); foreach (var kvp in projection) { documentFromFields[kvp.Key] = kvp.Value; } projection = documentFromFields; } return(new IndexQueryResult { Projection = projection, Score = score.Score, ReduceVal = field.StringValue }); }
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch) { return(new IndexQueryResult { Key = document.Get(Constants.DocumentIdFieldName), Projection = fieldsToFetch.IsProjection ? CreateDocumentFromFields(document, fieldsToFetch) : null }); }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch) { if (fieldsToFetch.IsProjection == false) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().OfType <Fieldable>().Select(x => x.Name()).ToArray()); } fieldsToFetch.EnsureHasField(Raven.Abstractions.Data.Constants.ReduceKeyFieldName); return(base.RetrieveDocument(document, fieldsToFetch)); }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { if (fieldsToFetch.IsProjection == false) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); } fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); return(base.RetrieveDocument(document, fieldsToFetch, score)); }
public IndexQueryOperation(Index parent, IndexQuery indexQuery, Func <IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers) { this.parent = parent; this.indexQuery = indexQuery; this.shouldIncludeInResults = shouldIncludeInResults; this.fieldsToFetch = fieldsToFetch; this.indexQueryTriggers = indexQueryTriggers; if (fieldsToFetch.IsDistinctQuery) { alreadyReturned = new HashSet <RavenJObject>(new RavenJTokenEqualityComparer()); } }
public Query GetLuceneQuery(string index, IndexQuery query, OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.Debug("Query on non existing index {0}", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } var fieldsToFetch = new FieldsToFetch(new string[0], AggregationOperation.None, null); return(new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetLuceneQuery()); }
public IEnumerable <IndexQueryResult> Query( string index, IndexQuery query, Func <IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.DebugFormat("Query on non existing index {0}", index); throw new InvalidOperationException("Index " + index + " does not exists"); } return(new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch).Query()); }
public IndexQueryOperation( Index parent, IndexQuery indexQuery, Func <IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch) { this.parent = parent; this.indexQuery = indexQuery; this.shouldIncludeInResults = shouldIncludeInResults; this.fieldsToFetch = fieldsToFetch; if (fieldsToFetch.IsDistinctQuery) { alreadyReturned = new HashSet <JObject>(new JTokenEqualityComparer()); } }
public IEnumerable <IndexQueryResult> Query( string index, IndexQuery query, Func <IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.Debug("Query on non existing index '{0}'", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } return(new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers).Query()); }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.IsProjection) { return(base.RetrieveDocument(document, fieldsToFetch, score)); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return(base.RetrieveDocument(document, fieldsToFetch, score)); } return(new IndexQueryResult { Projection = RavenJObject.Parse(field.StringValue), Score = score.Score }); }
public static RavenJObject CreateDocumentFromFields(Document document, FieldsToFetch fieldsToFetch) { var documentFromFields = new RavenJObject(); var fields = fieldsToFetch.Fields; if (fieldsToFetch.FetchAllStoredFields) fields = fields.Concat(document.GetFields().Select(x => x.Name)); var q = fields .Distinct() .SelectMany(name => document.GetFields(name) ?? new Field[0]) .Where(x => x != null) .Where( x => x.Name.EndsWith("_IsArray") == false && x.Name.EndsWith("_Range") == false && x.Name.EndsWith("_ConvertToJson") == false) .Select(fld => CreateProperty(fld, document)) .GroupBy(x => x.Key) .Select(g => { if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null) { return g.First(); } var ravenJTokens = g.Select(x => x.Value).ToArray(); return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray((IEnumerable)ravenJTokens)); }); foreach (var keyValuePair in q) { documentFromFields.Add(keyValuePair.Key, keyValuePair.Value); } return documentFromFields; }
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { return new IndexQueryResult { Score = score.Score, Key = document.Get(Constants.DocumentIdFieldName), Projection = (fieldsToFetch.IsProjection || fieldsToFetch.FetchAllStoredFields) ? CreateDocumentFromFields(document, fieldsToFetch) : null }; }
public DuplicateDocumentRecorder(Searchable indexSearcher, Index parent, HashSet<string> documentsAlreadySeenInPreviousPage, HashSet<RavenJObject> alreadyReturned, FieldsToFetch fieldsToFetch, bool isProjectionOrMapReduce) { this.indexSearcher = indexSearcher; this.parent = parent; this.isProjectionOrMapReduce = isProjectionOrMapReduce; this.alreadyReturned = alreadyReturned; this.fieldsToFetch = fieldsToFetch; this.documentsAlreadySeenInPreviousPage = documentsAlreadySeenInPreviousPage; }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.HasExplicitFieldsToFetch) { return base.RetrieveDocument(document, fieldsToFetch, score); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return base.RetrieveDocument(document, fieldsToFetch, score); } return new IndexQueryResult { Projection = RavenJObject.Parse(field.StringValue), Score = score.Score }; }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.HasExplicitFieldsToFetch) { return base.RetrieveDocument(document, fieldsToFetch, score); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return base.RetrieveDocument(document, fieldsToFetch, score); } var projection = RavenJObject.Parse(field.StringValue); if (fieldsToFetch.FetchAllStoredFields) { var fields = new HashSet<string>(document.GetFields().Select(x => x.Name)); fields.Remove(Constants.ReduceKeyFieldName); var documentFromFields = new RavenJObject(); AddFieldsToDocument(document, fields, documentFromFields); foreach (var kvp in projection) { documentFromFields[kvp.Key] = kvp.Value; } projection = documentFromFields; } return new IndexQueryResult { Projection = projection, Score = score.Score, ReduceVal = field.StringValue }; }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, float score) { if (fieldsToFetch.IsProjection == false) fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().OfType<Fieldable>().Select(x => x.Name()).ToArray()); fieldsToFetch.EnsureHasField(Abstractions.Data.Constants.ReduceKeyFieldName); return base.RetrieveDocument(document, fieldsToFetch, score); }
public IEnumerable<IndexQueryResult> Query(string index, IndexQuery query, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers, CancellationToken token) { Index value = TryIndexByName(index); if (value == null) { log.Debug("Query on non existing index '{0}'", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } if ((value.Priority.HasFlag(IndexingPriority.Idle) || value.Priority.HasFlag(IndexingPriority.Abandoned)) && value.Priority.HasFlag(IndexingPriority.Forced) == false) { documentDatabase.TransactionalStorage.Batch(accessor => { value.Priority = IndexingPriority.Normal; try { accessor.Indexing.SetIndexPriority(value.indexId, IndexingPriority.Normal); } catch (Exception e) { if (accessor.IsWriteConflict(e) == false) throw; // we explciitly ignore write conflicts here, it is okay if we got set twice (two concurrent queries, or setting while indexing). } documentDatabase.WorkContext.ShouldNotifyAboutWork(() => "Idle index queried"); documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification() { Name = value.PublicName, Type = IndexChangeTypes.IndexPromotedFromIdle }); }); } var indexQueryOperation = new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers); if (query.Query != null && query.Query.Contains(Constants.IntersectSeparator)) return indexQueryOperation.IntersectionQuery(token); return indexQueryOperation.Query(token); }
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch) { return new IndexQueryResult { Key = document.Get(Constants.DocumentIdFieldName), Projection = fieldsToFetch.IsProjection ? CreateDocumentFromFields(document, fieldsToFetch) : null }; }
public IEnumerable<IndexQueryResult> Query( string index, IndexQuery query, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.Debug("Query on non existing index '{0}'", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } var indexQueryOperation = new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers); if (query.Query != null && query.Query.Contains(Constants.IntersectSeperator)) return indexQueryOperation.IntersectionQuery(); return indexQueryOperation.Query(); }
public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult) { index = IndexDefinitionStorage.FixupIndexName(index); var highlightings = new Dictionary<string, Dictionary<string, string[]>>(); Func<IndexQueryResult, object> tryRecordHighlighting = queryResult => { if (queryResult.Highligtings != null && queryResult.Key != null) highlightings.Add(queryResult.Key, queryResult.Highligtings); return null; }; var stale = false; Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty); Etag resultEtag = Etag.Empty; var nonAuthoritativeInformation = false; if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { query.FieldsToFetch = new[] { Constants.AllFields }; } var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + index); resultEtag = GetIndexEtag(index, null); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch); var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers); indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults); var transformerErrors = new List<string>(); var results = GetQueryResults(query, viewGenerator, docRetriever, from queryResult in indexQueryResults let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) where doc != null let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false) let __ = tryRecordHighlighting(queryResult) select doc, transformerErrors); if (headerInfo != null) { headerInfo(new QueryHeaderInformation { Index = index, IsStable = stale, ResultEtag = resultEtag, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, TotalResults = query.TotalSize.Value }); } using (new CurrentTransformationScope(docRetriever)) { foreach (var result in results) { onResult(result); } if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } } }); return new QueryResultWithIncludes { IndexName = index, IsStale = stale, NonAuthoritativeInformation = nonAuthoritativeInformation, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, ResultEtag = resultEtag, IdsToInclude = idsToLoad, LastQueryTime = SystemTime.UtcNow, Highlightings = highlightings }; }
public QueryResult Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new InvalidOperationException("Could not find index named: " + index); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers) select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) into doc where doc != null select doc; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects, source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } if (query.PageSize > 0) // maybe they just want the query stats? list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResult { IndexName = index, Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2 }; }
public QueryResultWithIncludes Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var highlightings = new Dictionary<string, Dictionary<string, string[]>>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); Guid resultEtag = Guid.Empty; var nonAuthoritativeInformation = false; var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + index); resultEtag = GetIndexEtag(index, null); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch); var collection = from queryResult in IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers) select new { Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch), Fragments = queryResult.Highligtings } into docWithFragments where docWithFragments.Document != null let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false) select docWithFragments; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { var resultList = new List<RavenJObject>(); foreach (var docWithFragments in collection) { resultList.Add(docWithFragments.Document.ToJson()); if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null) highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments); } results = resultList; } list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResultWithIncludes { IndexName = index, Results = list, IsStale = stale, NonAuthoritativeInformation = nonAuthoritativeInformation, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, ResultEtag = resultEtag, IdsToInclude = idsToLoad, LastQueryTime = SystemTime.UtcNow, Highlightings = highlightings }; }
public IndexQueryOperation(Index parent, IndexQuery indexQuery, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers) { this.parent = parent; this.indexQuery = indexQuery; this.shouldIncludeInResults = shouldIncludeInResults; this.fieldsToFetch = fieldsToFetch; this.indexQueryTriggers = indexQueryTriggers; if (fieldsToFetch.IsDistinctQuery) alreadyReturned = new HashSet<RavenJObject>(new RavenJTokenEqualityComparer()); }
public IEnumerable<IndexQueryResult> Query( string index, IndexQuery query, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.DebugFormat("Query on non existing index {0}", index); throw new InvalidOperationException("Index " + index + " does not exists"); } return new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch).Query(); }
public Query GetDocumentQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers) { var value = TryIndexByName(index); if (value == null) { log.Debug("Query on non existing index {0}", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } var fieldsToFetch = new FieldsToFetch(new string[0], false, null); return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetDocumentQuery(); }
public void Init() { highlightings = new Dictionary<string, Dictionary<string, string[]>>(); scoreExplanations = new Dictionary<string, string>(); Func<IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult => { if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null)) highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings); if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null) scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation); return null; }; stale = false; indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty); resultEtag = Etag.Empty; nonAuthoritativeInformation = false; if (string.IsNullOrEmpty(query.ResultsTransformer) == false && (query.FieldsToFetch == null || query.FieldsToFetch.Length == 0)) { query.FieldsToFetch = new[] { Constants.AllFields }; } duration = Stopwatch.StartNew(); idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase); var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + indexName); resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer); stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag); if (stale == false && query.Cutoff == null && query.CutoffEtag == null) { var indexInstance = database.IndexStorage.GetIndexInstance(indexName); stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress); } if (stale && actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false && actions.Staleness.IsReduceStale(index.IndexId) == false) { var forEntityNames = viewGenerator.ForEntityNames.ToList(); var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag; if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false) stale = false; } indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId); var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad); var fieldsToFetch = new FieldsToFetch(query, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking); var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults, fieldsToFetch, database.IndexQueryTriggers, cancellationToken); indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults); if (query.ShowTimings) indexQueryResults = new TimedEnumerable<IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] = timeInMilliseconds); var docs = from queryResult in indexQueryResults let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking) where doc != null let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false) let __ = tryRecordHighlightingAndScoreExplanation(queryResult) select doc; transformerErrors = new List<string>(); results = database .Queries .GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors, timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds, timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds, query.ShowTimings, cancellationToken); Header = new QueryHeaderInformation { Index = indexName, IsStale = stale, ResultEtag = resultEtag, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, TotalResults = query.TotalSize.Value }; }
public Query GetLuceneQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.Debug("Query on non existing index {0}", index); throw new InvalidOperationException("Index '" + index + "' does not exists"); } var fieldsToFetch = new FieldsToFetch(new string[0], AggregationOperation.None, null); return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetLuceneQuery(); }
public IndexQueryOperation( Index parent, IndexQuery indexQuery, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch) { this.parent = parent; this.indexQuery = indexQuery; this.shouldIncludeInResults = shouldIncludeInResults; this.fieldsToFetch = fieldsToFetch; if (fieldsToFetch.IsDistinctQuery) alreadyReturned = new HashSet<JObject>(new JTokenEqualityComparer()); }