public RemoteQueryResults Query(LinearQuery query) { var viewGenerator = queryCache.GetOrAdd(query.Query, s => new DynamicViewCompiler("query", new IndexDefinition { Map = query.Query, }, new AbstractDynamicCompilationExtension[0]) { RequiresSelectNewAnonymousType = false }.GenerateInstance()); var results = new List<string>(); var errors = new List<string>(); int lastResult = 0; int finalResult = 0; remoteStorage.Batch(actions => { var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds(); finalResult = firstAndLastDocumentIds.Item2; var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start); var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2); if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization { matchingDocs = matchingDocs.Where(x => x.Item1.Metadata.Value<string>("Raven-Entity-Name") == viewGenerator.ForEntityName); } var docs = matchingDocs .Select(x => { lastResult = x.Item2; return new DynamicJsonObject(x.Item1.ToJson()); }); var robustEnumerator = new RobustEnumerator { OnError = (exception, o) => errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results.AddRange( robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition) .Take(query.PageSize) .Select(result => JsonExtensions.ToJObject(result).ToString()) ); }); return new RemoteQueryResults { LastScannedResult = lastResult, TotalResults = finalResult, Errors = errors.ToArray(), QueryCacheSize = queryCache.Count, Results = results.ToArray() }; }
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, Action<double> loadingDocumentsFinish, Action<double> transformerFinish, bool showTimings, CancellationToken token) { if (query.PageSize <= 0) // maybe they just want the stats? { return Enumerable.Empty<RavenJObject>(); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; else throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } if (transformFunc == null) { var resultsWithoutTransformer = results.Select(x => { var ravenJObject = x.ToJson(); if (query.IsDistinct) { ravenJObject[Constants.DocumentIdFieldName] = x.Key; } return ravenJObject; }); return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer; } var dynamicJsonObjects = results.Select(x => { var ravenJObject = x.ToJson(); if (query.IsDistinct) { ravenJObject[Constants.DocumentIdFieldName] = x.Key; } return new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject); }); var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; var resultsWithTransformer = robustEnumerator .RobustEnumeration(dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer; }
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors) { if (query.PageSize <= 0) // maybe they just want the stats? { return Enumerable.Empty<RavenJObject>(); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (query.ResultsTransformer != null) { var transformGenerator = IndexDefinitionStorage.GetTransfomer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; } else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source); } if (transformFunc == null) return results.Select(x => x.ToJson()); var dynamicJsonObjects = results.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; return robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); }
public QueryResult Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new InvalidOperationException("Could not find index named: " + index); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers) select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) into doc where doc != null select doc; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects, source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } if (query.PageSize > 0) // maybe they just want the query stats? list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResult { IndexName = index, Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2 }; }
public QueryResultWithIncludes Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List<RavenJObject>(); var highlightings = new Dictionary<string, Dictionary<string, string[]>>(); var stale = false; Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty); Guid resultEtag = Guid.Empty; var nonAuthoritativeInformation = false; var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); TransactionalStorage.Batch( actions => { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new IndexDoesNotExistsException("Could not find index named: " + index); resultEtag = GetIndexEtag(index, null); stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Constants.DocumentIdFieldName : Constants.ReduceKeyFieldName); Func<IndexQueryResult, bool> shouldIncludeInResults = result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch); var collection = from queryResult in IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers) select new { Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch), Fragments = queryResult.Highligtings } into docWithFragments where docWithFragments.Document != null let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false) select docWithFragments; var transformerErrors = new List<string>(); IEnumerable<RavenJObject> results; if (query.SkipTransformResults == false && query.PageSize > 0 && // maybe they just want the stats? viewGenerator.TransformResultsDefinition != null) { var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray(); var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { var resultList = new List<RavenJObject>(); foreach (var docWithFragments in collection) { resultList.Add(docWithFragments.Document.ToJson()); if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null) highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments); } results = resultList; } list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResultWithIncludes { IndexName = index, Results = list, IsStale = stale, NonAuthoritativeInformation = nonAuthoritativeInformation, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2, ResultEtag = resultEtag, IdsToInclude = idsToLoad, LastQueryTime = SystemTime.UtcNow, Highlightings = highlightings }; }
private IEnumerable<RavenJObject> ProcessResults(MoreLikeThisQuery query, IEnumerable<JsonDocument> documents, CancellationToken token) { IndexingFunc transformFunc = null; if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = database.IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; else throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } IEnumerable<RavenJObject> results; var transformerErrors = new List<string>(); if (transformFunc == null) results = documents.Select(x => x.ToJson()); else { var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator .RobustEnumeration(documents.Select(x => new DynamicJsonObject(x.ToJson())).GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); } return results; }
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, CancellationToken token) { if (query.PageSize <= 0) // maybe they just want the stats? { return Enumerable.Empty<RavenJObject>(); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) transformFunc = transformGenerator.TransformResultsDefinition; else throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source); } if (transformFunc == null) return results.Select(x => x.ToJson()); var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson())); var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; return robustEnumerator.RobustEnumeration( dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); }
public QueryResult Query(string index, IndexQuery query) { var list = new List<JObject>(); var stale = false; DateTime indexTimestamp = DateTime.MinValue; TransactionalStorage.Batch( actions => { string entityName = null; var abstractViewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (abstractViewGenerator != null) entityName = abstractViewGenerator.ForEntityName; stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, query.FieldsToFetch)) select docRetriever.RetrieveDocumentForQuery(queryResult, query.FieldsToFetch) into doc where doc != null select doc; var transformerErrors = new List<string>(); IEnumerable<JObject> results; if (abstractViewGenerator != null && abstractViewGenerator.TransformResultsDefinition != null) { var robustEnumerator = new RobustEnumerator { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; results = robustEnumerator.RobustEnumeration( collection.Select(x => new DynamicJsonObject(x.ToJson())), source => abstractViewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } list.AddRange(results); if(transformerErrors.Count>0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return new QueryResult { Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp }; }