Ejemplo n.º 1
0
        public QueryResult Query(string index, IndexQuery query)
        {
            index = IndexDefinitionStorage.FixupIndexName(index);
            var list  = new List <JObject>();
            var stale = false;
            Tuple <DateTime, Guid> indexTimestamp = null;

            TransactionalStorage.Batch(
                actions =>
            {
                string entityName = null;


                var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
                if (viewGenerator == null)
                {
                    throw new InvalidOperationException("Could not find index named: " + index);
                }

                entityName = viewGenerator.ForEntityName;

                stale          = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
                indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
                var indexFailureInformation = actions.Indexing.GetFailureRate(index);
                if (indexFailureInformation.IsInvalidIndex)
                {
                    throw new IndexDisabledException(indexFailureInformation);
                }
                var docRetriever    = new DocumentRetriever(actions, ReadTriggers);
                var indexDefinition = GetIndexDefinition(index);
                var fieldsToFetch   = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
                                                        viewGenerator.ReduceDefinition == null
                                                                                ? Abstractions.Data.Constants.DocumentIdFieldName
                                                                                : Abstractions.Data.Constants.ReduceKeyFieldName);
                var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
                                 select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
                                 into doc
                                 where doc != null
                                 select doc;

                var transformerErrors = new List <string>();
                IEnumerable <JObject> results;
                if (viewGenerator != null &&
                    query.SkipTransformResults == false &&
                    viewGenerator.TransformResultsDefinition != null)
                {
                    var robustEnumerator = new RobustEnumerator
                    {
                        OnError =
                            (exception, o) =>
                            transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
                                                                exception.Message))
                    };
                    var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
                    results =
                        robustEnumerator.RobustEnumeration(
                            dynamicJsonObjects,
                            source => viewGenerator.TransformResultsDefinition(docRetriever, source))
                        .Select(JsonExtensions.ToJObject);
                }
                else
                {
                    results = collection.Select(x => x.ToJson());
                }

                list.AddRange(results);

                if (transformerErrors.Count > 0)
                {
                    throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
                }
            });
            return(new QueryResult
            {
                IndexName = index,
                Results = list,
                IsStale = stale,
                SkippedResults = query.SkippedResults.Value,
                TotalResults = query.TotalSize.Value,
                IndexTimestamp = indexTimestamp.Item1,
                IndexEtag = indexTimestamp.Item2
            });
        }
Ejemplo n.º 2
0
		public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult)
		{
			index = IndexDefinitionStorage.FixupIndexName(index);
			var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
			Func<IndexQueryResult, object> tryRecordHighlighting = queryResult =>
			{
				if (queryResult.Highligtings != null && queryResult.Key != null)
					highlightings.Add(queryResult.Key, queryResult.Highligtings);
				return null;
			};
			var stale = false;
			Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
			Etag resultEtag = Etag.Empty;
			var nonAuthoritativeInformation = false;

			if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
			{
				query.FieldsToFetch = new[] { Constants.AllFields };
			}

			var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
			TransactionalStorage.Batch(
				actions =>
				{
					var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
					if (viewGenerator == null)
						throw new IndexDoesNotExistsException("Could not find index named: " + index);

					resultEtag = GetIndexEtag(index, null);

					stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);

					indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
					var indexFailureInformation = actions.Indexing.GetFailureRate(index);
					if (indexFailureInformation.IsInvalidIndex)
					{
						throw new IndexDisabledException(indexFailureInformation);
					}
					var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad);
					var indexDefinition = GetIndexDefinition(index);
					var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
														  viewGenerator.ReduceDefinition == null
															? Constants.DocumentIdFieldName
															: Constants.ReduceKeyFieldName);
					Func<IndexQueryResult, bool> shouldIncludeInResults =
						result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
					var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers);
					indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);

					var transformerErrors = new List<string>();
					var results = GetQueryResults(query, viewGenerator, docRetriever,
					                              from queryResult in indexQueryResults
					                              let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
					                              where doc != null
					                              let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
					                              let __ = tryRecordHighlighting(queryResult)
					                              select doc, transformerErrors);

					if (headerInfo != null)
					{
						headerInfo(new QueryHeaderInformation
						{
							Index = index,
							IsStable = stale,
							ResultEtag = resultEtag,
							IndexTimestamp = indexTimestamp.Item1,
							IndexEtag = indexTimestamp.Item2,
							TotalResults = query.TotalSize.Value
						});
					}
					using (new CurrentTransformationScope(docRetriever))
					{
						foreach (var result in results)
						{
							onResult(result);
						}

						if (transformerErrors.Count > 0)
						{
							throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
						}

					}


				});
			return new QueryResultWithIncludes
			{
				IndexName = index,
				IsStale = stale,
				NonAuthoritativeInformation = nonAuthoritativeInformation,
				SkippedResults = query.SkippedResults.Value,
				TotalResults = query.TotalSize.Value,
				IndexTimestamp = indexTimestamp.Item1,
				IndexEtag = indexTimestamp.Item2,
				ResultEtag = resultEtag,
				IdsToInclude = idsToLoad,
				LastQueryTime = SystemTime.UtcNow,
				Highlightings = highlightings
			};
		}
Ejemplo n.º 3
0
	    public QueryResult Query(string index, IndexQuery query)
		{
			index = IndexDefinitionStorage.FixupIndexName(index);
			var list = new List<RavenJObject>();
			var stale = false;
	    	Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
			TransactionalStorage.Batch(
				actions =>
				{

					var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
					if (viewGenerator == null)
						throw new InvalidOperationException("Could not find index named: " + index);

					stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
					indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
					var indexFailureInformation = actions.Indexing.GetFailureRate(index);
					if (indexFailureInformation.IsInvalidIndex)
					{
						throw new IndexDisabledException(indexFailureInformation);
					}
					var docRetriever = new DocumentRetriever(actions, ReadTriggers);
					var indexDefinition = GetIndexDefinition(index);
					var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
					                                      viewGenerator.ReduceDefinition == null
					                                      	? Constants.DocumentIdFieldName
					                                      	: Constants.ReduceKeyFieldName);
					var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers)
									 select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
										 into doc
										 where doc != null
										 select doc;

					var transformerErrors = new List<string>();
					IEnumerable<RavenJObject> results;
					if (query.SkipTransformResults == false && 
						query.PageSize > 0 && // maybe they just want the stats?
						viewGenerator.TransformResultsDefinition != null)
					{
						var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
						var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length)
						{
							OnError =
								(exception, o) =>
								transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
														 exception.Message))
						};
						results =
							robustEnumerator.RobustEnumeration(
								dynamicJsonObjects,
								source => viewGenerator.TransformResultsDefinition(docRetriever, source))
								.Select(JsonExtensions.ToJObject);
					}
					else
					{
						results = collection.Select(x => x.ToJson());
					}

					if (query.PageSize > 0) // maybe they just want the query stats?
						list.AddRange(results);

					if (transformerErrors.Count > 0)
					{
						throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
					}

				});
			return new QueryResult
			{
				IndexName = index,
				Results = list,
				IsStale = stale,
				SkippedResults = query.SkippedResults.Value,
				TotalResults = query.TotalSize.Value,
				IndexTimestamp = indexTimestamp.Item1,
				IndexEtag = indexTimestamp.Item2
			};
		}
Ejemplo n.º 4
0
            public void Init()
            {
                highlightings     = new Dictionary <string, Dictionary <string, string[]> >();
                scoreExplanations = new Dictionary <string, string>();
                Func <IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
                {
                    if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
                    {
                        highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
                    }
                    if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
                    {
                        scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
                    }
                    return(null);
                };

                stale          = false;
                indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
                resultEtag     = Etag.Empty;
                nonAuthoritativeInformation = false;

                if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
                    (query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
                {
                    query.FieldsToFetch = new[] { Constants.AllFields };
                }

                duration  = Stopwatch.StartNew();
                idsToLoad = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

                var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                var index         = database.IndexDefinitionStorage.GetIndexDefinition(indexName);

                if (viewGenerator == null)
                {
                    throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
                }

                resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);

                stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);

                if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
                {
                    var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
                    stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
                }

                if (stale &&
                    actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
                    actions.Staleness.IsReduceStale(index.IndexId) == false)
                {
                    var forEntityNames  = viewGenerator.ForEntityNames.ToList();
                    var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;

                    if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
                    {
                        stale = false;
                    }
                }

                indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
                var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);

                if (indexFailureInformation.IsInvalidIndex)
                {
                    throw new IndexDisabledException(indexFailureInformation);
                }
                docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad);
                var fieldsToFetch = new FieldsToFetch(query,
                                                      viewGenerator.ReduceDefinition == null
                                                ? Constants.DocumentIdFieldName
                                                : Constants.ReduceKeyFieldName);
                Func <IndexQueryResult, bool> shouldIncludeInResults =
                    result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
                var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults, fieldsToFetch, database.IndexQueryTriggers, cancellationToken);

                indexQueryResults = new ActiveEnumerable <IndexQueryResult>(indexQueryResults);

                if (query.ShowTimings)
                {
                    indexQueryResults = new TimedEnumerable <IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] = timeInMilliseconds);
                }

                var docs = from queryResult in indexQueryResults
                           let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
                                     where doc != null
                                     let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
                                                                            let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
                                                                                     select doc;

                transformerErrors = new List <string>();
                results           = database
                                    .Queries
                                    .GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
                                                     timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments]    = timeInMilliseconds,
                                                     timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
                                                     query.ShowTimings, cancellationToken);

                Header = new QueryHeaderInformation
                {
                    Index          = indexName,
                    IsStale        = stale,
                    ResultEtag     = resultEtag,
                    IndexTimestamp = indexTimestamp.Item1,
                    IndexEtag      = indexTimestamp.Item2,
                    TotalResults   = query.TotalSize.Value
                };
            }
Ejemplo n.º 5
0
		public QueryResultWithIncludes Query(string index, IndexQuery query)
		{
			index = IndexDefinitionStorage.FixupIndexName(index);
			var list = new List<RavenJObject>();
			var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
			var stale = false;
			Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
			Guid resultEtag = Guid.Empty;
			var nonAuthoritativeInformation = false;
			var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
			TransactionalStorage.Batch(
				actions =>
				{
					var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
					if (viewGenerator == null)
						throw new IndexDoesNotExistsException("Could not find index named: " + index);

					resultEtag = GetIndexEtag(index, null);

					stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);

					indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
					var indexFailureInformation = actions.Indexing.GetFailureRate(index);
					if (indexFailureInformation.IsInvalidIndex)
					{
						throw new IndexDisabledException(indexFailureInformation);
					}
					var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad);
					var indexDefinition = GetIndexDefinition(index);
					var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
														  viewGenerator.ReduceDefinition == null
															? Constants.DocumentIdFieldName
															: Constants.ReduceKeyFieldName);
					Func<IndexQueryResult, bool> shouldIncludeInResults =
						result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
					var collection =
						from queryResult in
							IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers)
						select new
						{
							Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch),
							Fragments = queryResult.Highligtings
						}
						into docWithFragments
						where docWithFragments.Document != null
						let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false)
						select docWithFragments;

					var transformerErrors = new List<string>();
					IEnumerable<RavenJObject> results;
					if (query.SkipTransformResults == false &&
						query.PageSize > 0 && // maybe they just want the stats?
						viewGenerator.TransformResultsDefinition != null)
					{
						var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray();
						var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
						{
							OnError =
								(exception, o) =>
								transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
														 exception.Message))
						};
						results =
							robustEnumerator.RobustEnumeration(
								dynamicJsonObjects.Cast<object>().GetEnumerator(),
								source => viewGenerator.TransformResultsDefinition(docRetriever, source))
								.Select(JsonExtensions.ToJObject);
					}
					else
					{
						var resultList = new List<RavenJObject>();
						foreach (var docWithFragments in collection)
						{
							resultList.Add(docWithFragments.Document.ToJson());

							if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null)
								highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments);
						}
						results = resultList;
					}

					list.AddRange(results);

					if (transformerErrors.Count > 0)
					{
						throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
					}
				});
			return new QueryResultWithIncludes
			{
				IndexName = index,
				Results = list,
				IsStale = stale,
				NonAuthoritativeInformation = nonAuthoritativeInformation,
				SkippedResults = query.SkippedResults.Value,
				TotalResults = query.TotalSize.Value,
				IndexTimestamp = indexTimestamp.Item1,
				IndexEtag = indexTimestamp.Item2,
				ResultEtag = resultEtag,
				IdsToInclude = idsToLoad,
				LastQueryTime = SystemTime.UtcNow,
				Highlightings = highlightings
			};
		}
Ejemplo n.º 6
0
		public QueryResult Query(string index, IndexQuery query)
		{
			index = IndexDefinitionStorage.FixupIndexName(index);
			var list = new List<JObject>();
			var stale = false;
			Tuple<DateTime, Guid> indexTimestamp = null;
			TransactionalStorage.Batch(
				actions =>
				{
					string entityName = null;


					var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
					if (viewGenerator != null)
						entityName = viewGenerator.ForEntityName;

					stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
					indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
					var indexFailureInformation = actions.Indexing.GetFailureRate(index);
					if (indexFailureInformation.IsInvalidIndex)
					{
						throw new IndexDisabledException(indexFailureInformation);
					}
					var docRetriever = new DocumentRetriever(actions, ReadTriggers);
					var indexDefinition = GetIndexDefinition(index);
					var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, query.FieldsToFetch,query.AggregationOperation))
									 select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, query.FieldsToFetch, query.AggregationOperation)
										 into doc
										 where doc != null
										 select doc;

					var transformerErrors = new List<string>();
					IEnumerable<JObject> results;
					if (viewGenerator != null &&
						query.SkipTransformResults == false &&
						viewGenerator.TransformResultsDefinition != null)
					{
						var robustEnumerator = new RobustEnumerator
						{
							OnError =
								(exception, o) =>
								transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
														 exception.Message))
						};
						var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
						results =
							robustEnumerator.RobustEnumeration(
								dynamicJsonObjects,
								source => viewGenerator.TransformResultsDefinition(docRetriever, source))
								.Select(JsonExtensions.ToJObject);
					}
					else
					{
						results = collection.Select(x => x.ToJson());
					}

					list.AddRange(results);

					if (transformerErrors.Count > 0)
					{
						throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
					}

				});
			return new QueryResult
			{
				IndexName = index,
				Results = list,
				IsStale = stale,
				SkippedResults = query.SkippedResults.Value,
				TotalResults = query.TotalSize.Value,
				IndexTimestamp = indexTimestamp.Item1,
				IndexEtag = indexTimestamp.Item2
			};
		}