public static List<DynamicQueryOptimizer.Explanation> ExplainDynamicIndexSelection(this DocumentDatabase self, string entityName, IndexQuery query)
		{
			var explanations = new List<DynamicQueryOptimizer.Explanation>();
			new DynamicQueryOptimizer(self)
				.SelectAppropriateIndex(entityName, query.Clone(), explanations);
			return explanations;
		}
Example #2
0
		private QueryResult ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery)
		{
			// Perform the query until we have some results at least
			QueryResult result;
			var sp = Stopwatch.StartNew();
			while (true)
			{
				result = documentDatabase.Query(map.IndexName,
				                                new IndexQuery
				                                {
				                                	Cutoff = query.Cutoff,
				                                	PageSize = query.PageSize,
				                                	Query = realQuery,
				                                	Start = query.Start,
				                                	FieldsToFetch = query.FieldsToFetch,
				                                	GroupBy = query.GroupBy,
				                                	AggregationOperation = query.AggregationOperation,
				                                	SortedFields = query.SortedFields,
				                                });

				if (!touchTemporaryIndexResult.Item2 ||
				    !result.IsStale ||
				    result.Results.Count >= query.PageSize ||
				    sp.Elapsed.TotalSeconds > 15)
				{
					return result;
				}

				Thread.Sleep(100);
			}
		}
		public static string FindDynamicIndexName(this DocumentDatabase self, string entityName, IndexQuery query)
		{
            var result = new DynamicQueryOptimizer(self).SelectAppropriateIndex(entityName, query.Clone());
		    if (result.MatchType == DynamicQueryMatchType.Complete)
		        return result.IndexName;
		    return null;
		}
Example #4
0
        public QueryResultWithIncludes Query(string index, IndexQuery query, CancellationToken externalCancellationToken)
        {
            QueryResultWithIncludes result = null;
            using (var cts = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, WorkContext.CancellationToken))
            {
                var cancellationToken = cts.Token;

                TransactionalStorage.Batch(
                    accessor =>
                    {
                        using (var op = new DatabaseQueryOperation(Database, index, query, accessor, cancellationToken)
                        {
                            ShouldSkipDuplicateChecking = query.SkipDuplicateChecking
                        })
                        {
                            var list = new List<RavenJObject>();
                            op.Init();
                            op.Execute(list.Add);
                            op.Result.Results = list;
                            result = op.Result;
                        }
                    });
            }

            return result;
        }
Example #5
0
        public static IndexQuery GetIndexQueryFromHttpContext(this IHttpContext context, int maxPageSize)
        {
            var query = new IndexQuery
            {
                Query = context.Request.QueryString["query"] ?? "",
                Start = context.GetStart(),
                Cutoff = context.GetCutOff(),
                PageSize = context.GetPageSize(maxPageSize),
                SkipTransformResults = context.GetSkipTransformResults(),
                FieldsToFetch = context.Request.QueryString.GetValues("fetch"),
				GroupBy = context.Request.QueryString.GetValues("groupBy"),
				AggregationOperation = context.GetAggregationOperation(),
                SortedFields = context.Request.QueryString.GetValues("sort")
                    .EmptyIfNull()
                    .Select(x => new SortedField(x))
                    .ToArray()
            };

            double lat = context.GetLat(), lng = context.GetLng(), radius = context.GetRadius();
            if (lat != 0 || lng != 0 || radius != 0)
            {
                return new SpatialIndexQuery(query)
                {
                    Latitude = lat,
                    Longitude = lng,
                    Radius = radius,
                };
            }
            return query;
        }
Example #6
0
 public ExecutingQueryInfo(DateTime startTime, IndexQuery queryInfo, long queryId, CancellationTokenSource tokenSource)
 {
     StartTime = startTime;
     QueryInfo = queryInfo;
     QueryId = queryId;
     stopwatch = Stopwatch.StartNew();
     TokenSource = tokenSource;
 }
		public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, bool allowStale)
		{
			return PerformBulkOperation(indexName, queryToDelete, allowStale, (docId, tx) =>
			{
				database.Delete(docId, null, tx);
				return new { Document = docId, Deleted = true };
			});
		}
		public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, PatchRequest[] patchRequests, bool allowStale)
		{
			return PerformBulkOperation(indexName, queryToUpdate, allowStale, (docId, tx) =>
			{
				var patchResult = database.ApplyPatch(docId, null, patchRequests, tx);
				return new { Document = docId, Result = patchResult };
			});
		}
        public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null)
        {
            return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) =>
			{
				database.Documents.Delete(docId, null, tx);
				return new { Document = docId, Deleted = true };
			});
		}
        public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null)
		{
            return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) =>
			{
				var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx);
				return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 };
			});
		}
        // GET: Campaign
        public async Task<IActionResult> Index()
        {
            var query = new IndexQuery();
            if (User.IsUserType(UserType.OrgAdmin))
            {
                query.OrganizationId = User.GetOrganizationId();
            }

            return View(await _mediator.SendAsync(query));
        }
Example #12
0
		private static void UpdateFieldNamesForSortedFields(IndexQuery query, DynamicQueryMapping map)
		{
			if (query.SortedFields == null) return;
			foreach (var sortedField in query.SortedFields)
			{
				var item = map.Items.FirstOrDefault(x => x.From == sortedField.Field);
				if (item != null)
					sortedField.Field = item.To;
			}
		}
Example #13
0
        private static IndexQuery CreateIndexQuery(IndexQuery query, DynamicQueryMapping map, string realQuery)
        {
            var indexQuery = new IndexQuery
            {
                Cutoff = query.Cutoff,
                WaitForNonStaleResultsAsOfNow = query.WaitForNonStaleResultsAsOfNow,
                PageSize = query.PageSize,
                Query = realQuery,
                Start = query.Start,
                FieldsToFetch = query.FieldsToFetch,
                IsDistinct = query.IsDistinct,
                SortedFields = query.SortedFields,
                DefaultField = query.DefaultField,
                CutoffEtag = query.CutoffEtag,
                DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries,
                DefaultOperator = query.DefaultOperator,
                SkippedResults = query.SkippedResults,
                HighlighterPreTags = query.HighlighterPreTags,
                HighlighterPostTags = query.HighlighterPostTags,
                HighlightedFields = query.HighlightedFields,
                HighlighterKeyName = query.HighlighterKeyName,
                ResultsTransformer = query.ResultsTransformer,
                TransformerParameters = query.TransformerParameters,
                ExplainScores = query.ExplainScores,
                SortHints = query.SortHints,
                ShowTimings = query.ShowTimings,
                AllowMultipleIndexEntriesForSameDocumentToResultTransformer = query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer,
                DisableCaching = query.DisableCaching,
                SkipDuplicateChecking = query.SkipDuplicateChecking,
                WaitForNonStaleResults = query.WaitForNonStaleResults
                
            };
            if (indexQuery.SortedFields == null)
                return indexQuery;

            for (int index = 0; index < indexQuery.SortedFields.Length; index++)
            {
                var sortedField = indexQuery.SortedFields[index];
                var fieldName = sortedField.Field;
                bool hasRange = false;
                if (fieldName.EndsWith("_Range"))
                {
                    fieldName = fieldName.Substring(0, fieldName.Length - "_Range".Length);
                    hasRange = true;
                }
            
                var item = map.Items.FirstOrDefault(x => string.Equals(x.QueryFrom, fieldName, StringComparison.InvariantCultureIgnoreCase));
                if (item == null)
                    continue;

                indexQuery.SortedFields[index] = new SortedField(hasRange ? item.To + "_Range" : item.To);
                indexQuery.SortedFields[index].Descending = sortedField.Descending;
            }
            return indexQuery;
        }
Example #14
0
		public void can_encode_and_decode_IndexQuery_Start() 
		{
			int expected = Some.Integer();
			var indexQuery = new IndexQuery();

			indexQuery.Start = expected;

			IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery);

			Assert.Equal(expected, result.Start);
		}
Example #15
0
		public void can_encode_and_decode_IndexQuery_PageSize()
		{
			var expected = Some.Integer();
			var indexQuery = new IndexQuery();

			indexQuery.PageSize = expected;

			IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery);

			Assert.Equal(expected, result.PageSize);
		}
Example #16
0
		public IEnumerable<string> QueryDocumentIds(string index, IndexQuery query, CancellationTokenSource tokenSource, out bool stale)
		{
			var queryStat = AddToCurrentlyRunningQueryList(index, query, tokenSource);
			try
			{
				bool isStale = false;
				HashSet<string> loadedIds = null;
				TransactionalStorage.Batch(
					actions =>
					{
						var definition = IndexDefinitionStorage.GetIndexDefinition(index);
						if (definition == null)
							throw new ArgumentException("specified index definition was not found", "index");

						isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);

						if (isStale == false && query.Cutoff == null)
						{
							var indexInstance = Database.IndexStorage.GetIndexInstance(index);
							isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
						}

						if (isStale && actions.Staleness.IsIndexStaleByTask(definition.IndexId, query.Cutoff) == false &&
							actions.Staleness.IsReduceStale(definition.IndexId) == false)
						{
							var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
							if (viewGenerator == null)
								throw new ArgumentException("specified index definition was not found", "index");

							var forEntityNames = viewGenerator.ForEntityNames.ToList();
							var lastIndexedEtag = actions.Indexing.GetIndexStats(definition.IndexId).LastIndexedEtag;

							if (Database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
								isStale = false;
						}

						var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);

						if (indexFailureInformation.IsInvalidIndex)
						{
							throw new IndexDisabledException(indexFailureInformation);
						}
						loadedIds = new HashSet<string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, tokenSource.Token)
														select queryResult.Key);
					});
				stale = isStale;
				return loadedIds;
			}
			finally
			{
				RemoveFromCurrentlyRunningQueryList(index, queryStat);
			}
		}
Example #17
0
		public void can_encode_and_decode_IndexQuery_Query_pound() 
		{
			var expected = Some.String() + '#' + Some.String();

			var indexQuery = new IndexQuery();

			indexQuery.Query = expected;

			IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery);

			Assert.Equal(expected, result.Query);
		}
Example #18
0
		private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
		{
			var array = new RavenJArray();
			var bulkIndexQuery = new IndexQuery
			{
				Query = indexQuery.Query,
				Start = indexQuery.Start,
				Cutoff = indexQuery.Cutoff,
                WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
				PageSize = int.MaxValue,
				FieldsToFetch = new[] { Constants.DocumentIdFieldName },
				SortedFields = indexQuery.SortedFields,
				HighlighterPreTags = indexQuery.HighlighterPreTags,
				HighlighterPostTags = indexQuery.HighlighterPostTags,
				HighlightedFields = indexQuery.HighlightedFields,
				SortHints = indexQuery.SortHints
			};

			bool stale;
            var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);

			if (stale && allowStale == false)
			{
				throw new InvalidOperationException(
						"Bulk operation cancelled because the index is stale and allowStale is false");
			}

		    var token = tokenSource.Token;

			const int batchSize = 1024;
			using (var enumerator = queryResults.GetEnumerator())
			{
				while (true)
				{
					if (timeout != null)
						timeout.Delay();
					var batchCount = 0;
                    token.ThrowIfCancellationRequested();
					database.TransactionalStorage.Batch(actions =>
					{
						while (batchCount < batchSize && enumerator.MoveNext())
						{
							batchCount++;
							var result = batchOperation(enumerator.Current, transactionInformation);
							array.Add(RavenJObject.FromObject(result));
						}
					});
					if (batchCount < batchSize) break;
				}
			}
			return array;
		}
Example #19
0
		public QueryResult ExecuteDynamicQuery(string entityName, IndexQuery query)
		{
			// Create the map
			var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);
			
			var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);

			map.IndexName = touchTemporaryIndexResult.Item1;
			// Re-write the query
			string realQuery = map.Items.Aggregate(query.Query,
												   (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));

			return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery);
		}
Example #20
0
		private Tuple<string, bool> GetAppropriateIndexToQuery(string entityName, IndexQuery query, DynamicQueryMapping map)
		{
			var appropriateIndex = new DynamicQueryOptimizer(documentDatabase).SelectAppropriateIndex(entityName, query);
			if (appropriateIndex != null)
			{
				if (appropriateIndex.StartsWith("Temp/"))// temporary index, we need to increase its usage
				{
					return  TouchTemporaryIndex(appropriateIndex, "Auto/" + appropriateIndex.Substring(5),
					                                                () => documentDatabase.IndexDefinitionStorage.GetIndexDefinition(appropriateIndex));
				}
				return Tuple.Create(appropriateIndex, false);
			}
			return TouchTemporaryIndex(map.TemporaryIndexName, map.PermanentIndexName,
			                                                map.CreateIndexDefinition);
		}
        public async Task ReturnTheCampaignsTheUserIsAnOrgAdminFor_WhenOrganizationIdOnMessageIsNotNull()
        {
            const int orgId1 = 1;
            const int orgId2 = 2;

            Context.Campaigns.Add(new Campaign { Id = 1, ManagingOrganization = new Organization { Id = orgId1 }});
            Context.Campaigns.Add(new Campaign { Id = 2, ManagingOrganization = new Organization { Id = orgId2 }});
            Context.SaveChanges();

            var message = new IndexQuery { OrganizationId = orgId1 };

            var sut = new IndexQueryHandler(Context);
            var result = await sut.Handle(message);

            Assert.Equal(result.Single().Id, orgId1);
        }
Example #22
0
		public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query, CancellationToken token)
		{
			// Create the map
			var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);
			var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);

			string realQuery = map.Items.Aggregate(query.Query,
												   (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));

			UpdateFieldNamesForSortedFields(query, map);

			// We explicitly do NOT want to update the field names of FieldsToFetch - that reads directly from the document
			//UpdateFieldsInArray(map, query.FieldsToFetch);
			
			return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery, token);
		}
Example #23
0
		public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query)
		{
			// Create the map
			var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);

			var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);

			map.IndexName = touchTemporaryIndexResult.Item1;
			// Re-write the query
			string realQuery = map.Items.Aggregate(query.Query,
												   (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));

			UpdateFieldNamesForSortedFields(query, map);
			UpdateFieldsInArray(map, query.FieldsToFetch);
			UpdateFieldsInArray(map, query.GroupBy);

			return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery);
		}
Example #24
0
		public void can_encode_and_decode_IndexQuery_Query() 
		{
			//  Fails when at least '&' is in the Query, not sure if that is acceptable
			//  Fails because the value has not by url decoded, I couldn't find code doing the url decode
			//  after GetIndexQueryFromHttpContext() so there may be another bug.

			//var expected = new string(Enumerable.Range(0, 255).Select(i => (char)i)
			//    .Where(c => !Char.IsControl(c)).ToArray());

			var expected = Some.String();

			var indexQuery = new IndexQuery();

			indexQuery.Query = expected;

			IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery);

			Assert.Equal(expected, result.Query);
		}
Example #25
0
		private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery)
		{
			// Perform the query until we have some results at least
			QueryResultWithIncludes result;
			var sp = Stopwatch.StartNew();
			while (true)
			{
				result = documentDatabase.Query(map.IndexName,
												new IndexQuery
												{
													Cutoff = query.Cutoff,
													PageSize = query.PageSize,
													Query = realQuery,
													Start = query.Start,
													FieldsToFetch = query.FieldsToFetch,
													GroupBy = query.GroupBy,
													AggregationOperation = query.AggregationOperation,
													SortedFields = query.SortedFields,
													DefaultField = query.DefaultField,
													CutoffEtag = query.CutoffEtag,
													DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries,
													DefaultOperator = query.DefaultOperator,
													SkipTransformResults = query.SkipTransformResults,
													SkippedResults = query.SkippedResults,
													HighlighterPreTags = query.HighlighterPreTags,
													HighlighterPostTags = query.HighlighterPostTags,
													HighlightedFields = query.HighlightedFields,
                                                    ResultsTransformer = query.ResultsTransformer,
                                                    QueryInputs = query.QueryInputs
												});

				if (!touchTemporaryIndexResult.Item2 ||
					!result.IsStale ||
					(result.Results.Count >= query.PageSize && query.PageSize > 0) ||
					sp.Elapsed.TotalSeconds > 15)
				{
					return result;
				}

				Thread.Sleep(100);
			}
		}
		private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation)
		{
			var array = new RavenJArray();
			var bulkIndexQuery = new IndexQuery
			{
				Query = indexQuery.Query,
				Start = indexQuery.Start,
				Cutoff = indexQuery.Cutoff,
				PageSize = int.MaxValue,
				FieldsToFetch = new[] { Constants.DocumentIdFieldName },
				SortedFields = indexQuery.SortedFields
			};

			bool stale;
			var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale);

			if (stale && allowStale == false)
			{
				throw new InvalidOperationException(
						"Bulk operation cancelled because the index is stale and allowStale is false");
			}

			const int batchSize = 1024;
			using (var enumerator = queryResults.GetEnumerator())
			{
				while (true)
				{
					var batchCount = 0;
					database.TransactionalStorage.Batch(actions =>
					{
						while (batchCount < batchSize && enumerator.MoveNext())
						{
							batchCount++;
							var result = batchOperation(enumerator.Current, transactionInformation);
							array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer()));
						}
					});
					if (batchCount < batchSize) break;
				}
			}
			return array;
		}
Example #27
0
		private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery)
		{
			// Perform the query until we have some results at least
			QueryResultWithIncludes result;
			var sp = Stopwatch.StartNew();
			while (true)
			{
				var indexQuery = CreateIndexQuery(query, map, realQuery);
				result = documentDatabase.Query(map.IndexName, indexQuery);

				if (!touchTemporaryIndexResult.Item2 ||
					!result.IsStale ||
					(result.Results.Count >= query.PageSize && query.PageSize > 0) ||
					sp.Elapsed.TotalSeconds > 15)
				{
					return result;
				}

				Thread.Sleep(100);
			}
		}
Example #28
0
        public IEnumerable<string> QueryDocumentIds(string index, IndexQuery query, CancellationToken token, out bool stale)
        {
            var queryStat = AddToCurrentlyRunningQueryList(index, query);
            try
            {
                bool isStale = false;
                HashSet<string> loadedIds = null;
                TransactionalStorage.Batch(
                    actions =>
                    {
                        var definition = IndexDefinitionStorage.GetIndexDefinition(index);
                        if (definition == null)
                            throw new ArgumentException("specified index definition was not found", "index");

                        isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);

                        if (isStale == false && query.Cutoff == null)
                        {
                            var indexInstance = Database.IndexStorage.GetIndexInstance(index);
                            isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
                        }

                        var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);

                        if (indexFailureInformation.IsInvalidIndex)
                        {
                            throw new IndexDisabledException(indexFailureInformation);
                        }
                        loadedIds = new HashSet<string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, token)
                                                        select queryResult.Key);
                    });
                stale = isStale;
                return loadedIds;
            }
            finally
            {
                RemoveFromCurrentlyRunningQueryList(index, queryStat);
            }
        }
Example #29
0
        private void Create_DeleteByIndex_CreateWithTransactionScope_ReplicationToSecondaryFailsWithNonCurrentEtag(IDocumentStore primaryStore, IDocumentStore secondaryStore, string primaryDbName, string secondaryDbName)
        {
            var docId = "create-deletebyindex-createwithtransactionscope";

            Assert.Null(primaryStore.DatabaseCommands.ForDatabase(primaryDbName).Get(docId));
            Assert.Null(secondaryStore.DatabaseCommands.ForDatabase(secondaryDbName).Get(docId));

            using (var session = primaryStore.OpenSession(primaryDbName))
            {
                var doc = new Doc(docId, "first primaryStore");

                session.Store(doc, doc.Id);
                session.SaveChanges();
            }

            WaitForIndexing(primaryStore, primaryDbName);
            WaitForReplication(secondaryStore, docId, secondaryDbName);

            var databaseCommands = primaryStore.DatabaseCommands.ForDatabase(primaryDbName);

            using (var session = primaryStore.OpenSession(primaryDbName))
            {
                var deleteExistingDocsQuery = session.Advanced
                                              .DocumentQuery <Doc, DocsIndex>()
                                              .WhereIn("Id", new string[] { docId });

                var deleteExistingDocsIndexQuery = new IndexQuery {
                    Query = deleteExistingDocsQuery.ToString()
                };

                var deleteByIndex = databaseCommands.DeleteByIndex(new DocsIndex().IndexName, deleteExistingDocsIndexQuery, false);
                var array         = deleteByIndex.WaitForCompletion() as RavenJArray;

                Assert.Equal(1, array.Length);
                Assert.Equal(docId, array[0].Value <string>("Document"));
                Assert.Equal(true, array[0].Value <bool>("Deleted"));
            }

            WaitForReplication(secondaryStore, session => session.Load <Doc>(docId) == null, secondaryDbName);

            using (var transaction = new TransactionScope())
            {
                using (var session = primaryStore.OpenSession(primaryDbName))
                {
                    // insert with same id again
                    var docAgain = new Doc(docId, "second primaryStore with transaction scope");

                    session.Store(docAgain, docAgain.Id);

                    session.SaveChanges();

                    transaction.Complete();
                }
            }

            WaitForDocument(primaryStore.DatabaseCommands.ForDatabase(primaryDbName), docId);

            WaitForDocument(secondaryStore.DatabaseCommands.ForDatabase(secondaryDbName), docId);

            using (var session = secondaryStore.OpenSession(secondaryDbName))
            {
                Assert.Equal("second primaryStore with transaction scope", session.Load <Doc>(docId).Description);
            }
        }
Example #30
0
        public QueryResult Query(string index, IndexQuery query)
        {
            index = IndexDefinitionStorage.FixupIndexName(index);
            var list  = new List <JObject>();
            var stale = false;
            Tuple <DateTime, Guid> indexTimestamp = null;

            TransactionalStorage.Batch(
                actions =>
            {
                string entityName = null;


                var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
                if (viewGenerator == null)
                {
                    throw new InvalidOperationException("Could not find index named: " + index);
                }

                entityName = viewGenerator.ForEntityName;

                stale          = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
                indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
                var indexFailureInformation = actions.Indexing.GetFailureRate(index);
                if (indexFailureInformation.IsInvalidIndex)
                {
                    throw new IndexDisabledException(indexFailureInformation);
                }
                var docRetriever    = new DocumentRetriever(actions, ReadTriggers);
                var indexDefinition = GetIndexDefinition(index);
                var fieldsToFetch   = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
                                                        viewGenerator.ReduceDefinition == null
                                                                                ? Abstractions.Data.Constants.DocumentIdFieldName
                                                                                : Abstractions.Data.Constants.ReduceKeyFieldName);
                var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
                                 select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
                                 into doc
                                 where doc != null
                                 select doc;

                var transformerErrors = new List <string>();
                IEnumerable <JObject> results;
                if (viewGenerator != null &&
                    query.SkipTransformResults == false &&
                    viewGenerator.TransformResultsDefinition != null)
                {
                    var robustEnumerator = new RobustEnumerator
                    {
                        OnError =
                            (exception, o) =>
                            transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
                                                                exception.Message))
                    };
                    var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
                    results =
                        robustEnumerator.RobustEnumeration(
                            dynamicJsonObjects,
                            source => viewGenerator.TransformResultsDefinition(docRetriever, source))
                        .Select(JsonExtensions.ToJObject);
                }
                else
                {
                    results = collection.Select(x => x.ToJson());
                }

                list.AddRange(results);

                if (transformerErrors.Count > 0)
                {
                    throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
                }
            });
            return(new QueryResult
            {
                IndexName = index,
                Results = list,
                IsStale = stale,
                SkippedResults = query.SkippedResults.Value,
                TotalResults = query.TotalSize.Value,
                IndexTimestamp = indexTimestamp.Item1,
                IndexEtag = indexTimestamp.Item2
            });
        }
Example #31
0
        public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery)
        {
            // There isn't much point for query optimizer of aggregation indexes
            // the main reason is that we must always aggregate on the same items, and using the same
            // aggregation. Therefore we can't reuse one aggregate index for another query.
            // We decline to suggest an index here and choose to use the default index created for this
            // sort of query, which is what we would have to choose anyway.
            if (indexQuery.AggregationOperation != AggregationOperation.None)
            {
                return(null);
            }

            if (string.IsNullOrEmpty(indexQuery.Query) &&                                   // we optimize for empty queries to use Raven/DocumentsByEntityName
                (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested
                database.IndexDefinitionStorage.Contains("Raven/DocumentsByEntityName"))    // and Raven/DocumentsByEntityName exists
            {
                if (string.IsNullOrEmpty(entityName) == false)
                {
                    indexQuery.Query = "Tag:" + entityName;
                }
                return("Raven/DocumentsByEntityName");
            }

            var fieldsQueriedUpon           = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2).ToArray();
            var normalizedFieldsQueriedUpon =
                fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceIndavlidCharactersForFields).ToArray();
            var distinctSelectManyFields = new HashSet <string>();

            foreach (var field in fieldsQueriedUpon)
            {
                var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                for (int i = 1; i < parts.Length; i++)
                {
                    distinctSelectManyFields.Add(string.Join(",", parts.Take(i)));
                }
            }

            return(database.IndexDefinitionStorage.IndexNames
                   .Where(indexName =>
            {
                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                if (abstractViewGenerator == null)                                 // there is no matching view generator
                {
                    return false;
                }

                if (abstractViewGenerator.ReduceDefinition != null)                                 // we can't choose a map/reduce index
                {
                    return false;
                }

                if (abstractViewGenerator.TransformResultsDefinition != null)                                // we can't choose an index with transform results
                {
                    return false;
                }

                if (abstractViewGenerator.HasWhereClause)                                 // without a where clause
                {
                    return false;
                }

                // we can't select an index that has SelectMany in it, because it result in invalid results when
                // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250
                // for indexes with internal projections, we use the exact match based on the generated index name
                // rather than selecting the optimal one
                // in order to handle that, we count the number of select many that would happen because of the query
                // and match it to the number of select many in the index
                if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count)
                {
                    return false;
                }

                if (entityName == null)
                {
                    if (abstractViewGenerator.ForEntityNames.Count != 0)
                    {
                        return false;
                    }
                }
                else
                {
                    if (abstractViewGenerator.ForEntityNames.Count != 1 ||                                     // we only allow indexes with a single entity name
                        abstractViewGenerator.ForEntityNames.Contains(entityName) == false)                    // for the specified entity name
                    {
                        return false;
                    }
                }

                if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false)
                {
                    return false;
                }

                var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
                if (indexDefinition == null)
                {
                    return false;
                }

                if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0)
                {
                    var sortInfo = DynamicQueryMapping.GetSortInfo(s => { });

                    foreach (var sortedField in indexQuery.SortedFields)                                     // with matching sort options
                    {
                        if (sortedField.Field.StartsWith(Constants.RandomFieldName))
                        {
                            continue;
                        }

                        // if the field is not in the output, then we can't sort on it.
                        if (abstractViewGenerator.ContainsField(sortedField.Field) == false)
                        {
                            return false;
                        }

                        var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field);

                        if (dynamicSortInfo == null)                                        // no sort order specified, we don't care, probably
                        {
                            continue;
                        }

                        SortOptions value;
                        if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false)
                        {
                            switch (dynamicSortInfo.FieldType)                                            // if we can't find the value, we check if we asked for the default sorting
                            {
                            case SortOptions.String:
                            case SortOptions.None:
                                continue;

                            default:
                                return false;
                            }
                        }

                        if (value != dynamicSortInfo.FieldType)
                        {
                            return false;                                             // different sort order, there is a problem here
                        }
                    }
                }

                if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0)
                {
                    // none of the fields have custom analyzers
                    if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey))
                    {
                        return false;
                    }
                }

                return true;
            })
                   .OrderByDescending(indexName =>
            {
                // We select the widest index, because we want to encourage bigger indexes
                // Over time, it means that we smaller indexes would wither away and die, while
                // bigger indexes will be selected more often and then upgrade to permanent indexes
                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                if (abstractViewGenerator == null)                                 // there is a matching view generator
                {
                    return -1;
                }
                return abstractViewGenerator.CountOfFields;
            })
                   .FirstOrDefault());
        }
Example #32
0
        private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery)
        {
            var indexEtag = GetIndexEtag(index);

            if (context.MatchEtag(indexEtag))
            {
                context.SetStatusToNotModified();
                return(null);
            }

            var queryResult = Database.Query(index, indexQuery);

            queryResult.IndexEtag = indexEtag;
            return(queryResult);
        }
Example #33
0
        private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag)
        {
            string entityName = null;

            if (index.StartsWith("dynamic/"))
            {
                entityName = index.Substring("dynamic/".Length);
            }

            var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery.Query);


            if (Database.IndexStorage.HasIndex(dynamicIndexName))
            {
                indexEtag = GetIndexEtag(dynamicIndexName);
                if (context.MatchEtag(indexEtag))
                {
                    context.SetStatusToNotModified();
                    return(null);
                }
            }
            else
            {
                indexEtag = Guid.Empty;
            }

            var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery);

            // have to check here because we might be getting the index etag just
            // as we make a switch from temp to auto, and we need to refresh the etag
            // if that is the case. This can also happen when the optmizer
            // decided to switch indexes for a query.
            if (queryResult.IndexName != dynamicIndexName)
            {
                indexEtag = GetIndexEtag(queryResult.IndexName);
            }

            return(queryResult);
        }
Example #34
0
            public DatabaseQueryOperation(DocumentDatabase database, string indexName, IndexQuery query, IStorageActionsAccessor actions, CancellationTokenSource cancellationTokenSource)
            {
                this.database  = database;
                this.indexName = indexName != null?indexName.Trim() : null;

                this.query        = query;
                this.actions      = actions;
                cancellationToken = cancellationTokenSource.Token;
                queryStat         = database.Queries.AddToCurrentlyRunningQueryList(indexName, query, cancellationTokenSource);

                if (query.ShowTimings == false)
                {
                    return;
                }

                executionTimes[QueryTimings.Lucene]           = 0;
                executionTimes[QueryTimings.LoadDocuments]    = 0;
                executionTimes[QueryTimings.TransformResults] = 0;
            }
Example #35
0
        private void GetIndexEntriesForDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, Reference <int> totalResults)
        {
            string entityName;
            var    dynamicIndexName = GetDynamicIndexName(index, indexQuery, out entityName);

            if (dynamicIndexName == null)
            {
                context.SetStatusToNotFound();
                return;
            }
            GetIndexEntriesForExistingIndex(context, dynamicIndexName, indexQuery, totalResults);
        }
 public LazySuggestionQueryOperation(InMemoryDocumentSessionOperations session, IndexQuery indexQuery, Action <QueryResult> invokeAfterQueryExecuted, Func <QueryResult, Dictionary <string, SuggestionResult> > processResults)
 {
     _session    = session;
     _indexQuery = indexQuery;
     _invokeAfterQueryExecuted = invokeAfterQueryExecuted;
     _processResults           = processResults;
 }
Example #37
0
 protected internal virtual IEnumerable <object> ApplyReduceFunctionIfExists(IndexQuery indexQuery, IEnumerable <object> enumerable)
 {
     return(enumerable.Take(indexQuery.PageSize));
 }
        public static SortOptions?GetSortOption(this IndexDefinition self, string name, IndexQuery query)
        {
            SortOptions value;

            if (name.EndsWith("_Range"))
            {
                string nameWithoutRange = name.Substring(0, name.Length - "_Range".Length);
                if (self.SortOptions.TryGetValue(nameWithoutRange, out value))
                {
                    return(value);
                }

                if (self.SortOptions.TryGetValue(Constants.AllFields, out value))
                {
                    return(value);
                }

                if (query != null && query.SortHints != null && query.SortHints.ContainsKey("SortHint-" + nameWithoutRange))
                {
                    return(query.SortHints["SortHint-" + nameWithoutRange]);
                }
            }

            if (self.SortOptions.TryGetValue(name, out value))
            {
                return(value);
            }
            if (self.SortOptions.TryGetValue(Constants.AllFields, out value))
            {
                return(value);
            }

            if (query == null || query.SortHints == null)
            {
                return(value);
            }

            if (query.SortHints.ContainsKey("SortHint-" + name) == false)
            {
                return(value);
            }

            return(query.SortHints["SortHint-" + name]);
        }
        private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation)
        {
            options = options ?? new BulkOperationOptions();
            var array          = new RavenJArray();
            var bulkIndexQuery = new IndexQuery
            {
                Query  = indexQuery.Query,
                Start  = indexQuery.Start,
                Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
                WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
                PageSize              = int.MaxValue,
                FieldsToFetch         = new[] { Constants.DocumentIdFieldName },
                SortedFields          = indexQuery.SortedFields,
                HighlighterPreTags    = indexQuery.HighlighterPreTags,
                HighlighterPostTags   = indexQuery.HighlighterPostTags,
                HighlightedFields     = indexQuery.HighlightedFields,
                SortHints             = indexQuery.SortHints,
                HighlighterKeyName    = indexQuery.HighlighterKeyName,
                TransformerParameters = indexQuery.TransformerParameters,
                ResultsTransformer    = indexQuery.ResultsTransformer
            };

            bool stale;
            var  queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);

            if (stale && options.AllowStale == false)
            {
                if (options.StaleTimeout != null)
                {
                    var staleWaitTimeout = Stopwatch.StartNew();
                    while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
                    {
                        queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
                        if (stale)
                        {
                            SystemTime.Wait(100);
                        }
                    }
                }
                if (stale)
                {
                    if (options.StaleTimeout != null)
                    {
                        throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout  of " + options.StaleTimeout + "passed");
                    }

                    throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
                }
            }

            var       token        = tokenSource.Token;
            const int batchSize    = 1024;
            int       maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;

            using (var enumerator = queryResults.GetEnumerator())
            {
                var duration   = Stopwatch.StartNew();
                var operations = 0;
                while (true)
                {
                    database.WorkContext.UpdateFoundWork();
                    if (timeout != null)
                    {
                        timeout.Delay();
                    }
                    var batchCount    = 0;
                    var shouldWaitNow = false;
                    token.ThrowIfCancellationRequested();
                    using (database.DocumentLock.Lock())
                    {
                        database.TransactionalStorage.Batch(actions =>
                        {
                            while (batchCount < batchSize && enumerator.MoveNext())
                            {
                                batchCount++;
                                operations++;
                                var result = batchOperation(enumerator.Current, transactionInformation);

                                if (options.RetrieveDetails)
                                {
                                    array.Add(RavenJObject.FromObject(result));
                                }

                                if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
                                {
                                    shouldWaitNow = true;
                                    break;
                                }
                            }
                        });
                        if (shouldWaitNow)
                        {
                            SystemTime.Wait(500);
                            operations = 0;
                            duration.Restart();
                            continue;
                        }
                    }
                    if (batchCount < batchSize)
                    {
                        break;
                    }
                }
            }
            return(array);
        }
Example #40
0
 public Operation UpdateByIndex(string indexName, IndexQuery queryToUpdate, PatchRequest patch,
                                QueryOperationOptions options = null)
 {
     return(AsyncHelpers.RunSync(() => asyncServerClient.UpdateByIndexAsync(indexName, queryToUpdate, patch, options)));
 }
Example #41
0
		private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag)
		{
			indexEtag = GetIndexEtag(index);
			if (context.MatchEtag(indexEtag))
			{
				context.SetStatusToNotModified();
				return null;
			}

			return Database.Query(index, indexQuery);
		}
Example #42
0
        public QueryOperation(InMemoryDocumentSessionOperations sessionOperations, string indexName, IndexQuery indexQuery,
                              string[] projectionFields, bool waitForNonStaleResults, TimeSpan timeout,
                              Func <IndexQuery, IEnumerable <object>, IEnumerable <object> > transformResults,
                              HashSet <string> includes, bool disableEntitiesTracking)
        {
            this.indexQuery             = indexQuery;
            this.waitForNonStaleResults = waitForNonStaleResults;
            this.timeout                 = timeout;
            this.transformResults        = transformResults;
            this.includes                = includes;
            this.projectionFields        = projectionFields;
            this.sessionOperations       = sessionOperations;
            this.indexName               = indexName;
            this.disableEntitiesTracking = disableEntitiesTracking;

            AssertNotQueryById();
        }
Example #43
0
        public FacetResults GetFacets(string index, IndexQuery indexQuery, List <Facet> facets, int start = 0, int?pageSize = null)
        {
            var sp            = Stopwatch.StartNew();
            var results       = new FacetResults();
            var defaultFacets = new Dictionary <string, Facet>();
            var rangeFacets   = new Dictionary <string, List <ParsedRange> >();

            var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(index);

            Index.AssertQueryDoesNotContainFieldsThatAreNotIndexed(indexQuery, viewGenerator);

            foreach (var facet in facets)
            {
                var key = string.IsNullOrWhiteSpace(facet.DisplayName) ? facet.Name : facet.DisplayName;

                defaultFacets[key] = facet;
                if (facet.Aggregation != FacetAggregation.Count && facet.Aggregation != FacetAggregation.None)
                {
                    if (string.IsNullOrEmpty(facet.AggregationField))
                    {
                        throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " +
                                                            facet.Aggregation + " without having a value in AggregationField");
                    }

                    if (facet.AggregationField.EndsWith("_Range") == false)
                    {
                        if (QueryForFacets.IsAggregationTypeNumerical(facet.AggregationType))
                        {
                            facet.AggregationField = facet.AggregationField + "_Range";
                        }
                    }
                }


                switch (facet.Mode)
                {
                case FacetMode.Default:
                    results.Results[key] = new FacetResult();
                    break;

                case FacetMode.Ranges:
                    rangeFacets[key]     = facet.Ranges.Select(range => ParseRange(facet.Name, range)).ToList();
                    results.Results[key] = new FacetResult
                    {
                        Values = facet.Ranges.Select(range => new FacetValue
                        {
                            Range = range,
                        }).ToList()
                    };

                    break;

                default:
                    throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode));
                }
            }

            var queryForFacets = new QueryForFacets(database, index, defaultFacets, rangeFacets, indexQuery, results, start, pageSize);

            queryForFacets.Execute();
            results.Duration = sp.Elapsed;
            return(results);
        }
Example #44
0
        public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken token)
        {
            var stopwatch            = Stopwatch.StartNew();
            var items                = new List <ICommandData>(deletionBatchSize);
            var attachments          = new List <string>(deletionBatchSize);
            var itemsAndAttachements = Tuple.Create(items, attachments);

            try
            {
                var query = new IndexQuery
                {
                    Start          = 0,
                    PageSize       = deletionBatchSize,
                    Cutoff         = SystemTime.UtcNow,
                    DisableCaching = true,
                    Query          = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]",
                    FieldsToFetch  = new[]
                    {
                        "__document_id",
                        "MessageMetadata.MessageId",
                        "MessageMetadata.BodyNotStored"
                    },
                    SortedFields = new[]
                    {
                        new SortedField("ProcessedAt")
                        {
                            Field      = "ProcessedAt",
                            Descending = false
                        }
                    }
                };
                var indexName = new ExpiryProcessedMessageIndex().IndexName;
                database.Query(indexName, query, token,
                               (doc, state) =>
                {
                    var id = doc.Value <string>("__document_id");
                    if (string.IsNullOrEmpty(id))
                    {
                        return;
                    }

                    state.Item1.Add(new DeleteCommandData
                    {
                        Key = id
                    });

                    if (TryGetBodyId(doc, out var bodyId))
                    {
                        state.Item2.Add(bodyId);
                    }
                }, itemsAndAttachements);
            }
            catch (OperationCanceledException)
            {
                logger.Info("Cleanup operation cancelled");
                return;
            }

            if (token.IsCancellationRequested)
            {
                return;
            }

            var deletedAuditDocuments = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) =>
            {
                if (logger.IsDebugEnabled)
                {
                    logger.Debug($"Batching deletion of {s}-{e} audit documents.");
                }

                var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None);
                if (logger.IsDebugEnabled)
                {
                    logger.Debug($"Batching deletion of {s}-{e} audit documents completed.");
                }

                return(results.Count(x => x.Deleted == true));
            }, items, database, token);

            var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (att, db, s, e) =>
            {
                var deleted = 0;
                if (logger.IsDebugEnabled)
                {
                    logger.Debug($"Batching deletion of {s}-{e} attachment audit documents.");
                }

                db.TransactionalStorage.Batch(accessor =>
                {
                    for (var idx = s; idx <= e; idx++)
                    {
                        //We want to continue using attachments for now
#pragma warning disable 618
                        accessor.Attachments.DeleteAttachment(att[idx], null);
#pragma warning restore 618
                        deleted++;
                    }
                });
                if (logger.IsDebugEnabled)
                {
                    logger.Debug($"Batching deletion of {s}-{e} attachment audit documents completed.");
                }

                return(deleted);
            }, attachments, database, token);

            if (deletedAttachments + deletedAuditDocuments == 0)
            {
                logger.Info("No expired audit documents found");
            }
            else
            {
                logger.Info($"Deleted {deletedAuditDocuments} expired audit documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms");
            }
        }
Example #45
0
        public void Physical_store_test()
        {
            List <Car> cars;

            using (var documentStore = NewRemoteDocumentStore())
            {
                documentStore.Initialize();

                new CarIndex().Execute(documentStore);
                new CarTransformer().Execute(documentStore);

                using (var session = documentStore.OpenSession())
                {
                    foreach (CarLot doc in Docs)
                    {
                        session.Store(doc);
                    }
                    session.SaveChanges();
                }

                string targetLeasee = CarLeasees[0].Id;

                using (var session = documentStore.OpenSession())
                {
                    //Synchronize indexes
                    session.Query <CarLot, CarIndex>().Customize(x => x.WaitForNonStaleResults(TimeSpan.FromMinutes(2))).FirstOrDefault();

                    var query = session.Query <CarLot, CarIndex>()
                                .Where(carLot => carLot.Cars.Any(car => car.LeaseHistory.Any(leasee => leasee.Id == targetLeasee)))
                                .Take(1024);

                    var deserializer = session.Advanced.DocumentStore.Conventions.CreateSerializer();
                    var indexQuery   = new IndexQuery {
                        Query = query.ToString(), PageSize = 1024,
                    };

                    var queryResult = session.Advanced.DocumentStore.DatabaseCommands.Query(typeof(CarIndex).Name, indexQuery, new string[0]);

                    var carLots = queryResult
                                  .Results
                                  .Select(x => deserializer.Deserialize <CarLot>(new RavenJTokenReader(x)))
                                  .ToArray()
                    ;

                    foreach (var carLot in carLots)
                    {
                        Assert.NotNull(carLot.Cars);
                        Assert.NotEmpty(carLot.Cars);
                    }

                    cars = carLots
                           .SelectMany(x => x.Cars)
                           .Where(car => car.LeaseHistory.Any(leasee => leasee.Id == targetLeasee))
                           .ToList();
                }
            }

            Assert.NotNull(cars);
            Assert.NotEmpty(cars);

            foreach (Car car in cars)
            {
                Assert.NotNull(car.LeaseHistory);
                Assert.NotEmpty(car.LeaseHistory);
            }
        }
        public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(query.Query));
            writer.WriteString(query.Query);
            writer.WriteComma();

            if (query.PageSizeSet && query.PageSize >= 0)
            {
                writer.WritePropertyName(nameof(query.PageSize));
                writer.WriteInteger(query.PageSize);
                writer.WriteComma();
            }

            if (query.WaitForNonStaleResults)
            {
                writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
                writer.WriteBool(query.WaitForNonStaleResults);
                writer.WriteComma();
            }

            if (query.Start > 0)
            {
                writer.WritePropertyName(nameof(query.Start));
                writer.WriteInteger(query.Start);
                writer.WriteComma();
            }

            if (query.WaitForNonStaleResultsTimeout.HasValue)
            {
                writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
                writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString());
                writer.WriteComma();
            }

            if (query.DisableCaching)
            {
                writer.WritePropertyName(nameof(query.DisableCaching));
                writer.WriteBool(query.DisableCaching);
                writer.WriteComma();
            }

#if FEATURE_SHOW_TIMINGS
            if (query.ShowTimings)
            {
                writer.WritePropertyName(nameof(query.ShowTimings));
                writer.WriteBool(query.ShowTimings);
                writer.WriteComma();
            }
#endif

            if (query.SkipDuplicateChecking)
            {
                writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
                writer.WriteBool(query.SkipDuplicateChecking);
                writer.WriteComma();
            }

            writer.WritePropertyName(nameof(query.QueryParameters));
            if (query.QueryParameters != null)
            {
                writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query.QueryParameters, conventions, context, conventions.CreateSerializer(), documentInfo: null));
            }
            else
            {
                writer.WriteNull();
            }

            writer.WriteEndObject();
        }
Example #47
0
        private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable <JsonDocument> results, List <string> transformerErrors, Action <double> loadingDocumentsFinish, Action <double> transformerFinish, bool showTimings, CancellationToken token)
        {
            if (query.PageSize <= 0)             // maybe they just want the stats?
            {
                return(Enumerable.Empty <RavenJObject>());
            }

            IndexingFunc transformFunc = null;

            // Check an explicitly declared one first
            if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
            {
                var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);

                if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
                {
                    transformFunc = transformGenerator.TransformResultsDefinition;
                }
                else
                {
                    throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
                }
            }

            if (transformFunc == null)
            {
                var resultsWithoutTransformer = results.Select(x =>
                {
                    var ravenJObject = x.ToJson();
                    if (query.IsDistinct)
                    {
                        ravenJObject[Constants.DocumentIdFieldName] = x.Key;
                    }
                    return(ravenJObject);
                });
                return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer);
            }

            var dynamicJsonObjects = results.Select(x =>
            {
                var ravenJObject = x.ToJson();
                if (query.IsDistinct)
                {
                    ravenJObject[Constants.DocumentIdFieldName] = x.Key;
                }
                return(new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject));
            });
            var robustEnumerator = new RobustEnumerator(token, 100)
            {
                OnError =
                    (exception, o) =>
                    transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
                                                        exception.Message))
            };

            var resultsWithTransformer = robustEnumerator
                                         .RobustEnumeration(dynamicJsonObjects.Cast <object>().GetEnumerator(), transformFunc)
                                         .Select(JsonExtensions.ToJObject);

            return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer);
        }
Example #48
0
        private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery)
        {
            string entityName = null;

            if (index.StartsWith("dynamic/"))
            {
                entityName = index.Substring("dynamic/".Length);
            }

            var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery.Query);
            var indexEtag        = Guid.Empty;

            if (Database.IndexStorage.HasIndex(dynamicIndexName))
            {
                indexEtag = GetIndexEtag(dynamicIndexName);
                if (context.MatchEtag(indexEtag))
                {
                    context.SetStatusToNotModified();
                    return(null);
                }
            }

            var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery);

            if (indexEtag != Guid.Empty)
            {
                queryResult.IndexEtag = indexEtag;
            }
            return(queryResult);
        }
Example #49
0
 public Operation DeleteByIndex(string indexName, IndexQuery queryToDelete, QueryOperationOptions options = null)
 {
     return(AsyncHelpers.RunSync(() => asyncServerClient.DeleteByIndexAsync(indexName, queryToDelete, options)));
 }
        public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
        {
            var stopwatch = Stopwatch.StartNew();
            var items     = new List <ICommandData>(deletionBatchSize);

            try
            {
                var query = new IndexQuery
                {
                    Start          = 0,
                    DisableCaching = true,
                    Cutoff         = SystemTime.UtcNow,
                    PageSize       = deletionBatchSize,
                    Query          = $"LastModified:[* TO {expiryThreshold.Ticks}]",
                    FieldsToFetch  = new[]
                    {
                        "__document_id"
                    },
                    SortedFields = new[]
                    {
                        new SortedField("LastModified")
                        {
                            Field      = "LastModified",
                            Descending = false
                        }
                    }
                };
                var indexName = new ExpiryEventLogItemsIndex().IndexName;
                database.Query(indexName, query, database.WorkContext.CancellationToken,
                               (doc, commands) =>
                {
                    var id = doc.Value <string>("__document_id");
                    if (string.IsNullOrEmpty(id))
                    {
                        return;
                    }

                    commands.Add(new DeleteCommandData
                    {
                        Key = id
                    });
                }, items);
            }
            catch (OperationCanceledException)
            {
                //Ignore
            }

            var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) =>
            {
                logger.InfoFormat("Batching deletion of {0}-{1} eventlogitem documents.", s, e);
                var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None);
                logger.InfoFormat("Batching deletion of {0}-{1} eventlogitem documents completed.", s, e);

                return(results.Count(x => x.Deleted == true));
            }, items, database);

            if (deletionCount == 0)
            {
                logger.Info("No expired eventlogitem documents found");
            }
            else
            {
                logger.InfoFormat("Deleted {0} expired eventlogitem documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
            }
        }
        /// <summary>
        /// This test target a bug around minimal splitter in gbpTree and unique index populator. It goes like this:
        /// Given a set of updates (value,entityId):
        /// - ("A01",1), ("A90",3), ("A9",2)
        /// If ("A01",1) and ("A90",3) would cause a split to occur they would produce a minimal splitter ("A9",3).
        /// Note that the value in this minimal splitter is equal to our last update ("A9",2).
        /// When making insertions with the unique populator we don't compare entityId which would means ("A9",2)
        /// ends up to the right of ("A9",3), even though it belongs to the left because of entityId being smaller.
        /// At this point the tree is in an inconsistent (key on wrong side of splitter).
        ///
        /// To work around this problem the entityId is only kept in minimal splitter if strictly necessary to divide
        /// left from right. This means the minimal splitter between ("A01",1) and ("A90",3) is ("A9",-1) and ("A9",2)
        /// will correctly be placed on the right side of this splitter.
        ///
        /// To trigger this scenario this test first insert a bunch of values that are all unique and that will cause a
        /// split to happen. This is the firstBatch.
        /// The second batch are constructed so that at least one of them will have a value equal to the splitter key
        /// constructed during the firstBatch.
        /// It's important that the secondBatch has ids that are lower than the first batch to align with example described above.
        /// </summary>
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldPopulateAndRemoveEntriesWithSimilarMinimalSplitter() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        public virtual void ShouldPopulateAndRemoveEntriesWithSimilarMinimalSplitter()
        {
            string prefix     = "Work out your own salvation. Do not depend on others. ";
            int    nbrOfNodes = 200;
            long   nodeId     = 0;

            // Second batch has lower ids
            IList <NodeAndValue> secondBatch = new List <NodeAndValue>();

            for (int i = 0; i < nbrOfNodes; i++)
            {
                secondBatch.Add(new NodeAndValue(nodeId++, stringValue(prefix + i)));
            }

            // First batch has higher ids and minimal splitter among values in first batch will be found among second batch
            IList <NodeAndValue> firstBatch = new List <NodeAndValue>();

            for (int i = 0; i < nbrOfNodes; i++)
            {
                firstBatch.Add(new NodeAndValue(nodeId++, stringValue(prefix + i + " " + i)));
            }

            WithPopulator(IndexProvider.getPopulator(Descriptor, IndexSamplingConfig, heapBufferFactory(1024)), p =>
            {
                p.add(Updates(firstBatch));
                p.add(Updates(secondBatch));

                // Index should be consistent
            });

            IList <NodeAndValue> toRemove = new List <NodeAndValue>();

            ((IList <NodeAndValue>)toRemove).AddRange(firstBatch);
            ((IList <NodeAndValue>)toRemove).AddRange(secondBatch);
            Collections.shuffle(toRemove);

            // And we should be able to remove the entries in any order
            using (IndexAccessor accessor = IndexProvider.getOnlineAccessor(Descriptor, IndexSamplingConfig))
            {
                // WHEN
                using (IndexUpdater updater = accessor.NewUpdater(IndexUpdateMode.ONLINE))
                {
                    foreach (NodeAndValue nodeAndValue in toRemove)
                    {
                        updater.Process(IndexEntryUpdate.Remove(nodeAndValue.NodeId, Descriptor, nodeAndValue.Value));
                    }
                }

                // THEN
                using (IndexReader reader = new QueryResultComparingIndexReader(accessor.NewReader()))
                {
                    int propertyKeyId = Descriptor.schema().PropertyId;
                    foreach (NodeAndValue nodeAndValue in toRemove)
                    {
                        NodeValueIterator nodes = new NodeValueIterator();
                        reader.Query(nodes, IndexOrder.NONE, false, IndexQuery.exact(propertyKeyId, nodeAndValue.Value));
                        bool anyHits = false;

                        StringJoiner nodesStillLeft = new StringJoiner(", ", "[", "]");
                        while (nodes.HasNext())
                        {
                            anyHits = true;
                            nodesStillLeft.add(Convert.ToString(nodes.Next()));
                        }
                        assertFalse("Expected this query to have zero hits but found " + nodesStillLeft.ToString(), anyHits);
                    }
                }
            }
        }
Example #52
0
        private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag)
        {
            indexEtag = GetIndexEtag(index);
            if (context.MatchEtag(indexEtag))
            {
                context.SetStatusToNotModified();
                return(null);
            }

            return(Database.Query(index, indexQuery));
        }
Example #53
0
 public Lazy <Task <Dictionary <string, FacetResult> > > ExecuteLazyAsync(Action <Dictionary <string, FacetResult> > onEval = null, CancellationToken token = default)
 {
     _query = GetIndexQuery(isAsync: true);
     return(((AsyncDocumentSession)_session).AddLazyOperation(new LazyAggregationQueryOperation(_session.Conventions, _query, InvokeAfterQueryExecuted, ProcessResults), onEval, token));
 }
Example #54
0
        private QueryCommand GetCommand(bool isAsync)
        {
            _query = GetIndexQuery(isAsync);

            return(new QueryCommand(_session, _query));
        }
Example #55
0
        public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery)
        {
            // There isn't much point for query optimizer of aggregation indexes
            // the main reason is that we must always aggregate on the same items, and using the same
            // aggregation. Therefore we can't reuse one aggregate index for another query.
            // We decline to suggest an index here and choose to use the default index created for this
            // sort of query, which is what we would have to choose anyway.
            if (indexQuery.AggregationOperation != AggregationOperation.None)
            {
                return(null);
            }

            var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2);

            return(database.IndexDefinitionStorage.IndexNames
                   .Where(indexName =>
            {
                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                if (abstractViewGenerator == null)                                 // there is no matching view generator
                {
                    return false;
                }

                if (abstractViewGenerator.ReduceDefinition != null)                                 // we can't choose a map/reduce index
                {
                    return false;
                }

                if (abstractViewGenerator.TransformResultsDefinition != null)                                // we can't choose an index with transform results
                {
                    return false;
                }

                if (abstractViewGenerator.HasWhereClause)                                 // without a where clause
                {
                    return false;
                }

                // we can't select an index that has SelectMany in it, because it result in invalid results when
                // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250
                // for indexes with internal projections, we use the exact match based on the generated index name
                // rather than selecting the optimal one
                if (abstractViewGenerator.CountOfSelectMany > 1)
                {
                    return false;
                }

                if (abstractViewGenerator.ForEntityName != entityName)                                 // for the specified entity name
                {
                    return false;
                }

                return fieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap);
            })
                   .Where(indexName =>
            {
                var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
                if (indexDefinition == null)
                {
                    return false;
                }
                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                if (abstractViewGenerator == null)
                {
                    return false;
                }

                if (indexQuery.SortedFields != null)
                {
                    var sortInfo = DynamicQueryMapping.GetSortInfo(s => { });

                    foreach (var sortedField in indexQuery.SortedFields)                                     // with matching sort options
                    {
                        // if the field is not in the output, then we can't sort on it.
                        if (abstractViewGenerator.ContainsField(sortedField.Field) == false)
                        {
                            return false;
                        }

                        var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field);

                        if (dynamicSortInfo == null)                                        // no sort order specified, we don't care, probably
                        {
                            continue;
                        }

                        SortOptions value;
                        if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false)
                        {
                            switch (dynamicSortInfo.FieldType)                                            // if we can't find the value, we check if we asked for the default sorting
                            {
                            case SortOptions.String:
                            case SortOptions.None:
                                continue;

                            default:
                                return false;
                            }
                        }

                        if (value != dynamicSortInfo.FieldType)
                        {
                            return false;                                             // different sort order, there is a problem here
                        }
                    }
                }

                if (indexDefinition.Analyzers != null)
                {
                    // none of the fields have custom analyzers
                    if (fieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey))
                    {
                        return false;
                    }
                }

                return true;
            })
                   .OrderByDescending(indexName =>
            {
                // We select the widest index, because we want to encourage bigger indexes
                // Over time, it means that we smaller indexes would wither away and die, while
                // bigger indexes will be selected more often and then upgrade to permanent indexes
                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                if (abstractViewGenerator == null)                                 // there is a matching view generator
                {
                    return -1;
                }
                return abstractViewGenerator.CountOfFields;
            })
                   .FirstOrDefault());
        }
Example #56
0
        public IDictionary <string, IEnumerable <FacetValue> > GetFacets(string index, IndexQuery indexQuery, string facetSetupDoc)
        {
            var facetSetup = database.Get(facetSetupDoc, null);

            if (facetSetup == null)
            {
                throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc);
            }

            var facets = facetSetup.DataAsJson.JsonDeserialization <FacetSetup>().Facets;

            var results = new Dictionary <string, IEnumerable <FacetValue> >();

            IndexSearcher currentIndexSearcher;

            using (database.IndexStorage.GetCurrentIndexSearcher(index, out currentIndexSearcher))
            {
                foreach (var facet in facets)
                {
                    switch (facet.Mode)
                    {
                    case FacetMode.Default:
                        HandleTermsFacet(index, facet, indexQuery, currentIndexSearcher, results);
                        break;

                    case FacetMode.Ranges:
                        HandleRangeFacet(index, facet, indexQuery, currentIndexSearcher, results);
                        break;

                    default:
                        throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode));
                    }
                }
            }

            return(results);
        }
Example #57
0
        public DynamicQueryOptimizerResult SelectAppropriateIndex(
            string entityName,
            IndexQuery indexQuery,
            List <Explanation> explanations = null)
        {
            if (string.IsNullOrEmpty(indexQuery.Query) &&                                       // we optimize for empty queries to use Raven/DocumentsByEntityName
                (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) &&     // and no sorting was requested
                database.IndexDefinitionStorage.Contains(Constants.DocumentsByEntityNameIndex)) // and Raven/DocumentsByEntityName exists
            {
                if (string.IsNullOrEmpty(entityName) == false)
                {
                    indexQuery.Query = "Tag:" + entityName;
                }
                return(new DynamicQueryOptimizerResult(Constants.DocumentsByEntityNameIndex, DynamicQueryMatchType.Complete));
            }

            var fieldsQueriedUpon           = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery).Select(x => x.Item2).ToArray();
            var normalizedFieldsQueriedUpon =
                fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceInvalidCharactersForFields).ToArray();
            var distinctSelectManyFields = new HashSet <string>();

            foreach (var field in fieldsQueriedUpon)
            {
                var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                for (int i = 1; i < parts.Length; i++)
                {
                    distinctSelectManyFields.Add(string.Join(",", parts.Take(i)));
                }
            }

            ExplainDelegate explain = (index, rejectionReason) => { };

            if (explanations != null)
            {
                explain = (index, rejectionReason) => explanations.Add(new Explanation
                {
                    Index  = index,
                    Reason = rejectionReason()
                });
            }

            //;

            // there is no reason why we can't use indexes with transform results
            // we merely need to disable the transform results for this particular query
            var results = database.IndexDefinitionStorage.IndexDefinitions
                          .Select(indexDefinitionKvp =>
            {
                var indexName = indexDefinitionKvp.Value.Name;

                var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
                var currentBestState      = DynamicQueryMatchType.Complete;
                if (abstractViewGenerator == null)                                 // there is no matching view generator
                {
                    explain(indexName, () => "There is no matching view generator. Maybe the index in the process of being deleted?");
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }

                var indexingPriority = IndexingPriority.None;
                var isInvalidIndex   = false;
                database.TransactionalStorage.Batch(accessor =>
                {
                    var stats        = accessor.Indexing.GetIndexStats(indexDefinitionKvp.Key);
                    isInvalidIndex   = stats.IsInvalidIndex;
                    indexingPriority = stats.Priority;
                });


                if (entityName == null)
                {
                    if (abstractViewGenerator.ForEntityNames.Count != 0)
                    {
                        explain(indexName, () => "Query is not specific for entity name, but the index filter by entity names.");
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                }
                else
                {
                    if (indexingPriority == IndexingPriority.Error ||
                        indexingPriority == IndexingPriority.Disabled ||
                        isInvalidIndex)
                    {
                        explain(indexName, () => string.Format("Cannot do dynamic queries on disabled index or index with errors (index name = {0})", indexName));
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }

                    if (abstractViewGenerator.ForEntityNames.Count > 1)                                     // we only allow indexes with a single entity name
                    {
                        explain(indexName, () => "Index contains more than a single entity name, may result in a different type being returned.");
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                    if (abstractViewGenerator.ForEntityNames.Count == 0)
                    {
                        explain(indexName, () => "Query is specific for entity name, but the index searches across all of them, may result in a different type being returned.");
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                    if (abstractViewGenerator.ForEntityNames.Contains(entityName) == false)                                     // for the specified entity name
                    {
                        explain(indexName, () => string.Format("Index does not apply to entity name: {0}", entityName));
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                }

                if (abstractViewGenerator.ReduceDefinition != null)                                 // we can't choose a map/reduce index
                {
                    explain(indexName, () => "Can't choose a map/reduce index for dynamic queries.");
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }

                if (abstractViewGenerator.HasWhereClause)                                 // without a where clause
                {
                    explain(indexName, () => "Can't choose an index with a where clause, it might filter things that the query is looking for.");
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }

                // we can't select an index that has SelectMany in it, because it result in invalid results when
                // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250
                // for indexes with internal projections, we use the exact match based on the generated index name
                // rather than selecting the optimal one
                // in order to handle that, we count the number of select many that would happen because of the query
                // and match it to the number of select many in the index
                if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count)
                {
                    explain(indexName,
                            () => "Can't choose an index with a different number of from clauses / SelectMany, will affect queries like Count().");
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }

                if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false)
                {
                    explain(indexName, () =>
                    {
                        var missingFields =
                            normalizedFieldsQueriedUpon.Where(s => abstractViewGenerator.ContainsFieldOnMap(s) == false);
                        return("The following fields are missing: " + string.Join(", ", missingFields));
                    });
                    currentBestState = DynamicQueryMatchType.Partial;
                }

                var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
                if (indexDefinition == null)
                {
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }

                if (indexQuery.HighlightedFields != null && indexQuery.HighlightedFields.Length > 0)
                {
                    var nonHighlightableFields = indexQuery
                                                 .HighlightedFields
                                                 .Where(x =>
                                                        !indexDefinition.Stores.ContainsKey(x.Field) ||
                                                        indexDefinition.Stores[x.Field] != FieldStorage.Yes ||
                                                        !indexDefinition.Indexes.ContainsKey(x.Field) ||
                                                        indexDefinition.Indexes[x.Field] != FieldIndexing.Analyzed ||
                                                        !indexDefinition.TermVectors.ContainsKey(x.Field) ||
                                                        indexDefinition.TermVectors[x.Field] != FieldTermVector.WithPositionsAndOffsets)
                                                 .Select(x => x.Field)
                                                 .ToArray();

                    if (nonHighlightableFields.Any())
                    {
                        explain(indexName,
                                () => "The following fields could not be highlighted because they are not stored, analyzed and using term vectors with positions and offsets: " +
                                string.Join(", ", nonHighlightableFields));
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                }

                if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0)
                {
                    var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }, indexQuery);

                    foreach (var sortedField in indexQuery.SortedFields)                                     // with matching sort options
                    {
                        var normalizedFieldName = DynamicQueryMapping.ReplaceInvalidCharactersForFields(sortedField.Field);
                        if (normalizedFieldName.EndsWith("_Range"))
                        {
                            normalizedFieldName = normalizedFieldName.Substring(0, normalizedFieldName.Length - "_Range".Length);
                        }
                        if (normalizedFieldName.StartsWith(Constants.RandomFieldName))
                        {
                            continue;                                     // virtual field that we don't sort by
                        }
                        // if the field is not in the output, then we can't sort on it.
                        if (abstractViewGenerator.ContainsField(normalizedFieldName) == false)
                        {
                            explain(indexName,
                                    () =>
                                    "Rejected because index does not contains field '" + normalizedFieldName + "' which we need to sort on");
                            currentBestState = DynamicQueryMatchType.Partial;
                            continue;
                        }

                        var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == normalizedFieldName);

                        if (dynamicSortInfo == null)                                        // no sort order specified, we don't care, probably
                        {
                            continue;
                        }

                        SortOptions value;
                        if (indexDefinition.SortOptions.TryGetValue(normalizedFieldName, out value) == false)
                        {
                            switch (dynamicSortInfo.FieldType)                                            // if we can't find the value, we check if we asked for the default sorting
                            {
                            case SortOptions.String:
                            case SortOptions.None:
                                continue;

                            default:
                                explain(indexName,
                                        () => "The specified sort type is different than the default for field: " + normalizedFieldName);
                                return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                            }
                        }

                        if (value != dynamicSortInfo.FieldType)
                        {
                            explain(indexName,
                                    () =>
                                    "The specified sort type (" + dynamicSortInfo.FieldType + ") is different than the one specified for field '" +
                                    normalizedFieldName + "' (" + value + ")");
                            return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                        }
                    }
                }

                if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0)
                {
                    // none of the fields have custom analyzers
                    if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey))
                    {
                        explain(indexName, () =>
                        {
                            var fields = normalizedFieldsQueriedUpon.Where(indexDefinition.Analyzers.ContainsKey);
                            return("The following field have a custom analyzer: " + string.Join(", ", fields));
                        });
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                }

                if (indexDefinition.Indexes != null && indexDefinition.Indexes.Count > 0)
                {
                    //If any of the fields we want to query on are set to something other than the default, don't use the index
                    var anyFieldWithNonDefaultIndexing = normalizedFieldsQueriedUpon.Where(x =>
                    {
                        FieldIndexing analyzedInfo;
                        if (indexDefinition.Indexes.TryGetValue(x, out analyzedInfo))
                        {
                            if (analyzedInfo != FieldIndexing.Default)
                            {
                                return(true);
                            }
                        }
                        return(false);
                    });

                    var fieldWithNonDefaultIndexing = anyFieldWithNonDefaultIndexing.ToArray();                                     //prevent several enumerations
                    if (fieldWithNonDefaultIndexing.Any())
                    {
                        explain(indexName, () =>
                        {
                            var fields = fieldWithNonDefaultIndexing.Where(indexDefinition.Analyzers.ContainsKey);
                            return("The following field have aren't using default indexing: " + string.Join(", ", fields));
                        });
                        return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                    }
                }
                if (currentBestState != DynamicQueryMatchType.Complete && indexDefinition.Type != "Auto")
                {
                    return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure));
                }
                if (currentBestState == DynamicQueryMatchType.Complete &&
                    (indexingPriority == IndexingPriority.Idle ||
                     indexingPriority == IndexingPriority.Abandoned))
                {
                    currentBestState = DynamicQueryMatchType.Partial;
                    explain(indexName, () => String.Format("The index (name = {0}) is disabled or abandoned. The preference is for active indexes - making a partial match", indexName));
                }

                return(new DynamicQueryOptimizerResult(indexName, currentBestState));
            })
                          .Where(result => result.MatchType != DynamicQueryMatchType.Failure)
                          .GroupBy(x => x.MatchType)
                          .ToDictionary(x => x.Key, x => x.ToArray());


            DynamicQueryOptimizerResult[] optimizerResults;
            if (results.TryGetValue(DynamicQueryMatchType.Complete, out optimizerResults) && optimizerResults.Length > 0)
            {
                DynamicQueryOptimizerResult[] prioritizedResults = null;
                database.TransactionalStorage.Batch(accessor =>
                {
                    prioritizedResults = optimizerResults.OrderByDescending(result =>
                    {
                        var instance = this.database.IndexStorage.GetIndexInstance(result.IndexName);
                        var stats    = accessor.Indexing.GetIndexStats(instance.indexId);
                        if (stats == null)
                        {
                            return(Etag.Empty);
                        }

                        return(stats.LastIndexedEtag);
                    })
                                         .ThenByDescending(result =>
                    {
                        var abstractViewGenerator =
                            database.IndexDefinitionStorage.GetViewGenerator(result.IndexName);
                        if (abstractViewGenerator == null)
                        {
                            return(-1);
                        }
                        return(abstractViewGenerator.CountOfFields);
                    })
                                         .ToArray();
                });
                for (int i = 1; i < prioritizedResults.Length; i++)
                {
                    explain(prioritizedResults[i].IndexName,
                            () => "Wasn't the widest / most unstable index matching this query");
                }

                return(prioritizedResults[0]);
            }

            if (results.TryGetValue(DynamicQueryMatchType.Partial, out optimizerResults) && optimizerResults.Length > 0)
            {
                return(optimizerResults.OrderByDescending(x =>
                {
                    var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(x.IndexName);

                    if (viewGenerator == null)
                    {
                        return -1;
                    }
                    return viewGenerator.CountOfFields;
                }).First());
            }

            return(new DynamicQueryOptimizerResult("", DynamicQueryMatchType.Failure));
        }
Example #58
0
 public Lazy <Dictionary <string, FacetResult> > ExecuteLazy(Action <Dictionary <string, FacetResult> > onEval = null)
 {
     _query = GetIndexQuery(isAsync: false);
     return(((DocumentSession)_session).AddLazyOperation(new LazyAggregationQueryOperation(_session.Conventions, _query, InvokeAfterQueryExecuted, ProcessResults), onEval));
 }
Example #59
0
		private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag)
		{
			string entityName = null;
			if (index.StartsWith("dynamic/"))
				entityName = index.Substring("dynamic/".Length);

			var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery);


			if (dynamicIndexName != null && 
				Database.IndexStorage.HasIndex(dynamicIndexName))
			{
				indexEtag = GetIndexEtag(dynamicIndexName);
				if (context.MatchEtag(indexEtag))
				{
					context.SetStatusToNotModified();
					return null;
				}
			}
			else
			{
				indexEtag = Guid.Empty;
			}

			var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery);

			// have to check here because we might be getting the index etag just 
			// as we make a switch from temp to auto, and we need to refresh the etag
			// if that is the case. This can also happen when the optmizer
			// decided to switch indexes for a query.
			if (queryResult.IndexName != dynamicIndexName)
				indexEtag = GetIndexEtag(queryResult.IndexName);

			return queryResult;
		}
Example #60
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void concurrentIndexPopulationAndInsertsShouldNotProduceDuplicates() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        public virtual void ConcurrentIndexPopulationAndInsertsShouldNotProduceDuplicates()
        {
            // Given
            Config config           = Config.defaults();
            GraphDatabaseService db = NewEmbeddedGraphDatabaseWithSlowJobScheduler(config);

            try
            {
                // When
                using (Transaction tx = Db.beginTx())
                {
                    Db.schema().indexFor(label(LABEL)).on(KEY).create();
                    tx.Success();
                }
                Node node;
                using (Transaction tx = Db.beginTx())
                {
                    node = Db.createNode(label(LABEL));
                    node.SetProperty(KEY, VALUE);
                    tx.Success();
                }

                using (Transaction tx = Db.beginTx())
                {
                    Db.schema().awaitIndexesOnline(1, MINUTES);
                    tx.Success();
                }

                // Then
                using (Transaction tx = Db.beginTx())
                {
                    KernelTransaction    ktx    = (( GraphDatabaseAPI )db).DependencyResolver.resolveDependency(typeof(ThreadToStatementContextBridge)).getKernelTransactionBoundToThisThread(true);
                    IndexReference       index  = ktx.SchemaRead().index(ktx.TokenRead().nodeLabel(LABEL), ktx.TokenRead().propertyKey(KEY));
                    NodeValueIndexCursor cursor = ktx.Cursors().allocateNodeValueIndexCursor();
                    ktx.DataRead().nodeIndexSeek(index, cursor, IndexOrder.NONE, false, IndexQuery.exact(1, VALUE));
                    assertTrue(cursor.Next());
                    assertEquals(node.Id, cursor.NodeReference());
                    assertFalse(cursor.Next());
                    tx.Success();
                }
            }
            finally
            {
                Db.shutdown();
            }
        }