public static List<DynamicQueryOptimizer.Explanation> ExplainDynamicIndexSelection(this DocumentDatabase self, string entityName, IndexQuery query) { var explanations = new List<DynamicQueryOptimizer.Explanation>(); new DynamicQueryOptimizer(self) .SelectAppropriateIndex(entityName, query.Clone(), explanations); return explanations; }
private QueryResult ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery) { // Perform the query until we have some results at least QueryResult result; var sp = Stopwatch.StartNew(); while (true) { result = documentDatabase.Query(map.IndexName, new IndexQuery { Cutoff = query.Cutoff, PageSize = query.PageSize, Query = realQuery, Start = query.Start, FieldsToFetch = query.FieldsToFetch, GroupBy = query.GroupBy, AggregationOperation = query.AggregationOperation, SortedFields = query.SortedFields, }); if (!touchTemporaryIndexResult.Item2 || !result.IsStale || result.Results.Count >= query.PageSize || sp.Elapsed.TotalSeconds > 15) { return result; } Thread.Sleep(100); } }
public static string FindDynamicIndexName(this DocumentDatabase self, string entityName, IndexQuery query) { var result = new DynamicQueryOptimizer(self).SelectAppropriateIndex(entityName, query.Clone()); if (result.MatchType == DynamicQueryMatchType.Complete) return result.IndexName; return null; }
public QueryResultWithIncludes Query(string index, IndexQuery query, CancellationToken externalCancellationToken) { QueryResultWithIncludes result = null; using (var cts = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, WorkContext.CancellationToken)) { var cancellationToken = cts.Token; TransactionalStorage.Batch( accessor => { using (var op = new DatabaseQueryOperation(Database, index, query, accessor, cancellationToken) { ShouldSkipDuplicateChecking = query.SkipDuplicateChecking }) { var list = new List<RavenJObject>(); op.Init(); op.Execute(list.Add); op.Result.Results = list; result = op.Result; } }); } return result; }
public static IndexQuery GetIndexQueryFromHttpContext(this IHttpContext context, int maxPageSize) { var query = new IndexQuery { Query = context.Request.QueryString["query"] ?? "", Start = context.GetStart(), Cutoff = context.GetCutOff(), PageSize = context.GetPageSize(maxPageSize), SkipTransformResults = context.GetSkipTransformResults(), FieldsToFetch = context.Request.QueryString.GetValues("fetch"), GroupBy = context.Request.QueryString.GetValues("groupBy"), AggregationOperation = context.GetAggregationOperation(), SortedFields = context.Request.QueryString.GetValues("sort") .EmptyIfNull() .Select(x => new SortedField(x)) .ToArray() }; double lat = context.GetLat(), lng = context.GetLng(), radius = context.GetRadius(); if (lat != 0 || lng != 0 || radius != 0) { return new SpatialIndexQuery(query) { Latitude = lat, Longitude = lng, Radius = radius, }; } return query; }
public ExecutingQueryInfo(DateTime startTime, IndexQuery queryInfo, long queryId, CancellationTokenSource tokenSource) { StartTime = startTime; QueryInfo = queryInfo; QueryId = queryId; stopwatch = Stopwatch.StartNew(); TokenSource = tokenSource; }
public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, bool allowStale) { return PerformBulkOperation(indexName, queryToDelete, allowStale, (docId, tx) => { database.Delete(docId, null, tx); return new { Document = docId, Deleted = true }; }); }
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, PatchRequest[] patchRequests, bool allowStale) { return PerformBulkOperation(indexName, queryToUpdate, allowStale, (docId, tx) => { var patchResult = database.ApplyPatch(docId, null, patchRequests, tx); return new { Document = docId, Result = patchResult }; }); }
public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) => { database.Documents.Delete(docId, null, tx); return new { Document = docId, Deleted = true }; }); }
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) => { var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx); return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 }; }); }
// GET: Campaign public async Task<IActionResult> Index() { var query = new IndexQuery(); if (User.IsUserType(UserType.OrgAdmin)) { query.OrganizationId = User.GetOrganizationId(); } return View(await _mediator.SendAsync(query)); }
private static void UpdateFieldNamesForSortedFields(IndexQuery query, DynamicQueryMapping map) { if (query.SortedFields == null) return; foreach (var sortedField in query.SortedFields) { var item = map.Items.FirstOrDefault(x => x.From == sortedField.Field); if (item != null) sortedField.Field = item.To; } }
private static IndexQuery CreateIndexQuery(IndexQuery query, DynamicQueryMapping map, string realQuery) { var indexQuery = new IndexQuery { Cutoff = query.Cutoff, WaitForNonStaleResultsAsOfNow = query.WaitForNonStaleResultsAsOfNow, PageSize = query.PageSize, Query = realQuery, Start = query.Start, FieldsToFetch = query.FieldsToFetch, IsDistinct = query.IsDistinct, SortedFields = query.SortedFields, DefaultField = query.DefaultField, CutoffEtag = query.CutoffEtag, DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries, DefaultOperator = query.DefaultOperator, SkippedResults = query.SkippedResults, HighlighterPreTags = query.HighlighterPreTags, HighlighterPostTags = query.HighlighterPostTags, HighlightedFields = query.HighlightedFields, HighlighterKeyName = query.HighlighterKeyName, ResultsTransformer = query.ResultsTransformer, TransformerParameters = query.TransformerParameters, ExplainScores = query.ExplainScores, SortHints = query.SortHints, ShowTimings = query.ShowTimings, AllowMultipleIndexEntriesForSameDocumentToResultTransformer = query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer, DisableCaching = query.DisableCaching, SkipDuplicateChecking = query.SkipDuplicateChecking, WaitForNonStaleResults = query.WaitForNonStaleResults }; if (indexQuery.SortedFields == null) return indexQuery; for (int index = 0; index < indexQuery.SortedFields.Length; index++) { var sortedField = indexQuery.SortedFields[index]; var fieldName = sortedField.Field; bool hasRange = false; if (fieldName.EndsWith("_Range")) { fieldName = fieldName.Substring(0, fieldName.Length - "_Range".Length); hasRange = true; } var item = map.Items.FirstOrDefault(x => string.Equals(x.QueryFrom, fieldName, StringComparison.InvariantCultureIgnoreCase)); if (item == null) continue; indexQuery.SortedFields[index] = new SortedField(hasRange ? item.To + "_Range" : item.To); indexQuery.SortedFields[index].Descending = sortedField.Descending; } return indexQuery; }
public void can_encode_and_decode_IndexQuery_Start() { int expected = Some.Integer(); var indexQuery = new IndexQuery(); indexQuery.Start = expected; IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery); Assert.Equal(expected, result.Start); }
public void can_encode_and_decode_IndexQuery_PageSize() { var expected = Some.Integer(); var indexQuery = new IndexQuery(); indexQuery.PageSize = expected; IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery); Assert.Equal(expected, result.PageSize); }
public IEnumerable<string> QueryDocumentIds(string index, IndexQuery query, CancellationTokenSource tokenSource, out bool stale) { var queryStat = AddToCurrentlyRunningQueryList(index, query, tokenSource); try { bool isStale = false; HashSet<string> loadedIds = null; TransactionalStorage.Batch( actions => { var definition = IndexDefinitionStorage.GetIndexDefinition(index); if (definition == null) throw new ArgumentException("specified index definition was not found", "index"); isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null); if (isStale == false && query.Cutoff == null) { var indexInstance = Database.IndexStorage.GetIndexInstance(index); isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress); } if (isStale && actions.Staleness.IsIndexStaleByTask(definition.IndexId, query.Cutoff) == false && actions.Staleness.IsReduceStale(definition.IndexId) == false) { var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) throw new ArgumentException("specified index definition was not found", "index"); var forEntityNames = viewGenerator.ForEntityNames.ToList(); var lastIndexedEtag = actions.Indexing.GetIndexStats(definition.IndexId).LastIndexedEtag; if (Database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false) isStale = false; } var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } loadedIds = new HashSet<string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, tokenSource.Token) select queryResult.Key); }); stale = isStale; return loadedIds; } finally { RemoveFromCurrentlyRunningQueryList(index, queryStat); } }
public void can_encode_and_decode_IndexQuery_Query_pound() { var expected = Some.String() + '#' + Some.String(); var indexQuery = new IndexQuery(); indexQuery.Query = expected; IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery); Assert.Equal(expected, result.Query); }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation) { var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && allowStale == false) { throw new InvalidOperationException( "Bulk operation cancelled because the index is stale and allowStale is false"); } var token = tokenSource.Token; const int batchSize = 1024; using (var enumerator = queryResults.GetEnumerator()) { while (true) { if (timeout != null) timeout.Delay(); var batchCount = 0; token.ThrowIfCancellationRequested(); database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; var result = batchOperation(enumerator.Current, transactionInformation); array.Add(RavenJObject.FromObject(result)); } }); if (batchCount < batchSize) break; } } return array; }
public QueryResult ExecuteDynamicQuery(string entityName, IndexQuery query) { // Create the map var map = DynamicQueryMapping.Create(documentDatabase, query, entityName); var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map); map.IndexName = touchTemporaryIndexResult.Item1; // Re-write the query string realQuery = map.Items.Aggregate(query.Query, (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To)); return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery); }
private Tuple<string, bool> GetAppropriateIndexToQuery(string entityName, IndexQuery query, DynamicQueryMapping map) { var appropriateIndex = new DynamicQueryOptimizer(documentDatabase).SelectAppropriateIndex(entityName, query); if (appropriateIndex != null) { if (appropriateIndex.StartsWith("Temp/"))// temporary index, we need to increase its usage { return TouchTemporaryIndex(appropriateIndex, "Auto/" + appropriateIndex.Substring(5), () => documentDatabase.IndexDefinitionStorage.GetIndexDefinition(appropriateIndex)); } return Tuple.Create(appropriateIndex, false); } return TouchTemporaryIndex(map.TemporaryIndexName, map.PermanentIndexName, map.CreateIndexDefinition); }
public async Task ReturnTheCampaignsTheUserIsAnOrgAdminFor_WhenOrganizationIdOnMessageIsNotNull() { const int orgId1 = 1; const int orgId2 = 2; Context.Campaigns.Add(new Campaign { Id = 1, ManagingOrganization = new Organization { Id = orgId1 }}); Context.Campaigns.Add(new Campaign { Id = 2, ManagingOrganization = new Organization { Id = orgId2 }}); Context.SaveChanges(); var message = new IndexQuery { OrganizationId = orgId1 }; var sut = new IndexQueryHandler(Context); var result = await sut.Handle(message); Assert.Equal(result.Single().Id, orgId1); }
public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query, CancellationToken token) { // Create the map var map = DynamicQueryMapping.Create(documentDatabase, query, entityName); var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map); string realQuery = map.Items.Aggregate(query.Query, (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To)); UpdateFieldNamesForSortedFields(query, map); // We explicitly do NOT want to update the field names of FieldsToFetch - that reads directly from the document //UpdateFieldsInArray(map, query.FieldsToFetch); return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery, token); }
public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query) { // Create the map var map = DynamicQueryMapping.Create(documentDatabase, query, entityName); var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map); map.IndexName = touchTemporaryIndexResult.Item1; // Re-write the query string realQuery = map.Items.Aggregate(query.Query, (current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To)); UpdateFieldNamesForSortedFields(query, map); UpdateFieldsInArray(map, query.FieldsToFetch); UpdateFieldsInArray(map, query.GroupBy); return ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery); }
public void can_encode_and_decode_IndexQuery_Query() { // Fails when at least '&' is in the Query, not sure if that is acceptable // Fails because the value has not by url decoded, I couldn't find code doing the url decode // after GetIndexQueryFromHttpContext() so there may be another bug. //var expected = new string(Enumerable.Range(0, 255).Select(i => (char)i) // .Where(c => !Char.IsControl(c)).ToArray()); var expected = Some.String(); var indexQuery = new IndexQuery(); indexQuery.Query = expected; IndexQuery result = EncodeAndDecodeIndexQuery(indexQuery); Assert.Equal(expected, result.Query); }
private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery) { // Perform the query until we have some results at least QueryResultWithIncludes result; var sp = Stopwatch.StartNew(); while (true) { result = documentDatabase.Query(map.IndexName, new IndexQuery { Cutoff = query.Cutoff, PageSize = query.PageSize, Query = realQuery, Start = query.Start, FieldsToFetch = query.FieldsToFetch, GroupBy = query.GroupBy, AggregationOperation = query.AggregationOperation, SortedFields = query.SortedFields, DefaultField = query.DefaultField, CutoffEtag = query.CutoffEtag, DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries, DefaultOperator = query.DefaultOperator, SkipTransformResults = query.SkipTransformResults, SkippedResults = query.SkippedResults, HighlighterPreTags = query.HighlighterPreTags, HighlighterPostTags = query.HighlighterPostTags, HighlightedFields = query.HighlightedFields, ResultsTransformer = query.ResultsTransformer, QueryInputs = query.QueryInputs }); if (!touchTemporaryIndexResult.Item2 || !result.IsStale || (result.Results.Count >= query.PageSize && query.PageSize > 0) || sp.Elapsed.TotalSeconds > 15) { return result; } Thread.Sleep(100); } }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, bool allowStale, Func<string, TransactionInformation, object> batchOperation) { var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields }; bool stale; var queryResults = database.QueryDocumentIds(index, bulkIndexQuery, out stale); if (stale && allowStale == false) { throw new InvalidOperationException( "Bulk operation cancelled because the index is stale and allowStale is false"); } const int batchSize = 1024; using (var enumerator = queryResults.GetEnumerator()) { while (true) { var batchCount = 0; database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; var result = batchOperation(enumerator.Current, transactionInformation); array.Add(RavenJObject.FromObject(result, JsonExtensions.CreateDefaultJsonSerializer())); } }); if (batchCount < batchSize) break; } } return array; }
private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple<string, bool> touchTemporaryIndexResult, string realQuery) { // Perform the query until we have some results at least QueryResultWithIncludes result; var sp = Stopwatch.StartNew(); while (true) { var indexQuery = CreateIndexQuery(query, map, realQuery); result = documentDatabase.Query(map.IndexName, indexQuery); if (!touchTemporaryIndexResult.Item2 || !result.IsStale || (result.Results.Count >= query.PageSize && query.PageSize > 0) || sp.Elapsed.TotalSeconds > 15) { return result; } Thread.Sleep(100); } }
public IEnumerable<string> QueryDocumentIds(string index, IndexQuery query, CancellationToken token, out bool stale) { var queryStat = AddToCurrentlyRunningQueryList(index, query); try { bool isStale = false; HashSet<string> loadedIds = null; TransactionalStorage.Batch( actions => { var definition = IndexDefinitionStorage.GetIndexDefinition(index); if (definition == null) throw new ArgumentException("specified index definition was not found", "index"); isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null); if (isStale == false && query.Cutoff == null) { var indexInstance = Database.IndexStorage.GetIndexInstance(index); isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress); } var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } loadedIds = new HashSet<string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, token) select queryResult.Key); }); stale = isStale; return loadedIds; } finally { RemoveFromCurrentlyRunningQueryList(index, queryStat); } }
private void Create_DeleteByIndex_CreateWithTransactionScope_ReplicationToSecondaryFailsWithNonCurrentEtag(IDocumentStore primaryStore, IDocumentStore secondaryStore, string primaryDbName, string secondaryDbName) { var docId = "create-deletebyindex-createwithtransactionscope"; Assert.Null(primaryStore.DatabaseCommands.ForDatabase(primaryDbName).Get(docId)); Assert.Null(secondaryStore.DatabaseCommands.ForDatabase(secondaryDbName).Get(docId)); using (var session = primaryStore.OpenSession(primaryDbName)) { var doc = new Doc(docId, "first primaryStore"); session.Store(doc, doc.Id); session.SaveChanges(); } WaitForIndexing(primaryStore, primaryDbName); WaitForReplication(secondaryStore, docId, secondaryDbName); var databaseCommands = primaryStore.DatabaseCommands.ForDatabase(primaryDbName); using (var session = primaryStore.OpenSession(primaryDbName)) { var deleteExistingDocsQuery = session.Advanced .DocumentQuery <Doc, DocsIndex>() .WhereIn("Id", new string[] { docId }); var deleteExistingDocsIndexQuery = new IndexQuery { Query = deleteExistingDocsQuery.ToString() }; var deleteByIndex = databaseCommands.DeleteByIndex(new DocsIndex().IndexName, deleteExistingDocsIndexQuery, false); var array = deleteByIndex.WaitForCompletion() as RavenJArray; Assert.Equal(1, array.Length); Assert.Equal(docId, array[0].Value <string>("Document")); Assert.Equal(true, array[0].Value <bool>("Deleted")); } WaitForReplication(secondaryStore, session => session.Load <Doc>(docId) == null, secondaryDbName); using (var transaction = new TransactionScope()) { using (var session = primaryStore.OpenSession(primaryDbName)) { // insert with same id again var docAgain = new Doc(docId, "second primaryStore with transaction scope"); session.Store(docAgain, docAgain.Id); session.SaveChanges(); transaction.Complete(); } } WaitForDocument(primaryStore.DatabaseCommands.ForDatabase(primaryDbName), docId); WaitForDocument(secondaryStore.DatabaseCommands.ForDatabase(secondaryDbName), docId); using (var session = secondaryStore.OpenSession(secondaryDbName)) { Assert.Equal("second primaryStore with transaction scope", session.Load <Doc>(docId).Description); } }
public QueryResult Query(string index, IndexQuery query) { index = IndexDefinitionStorage.FixupIndexName(index); var list = new List <JObject>(); var stale = false; Tuple <DateTime, Guid> indexTimestamp = null; TransactionalStorage.Batch( actions => { string entityName = null; var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) { throw new InvalidOperationException("Could not find index named: " + index); } entityName = viewGenerator.ForEntityName; stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName); indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index); var indexFailureInformation = actions.Indexing.GetFailureRate(index); if (indexFailureInformation.IsInvalidIndex) { throw new IndexDisabledException(indexFailureInformation); } var docRetriever = new DocumentRetriever(actions, ReadTriggers); var indexDefinition = GetIndexDefinition(index); var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation, viewGenerator.ReduceDefinition == null ? Abstractions.Data.Constants.DocumentIdFieldName : Abstractions.Data.Constants.ReduceKeyFieldName); var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch) select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch) into doc where doc != null select doc; var transformerErrors = new List <string>(); IEnumerable <JObject> results; if (viewGenerator != null && query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null) { var robustEnumerator = new RobustEnumerator { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray(); results = robustEnumerator.RobustEnumeration( dynamicJsonObjects, source => viewGenerator.TransformResultsDefinition(docRetriever, source)) .Select(JsonExtensions.ToJObject); } else { results = collection.Select(x => x.ToJson()); } list.AddRange(results); if (transformerErrors.Count > 0) { throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors)); } }); return(new QueryResult { IndexName = index, Results = list, IsStale = stale, SkippedResults = query.SkippedResults.Value, TotalResults = query.TotalSize.Value, IndexTimestamp = indexTimestamp.Item1, IndexEtag = indexTimestamp.Item2 }); }
public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } if (string.IsNullOrEmpty(indexQuery.Query) && // we optimize for empty queries to use Raven/DocumentsByEntityName (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested database.IndexDefinitionStorage.Contains("Raven/DocumentsByEntityName")) // and Raven/DocumentsByEntityName exists { if (string.IsNullOrEmpty(entityName) == false) { indexQuery.Query = "Tag:" + entityName; } return("Raven/DocumentsByEntityName"); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2).ToArray(); var normalizedFieldsQueriedUpon = fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceIndavlidCharactersForFields).ToArray(); var distinctSelectManyFields = new HashSet <string>(); foreach (var field in fieldsQueriedUpon) { var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < parts.Length; i++) { distinctSelectManyFields.Add(string.Join(",", parts.Take(i))); } } return(database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is no matching view generator { return false; } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { return false; } if (abstractViewGenerator.TransformResultsDefinition != null) // we can't choose an index with transform results { return false; } if (abstractViewGenerator.HasWhereClause) // without a where clause { return false; } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one // in order to handle that, we count the number of select many that would happen because of the query // and match it to the number of select many in the index if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count) { return false; } if (entityName == null) { if (abstractViewGenerator.ForEntityNames.Count != 0) { return false; } } else { if (abstractViewGenerator.ForEntityNames.Count != 1 || // we only allow indexes with a single entity name abstractViewGenerator.ForEntityNames.Contains(entityName) == false) // for the specified entity name { return false; } } if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false) { return false; } var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return false; } if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { if (sortedField.Field.StartsWith(Constants.RandomFieldName)) { continue; } // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(sortedField.Field) == false) { return false; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: return false; } } if (value != dynamicSortInfo.FieldType) { return false; // different sort order, there is a problem here } } } if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0) { // none of the fields have custom analyzers if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { return false; } } return true; }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return -1; } return abstractViewGenerator.CountOfFields; }) .FirstOrDefault()); }
private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery) { var indexEtag = GetIndexEtag(index); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return(null); } var queryResult = Database.Query(index, indexQuery); queryResult.IndexEtag = indexEtag; return(queryResult); }
private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag) { string entityName = null; if (index.StartsWith("dynamic/")) { entityName = index.Substring("dynamic/".Length); } var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery.Query); if (Database.IndexStorage.HasIndex(dynamicIndexName)) { indexEtag = GetIndexEtag(dynamicIndexName); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return(null); } } else { indexEtag = Guid.Empty; } var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery); // have to check here because we might be getting the index etag just // as we make a switch from temp to auto, and we need to refresh the etag // if that is the case. This can also happen when the optmizer // decided to switch indexes for a query. if (queryResult.IndexName != dynamicIndexName) { indexEtag = GetIndexEtag(queryResult.IndexName); } return(queryResult); }
public DatabaseQueryOperation(DocumentDatabase database, string indexName, IndexQuery query, IStorageActionsAccessor actions, CancellationTokenSource cancellationTokenSource) { this.database = database; this.indexName = indexName != null?indexName.Trim() : null; this.query = query; this.actions = actions; cancellationToken = cancellationTokenSource.Token; queryStat = database.Queries.AddToCurrentlyRunningQueryList(indexName, query, cancellationTokenSource); if (query.ShowTimings == false) { return; } executionTimes[QueryTimings.Lucene] = 0; executionTimes[QueryTimings.LoadDocuments] = 0; executionTimes[QueryTimings.TransformResults] = 0; }
private void GetIndexEntriesForDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, Reference <int> totalResults) { string entityName; var dynamicIndexName = GetDynamicIndexName(index, indexQuery, out entityName); if (dynamicIndexName == null) { context.SetStatusToNotFound(); return; } GetIndexEntriesForExistingIndex(context, dynamicIndexName, indexQuery, totalResults); }
public LazySuggestionQueryOperation(InMemoryDocumentSessionOperations session, IndexQuery indexQuery, Action <QueryResult> invokeAfterQueryExecuted, Func <QueryResult, Dictionary <string, SuggestionResult> > processResults) { _session = session; _indexQuery = indexQuery; _invokeAfterQueryExecuted = invokeAfterQueryExecuted; _processResults = processResults; }
protected internal virtual IEnumerable <object> ApplyReduceFunctionIfExists(IndexQuery indexQuery, IEnumerable <object> enumerable) { return(enumerable.Take(indexQuery.PageSize)); }
public static SortOptions?GetSortOption(this IndexDefinition self, string name, IndexQuery query) { SortOptions value; if (name.EndsWith("_Range")) { string nameWithoutRange = name.Substring(0, name.Length - "_Range".Length); if (self.SortOptions.TryGetValue(nameWithoutRange, out value)) { return(value); } if (self.SortOptions.TryGetValue(Constants.AllFields, out value)) { return(value); } if (query != null && query.SortHints != null && query.SortHints.ContainsKey("SortHint-" + nameWithoutRange)) { return(query.SortHints["SortHint-" + nameWithoutRange]); } } if (self.SortOptions.TryGetValue(name, out value)) { return(value); } if (self.SortOptions.TryGetValue(Constants.AllFields, out value)) { return(value); } if (query == null || query.SortHints == null) { return(value); } if (query.SortHints.ContainsKey("SortHint-" + name) == false) { return(value); } return(query.SortHints["SortHint-" + name]); }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func <string, TransactionInformation, object> batchOperation) { options = options ?? new BulkOperationOptions(); var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints, HighlighterKeyName = indexQuery.HighlighterKeyName, TransformerParameters = indexQuery.TransformerParameters, ResultsTransformer = indexQuery.ResultsTransformer }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && options.AllowStale == false) { if (options.StaleTimeout != null) { var staleWaitTimeout = Stopwatch.StartNew(); while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout) { queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale) { SystemTime.Wait(100); } } } if (stale) { if (options.StaleTimeout != null) { throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed"); } throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false"); } } var token = tokenSource.Token; const int batchSize = 1024; int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue; using (var enumerator = queryResults.GetEnumerator()) { var duration = Stopwatch.StartNew(); var operations = 0; while (true) { database.WorkContext.UpdateFoundWork(); if (timeout != null) { timeout.Delay(); } var batchCount = 0; var shouldWaitNow = false; token.ThrowIfCancellationRequested(); using (database.DocumentLock.Lock()) { database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; operations++; var result = batchOperation(enumerator.Current, transactionInformation); if (options.RetrieveDetails) { array.Add(RavenJObject.FromObject(result)); } if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000) { shouldWaitNow = true; break; } } }); if (shouldWaitNow) { SystemTime.Wait(500); operations = 0; duration.Restart(); continue; } } if (batchCount < batchSize) { break; } } } return(array); }
public Operation UpdateByIndex(string indexName, IndexQuery queryToUpdate, PatchRequest patch, QueryOperationOptions options = null) { return(AsyncHelpers.RunSync(() => asyncServerClient.UpdateByIndexAsync(indexName, queryToUpdate, patch, options))); }
private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag) { indexEtag = GetIndexEtag(index); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return null; } return Database.Query(index, indexQuery); }
public QueryOperation(InMemoryDocumentSessionOperations sessionOperations, string indexName, IndexQuery indexQuery, string[] projectionFields, bool waitForNonStaleResults, TimeSpan timeout, Func <IndexQuery, IEnumerable <object>, IEnumerable <object> > transformResults, HashSet <string> includes, bool disableEntitiesTracking) { this.indexQuery = indexQuery; this.waitForNonStaleResults = waitForNonStaleResults; this.timeout = timeout; this.transformResults = transformResults; this.includes = includes; this.projectionFields = projectionFields; this.sessionOperations = sessionOperations; this.indexName = indexName; this.disableEntitiesTracking = disableEntitiesTracking; AssertNotQueryById(); }
public FacetResults GetFacets(string index, IndexQuery indexQuery, List <Facet> facets, int start = 0, int?pageSize = null) { var sp = Stopwatch.StartNew(); var results = new FacetResults(); var defaultFacets = new Dictionary <string, Facet>(); var rangeFacets = new Dictionary <string, List <ParsedRange> >(); var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(index); Index.AssertQueryDoesNotContainFieldsThatAreNotIndexed(indexQuery, viewGenerator); foreach (var facet in facets) { var key = string.IsNullOrWhiteSpace(facet.DisplayName) ? facet.Name : facet.DisplayName; defaultFacets[key] = facet; if (facet.Aggregation != FacetAggregation.Count && facet.Aggregation != FacetAggregation.None) { if (string.IsNullOrEmpty(facet.AggregationField)) { throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " + facet.Aggregation + " without having a value in AggregationField"); } if (facet.AggregationField.EndsWith("_Range") == false) { if (QueryForFacets.IsAggregationTypeNumerical(facet.AggregationType)) { facet.AggregationField = facet.AggregationField + "_Range"; } } } switch (facet.Mode) { case FacetMode.Default: results.Results[key] = new FacetResult(); break; case FacetMode.Ranges: rangeFacets[key] = facet.Ranges.Select(range => ParseRange(facet.Name, range)).ToList(); results.Results[key] = new FacetResult { Values = facet.Ranges.Select(range => new FacetValue { Range = range, }).ToList() }; break; default: throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode)); } } var queryForFacets = new QueryForFacets(database, index, defaultFacets, rangeFacets, indexQuery, results, start, pageSize); queryForFacets.Execute(); results.Duration = sp.Elapsed; return(results); }
public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken token) { var stopwatch = Stopwatch.StartNew(); var items = new List <ICommandData>(deletionBatchSize); var attachments = new List <string>(deletionBatchSize); var itemsAndAttachements = Tuple.Create(items, attachments); try { var query = new IndexQuery { Start = 0, PageSize = deletionBatchSize, Cutoff = SystemTime.UtcNow, DisableCaching = true, Query = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { "__document_id", "MessageMetadata.MessageId", "MessageMetadata.BodyNotStored" }, SortedFields = new[] { new SortedField("ProcessedAt") { Field = "ProcessedAt", Descending = false } } }; var indexName = new ExpiryProcessedMessageIndex().IndexName; database.Query(indexName, query, token, (doc, state) => { var id = doc.Value <string>("__document_id"); if (string.IsNullOrEmpty(id)) { return; } state.Item1.Add(new DeleteCommandData { Key = id }); if (TryGetBodyId(doc, out var bodyId)) { state.Item2.Add(bodyId); } }, itemsAndAttachements); } catch (OperationCanceledException) { logger.Info("Cleanup operation cancelled"); return; } if (token.IsCancellationRequested) { return; } var deletedAuditDocuments = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => { if (logger.IsDebugEnabled) { logger.Debug($"Batching deletion of {s}-{e} audit documents."); } var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); if (logger.IsDebugEnabled) { logger.Debug($"Batching deletion of {s}-{e} audit documents completed."); } return(results.Count(x => x.Deleted == true)); }, items, database, token); var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (att, db, s, e) => { var deleted = 0; if (logger.IsDebugEnabled) { logger.Debug($"Batching deletion of {s}-{e} attachment audit documents."); } db.TransactionalStorage.Batch(accessor => { for (var idx = s; idx <= e; idx++) { //We want to continue using attachments for now #pragma warning disable 618 accessor.Attachments.DeleteAttachment(att[idx], null); #pragma warning restore 618 deleted++; } }); if (logger.IsDebugEnabled) { logger.Debug($"Batching deletion of {s}-{e} attachment audit documents completed."); } return(deleted); }, attachments, database, token); if (deletedAttachments + deletedAuditDocuments == 0) { logger.Info("No expired audit documents found"); } else { logger.Info($"Deleted {deletedAuditDocuments} expired audit documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); } }
public void Physical_store_test() { List <Car> cars; using (var documentStore = NewRemoteDocumentStore()) { documentStore.Initialize(); new CarIndex().Execute(documentStore); new CarTransformer().Execute(documentStore); using (var session = documentStore.OpenSession()) { foreach (CarLot doc in Docs) { session.Store(doc); } session.SaveChanges(); } string targetLeasee = CarLeasees[0].Id; using (var session = documentStore.OpenSession()) { //Synchronize indexes session.Query <CarLot, CarIndex>().Customize(x => x.WaitForNonStaleResults(TimeSpan.FromMinutes(2))).FirstOrDefault(); var query = session.Query <CarLot, CarIndex>() .Where(carLot => carLot.Cars.Any(car => car.LeaseHistory.Any(leasee => leasee.Id == targetLeasee))) .Take(1024); var deserializer = session.Advanced.DocumentStore.Conventions.CreateSerializer(); var indexQuery = new IndexQuery { Query = query.ToString(), PageSize = 1024, }; var queryResult = session.Advanced.DocumentStore.DatabaseCommands.Query(typeof(CarIndex).Name, indexQuery, new string[0]); var carLots = queryResult .Results .Select(x => deserializer.Deserialize <CarLot>(new RavenJTokenReader(x))) .ToArray() ; foreach (var carLot in carLots) { Assert.NotNull(carLot.Cars); Assert.NotEmpty(carLot.Cars); } cars = carLots .SelectMany(x => x.Cars) .Where(car => car.LeaseHistory.Any(leasee => leasee.Id == targetLeasee)) .ToList(); } } Assert.NotNull(cars); Assert.NotEmpty(cars); foreach (Car car in cars) { Assert.NotNull(car.LeaseHistory); Assert.NotEmpty(car.LeaseHistory); } }
public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query) { writer.WriteStartObject(); writer.WritePropertyName(nameof(query.Query)); writer.WriteString(query.Query); writer.WriteComma(); if (query.PageSizeSet && query.PageSize >= 0) { writer.WritePropertyName(nameof(query.PageSize)); writer.WriteInteger(query.PageSize); writer.WriteComma(); } if (query.WaitForNonStaleResults) { writer.WritePropertyName(nameof(query.WaitForNonStaleResults)); writer.WriteBool(query.WaitForNonStaleResults); writer.WriteComma(); } if (query.Start > 0) { writer.WritePropertyName(nameof(query.Start)); writer.WriteInteger(query.Start); writer.WriteComma(); } if (query.WaitForNonStaleResultsTimeout.HasValue) { writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout)); writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString()); writer.WriteComma(); } if (query.DisableCaching) { writer.WritePropertyName(nameof(query.DisableCaching)); writer.WriteBool(query.DisableCaching); writer.WriteComma(); } #if FEATURE_SHOW_TIMINGS if (query.ShowTimings) { writer.WritePropertyName(nameof(query.ShowTimings)); writer.WriteBool(query.ShowTimings); writer.WriteComma(); } #endif if (query.SkipDuplicateChecking) { writer.WritePropertyName(nameof(query.SkipDuplicateChecking)); writer.WriteBool(query.SkipDuplicateChecking); writer.WriteComma(); } writer.WritePropertyName(nameof(query.QueryParameters)); if (query.QueryParameters != null) { writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query.QueryParameters, conventions, context, conventions.CreateSerializer(), documentInfo: null)); } else { writer.WriteNull(); } writer.WriteEndObject(); }
private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable <JsonDocument> results, List <string> transformerErrors, Action <double> loadingDocumentsFinish, Action <double> transformerFinish, bool showTimings, CancellationToken token) { if (query.PageSize <= 0) // maybe they just want the stats? { return(Enumerable.Empty <RavenJObject>()); } IndexingFunc transformFunc = null; // Check an explicitly declared one first if (string.IsNullOrEmpty(query.ResultsTransformer) == false) { var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer); if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null) { transformFunc = transformGenerator.TransformResultsDefinition; } else { throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found"); } } if (transformFunc == null) { var resultsWithoutTransformer = results.Select(x => { var ravenJObject = x.ToJson(); if (query.IsDistinct) { ravenJObject[Constants.DocumentIdFieldName] = x.Key; } return(ravenJObject); }); return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer); } var dynamicJsonObjects = results.Select(x => { var ravenJObject = x.ToJson(); if (query.IsDistinct) { ravenJObject[Constants.DocumentIdFieldName] = x.Key; } return(new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject)); }); var robustEnumerator = new RobustEnumerator(token, 100) { OnError = (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)) }; var resultsWithTransformer = robustEnumerator .RobustEnumeration(dynamicJsonObjects.Cast <object>().GetEnumerator(), transformFunc) .Select(JsonExtensions.ToJObject); return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer); }
private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery) { string entityName = null; if (index.StartsWith("dynamic/")) { entityName = index.Substring("dynamic/".Length); } var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery.Query); var indexEtag = Guid.Empty; if (Database.IndexStorage.HasIndex(dynamicIndexName)) { indexEtag = GetIndexEtag(dynamicIndexName); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return(null); } } var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery); if (indexEtag != Guid.Empty) { queryResult.IndexEtag = indexEtag; } return(queryResult); }
public Operation DeleteByIndex(string indexName, IndexQuery queryToDelete, QueryOperationOptions options = null) { return(AsyncHelpers.RunSync(() => asyncServerClient.DeleteByIndexAsync(indexName, queryToDelete, options))); }
public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold) { var stopwatch = Stopwatch.StartNew(); var items = new List <ICommandData>(deletionBatchSize); try { var query = new IndexQuery { Start = 0, DisableCaching = true, Cutoff = SystemTime.UtcNow, PageSize = deletionBatchSize, Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { "__document_id" }, SortedFields = new[] { new SortedField("LastModified") { Field = "LastModified", Descending = false } } }; var indexName = new ExpiryEventLogItemsIndex().IndexName; database.Query(indexName, query, database.WorkContext.CancellationToken, (doc, commands) => { var id = doc.Value <string>("__document_id"); if (string.IsNullOrEmpty(id)) { return; } commands.Add(new DeleteCommandData { Key = id }); }, items); } catch (OperationCanceledException) { //Ignore } var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => { logger.InfoFormat("Batching deletion of {0}-{1} eventlogitem documents.", s, e); var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); logger.InfoFormat("Batching deletion of {0}-{1} eventlogitem documents completed.", s, e); return(results.Count(x => x.Deleted == true)); }, items, database); if (deletionCount == 0) { logger.Info("No expired eventlogitem documents found"); } else { logger.InfoFormat("Deleted {0} expired eventlogitem documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds); } }
/// <summary> /// This test target a bug around minimal splitter in gbpTree and unique index populator. It goes like this: /// Given a set of updates (value,entityId): /// - ("A01",1), ("A90",3), ("A9",2) /// If ("A01",1) and ("A90",3) would cause a split to occur they would produce a minimal splitter ("A9",3). /// Note that the value in this minimal splitter is equal to our last update ("A9",2). /// When making insertions with the unique populator we don't compare entityId which would means ("A9",2) /// ends up to the right of ("A9",3), even though it belongs to the left because of entityId being smaller. /// At this point the tree is in an inconsistent (key on wrong side of splitter). /// /// To work around this problem the entityId is only kept in minimal splitter if strictly necessary to divide /// left from right. This means the minimal splitter between ("A01",1) and ("A90",3) is ("A9",-1) and ("A9",2) /// will correctly be placed on the right side of this splitter. /// /// To trigger this scenario this test first insert a bunch of values that are all unique and that will cause a /// split to happen. This is the firstBatch. /// The second batch are constructed so that at least one of them will have a value equal to the splitter key /// constructed during the firstBatch. /// It's important that the secondBatch has ids that are lower than the first batch to align with example described above. /// </summary> //JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldPopulateAndRemoveEntriesWithSimilarMinimalSplitter() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldPopulateAndRemoveEntriesWithSimilarMinimalSplitter() { string prefix = "Work out your own salvation. Do not depend on others. "; int nbrOfNodes = 200; long nodeId = 0; // Second batch has lower ids IList <NodeAndValue> secondBatch = new List <NodeAndValue>(); for (int i = 0; i < nbrOfNodes; i++) { secondBatch.Add(new NodeAndValue(nodeId++, stringValue(prefix + i))); } // First batch has higher ids and minimal splitter among values in first batch will be found among second batch IList <NodeAndValue> firstBatch = new List <NodeAndValue>(); for (int i = 0; i < nbrOfNodes; i++) { firstBatch.Add(new NodeAndValue(nodeId++, stringValue(prefix + i + " " + i))); } WithPopulator(IndexProvider.getPopulator(Descriptor, IndexSamplingConfig, heapBufferFactory(1024)), p => { p.add(Updates(firstBatch)); p.add(Updates(secondBatch)); // Index should be consistent }); IList <NodeAndValue> toRemove = new List <NodeAndValue>(); ((IList <NodeAndValue>)toRemove).AddRange(firstBatch); ((IList <NodeAndValue>)toRemove).AddRange(secondBatch); Collections.shuffle(toRemove); // And we should be able to remove the entries in any order using (IndexAccessor accessor = IndexProvider.getOnlineAccessor(Descriptor, IndexSamplingConfig)) { // WHEN using (IndexUpdater updater = accessor.NewUpdater(IndexUpdateMode.ONLINE)) { foreach (NodeAndValue nodeAndValue in toRemove) { updater.Process(IndexEntryUpdate.Remove(nodeAndValue.NodeId, Descriptor, nodeAndValue.Value)); } } // THEN using (IndexReader reader = new QueryResultComparingIndexReader(accessor.NewReader())) { int propertyKeyId = Descriptor.schema().PropertyId; foreach (NodeAndValue nodeAndValue in toRemove) { NodeValueIterator nodes = new NodeValueIterator(); reader.Query(nodes, IndexOrder.NONE, false, IndexQuery.exact(propertyKeyId, nodeAndValue.Value)); bool anyHits = false; StringJoiner nodesStillLeft = new StringJoiner(", ", "[", "]"); while (nodes.HasNext()) { anyHits = true; nodesStillLeft.add(Convert.ToString(nodes.Next())); } assertFalse("Expected this query to have zero hits but found " + nodesStillLeft.ToString(), anyHits); } } } }
private QueryResult PerformQueryAgainstExistingIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag) { indexEtag = GetIndexEtag(index); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return(null); } return(Database.Query(index, indexQuery)); }
public Lazy <Task <Dictionary <string, FacetResult> > > ExecuteLazyAsync(Action <Dictionary <string, FacetResult> > onEval = null, CancellationToken token = default) { _query = GetIndexQuery(isAsync: true); return(((AsyncDocumentSession)_session).AddLazyOperation(new LazyAggregationQueryOperation(_session.Conventions, _query, InvokeAfterQueryExecuted, ProcessResults), onEval, token)); }
private QueryCommand GetCommand(bool isAsync) { _query = GetIndexQuery(isAsync); return(new QueryCommand(_session, _query)); }
public string SelectAppropriateIndex(string entityName, IndexQuery indexQuery) { // There isn't much point for query optimizer of aggregation indexes // the main reason is that we must always aggregate on the same items, and using the same // aggregation. Therefore we can't reuse one aggregate index for another query. // We decline to suggest an index here and choose to use the default index created for this // sort of query, which is what we would have to choose anyway. if (indexQuery.AggregationOperation != AggregationOperation.None) { return(null); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery.Query).Select(x => x.Item2); return(database.IndexDefinitionStorage.IndexNames .Where(indexName => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is no matching view generator { return false; } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { return false; } if (abstractViewGenerator.TransformResultsDefinition != null) // we can't choose an index with transform results { return false; } if (abstractViewGenerator.HasWhereClause) // without a where clause { return false; } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one if (abstractViewGenerator.CountOfSelectMany > 1) { return false; } if (abstractViewGenerator.ForEntityName != entityName) // for the specified entity name { return false; } return fieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap); }) .Where(indexName => { var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return false; } var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) { return false; } if (indexQuery.SortedFields != null) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(sortedField.Field) == false) { return false; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == sortedField.Field); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(sortedField.Field, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: return false; } } if (value != dynamicSortInfo.FieldType) { return false; // different sort order, there is a problem here } } } if (indexDefinition.Analyzers != null) { // none of the fields have custom analyzers if (fieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { return false; } } return true; }) .OrderByDescending(indexName => { // We select the widest index, because we want to encourage bigger indexes // Over time, it means that we smaller indexes would wither away and die, while // bigger indexes will be selected more often and then upgrade to permanent indexes var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); if (abstractViewGenerator == null) // there is a matching view generator { return -1; } return abstractViewGenerator.CountOfFields; }) .FirstOrDefault()); }
public IDictionary <string, IEnumerable <FacetValue> > GetFacets(string index, IndexQuery indexQuery, string facetSetupDoc) { var facetSetup = database.Get(facetSetupDoc, null); if (facetSetup == null) { throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc); } var facets = facetSetup.DataAsJson.JsonDeserialization <FacetSetup>().Facets; var results = new Dictionary <string, IEnumerable <FacetValue> >(); IndexSearcher currentIndexSearcher; using (database.IndexStorage.GetCurrentIndexSearcher(index, out currentIndexSearcher)) { foreach (var facet in facets) { switch (facet.Mode) { case FacetMode.Default: HandleTermsFacet(index, facet, indexQuery, currentIndexSearcher, results); break; case FacetMode.Ranges: HandleRangeFacet(index, facet, indexQuery, currentIndexSearcher, results); break; default: throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode)); } } } return(results); }
public DynamicQueryOptimizerResult SelectAppropriateIndex( string entityName, IndexQuery indexQuery, List <Explanation> explanations = null) { if (string.IsNullOrEmpty(indexQuery.Query) && // we optimize for empty queries to use Raven/DocumentsByEntityName (indexQuery.SortedFields == null || indexQuery.SortedFields.Length == 0) && // and no sorting was requested database.IndexDefinitionStorage.Contains(Constants.DocumentsByEntityNameIndex)) // and Raven/DocumentsByEntityName exists { if (string.IsNullOrEmpty(entityName) == false) { indexQuery.Query = "Tag:" + entityName; } return(new DynamicQueryOptimizerResult(Constants.DocumentsByEntityNameIndex, DynamicQueryMatchType.Complete)); } var fieldsQueriedUpon = SimpleQueryParser.GetFieldsForDynamicQuery(indexQuery).Select(x => x.Item2).ToArray(); var normalizedFieldsQueriedUpon = fieldsQueriedUpon.Select(DynamicQueryMapping.ReplaceInvalidCharactersForFields).ToArray(); var distinctSelectManyFields = new HashSet <string>(); foreach (var field in fieldsQueriedUpon) { var parts = field.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < parts.Length; i++) { distinctSelectManyFields.Add(string.Join(",", parts.Take(i))); } } ExplainDelegate explain = (index, rejectionReason) => { }; if (explanations != null) { explain = (index, rejectionReason) => explanations.Add(new Explanation { Index = index, Reason = rejectionReason() }); } //; // there is no reason why we can't use indexes with transform results // we merely need to disable the transform results for this particular query var results = database.IndexDefinitionStorage.IndexDefinitions .Select(indexDefinitionKvp => { var indexName = indexDefinitionKvp.Value.Name; var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName); var currentBestState = DynamicQueryMatchType.Complete; if (abstractViewGenerator == null) // there is no matching view generator { explain(indexName, () => "There is no matching view generator. Maybe the index in the process of being deleted?"); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } var indexingPriority = IndexingPriority.None; var isInvalidIndex = false; database.TransactionalStorage.Batch(accessor => { var stats = accessor.Indexing.GetIndexStats(indexDefinitionKvp.Key); isInvalidIndex = stats.IsInvalidIndex; indexingPriority = stats.Priority; }); if (entityName == null) { if (abstractViewGenerator.ForEntityNames.Count != 0) { explain(indexName, () => "Query is not specific for entity name, but the index filter by entity names."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } else { if (indexingPriority == IndexingPriority.Error || indexingPriority == IndexingPriority.Disabled || isInvalidIndex) { explain(indexName, () => string.Format("Cannot do dynamic queries on disabled index or index with errors (index name = {0})", indexName)); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.ForEntityNames.Count > 1) // we only allow indexes with a single entity name { explain(indexName, () => "Index contains more than a single entity name, may result in a different type being returned."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.ForEntityNames.Count == 0) { explain(indexName, () => "Query is specific for entity name, but the index searches across all of them, may result in a different type being returned."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.ForEntityNames.Contains(entityName) == false) // for the specified entity name { explain(indexName, () => string.Format("Index does not apply to entity name: {0}", entityName)); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (abstractViewGenerator.ReduceDefinition != null) // we can't choose a map/reduce index { explain(indexName, () => "Can't choose a map/reduce index for dynamic queries."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (abstractViewGenerator.HasWhereClause) // without a where clause { explain(indexName, () => "Can't choose an index with a where clause, it might filter things that the query is looking for."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } // we can't select an index that has SelectMany in it, because it result in invalid results when // you query it for things like Count, see https://github.com/ravendb/ravendb/issues/250 // for indexes with internal projections, we use the exact match based on the generated index name // rather than selecting the optimal one // in order to handle that, we count the number of select many that would happen because of the query // and match it to the number of select many in the index if (abstractViewGenerator.CountOfSelectMany != distinctSelectManyFields.Count) { explain(indexName, () => "Can't choose an index with a different number of from clauses / SelectMany, will affect queries like Count()."); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (normalizedFieldsQueriedUpon.All(abstractViewGenerator.ContainsFieldOnMap) == false) { explain(indexName, () => { var missingFields = normalizedFieldsQueriedUpon.Where(s => abstractViewGenerator.ContainsFieldOnMap(s) == false); return("The following fields are missing: " + string.Join(", ", missingFields)); }); currentBestState = DynamicQueryMatchType.Partial; } var indexDefinition = database.IndexDefinitionStorage.GetIndexDefinition(indexName); if (indexDefinition == null) { return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (indexQuery.HighlightedFields != null && indexQuery.HighlightedFields.Length > 0) { var nonHighlightableFields = indexQuery .HighlightedFields .Where(x => !indexDefinition.Stores.ContainsKey(x.Field) || indexDefinition.Stores[x.Field] != FieldStorage.Yes || !indexDefinition.Indexes.ContainsKey(x.Field) || indexDefinition.Indexes[x.Field] != FieldIndexing.Analyzed || !indexDefinition.TermVectors.ContainsKey(x.Field) || indexDefinition.TermVectors[x.Field] != FieldTermVector.WithPositionsAndOffsets) .Select(x => x.Field) .ToArray(); if (nonHighlightableFields.Any()) { explain(indexName, () => "The following fields could not be highlighted because they are not stored, analyzed and using term vectors with positions and offsets: " + string.Join(", ", nonHighlightableFields)); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (indexQuery.SortedFields != null && indexQuery.SortedFields.Length > 0) { var sortInfo = DynamicQueryMapping.GetSortInfo(s => { }, indexQuery); foreach (var sortedField in indexQuery.SortedFields) // with matching sort options { var normalizedFieldName = DynamicQueryMapping.ReplaceInvalidCharactersForFields(sortedField.Field); if (normalizedFieldName.EndsWith("_Range")) { normalizedFieldName = normalizedFieldName.Substring(0, normalizedFieldName.Length - "_Range".Length); } if (normalizedFieldName.StartsWith(Constants.RandomFieldName)) { continue; // virtual field that we don't sort by } // if the field is not in the output, then we can't sort on it. if (abstractViewGenerator.ContainsField(normalizedFieldName) == false) { explain(indexName, () => "Rejected because index does not contains field '" + normalizedFieldName + "' which we need to sort on"); currentBestState = DynamicQueryMatchType.Partial; continue; } var dynamicSortInfo = sortInfo.FirstOrDefault(x => x.Field == normalizedFieldName); if (dynamicSortInfo == null) // no sort order specified, we don't care, probably { continue; } SortOptions value; if (indexDefinition.SortOptions.TryGetValue(normalizedFieldName, out value) == false) { switch (dynamicSortInfo.FieldType) // if we can't find the value, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; default: explain(indexName, () => "The specified sort type is different than the default for field: " + normalizedFieldName); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (value != dynamicSortInfo.FieldType) { explain(indexName, () => "The specified sort type (" + dynamicSortInfo.FieldType + ") is different than the one specified for field '" + normalizedFieldName + "' (" + value + ")"); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } } if (indexDefinition.Analyzers != null && indexDefinition.Analyzers.Count > 0) { // none of the fields have custom analyzers if (normalizedFieldsQueriedUpon.Any(indexDefinition.Analyzers.ContainsKey)) { explain(indexName, () => { var fields = normalizedFieldsQueriedUpon.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have a custom analyzer: " + string.Join(", ", fields)); }); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (indexDefinition.Indexes != null && indexDefinition.Indexes.Count > 0) { //If any of the fields we want to query on are set to something other than the default, don't use the index var anyFieldWithNonDefaultIndexing = normalizedFieldsQueriedUpon.Where(x => { FieldIndexing analyzedInfo; if (indexDefinition.Indexes.TryGetValue(x, out analyzedInfo)) { if (analyzedInfo != FieldIndexing.Default) { return(true); } } return(false); }); var fieldWithNonDefaultIndexing = anyFieldWithNonDefaultIndexing.ToArray(); //prevent several enumerations if (fieldWithNonDefaultIndexing.Any()) { explain(indexName, () => { var fields = fieldWithNonDefaultIndexing.Where(indexDefinition.Analyzers.ContainsKey); return("The following field have aren't using default indexing: " + string.Join(", ", fields)); }); return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } } if (currentBestState != DynamicQueryMatchType.Complete && indexDefinition.Type != "Auto") { return(new DynamicQueryOptimizerResult(indexName, DynamicQueryMatchType.Failure)); } if (currentBestState == DynamicQueryMatchType.Complete && (indexingPriority == IndexingPriority.Idle || indexingPriority == IndexingPriority.Abandoned)) { currentBestState = DynamicQueryMatchType.Partial; explain(indexName, () => String.Format("The index (name = {0}) is disabled or abandoned. The preference is for active indexes - making a partial match", indexName)); } return(new DynamicQueryOptimizerResult(indexName, currentBestState)); }) .Where(result => result.MatchType != DynamicQueryMatchType.Failure) .GroupBy(x => x.MatchType) .ToDictionary(x => x.Key, x => x.ToArray()); DynamicQueryOptimizerResult[] optimizerResults; if (results.TryGetValue(DynamicQueryMatchType.Complete, out optimizerResults) && optimizerResults.Length > 0) { DynamicQueryOptimizerResult[] prioritizedResults = null; database.TransactionalStorage.Batch(accessor => { prioritizedResults = optimizerResults.OrderByDescending(result => { var instance = this.database.IndexStorage.GetIndexInstance(result.IndexName); var stats = accessor.Indexing.GetIndexStats(instance.indexId); if (stats == null) { return(Etag.Empty); } return(stats.LastIndexedEtag); }) .ThenByDescending(result => { var abstractViewGenerator = database.IndexDefinitionStorage.GetViewGenerator(result.IndexName); if (abstractViewGenerator == null) { return(-1); } return(abstractViewGenerator.CountOfFields); }) .ToArray(); }); for (int i = 1; i < prioritizedResults.Length; i++) { explain(prioritizedResults[i].IndexName, () => "Wasn't the widest / most unstable index matching this query"); } return(prioritizedResults[0]); } if (results.TryGetValue(DynamicQueryMatchType.Partial, out optimizerResults) && optimizerResults.Length > 0) { return(optimizerResults.OrderByDescending(x => { var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(x.IndexName); if (viewGenerator == null) { return -1; } return viewGenerator.CountOfFields; }).First()); } return(new DynamicQueryOptimizerResult("", DynamicQueryMatchType.Failure)); }
public Lazy <Dictionary <string, FacetResult> > ExecuteLazy(Action <Dictionary <string, FacetResult> > onEval = null) { _query = GetIndexQuery(isAsync: false); return(((DocumentSession)_session).AddLazyOperation(new LazyAggregationQueryOperation(_session.Conventions, _query, InvokeAfterQueryExecuted, ProcessResults), onEval)); }
private QueryResult PerformQueryAgainstDynamicIndex(IHttpContext context, string index, IndexQuery indexQuery, out Guid indexEtag) { string entityName = null; if (index.StartsWith("dynamic/")) entityName = index.Substring("dynamic/".Length); var dynamicIndexName = Database.FindDynamicIndexName(entityName, indexQuery); if (dynamicIndexName != null && Database.IndexStorage.HasIndex(dynamicIndexName)) { indexEtag = GetIndexEtag(dynamicIndexName); if (context.MatchEtag(indexEtag)) { context.SetStatusToNotModified(); return null; } } else { indexEtag = Guid.Empty; } var queryResult = Database.ExecuteDynamicQuery(entityName, indexQuery); // have to check here because we might be getting the index etag just // as we make a switch from temp to auto, and we need to refresh the etag // if that is the case. This can also happen when the optmizer // decided to switch indexes for a query. if (queryResult.IndexName != dynamicIndexName) indexEtag = GetIndexEtag(queryResult.IndexName); return queryResult; }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void concurrentIndexPopulationAndInsertsShouldNotProduceDuplicates() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ConcurrentIndexPopulationAndInsertsShouldNotProduceDuplicates() { // Given Config config = Config.defaults(); GraphDatabaseService db = NewEmbeddedGraphDatabaseWithSlowJobScheduler(config); try { // When using (Transaction tx = Db.beginTx()) { Db.schema().indexFor(label(LABEL)).on(KEY).create(); tx.Success(); } Node node; using (Transaction tx = Db.beginTx()) { node = Db.createNode(label(LABEL)); node.SetProperty(KEY, VALUE); tx.Success(); } using (Transaction tx = Db.beginTx()) { Db.schema().awaitIndexesOnline(1, MINUTES); tx.Success(); } // Then using (Transaction tx = Db.beginTx()) { KernelTransaction ktx = (( GraphDatabaseAPI )db).DependencyResolver.resolveDependency(typeof(ThreadToStatementContextBridge)).getKernelTransactionBoundToThisThread(true); IndexReference index = ktx.SchemaRead().index(ktx.TokenRead().nodeLabel(LABEL), ktx.TokenRead().propertyKey(KEY)); NodeValueIndexCursor cursor = ktx.Cursors().allocateNodeValueIndexCursor(); ktx.DataRead().nodeIndexSeek(index, cursor, IndexOrder.NONE, false, IndexQuery.exact(1, VALUE)); assertTrue(cursor.Next()); assertEquals(node.Id, cursor.NodeReference()); assertFalse(cursor.Next()); tx.Success(); } } finally { Db.shutdown(); } }