public Task <FacetedQueryResult> ExecuteFacetedQuery(string indexName, FacetQuery query, long?facetsEtag, long?existingResultEtag, OperationCancelToken token) { if (query.FacetSetupDoc != null) { FacetSetup facetSetup; using (_documentsContext.OpenReadTransaction()) { var facetSetupAsJson = _database.DocumentsStorage.Get(_documentsContext, query.FacetSetupDoc); if (facetSetupAsJson == null) { throw new DocumentDoesNotExistException(query.FacetSetupDoc); } try { facetSetup = JsonDeserializationServer.FacetSetup(facetSetupAsJson.Data); } catch (Exception e) { throw new DocumentParseException(query.FacetSetupDoc, typeof(FacetSetup), e); } facetsEtag = facetSetupAsJson.Etag; } query.Facets = facetSetup.Facets; } return(ExecuteFacetedQuery(indexName, query, facetsEtag.Value, existingResultEtag, token)); }
private async Task ApplyRetention( DocumentsOperationContext context, TimeSeriesCollectionConfiguration config, CollectionName collectionName, TimeSeriesPolicy policy, DateTime now) { var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage; if (policy.RetentionTime == TimeValue.MaxValue) { return; } var to = now.Add(-policy.RetentionTime); var list = new List <Slice>(); while (true) { Cts.Token.ThrowIfCancellationRequested(); context.Reset(); context.Renew(); list.Clear(); using (context.OpenReadTransaction()) { foreach (var item in tss.Stats.GetTimeSeriesByPolicyFromStartDate(context, collectionName, policy.Name, to, TimeSeriesRollups.TimeSeriesRetentionCommand.BatchSize)) { if (RequiredForNextPolicy(context, config, policy, item, to)) { continue; } if (tss.Rollups.HasPendingRollupFrom(context, item, to) == false) { list.Add(item); } } if (list.Count == 0) { return; } if (Logger.IsInfoEnabled) { Logger.Info($"Found {list.Count} time-series for retention in policy {policy.Name} with collection '{collectionName.Name}' up-to {to}" #if DEBUG + $"{Environment.NewLine}{string.Join(Environment.NewLine, list)}" #endif ); } var cmd = new TimeSeriesRollups.TimeSeriesRetentionCommand(list, collectionName.Name, to); await _database.TxMerger.Enqueue(cmd); } } }
public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(string indexName, MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { if (query == null) { throw new ArgumentNullException(nameof(query)); } if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0) { throw new InvalidOperationException("The document id or map group fields are mandatory"); } var index = GetIndex(indexName); var etag = index.GetIndexEtag(); if (etag == existingResultEtag) { return(MoreLikeThisQueryResultServerSide.NotModifiedResult); } context.OpenReadTransaction(); return(index.MoreLikeThisQuery(query, context, token)); }
public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { if (query == null) { throw new ArgumentNullException(nameof(query)); } if (string.IsNullOrWhiteSpace(query.DocumentId) && query.MapGroupFields.Count == 0 && string.IsNullOrWhiteSpace(query.Document)) { throw new InvalidOperationException("The document id, document (artificial document property) or map group fields are mandatory"); } var sw = Stopwatch.StartNew(); var index = GetIndex(query.Metadata.IndexName); if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag(); if (etag == existingResultEtag.Value) { return(MoreLikeThisQueryResultServerSide.NotModifiedResult); } } context.OpenReadTransaction(); var result = index.MoreLikeThisQuery(query, context, token); result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds; return(result); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.Counters)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); } buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); })); }
public override Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) { var result = new DocumentQueryResult(); if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed) { documentsContext.OpenReadTransaction(); } FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext); if (query.Metadata.HasOrderByRandom == false && existingResultEtag.HasValue) { if (result.ResultEtag == existingResultEtag) { return(Task.FromResult(DocumentQueryResult.NotModifiedResult)); } } var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName); using (QueryRunner.MarkQueryAsRunning(indexName, query, token)) { result.IndexName = indexName; ExecuteCollectionQuery(result, query, collection, documentsContext, pulseReadingTransaction: false, token.Token); return(Task.FromResult(result)); } }
public async Task CreateAutoIndexesAndWaitIfNecessary() { var queryStepsGatherer = new QueryQueryStepGatherer(); queryStepsGatherer.Visit(RootQueryStep); if (_context.Transaction == null || _context.Transaction.Disposed) { _context.OpenReadTransaction(); } try { var etag = DocumentsStorage.ReadLastEtag(_context.Transaction.InnerTransaction); var queryDuration = Stopwatch.StartNew(); var indexes = new List <Index>(); var indexWaiters = new Dictionary <Index, (IndexQueryServerSide, AsyncWaitForIndexing)>(); foreach (var queryStepInfo in queryStepsGatherer.QuerySteps) { if (string.IsNullOrWhiteSpace(queryStepInfo.QueryStep.Query.From.From.FieldValue) || queryStepInfo.IsIndexQuery) { continue; } var indexQuery = new IndexQueryServerSide(queryStepInfo.QueryStep.GetQueryString, queryStepInfo.QueryStep.QueryParameters); //No sense creating an index for collection queries if (indexQuery.Metadata.IsCollectionQuery) { continue; } var indexCreationInfo = await _dynamicQueryRunner.CreateAutoIndexIfNeeded(indexQuery, true, null, _database.DatabaseShutdown); if (indexCreationInfo.HasCreatedAutoIndex) //wait for non-stale only IF we just created an auto-index { indexes.Add(indexCreationInfo.Index); var queryTimeout = indexQuery.WaitForNonStaleResultsTimeout ?? Index.DefaultWaitForNonStaleResultsTimeout; indexWaiters.Add(indexCreationInfo.Index, (indexQuery, new AsyncWaitForIndexing(queryDuration, queryTimeout, indexCreationInfo.Index))); } } await WaitForNonStaleResultsInternal(etag, indexes, indexWaiters); } finally { //The rest of the code assumes that a Tx is not opened _context.Transaction.Dispose(); } }
public static FacetQuery Create(DocumentsOperationContext context, IndexQueryServerSide query) { long?facetsEtag = null; DocumentsTransaction tx = null; try { var facets = new Dictionary <string, FacetSetup>(StringComparer.OrdinalIgnoreCase); foreach (var selectField in query.Metadata.SelectFields) { if (selectField.IsFacet == false) { continue; } var facetField = (FacetField)selectField; if (facetField.FacetSetupDocumentId == null || facets.ContainsKey(facetField.FacetSetupDocumentId)) { continue; } if (tx == null) { tx = context.OpenReadTransaction(); } var documentJson = context.DocumentDatabase.DocumentsStorage.Get(context, facetField.FacetSetupDocumentId); if (documentJson == null) { throw new DocumentDoesNotExistException(facetField.FacetSetupDocumentId); } if (facetsEtag.HasValue == false) { facetsEtag = documentJson.Etag; } else { facetsEtag = facetsEtag.Value ^ documentJson.Etag; } var document = FacetSetup.Create(facetField.FacetSetupDocumentId, documentJson.Data); facets[facetField.FacetSetupDocumentId] = document; } return(new FacetQuery(query, facets, facetsEtag ?? 0) { Legacy = string.IsNullOrEmpty(query.ClientVersion) == false && query.ClientVersion[0] == '4' }); } finally { tx?.Dispose(); } }
public IDisposable OpenReadTransaction() { var documentsTx = Documents.OpenReadTransaction(); RavenTransaction serverTx = null; if (Server != null) { serverTx = Server.OpenReadTransaction(); } return(new DisposeTransactions(documentsTx, serverTx)); }
public static FacetQuery Create(DocumentsOperationContext context, IndexQueryServerSide query) { long?facetsEtag = null; DocumentsTransaction tx = null; try { var facets = new Dictionary <string, FacetSetup>(StringComparer.OrdinalIgnoreCase); foreach (var selectField in query.Metadata.SelectFields) { if (selectField.IsFacet == false) { continue; } var facetField = (FacetField)selectField; if (facetField.FacetSetupDocumentId == null || facets.ContainsKey(facetField.FacetSetupDocumentId)) { continue; } if (tx == null) { tx = context.OpenReadTransaction(); } var documentJson = context.DocumentDatabase.DocumentsStorage.Get(context, facetField.FacetSetupDocumentId); if (documentJson == null) { throw new DocumentDoesNotExistException(facetField.FacetSetupDocumentId); } if (facetsEtag.HasValue == false) { facetsEtag = documentJson.Etag; } else { facetsEtag = facetsEtag.Value ^ documentJson.Etag; } var document = (FacetSetup)EntityToBlittable.ConvertToEntity(typeof(FacetSetup), facetField.FacetSetupDocumentId, documentJson.Data, DocumentConventions.DefaultForServer); facets[facetField.FacetSetupDocumentId] = document; } return(new FacetQuery(query, facets, facetsEtag ?? 0)); } finally { tx?.Dispose(); } }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeTransaction.Dispose(); _returnContext.Dispose(); })); }
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamDocumentQueryResultWriter writer, OperationCancelToken token) { using (var result = new StreamDocumentQueryResult(response, writer)) { documentsContext.OpenReadTransaction(); FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext); ExecuteCollectionQuery(result, query, query.Metadata.CollectionName, documentsContext, token.Token); } return(Task.CompletedTask); }
public async Task ExecuteStream(HttpResponse response, BlittableJsonTextWriter writer, IndexQueryServerSide query) { var tuple = await MatchIndex(query, false); var index = tuple.Index; var collection = tuple.Collection; if (index == null) { using (var result = new StreamDocumentQueryResult(response, writer, _context)) { _context.OpenReadTransaction(); FillCountOfResultsAndIndexEtag(result, collection); ExecuteCollectionQuery(result, query, collection); } return; } await index.StreamQuery(response, writer, query, _context, _token); }
public async Task <FacetedQueryResult> ExecuteFacetedQuery(FacetQueryServerSide query, long?facetsEtag, long?existingResultEtag, DocumentsOperationContext documentsContext, OperationCancelToken token) { if (query.Metadata.IsDynamic) { throw new InvalidQueryException("Facet query must be executed against static index.", query.Metadata.QueryText, query.QueryParameters); } if (query.FacetSetupDoc != null) { FacetSetup facetSetup; using (documentsContext.OpenReadTransaction()) { var facetSetupAsJson = Database.DocumentsStorage.Get(documentsContext, query.FacetSetupDoc); if (facetSetupAsJson == null) { throw new DocumentDoesNotExistException(query.FacetSetupDoc); } try { facetSetup = JsonDeserializationServer.FacetSetup(facetSetupAsJson.Data); } catch (Exception e) { throw new DocumentParseException(query.FacetSetupDoc, typeof(FacetSetup), e); } facetsEtag = facetSetupAsJson.Etag; } query.Facets = facetSetup.Facets; } var index = GetIndex(query.Metadata.IndexName); if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag() ^ facetsEtag.Value; if (etag == existingResultEtag) { return(FacetedQueryResult.NotModifiedResult); } } return(await index.FacetedQuery(query, facetsEtag.Value, documentsContext, token)); }
private LegacySourceReplicationInformation GetSourceReplicationInformation(DocumentsOperationContext context, Guid remoteServerInstanceId, out string documentId) { documentId = $"Raven/Replication/Sources/{remoteServerInstanceId}"; using (context.OpenReadTransaction()) { var document = Database.DocumentsStorage.Get(context, documentId); if (document == null) { return(new LegacySourceReplicationInformation { ServerInstanceId = Database.DbId }); } return(JsonDeserializationServer.LegacySourceReplicationInformation(document.Data)); } }
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) || options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) || options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.EtagForBackup; } } var disposable = new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); }); return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build))); }
public override Task <DocumentQueryResult> ExecuteQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token) { var result = new DocumentQueryResult(); documentsContext.OpenReadTransaction(); FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext); if (query.Metadata.HasOrderByRandom == false && existingResultEtag.HasValue) { if (result.ResultEtag == existingResultEtag) { return(Task.FromResult(DocumentQueryResult.NotModifiedResult)); } } ExecuteCollectionQuery(result, query, query.Metadata.CollectionName, documentsContext, token.Token); return(Task.FromResult(result)); }
public override Task ExecuteStreamQuery(IndexQueryServerSide query, DocumentsOperationContext documentsContext, HttpResponse response, IStreamQueryResultWriter<Document> writer, OperationCancelToken token) { var result = new StreamDocumentQueryResult(response, writer, token); documentsContext.OpenReadTransaction(); FillCountOfResultsAndIndexEtag(result, query.Metadata, documentsContext); var collection = GetCollectionName(query.Metadata.CollectionName, out var indexName); using (QueryRunner.MarkQueryAsRunning(indexName, query, token)) { result.IndexName = indexName; ExecuteCollectionQuery(result, query, collection, documentsContext, token.Token); result.Flush(); return Task.CompletedTask; } }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.GetEtagForBackup(); } } buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); })); }
private long GetStartEtagForSubscription(DocumentsOperationContext docsContext, SubscriptionState subscription) { using (docsContext.OpenReadTransaction()) { long startEtag = 0; if (string.IsNullOrEmpty(subscription.ChangeVectorForNextBatchStartingPoint)) { return(startEtag); } var changeVector = subscription.ChangeVectorForNextBatchStartingPoint.ToChangeVector(); var cv = changeVector.FirstOrDefault(x => x.DbId == TcpConnection.DocumentDatabase.DbBase64Id); if (cv.DbId == "" && cv.Etag == 0 && cv.NodeTag == 0) { return(startEtag); } return(cv.Etag); } }
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report, DocumentsOperationContext context, DocumentsStorage documentsStorage) { if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId) { report.LastEtag = prevDatabaseReport.LastEtag; report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag; report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts; report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments; report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector; } else { using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } } }
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { const int pageSize = int.MaxValue; var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan : TimeSpan.FromMinutes(15); var moreWorkFound = false; foreach (var collection in _index.Collections) { using (var collectionStats = stats.For("Collection_" + collection)) { var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection); if (_logger.IsInfoEnabled) { _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}."); } var inMemoryStats = _index.GetStats(collection); var lastEtag = lastTombstoneEtag; var count = 0; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var batchCount = 0; using (databaseContext.OpenReadTransaction()) { sw.Restart(); if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection ? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize) : _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize); foreach (var tombstone in tombstones) { token.ThrowIfCancellationRequested(); if (indexWriter == null) { indexWriter = writeOperation.Value; } count++; batchCount++; lastEtag = tombstone.Etag; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true); if (_logger.IsInfoEnabled && count % 2048 == 0) { _logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}."); } if (tombstone.Type != Tombstone.TombstoneType.Document) { continue; // this can happen when we have '@all_docs' } _index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats); if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (batchCount == 0 || batchCount >= pageSize) { break; } } } if (count == 0) { continue; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } if (_index.Type.IsMap()) { _indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag); } else { _mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag; } moreWorkFound = true; } } return(moreWorkFound); }
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan : TimeSpan.FromMinutes(15); var moreWorkFound = false; foreach (var collection in _index.Collections) { using (var collectionStats = stats.For("Collection_" + collection)) { var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}."); } var inMemoryStats = _index.GetStats(collection); var lastEtag = lastMappedEtag; var count = 0; var resultsCount = 0; var pageSize = int.MaxValue; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { using (databaseContext.OpenReadTransaction()) { sw.Restart(); if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize); using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type)) { while (true) { if (docsEnumerator.MoveNext(out IEnumerable mapResults) == false) { collectionStats.RecordMapCompletedReason("No more documents to index"); keepRunning = false; break; } token.ThrowIfCancellationRequested(); if (indexWriter == null) { indexWriter = writeOperation.Value; } var current = docsEnumerator.Current; count++; collectionStats.RecordMapAttempt(); stats.RecordDocumentSize(current.Data.Size); if (_logger.IsInfoEnabled && count % 8192 == 0) { _logger.Info($"Executing map for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag:#,#;;0}."); } lastEtag = current.Etag; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false); try { var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, indexWriter, indexContext, collectionStats); _index.MapsPerSec.MarkSingleThreaded(numberOfResults); resultsCount += numberOfResults; collectionStats.RecordMapSuccess(); } catch (Exception e) when(e.IsIndexError()) { docsEnumerator.OnError(); _index.ErrorIndexIfCriticalException(e); collectionStats.RecordMapError(); if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " + $"Exception: {e}"); } if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, count) == false) { keepRunning = false; break; } if (count >= pageSize) { keepRunning = false; break; } if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (_logger.IsInfoEnabled) { _logger.Info($"Done executing map for '{_index.Name}'. Processed count: {count:#,#;;0} LastEtag {lastEtag}."); } } } } if (count > 0) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name}'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } moreWorkFound = true; } if (_index.Type.IsMap()) { _index.SaveLastState(); _indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag); } else { _mapReduceContext.ProcessedDocEtags[collection] = lastEtag; } } } return(moreWorkFound); }
private void FillDatabaseStatistics(DatabaseStatistics stats, DocumentsOperationContext context) { using (context.OpenReadTransaction()) { var indexes = Database.IndexStore.GetIndexes().ToList(); var size = Database.GetSizeOnDisk(); stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction); stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context); stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context); stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context); stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context); stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount; stats.SizeOnDisk = size.Data; stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations; stats.TempBuffersSizeOnDisk = size.TempBuffers; stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context); var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context); stats.CountOfAttachments = attachments.AttachmentCount; stats.CountOfUniqueAttachments = attachments.StreamsCount; stats.CountOfIndexes = indexes.Count; var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); stats.DatabaseChangeVector = statsDatabaseChangeVector; stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id; stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long); stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString(); stats.Indexes = new IndexInformation[indexes.Count]; for (var i = 0; i < indexes.Count; i++) { var index = indexes[i]; bool isStale; try { isStale = index.IsStale(context); } catch (OperationCanceledException) { // if the index has just been removed, let us consider it stale // until it can be safely removed from the list of indexes in the // database isStale = true; } stats.Indexes[i] = new IndexInformation { State = index.State, IsStale = isStale, Name = index.Name, LockMode = index.Definition.LockMode, Priority = index.Definition.Priority, Type = index.Type, LastIndexingTime = index.LastIndexingTime }; if (stats.LastIndexingTime.HasValue) { stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime; } else { stats.LastIndexingTime = index.LastIndexingTime; } } } }
private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token) { var moreWorkFound = false; Dictionary <string, long> lastIndexedEtagsByCollection = null; foreach (var collection in _index.Collections) { if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } if (lastIndexedEtagsByCollection == null) { lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase); } if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false) { lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); } if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now { continue; } foreach (var referencedCollection in referencedCollections) { using (var collectionStats = stats.For("Collection_" + referencedCollection.Name)) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}."); } long lastReferenceEtag; switch (actionType) { case ActionType.Document: lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection); break; case ActionType.Tombstone: lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection); break; default: throw new NotSupportedException(); } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}."); } var lastEtag = lastReferenceEtag; var count = 0; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var batchCount = 0; using (databaseContext.OpenReadTransaction()) { sw.Restart(); IEnumerable <Reference> references; switch (actionType) { case ActionType.Document: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(document => { _reference.Key = document.Id; _reference.Etag = document.Etag; return(_reference); }); break; case ActionType.Tombstone: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(tombstone => { _reference.Key = tombstone.LowerId; _reference.Etag = tombstone.Etag; return(_reference); }); break; default: throw new NotSupportedException(); } foreach (var referencedDocument in references) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}."); } lastEtag = referencedDocument.Etag; count++; batchCount++; var documents = new List <Document>(); foreach (var key in _indexStorage .GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction)) { using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey)) { // when there is conflict, we need to apply same behavior as if the document would not exist var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false); if (doc != null && doc.Etag <= lastIndexedEtag) { documents.Add(doc); } } } using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats)) { while (docsEnumerator.MoveNext(out IEnumerable mapResults)) { token.ThrowIfCancellationRequested(); var current = docsEnumerator.Current; if (indexWriter == null) { indexWriter = writeOperation.Value; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}."); } try { _index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats); } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } } } } if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (batchCount == 0 || batchCount >= pageSize) { break; } } } if (count == 0) { continue; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } switch (actionType) { case ActionType.Document: _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; case ActionType.Tombstone: _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; default: throw new NotSupportedException(); } moreWorkFound = true; } } } return(moreWorkFound); }
private bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token) { var moreWorkFound = false; Dictionary <string, long> lastIndexedEtagsByCollection = null; foreach (var collection in _index.Collections) { if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } if (lastIndexedEtagsByCollection == null) { lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase); } if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false) { lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); } if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now { continue; } var totalProcessedCount = 0; foreach (var referencedCollection in referencedCollections) { var inMemoryStats = _index.GetReferencesStats(referencedCollection.Name); using (var collectionStats = stats.For("Collection_" + referencedCollection.Name)) { long lastReferenceEtag; switch (actionType) { case ActionType.Document: lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); break; case ActionType.Tombstone: lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); break; default: throw new NotSupportedException(); } var lastEtag = lastReferenceEtag; var resultsCount = 0; var sw = new Stopwatch(); var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var hasChanges = false; using (databaseContext.OpenReadTransaction()) { sw.Restart(); IEnumerable <Reference> references; switch (actionType) { case ActionType.Document: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize, DocumentFields.Id | DocumentFields.Etag) .Select(document => { _reference.Key = document.Id; _reference.Etag = document.Etag; return(_reference); }); break; case ActionType.Tombstone: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(tombstone => { _reference.Key = tombstone.LowerId; _reference.Etag = tombstone.Etag; return(_reference); }); break; default: throw new NotSupportedException(); } var isTombstone = actionType == ActionType.Tombstone; foreach (var referencedDocument in references) { lastEtag = referencedDocument.Etag; hasChanges = true; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone); var documents = GetDocumentFromCollectionThatReference(databaseContext, indexContext, collection, referencedDocument, lastIndexedEtag); using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type)) { while (docsEnumerator.MoveNext(out IEnumerable mapResults)) { token.ThrowIfCancellationRequested(); totalProcessedCount++; collectionStats.RecordMapReferenceAttempt(); var current = docsEnumerator.Current; stats.RecordDocumentSize(current.Data.Size); try { var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, writeOperation, indexContext, collectionStats); resultsCount += numberOfResults; collectionStats.RecordMapReferenceSuccess(); _index.MapsPerSec.MarkSingleThreaded(numberOfResults); } catch (Exception e) when(e.IsIndexError()) { docsEnumerator.OnError(); _index.ErrorIndexIfCriticalException(e); collectionStats.RecordMapReferenceError(); if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } collectionStats.AddMapReferenceError(current.Id, $"Failed to execute mapping function on {current.Id}. Exception: {e}"); } _index.UpdateThreadAllocations(indexContext, writeOperation, stats, updateReduceStats: false); } } if (CanContinueBatch(databaseContext, indexContext, collectionStats, writeOperation, lastEtag, lastCollectionEtag, totalProcessedCount) == false) { keepRunning = false; break; } if (totalProcessedCount >= pageSize) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (hasChanges == false) { break; } } } if (lastReferenceEtag == lastEtag) { // the last referenced etag hasn't changed continue; } moreWorkFound = true; if (_logger.IsInfoEnabled) { _logger.Info($"Executed handle references for '{_index.Name}' index and '{referencedCollection.Name}' collection. " + $"Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } switch (actionType) { case ActionType.Document: _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; case ActionType.Tombstone: _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; default: throw new NotSupportedException(); } } } } return(moreWorkFound); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; var alreadySeenIdsCount = new Reference <long>(); string startAfterId = null; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; var ids = new Queue <string>(OperationBatchSize); while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); ids.Clear(); Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke(); using (context.OpenReadTransaction()) { foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery)) { using (document) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase)) { continue; } // start with and id queries aren't ordered by the etag if (isStartsWithOrIdQuery == false && document.Etag > lastEtag) { // we don't want to go over the documents that we have patched end = true; break; } startEtag = document.Etag + 1; if (start > 0) { start--; continue; } if (take-- <= 0) { end = true; break; } startAfterId = document.Id; ids.Enqueue(document.Id); context.Transaction.ForgetAbout(document); } } } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token, maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte, batchSize: OperationBatchSize); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } token.Delay(); } while (ids.Count > 0); if (end) { break; } } } return(new BulkOperationResult { Total = progress.Processed }); }
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false) { var q = query.Metadata.Query; var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database) { CollectIntermediateResults = collectIntermediateResults }; qp.BuildQueryPlan(); qp.OptimizeQueryPlan(); //TODO: audit optimization if (query.WaitForNonStaleResults) { qp.IsStale = await qp.WaitForNonStaleResults(); } else { await qp.CreateAutoIndexesAndWaitIfNecessary(); } //for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed) { documentsContext.OpenReadTransaction(); } qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction); if (existingResultEtag.HasValue) { if (qp.ResultEtag == existingResultEtag) { return(null, null, true); } } await qp.Initialize(); var matchResults = qp.Execute(); if (query.Metadata.OrderBy != null) { Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query); } var filter = q.GraphQuery.Where; if (filter != null) { for (int i = 0; i < matchResults.Count; i++) { var resultAsJson = new DynamicJsonValue(); matchResults[i].PopulateVertices(resultAsJson); using (var result = documentsContext.ReadObject(resultAsJson, "graph/result")) { if (filter.IsMatchedBy(result, query.QueryParameters) == false) { matchResults[i] = default; } } } } if (query.Start > 0) { matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count)); } if (query.PageSize < matchResults.Count) { matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize); } return(matchResults, qp, false); }
public SuggestionQueryResultServerSide ExecuteSuggestionQuery(SuggestionQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token) { if (query == null) { throw new ArgumentNullException(nameof(query)); } // Check pre-requisites for the query to happen. if (string.IsNullOrWhiteSpace(query.Term)) { throw new InvalidOperationException("Suggestions queries require a term."); } if (string.IsNullOrWhiteSpace(query.Field)) { throw new InvalidOperationException("Suggestions queries require a field."); } var sw = Stopwatch.StartNew(); // Check definition for the index. var index = GetIndex(query.IndexName); var indexDefinition = index.GetIndexDefinition(); if (indexDefinition == null) { throw new InvalidOperationException($"Could not find specified index '{this}'."); } if (indexDefinition.Fields.TryGetValue(query.Field, out IndexFieldOptions field) == false) { throw new InvalidOperationException($"Index '{this}' does not have a field '{query.Field}'."); } if (field.Suggestions == null) { throw new InvalidOperationException($"Index '{this}' does not have suggestions configured for field '{query.Field}'."); } if (field.Suggestions.Value == false) { throw new InvalidOperationException($"Index '{this}' have suggestions explicitly disabled for field '{query.Field}'."); } if (existingResultEtag.HasValue) { var etag = index.GetIndexEtag(); if (etag == existingResultEtag.Value) { return(SuggestionQueryResultServerSide.NotModifiedResult); } } context.OpenReadTransaction(); var result = index.SuggestionsQuery(query, context, token); result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds; return(result); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { const int batchSize = 1024; var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); using (context.OpenReadTransaction()) { var ids = new Queue <LazyStringValue>(batchSize); foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs)) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && IsHiLoDocument(document)) { continue; } if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched { end = true; break; } startEtag = document.Etag + 1; ids.Enqueue(document.Id); } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } } while (ids.Count > 0); if (end) { break; } } } } return(new BulkOperationResult { Total = progress.Processed }); }