public JArray GetDocuments(int start, int pageSize, Guid?etag) { var list = new JArray(); TransactionalStorage.Batch(actions => { IEnumerable <JsonDocument> documents; if (etag == null) { documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); } else { documents = actions.Documents.GetDocumentsAfter(etag.Value); } var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } list.Add(document.ToJson()); } }); return(list); }
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument) { Etag lastDocumentReadEtag = null; TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, token); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) { etag = doc.Etag; } JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } returnedDocs = true; Database.WorkContext.UpdateFoundWork(); bool canContinue = addDocument(document); if (!canContinue) { break; } lastDocumentReadEtag = etag; } if (returnedDocs || docCount == 0) { break; } start += docCount; } }); return(lastDocumentReadEtag); }
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) { return; // index was deleted, probably } var dateTime = jsonDocs.Min(x => x.LastModified); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); } catch (Exception e) { if (actions.IsWriteConflict(e)) { return; } log.WarnFormat(e, "Failed to index documents for index: {0}", index); } }
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Action <JsonDocument> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2)); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); etag = doc.Etag; JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } addDocument(document); returnedDocs = true; Database.WorkContext.UpdateFoundWork(); } if (returnedDocs || docCount == 0) { break; } } }); return(etag); }
public static int UpdateDocumentsAfter(this DocumentDatabase db, Guid eTag, Func <JsonDocument, bool> updateDoc, CancellationToken cancellationToken, TransactionInformation transactionInformation) { var initialTime = DateTime.UtcNow; log.Trace("UpdateDocumentsAfter started whith Etag {0}", eTag); int updatedDocCount = 0; db.TransactionalStorage.Batch(action => { var documentRetriever = new DocumentRetriever(action, db.ReadTriggers); foreach (var doc in action.Documents.GetDocumentsAfter(eTag, int.MaxValue)) { DocumentRetriever.EnsureIdInMetadata(doc); if (cancellationToken.IsCancellationRequested) { log.Trace("UpdateDocumentsAfter has been cancelled"); } cancellationToken.ThrowIfCancellationRequested(); var document = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document != null && updateDoc(document)) { try { db.Put(document.Key, document.Etag, document.DataAsJson, document.Metadata, transactionInformation); updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } catch (ConcurrencyException) { log.Trace("ConcurrencyException caught in UpdateDocumentsAfter"); if (db.UpdateDocumentWithRetries(document.Key, updateDoc, transactionInformation)) { updatedDocCount++; log.Debug("{0} document has been updated in UpdateDocumentsAfter. {1} documents updated so far", document.Key, updatedDocCount); } } } } }); log.Trace("UpdateDocumentsAfter ETag {0} completed successfully. {1} documents updated in {2}", eTag, updatedDocCount, DateTime.UtcNow.Subtract(initialTime)); return(updatedDocCount); }
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) { etag = doc.Etag; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever .ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) { break; } start += docCount; } }); }
public static IEnumerable <JsonDocument> GetDocsWithIdStartingWith(this DocumentDatabase db, string idPrefix, int start, int pageSize) { if (idPrefix == null) { throw new ArgumentNullException("idPrefix"); } idPrefix = idPrefix.Trim(); var docs = new List <JsonDocument>(pageSize); db.TransactionalStorage.Batch(actions => { var retriever = new DocumentRetriever(actions, db.ReadTriggers); foreach (var doc in actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize)) { var document = retriever.ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } docs.Add(document); } }); return(docs); }
public JArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize) { var list = new JArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start) .Take(pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) { continue; } list.Add(document.ToJson()); } }); return(list); }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { IEnumerable<JsonDocument> documents; if (etag == null) documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); else documents = actions.Documents.GetDocumentsAfter(etag.Value); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, string skipAfter = null) { if (idPrefix == null) { throw new ArgumentNullException("idPrefix"); } idPrefix = idPrefix.Trim(); var canPerformRapidPagination = nextStart > 0 && start == nextStart; var actualStart = canPerformRapidPagination ? start : 0; var addedDocs = 0; var matchedDocs = 0; TransactionalStorage.Batch( actions => { var docsToSkip = canPerformRapidPagination ? 0 : start; int docCount; AbstractTransformer storedTransformer = null; if (transformer != null) { storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } } do { docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) { continue; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } matchedDocs++; if (matchedDocs <= docsToSkip) { continue; } token.ThrowIfCancellationRequested(); if (storedTransformer != null) { using (new CurrentTransformationScope(Database, documentRetriever)) { var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) { throw new InvalidOperationException("The transform results function failed on a document: " + document.Key); } var transformedJsonDocument = new JsonDocument { Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag), NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }, }; addDoc(transformedJsonDocument.ToJson()); } } else { addDoc(document.ToJson()); } addedDocs++; if (addedDocs >= pageSize) { break; } } actualStart += pageSize; }while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue); }); if (addedDocs != pageSize) { nextStart = start; // will mark as last page } else if (canPerformRapidPagination) { nextStart = start + matchedDocs; } else { nextStart = actualStart; } }
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument) { Etag lastDocumentReadEtag = null; TransactionalStorage.Batch(actions => { var returnedDocs = false; while (true) { var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2), lastProcessedDocument: x => lastDocumentReadEtag = x); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers); var docCount = 0; var docCountOnLastAdd = 0; foreach (var doc in documents) { docCount++; if (docCount - docCountOnLastAdd > 1000) { addDocument(null); // heartbeat } token.ThrowIfCancellationRequested(); etag = doc.Etag; JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } returnedDocs = true; Database.WorkContext.UpdateFoundWork(); var canContinue = addDocument(document); docCountOnLastAdd = docCount; if (!canContinue) { break; } } if (returnedDocs) { break; } // No document was found that matches the requested criteria if (docCount == 0) { // If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways). if (lastDocumentReadEtag != null) { etag = lastDocumentReadEtag; } break; } } }); return(etag); }
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, long?maxSize = null, TimeSpan?timeout = null) { Etag lastDocumentReadEtag = null; using (DocumentCacher.SkipSetDocumentsInDocumentCache()) TransactionalStorage.Batch(actions => { AbstractTransformer storedTransformer = null; if (transformer != null) { storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } } var returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters); var docCount = 0; var docCountOnLastAdd = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (docCount - docCountOnLastAdd > 1000) { addDocument(null); // heartbeat } if (etag != null) { etag = doc.Etag; } JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } returnedDocs = true; Database.WorkContext.UpdateFoundWork(); document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever); var canContinue = addDocument(document); if (!canContinue) { break; } lastDocumentReadEtag = etag; docCountOnLastAdd = docCount; } if (returnedDocs || docCount == 0) { break; } // No document was found that matches the requested criteria // If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways). if (lastDocumentReadEtag != null) { etag = lastDocumentReadEtag; } start += docCount; } }); return(lastDocumentReadEtag); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, string skipAfter = null) { if (idPrefix == null) { throw new ArgumentNullException("idPrefix"); } idPrefix = idPrefix.Trim(); var canPerformRapidPagination = nextStart > 0 && start == nextStart; var actualStart = canPerformRapidPagination ? start : 0; var addedDocs = 0; var docCountOnLastAdd = 0; var matchedDocs = 0; TransactionalStorage.Batch( actions => { var docsToSkip = canPerformRapidPagination ? 0 : start; int docCount; AbstractTransformer storedTransformer = null; if (transformer != null) { storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } } do { Database.WorkContext.UpdateFoundWork(); docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter); var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; if (docCount - docCountOnLastAdd > 1000) { addDoc(null); // heartbeat } var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) { continue; } JsonDocument.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } matchedDocs++; if (matchedDocs <= docsToSkip) { continue; } token.ThrowIfCancellationRequested(); document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever); addDoc(document); addedDocs++; docCountOnLastAdd = docCount; if (addedDocs >= pageSize) { break; } } actualStart += pageSize; } while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue); }); if (addedDocs != pageSize) { nextStart = start; // will mark as last page } else if (canPerformRapidPagination) { nextStart = start + matchedDocs; } else { nextStart = actualStart; } }
private IEnumerable <Tuple <IndexToWorkOn, IndexingBatch> > FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); var filteredDocs = BackgroundTaskExecuter.Instance.Apply(jsonDocs, doc => { doc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index); return(doc == null ? null : new { Doc = doc, Json = JsonToExpando.Convert(doc.ToJson()) }); }); log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count); var results = new Tuple <IndexToWorkOn, IndexingBatch> [indexesToWorkOn.Count]; var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count]; BackgroundTaskExecuter.Instance.ExecuteAll(context.Configuration, scheduler, indexesToWorkOn, (indexToWorkOn, i) => { var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()); if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0) { return; } var indexName = indexToWorkOn.IndexName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) { return; // probably deleted } var batch = new IndexingBatch(); foreach (var item in filteredDocs) { // did we already indexed this document in this index? if (indexLastInedexEtag.CompareTo(new ComparableByteArray(item.Doc.Etag.Value.ToByteArray())) >= 0) { continue; } // is the Raven-Entity-Name a match for the things the index executes on? if (viewGenerator.ForEntityNames.Count != 0 && viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false) { continue; } batch.Add(item.Doc, item.Json); if (batch.DateTime == null) { batch.DateTime = item.Doc.LastModified; } else { batch.DateTime = batch.DateTime > item.Doc.LastModified ? item.Doc.LastModified : batch.DateTime; } } if (batch.Docs.Count == 0) { log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified); // we use it this way to batch all the updates together actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified); return; } log.Debug("Going to index {0} documents in {1}", batch.Ids.Count, indexToWorkOn); results[i] = Tuple.Create(indexToWorkOn, batch); }); transactionalStorage.Batch(actionsAccessor => { foreach (var action in actions) { if (action != null) { action(actionsAccessor); } } }); return(results.Where(x => x != null)); }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); var list = new RavenJArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false) continue; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public void GetDocuments(int start, int pageSize, Etag etag, Action<RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); int docCount = 0; foreach (var doc in documents) { docCount++; if (etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public JArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new JArray(); TransactionalStorage.Batch(actions => { IEnumerable<JsonDocument> documents; if (etag == null) documents = actions.Documents.GetDocumentsByReverseUpdateOrder(start); else documents = actions.Documents.GetDocumentsAfter(etag.Value); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents.Take(pageSize)) { var document = documentRetriever.ExecuteReadTriggers(doc, null, // here we want to have the Load semantic, not Query, because we need this to be // as close as possible to the full database contents ReadOperation.Load); if (document == null) continue; if (document.Metadata.Property("@id") == null) document.Metadata.Add("@id", new JValue(doc.Key)); list.Add(document.ToJson()); } }); return list; }
private IEnumerable <IndexingBatchForIndex> FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, Etag highestETagInBatch) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag; var lastModified = last.LastModified.Value; var documentRetriever = new DocumentRetriever(null, null, context.ReadTriggers, context.Database.InFlightTransactionalState); var filteredDocs = BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc => { var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index); return(filteredDoc == null ? new { Doc = doc, Json = (object)new FilteredDocument(doc) } : new { Doc = filteredDoc, Json = JsonToExpando.Convert(doc.ToJson()) }); }); Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count); var results = new IndexingBatchForIndex[indexesToWorkOn.Count]; var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count]; BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) => { var indexName = indexToWorkOn.Index.PublicName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) { return; // probably deleted } var batch = new IndexingBatch(highestETagInBatch); foreach (var item in filteredDocs) { if (defaultPrefetchingBehavior.FilterDocuments(item.Doc) == false) { continue; } // did we already indexed this document in this index? var etag = item.Doc.Etag; if (etag == null) { continue; } // is the Raven-Entity-Name a match for the things the index executes on? if (viewGenerator.ForEntityNames.Count != 0 && viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false) { continue; } batch.Add(item.Doc, item.Json, defaultPrefetchingBehavior.ShouldSkipDeleteFromIndex(item.Doc)); if (batch.DateTime == null) { batch.DateTime = item.Doc.LastModified; } else { batch.DateTime = batch.DateTime > item.Doc.LastModified ? item.Doc.LastModified : batch.DateTime; } } if (batch.Docs.Count == 0) { Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified); // we use it this way to batch all the updates together actions[i] = accessor => { accessor.Indexing.UpdateLastIndexed(indexToWorkOn.Index.indexId, lastEtag, lastModified); accessor.AfterStorageCommit += () => { indexToWorkOn.Index.EnsureIndexWriter(); indexToWorkOn.Index.Flush(lastEtag); }; }; return; } if (Log.IsDebugEnabled) { Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids)); } results[i] = new IndexingBatchForIndex { Batch = batch, IndexId = indexToWorkOn.IndexId, Index = indexToWorkOn.Index, LastIndexedEtag = indexToWorkOn.LastIndexedEtag }; }); transactionalStorage.Batch(actionsAccessor => { foreach (var action in actions) { if (action != null) { action(actionsAccessor); } } }); return(results.Where(x => x != null)); }
private IEnumerable<IndexingBatchForIndex> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, List<JsonDocument> jsonDocs, Etag highestETagInBatch, out List<IndexToWorkOn> filteredOutIndexes) { var innerFilteredOutIndexes = new ConcurrentStack<IndexToWorkOn>(); var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag; var lastModified = last.LastModified.Value; var documentRetriever = new DocumentRetriever(null, null, context.ReadTriggers); var filteredDocs = BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc => { var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Index); return filteredDoc == null ? new { Doc = doc, Json = (object)new FilteredDocument(doc) } : new { Doc = filteredDoc, Json = JsonToExpando.Convert(doc.ToJson()) }; }); if ( Log.IsDebugEnabled ) Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count); var results = new ConcurrentQueue<IndexingBatchForIndex>(); var actions = new ConcurrentQueue<Action<IStorageActionsAccessor>>(); context.Database.MappingThreadPool.ExecuteBatch(indexesToWorkOn, indexToWorkOn => { var indexName = indexToWorkOn.Index.PublicName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) return; // probably deleted var batch = new IndexingBatch(highestETagInBatch); foreach (var filteredDoc in filteredDocs) { var doc = filteredDoc.Doc; var json = filteredDoc.Json; if (defaultPrefetchingBehavior.FilterDocuments(doc) == false || doc.Etag.CompareTo(indexToWorkOn.LastIndexedEtag) <= 0) continue; // did we already indexed this document in this index? var etag = doc.Etag; if (etag == null) continue; // is the Raven-Entity-Name a match for the things the index executes on? if (viewGenerator.ForEntityNames.Count != 0 && viewGenerator.ForEntityNames.Contains(doc.Metadata.Value<string>(Constants.RavenEntityName)) == false) { continue; } batch.Add(doc, json, defaultPrefetchingBehavior.ShouldSkipDeleteFromIndex(doc)); if (batch.DateTime == null) batch.DateTime = doc.LastModified; else batch.DateTime = batch.DateTime > doc.LastModified ? doc.LastModified : batch.DateTime; } if (batch.Docs.Count == 0) { if ( Log.IsDebugEnabled ) Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified); // we use it this way to batch all the updates together if (indexToWorkOn.LastIndexedEtag.CompareTo(lastEtag) < 0) actions.Enqueue(accessor => { accessor.Indexing.UpdateLastIndexed(indexToWorkOn.Index.indexId, lastEtag, lastModified); accessor.AfterStorageCommit += () => { indexToWorkOn.Index.EnsureIndexWriter(); indexToWorkOn.Index.Flush(lastEtag); }; }); innerFilteredOutIndexes.Push(indexToWorkOn); context.MarkIndexFilteredOut(indexName); return; } if (Log.IsDebugEnabled) Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids)); results.Enqueue(new IndexingBatchForIndex { Batch = batch, IndexId = indexToWorkOn.IndexId, Index = indexToWorkOn.Index, LastIndexedEtag = indexToWorkOn.LastIndexedEtag }); }, description: string.Format("Filtering documents for {0} indexes", indexesToWorkOn.Count)); filteredOutIndexes = innerFilteredOutIndexes.ToList(); foreach (var action in actions) { bool keepTrying = true; for (int i = 0; i < 10 && keepTrying; i++) { keepTrying = false; transactionalStorage.Batch(actionsAccessor => { if (action != null) { try { action(actionsAccessor); } catch (Exception e) { if (actionsAccessor.IsWriteConflict(e)) { keepTrying = true; return; } throw; } } }); if (keepTrying) Thread.Sleep(11); } } return results.Where(x => x != null); }
public RavenJArray GetDocuments(int start, int pageSize, Guid? etag) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag.Value, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); int docCount = 0; foreach (var doc in documents) { docCount++; if(etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } if (list.Length != 0 || docCount == 0) break; start += docCount; } }); return list; }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { int docCount = 0; var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); foreach (var doc in documents) { docCount++; string keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) continue; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); JsonDocument document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDoc(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action<RavenJObject> addDocument) { TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { var documents = etag == null ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize) : actions.Documents.GetDocumentsAfter(etag, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); int docCount = 0; foreach (var doc in documents) { docCount++; token.ThrowIfCancellationRequested(); if (etag != null) etag = doc.Etag; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc); document = documentRetriever .ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) continue; addDocument(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch( actions => { var docsToSkip = start; var addedDocs = 0; var matchedDocs = 0; int docCount; start = 0; do { docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers, inFlightTransactionalState); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) continue; DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) continue; matchedDocs++; if (matchedDocs <= docsToSkip) continue; token.ThrowIfCancellationRequested(); addDoc(document.ToJson()); addedDocs++; if (addedDocs >= pageSize) break; } start += pageSize; } while (docCount > 0 && addedDocs < pageSize && start >= 0 && start < int.MaxValue); }); }
public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, int start, int pageSize) { var list = new RavenJArray(); TransactionalStorage.Batch(actions => { var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start) .Take(pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; list.Add(document.ToJson()); } }); return list; }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, int start, int pageSize, Action<RavenJObject> addDoc) { if (idPrefix == null) throw new ArgumentNullException("idPrefix"); idPrefix = idPrefix.Trim(); TransactionalStorage.Batch(actions => { bool returnedDocs = false; while (true) { int docCount = 0; var documents = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, start, pageSize); var documentRetriever = new DocumentRetriever(actions, ReadTriggers); foreach (var doc in documents) { docCount++; if (WildcardMatcher.Matches(matches, doc.Key.Substring(idPrefix.Length)) == false) continue; DocumentRetriever.EnsureIdInMetadata(doc); var document = documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Load); if (document == null) continue; addDoc(document.ToJson()); returnedDocs = true; } if (returnedDocs || docCount == 0) break; start += docCount; } }); }