private List <JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag) { var result = new List <JsonDocument>(); bool docsLoaded; do { var nextEtagToIndex = GetNextDocEtag(etag); var firstEtagInQueue = prefetchingQueue.NextDocumentETag(); if (nextEtagToIndex != firstEtagInQueue) { if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false) { LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one } } docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, result); if (docsLoaded) { etag = result[result.Count - 1].Etag; } } while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && docsLoaded && (prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) + autoTuner.CurrentlyUsedBatchSizes.Values.Sum()) < context.Configuration.MemoryLimitForIndexingInMB); return(result); }
private List <JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag) { var result = new List <JsonDocument>(); bool docsLoaded; do { var nextEtagToIndex = GetNextDocEtag(etag); var firstEtagInQueue = prefetchingQueue.NextDocumentETag(); if (nextEtagToIndex != firstEtagInQueue) { if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false) { LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one } } docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, ref result); if (docsLoaded) { etag = result[result.Count - 1].Etag; } } while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && docsLoaded); return(result); }
public bool CanUsePrefetcherToLoadFrom(Etag fromEtag) { var nextEtagToIndex = GetNextDocEtag(fromEtag); var firstEtagInQueue = prefetchingQueue.NextDocumentETag(); if (firstEtagInQueue == null) // queue is empty, let it use this prefetcher { return(true); } if (nextEtagToIndex == firstEtagInQueue) // docs for requested etag are already in queue { return(true); } if (CanLoadDocumentsFromFutureBatches(nextEtagToIndex)) { return(true); } return(false); }