private List <JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
{
var result = new List <JsonDocument>();
bool docsLoaded;
do
{
var nextEtagToIndex = GetNextDocEtag(etag);
var firstEtagInQueue = prefetchingQueue.NextDocumentETag();
if (nextEtagToIndex != firstEtagInQueue)
{
if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false)
{
LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one
}
}
docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, result);
if (docsLoaded)
{
etag = result[result.Count - 1].Etag;
}
} while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch &&
docsLoaded &&
(prefetchingQueue.Aggregate(0, (acc, doc) => acc + doc.SerializedSizeOnDisk) +
autoTuner.CurrentlyUsedBatchSizes.Values.Sum()) < context.Configuration.MemoryLimitForIndexingInMB);
return(result);
}
private List <JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
{
var result = new List <JsonDocument>();
bool docsLoaded;
do
{
var nextEtagToIndex = GetNextDocEtag(etag);
var firstEtagInQueue = prefetchingQueue.NextDocumentETag();
if (nextEtagToIndex != firstEtagInQueue)
{
if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false)
{
LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one
}
}
docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, ref result);
if (docsLoaded)
{
etag = result[result.Count - 1].Etag;
}
} while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && docsLoaded);
return(result);
}
public bool CanUsePrefetcherToLoadFrom(Etag fromEtag)
{
var nextEtagToIndex = GetNextDocEtag(fromEtag);
var firstEtagInQueue = prefetchingQueue.NextDocumentETag();
if (firstEtagInQueue == null) // queue is empty, let it use this prefetcher
{
return(true);
}
if (nextEtagToIndex == firstEtagInQueue) // docs for requested etag are already in queue
{
return(true);
}
if (CanLoadDocumentsFromFutureBatches(nextEtagToIndex))
{
return(true);
}
return(false);
}
