public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit)
{
return;
}
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetch ||
prefetchingQueue.LoadedSize > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit)
{
return;
}
Etag lowestEtag = null;
using (prefetchingQueue.EnterWriteLock())
{
foreach (var jsonDocument in docs)
{
JsonDocument.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0))
{
lowestEtag = jsonDocument.Etag;
}
}
}
if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null)
{
if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0)
{
lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit
{
Etag = lowestEtag,
AddedAt = SystemTime.UtcNow
};
}
}
}
private void HandleCollectingDocumentsAfterCommit(Etag reqestedEtag)
{
if (ShouldHandleUnusedDocumentsAddedAfterCommit == false)
{
return;
}
if (DisableCollectingDocumentsAfterCommit)
{
if (lowestInMemoryDocumentAddedAfterCommit != null && reqestedEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) > 0)
{
lowestInMemoryDocumentAddedAfterCommit = null;
DisableCollectingDocumentsAfterCommit = false;
}
}
else
{
if (lowestInMemoryDocumentAddedAfterCommit != null && SystemTime.UtcNow - lowestInMemoryDocumentAddedAfterCommit.AddedAt > TimeSpan.FromMinutes(10))
{
DisableCollectingDocumentsAfterCommit = true;
}
}
}
private void HandleCollectingDocumentsAfterCommit(Etag requestedEtag)
{
if (ShouldHandleUnusedDocumentsAddedAfterCommit == false)
return;
var current = lowestInMemoryDocumentAddedAfterCommit;
if (DisableCollectingDocumentsAfterCommit)
{
if (current != null && requestedEtag.CompareTo(current.Etag) > 0)
{
lowestInMemoryDocumentAddedAfterCommit = null;
DisableCollectingDocumentsAfterCommit = false;
numberOfTimesIngestRateWasTooHigh = 0;
}
}
else if (current != null)
{
var oldestTimeInQueue = SystemTime.UtcNow - current.AddedAt;
if (
// this can happen if we have a very long indexing time, which takes
// us a long while to process, so we might as might as well stop keeping
// stuff in memory, because we lag
oldestTimeInQueue > TimeSpan.FromMinutes(10)
)
{
DisableCollectingDocumentsAfterCommit = true;
// If we disable in memory collection of data, we need to also remove all the
// items in the prefetching queue that are after the last in memory docs
// immediately, not wait for them to be indexed. They have already been in
// memory for ten minutes
// But not if our time has finally been reached
if (requestedEtag.CompareTo(current.Etag) >= 0)
prefetchingQueue.RemoveAfter(current.Etag);
}
}
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (ShouldHandleUnusedDocumentsAddedAfterCommit == false ||
context.Configuration.DisableDocumentPreFetching ||
docs.Length == 0 ||
DisableCollectingDocumentsAfterCommit ||
context.RunIndexing == false)
return;
// don't use too much, this is an optimization and we need to be careful about using too much memory
if (CanPrefetchMoreDocs(isFutureBatch: false) == false)
return;
if (prefetchingQueue.Count >= context.Configuration.MaxNumberOfItemsToPreFetch)
return;
ingestMeter.Mark(docs.Length);
var current = lowestInMemoryDocumentAddedAfterCommit;
if (current != null &&
// ingest rate is too high, maybe we need to protect ourselves from
// ingest overflow that can cause high memory usage
ingestMeter.OneMinuteRate > returnedDocsMeter.OneMinuteRate * 1.5
)
{
// we don't want to trigger it too soon, so we need a few commits
// with the high ingest rate before we'll decide that we need
// to take such measures
if (numberOfTimesIngestRateWasTooHigh++ > 3)
{
DisableCollectingDocumentsAfterCommit = true;
// remove everything after this
prefetchingQueue.RemoveAfter(current.Etag);
}
return;
}
else
{
numberOfTimesIngestRateWasTooHigh = 0;
}
Etag lowestEtag = null;
using (prefetchingQueue.EnterWriteLock())
{
foreach (var jsonDocument in docs)
{
JsonDocument.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
if (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0)
{
lowestEtag = jsonDocument.Etag;
}
}
}
if (lowestEtag != null)
{
if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0)
{
lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit
{
Etag = lowestEtag,
AddedAt = SystemTime.UtcNow
};
}
}
}
public void AfterStorageCommitBeforeWorkNotifications(JsonDocument[] docs)
{
if (context.Configuration.DisableDocumentPreFetching || docs.Length == 0 || DisableCollectingDocumentsAfterCommit)
return;
if (prefetchingQueue.Count >= // don't use too much, this is an optimization and we need to be careful about using too much mem
context.Configuration.MaxNumberOfItemsToPreFetch ||
prefetchingQueue.Aggregate(0, (x,c) => x + SelectSerializedSizeOnDiskIfNotNull(c)) > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit)
return;
Etag lowestEtag = null;
foreach (var jsonDocument in docs)
{
DocumentRetriever.EnsureIdInMetadata(jsonDocument);
prefetchingQueue.Add(jsonDocument);
if (ShouldHandleUnusedDocumentsAddedAfterCommit && (lowestEtag == null || jsonDocument.Etag.CompareTo(lowestEtag) < 0))
{
lowestEtag = jsonDocument.Etag;
}
}
if (ShouldHandleUnusedDocumentsAddedAfterCommit && lowestEtag != null)
{
if (lowestInMemoryDocumentAddedAfterCommit == null || lowestEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) < 0)
{
lowestInMemoryDocumentAddedAfterCommit = new DocAddedAfterCommit
{
Etag = lowestEtag,
AddedAt = SystemTime.UtcNow
};
}
}
}
private void HandleCollectingDocumentsAfterCommit(Etag reqestedEtag)
{
if(ShouldHandleUnusedDocumentsAddedAfterCommit == false)
return;
if (DisableCollectingDocumentsAfterCommit)
{
if (lowestInMemoryDocumentAddedAfterCommit != null && reqestedEtag.CompareTo(lowestInMemoryDocumentAddedAfterCommit.Etag) > 0)
{
lowestInMemoryDocumentAddedAfterCommit = null;
DisableCollectingDocumentsAfterCommit = false;
}
}
else
{
if (lowestInMemoryDocumentAddedAfterCommit != null && SystemTime.UtcNow - lowestInMemoryDocumentAddedAfterCommit.AddedAt > TimeSpan.FromMinutes(10))
{
DisableCollectingDocumentsAfterCommit = true;
}
}
}
