private bool ReduceBatchSizeIfCloseToMemoryCeiling(bool forceReducing = false) { if (MemoryStatistics.AvailableMemoryInMb >= context.Configuration.AvailableMemoryForRaisingBatchSizeLimit && forceReducing == false && IsProcessingUsingTooMuchMemory == false) { // there is enough memory available for the next indexing run return(false); } // we are using too much memory, let us use a less next time... // maybe it is us? we generate a lot of garbage when doing indexing, so we ask the GC if it would kindly try to // do something about it. // Note that this order for this to happen we need: // * We had two full run when we were doing nothing but indexing at full throttle // * The system is over the configured limit, and there is a strong likelihood that this is us causing this // * By forcing a GC, we ensure that we use less memory, and it is not frequent enough to cause perf problems RavenGC.ConsiderRunningGC(); // let us check again after the GC call, do we still need to reduce the batch size? if (MemoryStatistics.AvailableMemoryInMb > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit && forceReducing == false) { // we don't want to try increasing things, we just hit the ceiling, maybe on the next try return(true); } // we are still too high, let us reduce the size and see what is going on. NumberOfItemsToProcessInSingleBatch = CalculateReductionOfItemsInSingleBatch(); return(true); }
private bool ConsiderDecreasingBatchSize(int amountOfItemsToProcess, TimeSpan processingDuration) { var isIndexingUsingTooMuchMemory = IsProcessingUsingTooMuchMemory; if (isIndexingUsingTooMuchMemory == false) //skip all other heuristics if indexing takes too much memory { if ( // we had as much work to do as we are currently capable of handling, // we might need to increase, but certainly not decrease the batch size amountOfItemsToProcess >= NumberOfItemsToProcessInSingleBatch || // we haven't gone over the max latency limit, no reason to decrease yet processingDuration < context.Configuration.MaxProcessingRunLatency) { return(false); } if ((SystemTime.UtcNow - lastIncrease).TotalMinutes < 3) { return(true); } // we didn't have a lot of work to do, so let us see if we can reduce the batch size // we are at the configured minimum, nothing to do if (NumberOfItemsToProcessInSingleBatch == InitialNumberOfItems) { return(true); } // we were above the max/2 the last few times, we can't reduce the work load now if (GetLastAmountOfItems().Any(x => x > NumberOfItemsToProcessInSingleBatch / 2)) { return(true); } } var old = NumberOfItemsToProcessInSingleBatch; NumberOfItemsToProcessInSingleBatch = CalculateReductionOfItemsInSingleBatch(); // we just reduced the batch size because we have two concurrent runs where we had // less to do than the previous runs. That indicate the the busy period is over, maybe we // run out of data? Or the rate of data entry into the system was just reduce? // At any rate, there is a strong likelihood of having a lot of garbage in the system // let us ask the GC nicely to clean it // but we only want to do it if the change was significant if (NumberOfItemsToProcessInSingleBatch - old > 4096) { RavenGC.ConsiderRunningGC(); } return(true); }