private bool ReduceBatchSizeIfCloseToMemoryCeiling(bool forceReducing = false) { if (MemoryStatistics.AvailableMemory >= context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit && forceReducing == false && IsIndexingUsingTooMuchMemory == false) { // there is enough memory available for the next indexing run return(false); } // we are using too much memory, let us use a less next time... // maybe it is us? we generate a lot of garbage when doing indexing, so we ask the GC if it would kindly try to // do something about it. // Note that this order for this to happen we need: // * We had two full run when we were doing nothing but indexing at full throttle // * The system is over the configured limit, and there is a strong likelihood that this is us causing this // * By forcing a GC, we ensure that we use less memory, and it is not frequent enough to cause perf problems RavenGC.CollectGarbage(compactLoh: true, afterCollect: null); // let us check again after the GC call, do we still need to reduce the batch size? if (MemoryStatistics.AvailableMemory > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit && forceReducing == false) { // we don't want to try increasing things, we just hit the ceiling, maybe on the next try return(true); } // we are still too high, let us reduce the size and see what is going on. NumberOfItemsToIndexInSingleBatch = CalculateReductionOfItemsInSingleBatch(); return(true); }
private static void TryHandlingError(string directory, int i, Exception e) { if (i == retries - 1) // last try also failed { foreach (var file in Directory.GetFiles(directory, "*", SearchOption.AllDirectories)) { var path = Path.GetFullPath(file); try { File.Delete(path); } catch (UnauthorizedAccessException) { throw new IOException(WhoIsLocking.ThisFile(path)); } catch (IOException) { var processesUsingFiles = WhoIsLocking.GetProcessesUsingFile(path); var stringBuilder = new StringBuilder(); stringBuilder.Append("The following processes are locking ").Append(path).AppendLine(); foreach (var processesUsingFile in processesUsingFiles) { stringBuilder.Append(" ").Append(processesUsingFile.ProcessName).Append(' ').Append(processesUsingFile.Id). AppendLine(); } throw new IOException(stringBuilder.ToString()); } } throw new IOException("Could not delete " + Path.GetFullPath(directory), e); } RavenGC.CollectGarbage(true); Thread.Sleep(100); }
private bool ConsiderDecreasingBatchSize(int amountOfItemsToIndex, TimeSpan indexingDuration) { var isIndexingUsingTooMuchMemory = IsIndexingUsingTooMuchMemory; if (isIndexingUsingTooMuchMemory == false) //skip all other heuristics if indexing takes too much memory { if ( // we had as much work to do as we are currently capable of handling, // we might need to increase, but certainly not decrease the batch size amountOfItemsToIndex >= NumberOfItemsToIndexInSingleBatch || // we haven't gone over the max latency limit, no reason to decrease yet indexingDuration < context.Configuration.MaxIndexingRunLatency) { return(false); } if ((SystemTime.UtcNow - lastIncrease).TotalMinutes < 3) { return(true); } // we didn't have a lot of work to do, so let us see if we can reduce the batch size // we are at the configured minimum, nothing to do if (NumberOfItemsToIndexInSingleBatch == InitialNumberOfItems) { return(true); } // we were above the max/2 the last few times, we can't reduce the work load now if (GetLastAmountOfItems().Any(x => x > NumberOfItemsToIndexInSingleBatch / 2)) { return(true); } } var old = NumberOfItemsToIndexInSingleBatch; NumberOfItemsToIndexInSingleBatch = CalculateReductionOfItemsInSingleBatch(); // we just reduced the batch size because we have two concurrent runs where we had // less to do than the previous runs. That indicate the the busy period is over, maybe we // run out of data? Or the rate of data entry into the system was just reduce? // At any rate, there is a strong likelyhood of having a lot of garbage in the system // let us ask the GC nicely to clean it // but we only want to do it if the change was significant if (NumberOfItemsToIndexInSingleBatch - old > 4096) { RavenGC.CollectGarbage(1, GCCollectionMode.Optimized); } return(true); }
private void HandleOutOfMemoryException(Exception oome) { Log.WarnException( @"Failed to execute indexing because of an out of memory exception. Will force a full GC cycle and then become more conservative with regards to memory", oome); // On the face of it, this is stupid, because OOME will not be thrown if the GC could release // memory, but we are actually aware that during indexing, the GC couldn't find garbage to clean, // but in here, we are AFTER the index was done, so there is likely to be a lot of garbage. RavenGC.CollectGarbage(GC.MaxGeneration); autoTuner.HandleOutOfMemory(); }
public HttpResponseMessage Gc() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Garbage Collection is only possible from the system database", HttpStatusCode.BadRequest)); } Action <DocumentDatabase> clearCaches = documentDatabase => documentDatabase.TransactionalStorage.ClearCaches(); Action afterCollect = () => DatabasesLandlord.ForAllDatabases(clearCaches); RavenGC.CollectGarbage(false, afterCollect, true); return(GetMessageWithString("GC Done")); }
public void Before_and_after_memory_allocation_should_be_recorded_correctly() { WeakReference reference; new Action(() => { var bytes = new byte[4096 * 1024]; new Random().NextBytes(bytes); reference = new WeakReference(bytes, true); })(); RavenGC.CollectGarbage(true, () => { }, true); Assert.True(RavenGC.MemoryBeforeLastForcedGC > RavenGC.MemoryAfterLastForcedGC); }
public HttpResponseMessage LohCompaction() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Large Object Heap Garbage Collection is only possiable from the system database", HttpStatusCode.BadRequest)); } Action <DocumentDatabase> clearCaches = documentDatabase => documentDatabase.TransactionalStorage.ClearCaches(); Action afterCollect = () => DatabasesLandlord.ForAllDatabases(clearCaches); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; RavenGC.CollectGarbage(true, afterCollect); return(GetMessageWithString("LOH GC Done")); }
private static bool InteractiveRun(RavenDbServer server) { bool?done = null; var actions = new Dictionary <string, Action> { { "cls", TryClearingConsole }, { "reset", () => { TryClearingConsole(); done = true; } }, { "gc", () => { long before = Process.GetCurrentProcess().WorkingSet64; Console.WriteLine( "Starting garbage collection (without LOH compaction), current memory is: {0:#,#.##;;0} MB", before / 1024d / 1024d); RavenGC.CollectGarbage(false, () => server.SystemDatabase.TransactionalStorage.ClearCaches()); var after = Process.GetCurrentProcess().WorkingSet64; Console.WriteLine( "Done garbage collection, current memory is: {0:#,#.##;;0} MB, saved: {1:#,#.##;;0} MB", after / 1024d / 1024d, (before - after) / 1024d / 1024d); } }, { "loh-compaction", () => { long before = Process.GetCurrentProcess().WorkingSet64; Console.WriteLine( "Starting garbage collection (with LOH compaction), current memory is: {0:#,#.##;;0} MB", before / 1024d / 1024d); RavenGC.CollectGarbage(true, () => server.SystemDatabase.TransactionalStorage.ClearCaches()); var after = Process.GetCurrentProcess().WorkingSet64; Console.WriteLine( "Done garbage collection, current memory is: {0:#,#.##;;0} MB, saved: {1:#,#.##;;0} MB", after / 1024d / 1024d, (before - after) / 1024d / 1024d); } }, { "q", () => done = false } }; WriteInteractiveOptions(actions); while (true) { var readLine = Console.ReadLine() ?? ""; Action value; if (actions.TryGetValue(readLine, out value) == false) { Console.WriteLine("Could not understand: {0}", readLine); WriteInteractiveOptions(actions); continue; } value(); if (done != null) { return(done.Value); } } }