private static void TryHandlingError(string directory, int i, Exception e)
{
if (i == retries - 1) // last try also failed
{
foreach (var file in Directory.GetFiles(directory, "*", SearchOption.AllDirectories))
{
var path = Path.GetFullPath(file);
try
{
File.Delete(path);
}
catch (UnauthorizedAccessException)
{
throw new IOException(WhoIsLocking.ThisFile(path));
}
catch (IOException)
{
var processesUsingFiles = WhoIsLocking.GetProcessesUsingFile(path);
var stringBuilder = new StringBuilder();
stringBuilder.Append("The following processes are locking ").Append(path).AppendLine();
foreach (var processesUsingFile in processesUsingFiles)
{
stringBuilder.Append(" ").Append(processesUsingFile.ProcessName).Append(' ').Append(processesUsingFile.Id).
AppendLine();
}
throw new IOException(stringBuilder.ToString());
}
}
throw new IOException("Could not delete " + Path.GetFullPath(directory), e);
}
RavenGC.CollectGarbage(true);
Thread.Sleep(100);
}
private bool ReduceBatchSizeIfCloseToMemoryCeiling(bool forceReducing = false)
{
if (MemoryStatistics.AvailableMemory >= context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit && forceReducing == false &&
IsIndexingUsingTooMuchMemory == false)
{
// there is enough memory available for the next indexing run
return(false);
}
// we are using too much memory, let us use a less next time...
// maybe it is us? we generate a lot of garbage when doing indexing, so we ask the GC if it would kindly try to
// do something about it.
// Note that this order for this to happen we need:
// * We had two full run when we were doing nothing but indexing at full throttle
// * The system is over the configured limit, and there is a strong likelihood that this is us causing this
// * By forcing a GC, we ensure that we use less memory, and it is not frequent enough to cause perf problems
RavenGC.CollectGarbage(compactLoh: true, afterCollect: null);
// let us check again after the GC call, do we still need to reduce the batch size?
if (MemoryStatistics.AvailableMemory > context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit && forceReducing == false)
{
// we don't want to try increasing things, we just hit the ceiling, maybe on the next try
return(true);
}
// we are still too high, let us reduce the size and see what is going on.
NumberOfItemsToIndexInSingleBatch = CalculateReductionOfItemsInSingleBatch();
return(true);
}
public void Dispose()
{
RavenGC.Unregister(ClearPrefetchingBehaviors);
foreach (var prefetchingBehavior in prefetchingBehaviors)
{
prefetchingBehavior.Dispose();
}
}
private bool ConsiderDecreasingBatchSize(int amountOfItemsToProcess, TimeSpan processingDuration)
{
var isIndexingUsingTooMuchMemory = IsProcessingUsingTooMuchMemory;
if (isIndexingUsingTooMuchMemory == false) //skip all other heuristics if indexing takes too much memory
{
if (
// we had as much work to do as we are currently capable of handling,
// we might need to increase, but certainly not decrease the batch size
amountOfItemsToProcess >= NumberOfItemsToProcessInSingleBatch ||
// we haven't gone over the max latency limit, no reason to decrease yet
processingDuration < context.Configuration.MaxProcessingRunLatency)
{
return(false);
}
if ((SystemTime.UtcNow - lastIncrease).TotalMinutes < 3)
{
return(true);
}
// we didn't have a lot of work to do, so let us see if we can reduce the batch size
// we are at the configured minimum, nothing to do
if (NumberOfItemsToProcessInSingleBatch == InitialNumberOfItems)
{
return(true);
}
// we were above the max/2 the last few times, we can't reduce the work load now
if (GetLastAmountOfItems().Any(x => x > NumberOfItemsToProcessInSingleBatch / 2))
{
return(true);
}
}
var old = NumberOfItemsToProcessInSingleBatch;
NumberOfItemsToProcessInSingleBatch = CalculateReductionOfItemsInSingleBatch();
// we just reduced the batch size because we have two concurrent runs where we had
// less to do than the previous runs. That indicate the the busy period is over, maybe we
// run out of data? Or the rate of data entry into the system was just reduce?
// At any rate, there is a strong likelihood of having a lot of garbage in the system
// let us ask the GC nicely to clean it
// but we only want to do it if the change was significant
if (NumberOfItemsToProcessInSingleBatch - old > 4096)
{
RavenGC.ConsiderRunningGC();
}
return(true);
}
private void HandleOutOfMemoryException(Exception oome)
{
Log.WarnException(
@"Failed to execute indexing because of an out of memory exception. Will force a full GC cycle and then become more conservative with regards to memory",
oome);
// On the face of it, this is stupid, because OOME will not be thrown if the GC could release
// memory, but we are actually aware that during indexing, the GC couldn't find garbage to clean,
// but in here, we are AFTER the index was done, so there is likely to be a lot of garbage.
RavenGC.CollectGarbage(GC.MaxGeneration);
autoTuner.HandleOutOfMemory();
}
public HttpResponseMessage Gc()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Garbage Collection is only possible from the system database", HttpStatusCode.BadRequest));
}
Action <DocumentDatabase> clearCaches = documentDatabase => documentDatabase.TransactionalStorage.ClearCaches();
Action afterCollect = () => DatabasesLandlord.ForAllDatabases(clearCaches);
RavenGC.CollectGarbage(false, afterCollect, true);
return(GetMessageWithString("GC Done"));
}
public void Before_and_after_memory_allocation_should_be_recorded_correctly()
{
WeakReference reference;
new Action(() =>
{
var bytes = new byte[4096 * 1024];
new Random().NextBytes(bytes);
reference = new WeakReference(bytes, true);
})();
RavenGC.CollectGarbage(true, () => { }, true);
Assert.True(RavenGC.MemoryBeforeLastForcedGC > RavenGC.MemoryAfterLastForcedGC);
}
public HttpResponseMessage LohCompaction()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Large Object Heap Garbage Collection is only possiable from the system database", HttpStatusCode.BadRequest));
}
Action <DocumentDatabase> clearCaches = documentDatabase => documentDatabase.TransactionalStorage.ClearCaches();
Action afterCollect = () => DatabasesLandlord.ForAllDatabases(clearCaches);
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
RavenGC.CollectGarbage(true, afterCollect);
return(GetMessageWithString("LOH GC Done"));
}
private bool ReduceBatchSizeIfCloseToMemoryCeiling(bool forceReducing = false)
{
if (MemoryStatistics.AvailableMemoryInMb >= context.Configuration.AvailableMemoryForRaisingBatchSizeLimit && forceReducing == false &&
IsProcessingUsingTooMuchMemory == false)
{
// there is enough memory available for the next indexing run
return(false);
}
// we are using too much memory, let us use a less next time...
// maybe it is us? we generate a lot of garbage when doing indexing, so we ask the GC if it would kindly try to
// do something about it.
// Note that this order for this to happen we need:
// * We had two full run when we were doing nothing but indexing at full throttle
// * The system is over the configured limit, and there is a strong likelihood that this is us causing this
// * By forcing a GC, we ensure that we use less memory, and it is not frequent enough to cause perf problems
RavenGC.ConsiderRunningGC();
// let us check again after the GC call, do we still need to reduce the batch size?
if (MemoryStatistics.AvailableMemoryInMb > context.Configuration.AvailableMemoryForRaisingBatchSizeLimit && forceReducing == false)
{
// we don't want to try increasing things, we just hit the ceiling, maybe on the next try
return(true);
}
// we are still too high, let us reduce the size and see what is going on.
var prev = NumberOfItemsToProcessInSingleBatch;
NumberOfItemsToProcessInSingleBatch = CalculateReductionOfItemsInSingleBatch();
var reason = string.Format("Batch size is close to memory ceiling, NumberOfItemsToProcessInSingleBatch was reduced from {0:##,###;;0} to {1:##,###;;0}", prev, NumberOfItemsToProcessInSingleBatch);
context.Database.AutoTuningTrace.Enqueue(new AutoTunerDecisionDescription(Name, context.DatabaseName, reason));
return(true);
}
public RavenGCTests()
{
RavenGC.ResetHistory();
}
private static bool InteractiveRun(RavenDbServer server)
{
bool?done = null;
var actions = new Dictionary <string, Action>
{
{ "cls", TryClearingConsole },
{
"reset", () =>
{
TryClearingConsole();
done = true;
}
},
{
"gc", () =>
{
long before = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine(
"Starting garbage collection (without LOH compaction), current memory is: {0:#,#.##;;0} MB",
before / 1024d / 1024d);
RavenGC.CollectGarbage(false, () => server.SystemDatabase.TransactionalStorage.ClearCaches());
var after = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine(
"Done garbage collection, current memory is: {0:#,#.##;;0} MB, saved: {1:#,#.##;;0} MB",
after / 1024d / 1024d,
(before - after) / 1024d / 1024d);
}
},
{
"loh-compaction", () =>
{
long before = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine(
"Starting garbage collection (with LOH compaction), current memory is: {0:#,#.##;;0} MB",
before / 1024d / 1024d);
RavenGC.CollectGarbage(true, () => server.SystemDatabase.TransactionalStorage.ClearCaches());
var after = Process.GetCurrentProcess().WorkingSet64;
Console.WriteLine(
"Done garbage collection, current memory is: {0:#,#.##;;0} MB, saved: {1:#,#.##;;0} MB",
after / 1024d / 1024d,
(before - after) / 1024d / 1024d);
}
},
{ "q", () => done = false }
};
WriteInteractiveOptions(actions);
while (true)
{
var readLine = Console.ReadLine() ?? "";
Action value;
if (actions.TryGetValue(readLine, out value) == false)
{
Console.WriteLine("Could not understand: {0}", readLine);
WriteInteractiveOptions(actions);
continue;
}
value();
if (done != null)
{
return(done.Value);
}
}
}
static WebSocketTransportFactory()
{
RavenGC.Register(DisconnectWebSockets);
AppDomain.CurrentDomain.DomainUnload += (s, e) => RavenGC.Unregister(DisconnectWebSockets);
}
public Prefetcher(WorkContext workContext)
{
this.workContext = workContext;
RavenGC.Register(ClearPrefetchingBehaviors);
}
