protected void HandleRavenOutOfMemoryException(Exception exception, OutOfMemoryDetails details)
{
var message = $"Failed to execute indexing because of {context.Database.TransactionalStorage.FriendlyName} " +
"out of memory exception. Will try to reduce batch size";
if (details == null)
{
Log.WarnException(message, exception);
autoTuner.DecreaseBatchSize();
return;
}
var errorCount = details.Index.IncrementOutOfMemoryErrors(details.IsReducing);
//if the current number of items to process in single batch is more than
//the initial number to index/reduce, than we can reduce the batch size and try again.
//If we can't reduce anymore, we'll try to disable the index
if (autoTuner.NumberOfItemsToProcessInSingleBatch > autoTuner.InitialNumberOfItems)
{
//will try to reduce the batch size
Log.WarnException(message, exception);
autoTuner.DecreaseBatchSize();
details.Index.AddOutOfMemoryDatabaseAlert(exception);
return;
}
details.Index.TryDisable(exception, errorCount, details.FailedItemsToProcessCount);
}
protected bool HandleIfOutOfMemory(Exception exception, OutOfMemoryDetails details)
{
try
{
var ae = exception as AggregateException;
if (ae == null)
{
if (exception is OutOfMemoryException)
{
HandleSystemOutOfMemoryException(exception);
return(true);
}
if (TransactionalStorageHelper.IsOutOfMemoryException(exception))
{
HandleRavenOutOfMemoryException(exception, details);
return(true);
}
return(false);
}
var isSystemOutOfMemory = false;
var isRavenOutOfMemoryException = false;
Exception oome = null;
Exception ravenOutOfMemoryException = null;
foreach (var innerException in ae.Flatten().InnerExceptions)
{
if (innerException is OutOfMemoryException)
{
isSystemOutOfMemory = true;
oome = innerException;
}
if (TransactionalStorageHelper.IsOutOfMemoryException(innerException))
{
isRavenOutOfMemoryException = true;
ravenOutOfMemoryException = innerException;
}
if (isSystemOutOfMemory && isRavenOutOfMemoryException)
{
break;
}
}
if (isSystemOutOfMemory)
{
HandleSystemOutOfMemoryException(oome);
}
if (isRavenOutOfMemoryException)
{
HandleRavenOutOfMemoryException(ravenOutOfMemoryException, details);
}
return(isRavenOutOfMemoryException);
}
catch (Exception e)
{
const string error = "Couldn't execute out of memory exception handler";
Log.WarnException(error, e);
context.Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = DateTime.UtcNow,
Title = error,
UniqueKey = error,
Message = e.ToString(),
Exception = e.Message
});
return(false);
}
}