OutOfMemoryExceptionHappened() public method

This let us know that an OOME has happened, and we need to be much more conservative with regards to how fast we can grow memory.
public OutOfMemoryExceptionHappened ( ) : void
return void
Example #1
0
        private void HandleOutOfMemoryException(OutOfMemoryException oome)
        {
            log.WarnException(
                @"Failed to execute indexing because of an out of memory exception. Will force a full GC cycle and then become more conservative with regards to memory",
                oome);

            // On the face of it, this is stupid, because OOME will not be thrown if the GC could release
            // memory, but we are actually aware that during indexing, the GC couldn't find garbage to clean,
            // but in here, we are AFTER the index was done, so there is likely to be a lot of garbage.
            GC.Collect(GC.MaxGeneration);
            autoTuner.OutOfMemoryExceptionHappened();
        }
Example #2
0
        public void Execute()
        {
            using (LogContext.WithDatabase(context.DatabaseName))
            {
                Init();
                var  name        = GetType().Name;
                var  workComment = "WORK BY " + name;
                bool isIdle      = false;
                while (context.RunIndexing)
                {
                    bool foundWork;
                    try
                    {
                        bool onlyFoundIdleWork;
                        foundWork = ExecuteIndexing(isIdle, out onlyFoundIdleWork);
                        if (foundWork && onlyFoundIdleWork == false)
                        {
                            isIdle = false;
                        }

                        while (context.RunIndexing) // we want to drain all of the pending tasks before the next run
                        {
                            if (ExecuteTasks() == false)
                            {
                                break;
                            }
                            foundWork = true;
                        }
                    }
                    catch (OutOfMemoryException oome)
                    {
                        foundWork = true;
                        HandleOutOfMemoryException(oome);
                    }
                    catch (AggregateException ae)
                    {
                        foundWork = true;
                        var actual = ae.ExtractSingleInnerException();
                        var oome   = actual as OutOfMemoryException;
                        if (oome == null)
                        {
                            if (IsEsentOutOfMemory(actual))
                            {
                                autoTuner.OutOfMemoryExceptionHappened();
                            }
                            Log.ErrorException("Failed to execute indexing", ae);
                        }
                        else
                        {
                            HandleOutOfMemoryException(oome);
                        }
                    }
                    catch (OperationCanceledException)
                    {
                        Log.Info("Got rude cancellation of indexing as a result of shutdown, aborting current indexing run");
                        return;
                    }
                    catch (Exception e)
                    {
                        foundWork = true; // we want to keep on trying, anyway, not wait for the timeout or more work
                        Log.ErrorException("Failed to execute indexing", e);
                        if (IsEsentOutOfMemory(e))
                        {
                            autoTuner.OutOfMemoryExceptionHappened();
                        }
                    }
                    if (foundWork == false && context.RunIndexing)
                    {
                        isIdle = context.WaitForWork(context.Configuration.TimeToWaitBeforeRunningIdleIndexes, ref workCounter, () =>
                        {
                            try
                            {
                                FlushIndexes();
                            }
                            catch (Exception e)
                            {
                                Log.WarnException("Could not flush indexes properly", e);
                            }
                        }, name);
                    }
                    else // notify the tasks executer that it has work to do
                    {
                        context.ShouldNotifyAboutWork(() => workComment);
                        context.NotifyAboutWork();
                    }
                }
                Dispose();
            }
        }
Example #3
0
 public void Execute()
 {
     using (LogManager.OpenMappedContext("database", context.DatabaseName ?? Constants.SystemDatabase))
         using (new DisposableAction(() => LogContext.DatabaseName.Value = null))
         {
             Init();
             LogContext.DatabaseName.Value = context.DatabaseName;
             var name        = GetType().Name;
             var workComment = "WORK BY " + name;
             while (context.RunIndexing)
             {
                 bool foundWork;
                 try
                 {
                     foundWork = ExecuteIndexing();
                     while (context.RunIndexing)                     // we want to drain all of the pending tasks before the next run
                     {
                         if (ExecuteTasks() == false)
                         {
                             break;
                         }
                         foundWork = true;
                     }
                 }
                 catch (OutOfMemoryException oome)
                 {
                     foundWork = true;
                     HandleOutOfMemoryException(oome);
                 }
                 catch (AggregateException ae)
                 {
                     foundWork = true;
                     var actual = ae.ExtractSingleInnerException();
                     var oome   = actual as OutOfMemoryException;
                     if (oome == null)
                     {
                         if (IsEsentOutOfMemory(actual))
                         {
                             autoTuner.OutOfMemoryExceptionHappened();
                         }
                         Log.ErrorException("Failed to execute indexing", ae);
                     }
                     else
                     {
                         HandleOutOfMemoryException(oome);
                     }
                 }
                 catch (OperationCanceledException)
                 {
                     Log.Info("Got rude cancellation of indexing as a result of shutdown, aborting current indexing run");
                     return;
                 }
                 catch (Exception e)
                 {
                     foundWork = true;                     // we want to keep on trying, anyway, not wait for the timeout or more work
                     Log.ErrorException("Failed to execute indexing", e);
                     if (IsEsentOutOfMemory(e))
                     {
                         autoTuner.OutOfMemoryExceptionHappened();
                     }
                 }
                 if (foundWork == false)
                 {
                     context.WaitForWork(TimeSpan.FromHours(1), ref workCounter, FlushIndexes, name);
                 }
                 else                 // notify the tasks executer that it has work to do
                 {
                     context.ShouldNotifyAboutWork(() => workComment);
                     context.NotifyAboutWork();
                 }
             }
             Dispose();
         }
 }