private void RunEachOperationIndependently(List <MergedTransactionCommand> pendingOps)
 {
     try
     {
         foreach (var op in pendingOps)
         {
             try
             {
                 using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                 {
                     using (var tx = context.OpenWriteTransaction())
                     {
                         op.Execute(context);
                         tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
                         tx.Commit();
                         SlowWriteNotification.Notify(stats, _parent);
                     }
                 }
                 DoCommandNotification(op);
             }
             catch (Exception e)
             {
                 op.Exception = e;
                 NotifyOnThreadPool(op);
             }
         }
     }
     finally
     {
         pendingOps.Clear();
         _opsBuffers.Enqueue(pendingOps);
     }
 }
        private void RunEachOperationIndependently(List <MergedTransactionCommand> pendingOps)
        {
            try
            {
                foreach (var op in pendingOps)
                {
                    bool alreadyRetried = false;
                    while (true)
                    {
                        try
                        {
                            using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                            {
                                _recording.State?.Record(context, TxInstruction.BeginTx);
                                using (var tx = context.OpenWriteTransaction())
                                {
                                    op.RetryOnError = false;

                                    op.Execute(context, _recording.State);

                                    tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);

                                    _recording.State?.Record(context, TxInstruction.Commit);
                                    tx.Commit();
                                    SlowWriteNotification.Notify(stats, _parent);
                                }
                            }
                            DoCommandNotification(op);
                        }
                        catch (Exception e)
                        {
                            if (alreadyRetried == false && op.RetryOnError)
                            {
                                alreadyRetried = true;
                                continue;
                            }

                            op.Exception = e;
                            NotifyOnThreadPool(op);
                        }
                        break;
                    }
                }
            }
            finally
            {
                pendingOps.Clear();
                _opsBuffers.Enqueue(pendingOps);
            }
        }
        private void CompletePreviousTransaction(
            DocumentsOperationContext context,
            RavenTransaction previous,
            CommitStats commitStats,
            ref List <MergedTransactionCommand> previousPendingOps,
            bool throwOnError)
        {
            try
            {
                _recording.State?.Record(context, TxInstruction.EndAsyncCommit);
                previous.EndAsyncCommit();

                //not sure about this 'if'
                if (commitStats != null)
                {
                    SlowWriteNotification.Notify(commitStats, _parent);
                }

                if (_log.IsInfoEnabled)
                {
                    _log.Info($"EndAsyncCommit on {previous.InnerTransaction.LowLevelTransaction.Id}");
                }

                NotifyOnThreadPool(previousPendingOps);
            }
            catch (Exception e)
            {
                foreach (var op in previousPendingOps)
                {
                    op.Exception = e;
                }
                NotifyOnThreadPool(previousPendingOps);
                previousPendingOps = null; // RavenDB-7417
                if (throwOnError)
                {
                    throw;
                }
            }
        }
        private void MergeTransactionsWithAsyncCommit(
            ref DocumentsOperationContext previous,
            ref IDisposable returnPreviousContext,
            List <MergedTransactionCommand> previousPendingOps)
        {
            DocumentsOperationContext current = null;
            IDisposable currentReturnContext  = null;

            try
            {
                while (true)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info($"BeginAsyncCommit on {previous.Transaction.InnerTransaction.LowLevelTransaction.Id} with {_operations.Count} additional operations pending");
                    }

                    currentReturnContext = _parent.DocumentsStorage.ContextPool.AllocateOperationContext(out current);
                    CommitStats commitStats = null;
                    try
                    {
                        previous.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out commitStats);
                        _recording.State?.Record(current, TxInstruction.BeginAsyncCommitAndStartNewTransaction);
                        current.Transaction = previous.Transaction.BeginAsyncCommitAndStartNewTransaction(current);
                    }
                    catch (Exception e)
                    {
                        foreach (var op in previousPendingOps)
                        {
                            op.Exception = e;
                        }
                        NotifyOnThreadPool(previousPendingOps);

                        if (e is OutOfMemoryException)
                        {
                            try
                            {
                                //already throwing, attempt to complete previous tx
                                CompletePreviousTransaction(previous, previous.Transaction, commitStats, ref previousPendingOps, throwOnError: false);
                            }
                            finally
                            {
                                current.Transaction?.Dispose();
                                currentReturnContext.Dispose();
                            }
                        }

                        return;
                    }

                    var currentPendingOps = GetBufferForPendingOps();
                    PendingOperations result;
                    bool calledCompletePreviousTx = false;
                    try
                    {
                        var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
                        try
                        {
                            result = ExecutePendingOperationsInTransaction(
                                currentPendingOps, current,
                                previous.Transaction.InnerTransaction.LowLevelTransaction.AsyncCommit, ref transactionMeter);
                            UpdateGlobalReplicationInfoBeforeCommit(current);
                        }
                        finally
                        {
                            transactionMeter.Dispose();
                        }
                        calledCompletePreviousTx = true;
                        CompletePreviousTransaction(previous, previous.Transaction, commitStats, ref previousPendingOps, throwOnError: true);
                    }
                    catch (Exception e)
                    {
                        using (current.Transaction)
                            using (currentReturnContext)
                            {
                                if (calledCompletePreviousTx == false)
                                {
                                    CompletePreviousTransaction(
                                        previous,
                                        previous.Transaction,
                                        commitStats,
                                        ref previousPendingOps,
                                        // if this previous threw, it won't throw again
                                        throwOnError: false);
                                }
                                else
                                {
                                    throw;
                                }
                            }

                        if (e is HighDirtyMemoryException highDirtyMemoryException)
                        {
                            if (_log.IsInfoEnabled)
                            {
                                var errorMessage = $"{currentPendingOps.Count:#,#0} operations were cancelled because of high dirty memory, details: {highDirtyMemoryException.Message}";
                                _log.Info(errorMessage, highDirtyMemoryException);
                            }

                            NotifyHighDirtyMemoryFailure(currentPendingOps, highDirtyMemoryException);
                        }
                        else
                        {
                            if (_log.IsInfoEnabled)
                            {
                                _log.Info($"Failed to run merged transaction with {currentPendingOps.Count:#,#0} operations in async manner, will retry independently", e);
                            }

                            NotifyTransactionFailureAndRerunIndependently(currentPendingOps, e);
                        }

                        return;
                    }

                    _recording.State?.Record(previous, TxInstruction.DisposePrevTx, previous.Transaction.Disposed == false);

                    previous.Transaction.Dispose();
                    returnPreviousContext.Dispose();

                    previous = current;
                    returnPreviousContext = currentReturnContext;

                    switch (result)
                    {
                    case PendingOperations.CompletedAll:
                        try
                        {
                            previous.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
                            _recording.State?.Record(current, TxInstruction.Commit);
                            previous.Transaction.Commit();

                            SlowWriteNotification.Notify(stats, _parent);
                        }
                        catch (Exception e)
                        {
                            foreach (var op in currentPendingOps)
                            {
                                op.Exception = e;
                            }
                        }
                        NotifyOnThreadPool(currentPendingOps);
                        return;

                    case PendingOperations.HasMore:
                        previousPendingOps = currentPendingOps;
                        break;

                    default:
                        Debug.Assert(false);
                        return;
                    }
                }
            }
            catch
            {
                if (current.Transaction != null)
                {
                    _recording.State?.Record(current, TxInstruction.DisposeTx, current.Transaction.Disposed == false);
                    current.Transaction.Dispose();
                }
                currentReturnContext?.Dispose();
                throw;
            }
        }
        private void MergeTransactionsOnce()
        {
            DocumentsOperationContext context  = null;
            IDisposable          returnContext = null;
            DocumentsTransaction tx            = null;

            try
            {
                var pendingOps = GetBufferForPendingOps();
                returnContext = _parent.DocumentsStorage.ContextPool.AllocateOperationContext(out context);
                {
                    try
                    {
                        _recording.State?.Record(context, TxInstruction.BeginTx);
                        tx = context.OpenWriteTransaction();
                    }
                    catch (Exception e)
                    {
                        try
                        {
                            if (_operations.TryDequeue(out MergedTransactionCommand command))
                            {
                                command.Exception = e;
                                DoCommandNotification(command);
                            }
                            return;
                        }
                        finally
                        {
                            if (tx != null)
                            {
                                _recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
                                tx.Dispose();
                            }
                        }
                    }
                    PendingOperations result;
                    try
                    {
                        var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
                        try
                        {
                            result = ExecutePendingOperationsInTransaction(pendingOps, context, null, ref transactionMeter);
                            UpdateGlobalReplicationInfoBeforeCommit(context);
                        }
                        finally
                        {
                            transactionMeter.Dispose();
                        }
                    }
                    catch (Exception e)
                    {
                        // need to dispose here since we are going to open new tx for each operation
                        if (tx != null)
                        {
                            _recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
                            tx.Dispose();
                        }

                        if (e is HighDirtyMemoryException highDirtyMemoryException)
                        {
                            if (_log.IsInfoEnabled)
                            {
                                var errorMessage = $"{pendingOps.Count:#,#0} operations were cancelled because of high dirty memory, details: {highDirtyMemoryException.Message}";
                                _log.Info(errorMessage, highDirtyMemoryException);
                            }

                            NotifyHighDirtyMemoryFailure(pendingOps, highDirtyMemoryException);
                        }
                        else
                        {
                            if (_log.IsInfoEnabled)
                            {
                                _log.Info($"Failed to run merged transaction with {pendingOps.Count:#,#0}, will retry independently", e);
                            }

                            NotifyTransactionFailureAndRerunIndependently(pendingOps, e);
                        }

                        return;
                    }

                    switch (result)
                    {
                    case PendingOperations.CompletedAll:
                        try
                        {
                            tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);

                            _recording.State?.Record(context, TxInstruction.Commit);
                            tx.Commit();

                            SlowWriteNotification.Notify(stats, _parent);
                            _recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
                            tx.Dispose();
                        }
                        catch (Exception e)
                        {
                            foreach (var op in pendingOps)
                            {
                                op.Exception = e;
                            }
                        }
                        finally
                        {
                            NotifyOnThreadPool(pendingOps);
                        }
                        return;

                    case PendingOperations.HasMore:
                        MergeTransactionsWithAsyncCommit(ref context, ref returnContext, pendingOps);
                        return;

                    default:
                        Debug.Assert(false, "Should never happen");
                        return;
                    }
                }
            }
            finally
            {
                if (context?.Transaction != null)
                {
                    using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
                    {
                        _recording.State?.Record(ctx, TxInstruction.DisposeTx, context.Transaction.Disposed == false);
                    }
                    context.Transaction.Dispose();
                }
                returnContext?.Dispose();
            }
        }
        private void MergeTransactionsWithAsyncCommit(
            DocumentsOperationContext context,
            List <MergedTransactionCommand> previousPendingOps)
        {
            var previous = context.Transaction;

            try
            {
                while (true)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info($"BeginAsyncCommit on {previous.InnerTransaction.LowLevelTransaction.Id} with {_operations.Count} additional operations pending");
                    }

                    CommitStats commitStats = null;
                    try
                    {
                        previous.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out commitStats);
                        context.Transaction = previous.BeginAsyncCommitAndStartNewTransaction();
                    }
                    catch (Exception e)
                    {
                        foreach (var op in previousPendingOps)
                        {
                            op.Exception = e;
                        }
                        NotifyOnThreadPool(previousPendingOps);

                        if (e is OutOfMemoryException)
                        {
                            try
                            {
                                //already throwing, attempt to complete previous tx
                                CompletePreviousTransaction(previous, commitStats, ref previousPendingOps, throwOnError: false);
                            }
                            finally
                            {
                                context.Transaction?.Dispose();
                            }
                        }

                        return;
                    }
                    try
                    {
                        var currentPendingOps = GetBufferForPendingOps();
                        PendingOperations result;
                        bool calledCompletePreviousTx = false;
                        try
                        {
                            var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
                            try
                            {
                                result = ExecutePendingOperationsInTransaction(
                                    currentPendingOps, context,
                                    previous.InnerTransaction.LowLevelTransaction.AsyncCommit, ref transactionMeter);
                                UpdateGlobalReplicationInfoBeforeCommit(context);
                            }
                            finally
                            {
                                transactionMeter.Dispose();
                            }
                            calledCompletePreviousTx = true;
                            CompletePreviousTransaction(previous, commitStats, ref previousPendingOps, throwOnError: true);
                        }
                        catch (Exception e)
                        {
                            if (_log.IsInfoEnabled)
                            {
                                _log.Info(
                                    $"Failed to run merged transaction with {currentPendingOps.Count:#,#0} operations in async manner, will retry independently",
                                    e);
                            }
                            using (previous)
                                using (context.Transaction)
                                {
                                    if (calledCompletePreviousTx == false)
                                    {
                                        CompletePreviousTransaction(previous,
                                                                    commitStats,
                                                                    ref previousPendingOps,
                                                                    // if this previous threw, it won't throw again
                                                                    throwOnError: false);
                                    }
                                    else
                                    {
                                        throw;
                                    }
                                }
                            NotifyTransactionFailureAndRerunIndependently(currentPendingOps, e);
                            return;
                        }
                        previous.Dispose();

                        switch (result)
                        {
                        case PendingOperations.CompletedAll:
                            try
                            {
                                context.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
                                context.Transaction.Commit();
                                SlowWriteNotification.Notify(stats, _parent);
                                context.Transaction.Dispose();
                            }
                            catch (Exception e)
                            {
                                foreach (var op in currentPendingOps)
                                {
                                    op.Exception = e;
                                }
                            }
                            NotifyOnThreadPool(currentPendingOps);
                            return;

                        case PendingOperations.HasMore:
                            previousPendingOps  = currentPendingOps;
                            previous            = context.Transaction;
                            context.Transaction = null;
                            break;

                        default:
                            Debug.Assert(false);
                            return;
                        }
                    }
                    finally
                    {
                        context.Transaction?.Dispose();
                    }
                }
            }
            finally
            {
                previous.Dispose();
            }
        }
        private void MergeTransactionsOnce()
        {
            DocumentsTransaction tx = null;

            try
            {
                var pendingOps = GetBufferForPendingOps();
                using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                {
                    try
                    {
                        tx = context.OpenWriteTransaction();
                    }
                    catch (Exception e)
                    {
                        try
                        {
                            if (_operations.TryDequeue(out MergedTransactionCommand command))
                            {
                                command.Exception = e;
                                DoCommandNotification(command);
                            }
                            return;
                        }
                        finally
                        {
                            tx?.Dispose();
                        }
                    }
                    PendingOperations result;
                    try
                    {
                        var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
                        try
                        {
                            result = ExecutePendingOperationsInTransaction(pendingOps, context, null, ref transactionMeter);
                            UpdateGlobalReplicationInfoBeforeCommit(context);
                        }
                        finally
                        {
                            transactionMeter.Dispose();
                        }
                    }
                    catch (Exception e)
                    {
                        if (_log.IsInfoEnabled)
                        {
                            _log.Info(
                                $"Failed to run merged transaction with {pendingOps.Count:#,#0}, will retry independently",
                                e);
                        }

                        // need to dispose here since we are going to open new tx for each operation
                        tx?.Dispose();

                        NotifyTransactionFailureAndRerunIndependently(pendingOps, e);
                        return;
                    }

                    switch (result)
                    {
                    case PendingOperations.CompletedAll:
                        try
                        {
                            tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
                            tx.Commit();
                            SlowWriteNotification.Notify(stats, _parent);
                            tx.Dispose();
                        }
                        catch (Exception e)
                        {
                            foreach (var op in pendingOps)
                            {
                                op.Exception = e;
                            }
                        }
                        finally
                        {
                            NotifyOnThreadPool(pendingOps);
                        }
                        return;

                    case PendingOperations.HasMore:
                        MergeTransactionsWithAsyncCommit(context, pendingOps);
                        return;

                    default:
                        Debug.Assert(false, "Should never happen");
                        return;
                    }
                }
            }
            finally
            {
                tx?.Dispose();
            }
        }