private void RunEachOperationIndependently(List <MergedTransactionCommand> pendingOps)
{
try
{
foreach (var op in pendingOps)
{
try
{
using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (var tx = context.OpenWriteTransaction())
{
op.Execute(context);
tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
tx.Commit();
SlowWriteNotification.Notify(stats, _parent);
}
}
DoCommandNotification(op);
}
catch (Exception e)
{
op.Exception = e;
NotifyOnThreadPool(op);
}
}
}
finally
{
pendingOps.Clear();
_opsBuffers.Enqueue(pendingOps);
}
}
private void RunEachOperationIndependently(List <MergedTransactionCommand> pendingOps)
{
try
{
foreach (var op in pendingOps)
{
bool alreadyRetried = false;
while (true)
{
try
{
using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
_recording.State?.Record(context, TxInstruction.BeginTx);
using (var tx = context.OpenWriteTransaction())
{
op.RetryOnError = false;
op.Execute(context, _recording.State);
tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
_recording.State?.Record(context, TxInstruction.Commit);
tx.Commit();
SlowWriteNotification.Notify(stats, _parent);
}
}
DoCommandNotification(op);
}
catch (Exception e)
{
if (alreadyRetried == false && op.RetryOnError)
{
alreadyRetried = true;
continue;
}
op.Exception = e;
NotifyOnThreadPool(op);
}
break;
}
}
}
finally
{
pendingOps.Clear();
_opsBuffers.Enqueue(pendingOps);
}
}
private void CompletePreviousTransaction(
DocumentsOperationContext context,
RavenTransaction previous,
CommitStats commitStats,
ref List <MergedTransactionCommand> previousPendingOps,
bool throwOnError)
{
try
{
_recording.State?.Record(context, TxInstruction.EndAsyncCommit);
previous.EndAsyncCommit();
//not sure about this 'if'
if (commitStats != null)
{
SlowWriteNotification.Notify(commitStats, _parent);
}
if (_log.IsInfoEnabled)
{
_log.Info($"EndAsyncCommit on {previous.InnerTransaction.LowLevelTransaction.Id}");
}
NotifyOnThreadPool(previousPendingOps);
}
catch (Exception e)
{
foreach (var op in previousPendingOps)
{
op.Exception = e;
}
NotifyOnThreadPool(previousPendingOps);
previousPendingOps = null; // RavenDB-7417
if (throwOnError)
{
throw;
}
}
}
private void MergeTransactionsWithAsyncCommit(
ref DocumentsOperationContext previous,
ref IDisposable returnPreviousContext,
List <MergedTransactionCommand> previousPendingOps)
{
DocumentsOperationContext current = null;
IDisposable currentReturnContext = null;
try
{
while (true)
{
if (_log.IsInfoEnabled)
{
_log.Info($"BeginAsyncCommit on {previous.Transaction.InnerTransaction.LowLevelTransaction.Id} with {_operations.Count} additional operations pending");
}
currentReturnContext = _parent.DocumentsStorage.ContextPool.AllocateOperationContext(out current);
CommitStats commitStats = null;
try
{
previous.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out commitStats);
_recording.State?.Record(current, TxInstruction.BeginAsyncCommitAndStartNewTransaction);
current.Transaction = previous.Transaction.BeginAsyncCommitAndStartNewTransaction(current);
}
catch (Exception e)
{
foreach (var op in previousPendingOps)
{
op.Exception = e;
}
NotifyOnThreadPool(previousPendingOps);
if (e is OutOfMemoryException)
{
try
{
//already throwing, attempt to complete previous tx
CompletePreviousTransaction(previous, previous.Transaction, commitStats, ref previousPendingOps, throwOnError: false);
}
finally
{
current.Transaction?.Dispose();
currentReturnContext.Dispose();
}
}
return;
}
var currentPendingOps = GetBufferForPendingOps();
PendingOperations result;
bool calledCompletePreviousTx = false;
try
{
var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
try
{
result = ExecutePendingOperationsInTransaction(
currentPendingOps, current,
previous.Transaction.InnerTransaction.LowLevelTransaction.AsyncCommit, ref transactionMeter);
UpdateGlobalReplicationInfoBeforeCommit(current);
}
finally
{
transactionMeter.Dispose();
}
calledCompletePreviousTx = true;
CompletePreviousTransaction(previous, previous.Transaction, commitStats, ref previousPendingOps, throwOnError: true);
}
catch (Exception e)
{
using (current.Transaction)
using (currentReturnContext)
{
if (calledCompletePreviousTx == false)
{
CompletePreviousTransaction(
previous,
previous.Transaction,
commitStats,
ref previousPendingOps,
// if this previous threw, it won't throw again
throwOnError: false);
}
else
{
throw;
}
}
if (e is HighDirtyMemoryException highDirtyMemoryException)
{
if (_log.IsInfoEnabled)
{
var errorMessage = $"{currentPendingOps.Count:#,#0} operations were cancelled because of high dirty memory, details: {highDirtyMemoryException.Message}";
_log.Info(errorMessage, highDirtyMemoryException);
}
NotifyHighDirtyMemoryFailure(currentPendingOps, highDirtyMemoryException);
}
else
{
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to run merged transaction with {currentPendingOps.Count:#,#0} operations in async manner, will retry independently", e);
}
NotifyTransactionFailureAndRerunIndependently(currentPendingOps, e);
}
return;
}
_recording.State?.Record(previous, TxInstruction.DisposePrevTx, previous.Transaction.Disposed == false);
previous.Transaction.Dispose();
returnPreviousContext.Dispose();
previous = current;
returnPreviousContext = currentReturnContext;
switch (result)
{
case PendingOperations.CompletedAll:
try
{
previous.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
_recording.State?.Record(current, TxInstruction.Commit);
previous.Transaction.Commit();
SlowWriteNotification.Notify(stats, _parent);
}
catch (Exception e)
{
foreach (var op in currentPendingOps)
{
op.Exception = e;
}
}
NotifyOnThreadPool(currentPendingOps);
return;
case PendingOperations.HasMore:
previousPendingOps = currentPendingOps;
break;
default:
Debug.Assert(false);
return;
}
}
}
catch
{
if (current.Transaction != null)
{
_recording.State?.Record(current, TxInstruction.DisposeTx, current.Transaction.Disposed == false);
current.Transaction.Dispose();
}
currentReturnContext?.Dispose();
throw;
}
}
private void MergeTransactionsOnce()
{
DocumentsOperationContext context = null;
IDisposable returnContext = null;
DocumentsTransaction tx = null;
try
{
var pendingOps = GetBufferForPendingOps();
returnContext = _parent.DocumentsStorage.ContextPool.AllocateOperationContext(out context);
{
try
{
_recording.State?.Record(context, TxInstruction.BeginTx);
tx = context.OpenWriteTransaction();
}
catch (Exception e)
{
try
{
if (_operations.TryDequeue(out MergedTransactionCommand command))
{
command.Exception = e;
DoCommandNotification(command);
}
return;
}
finally
{
if (tx != null)
{
_recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
tx.Dispose();
}
}
}
PendingOperations result;
try
{
var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
try
{
result = ExecutePendingOperationsInTransaction(pendingOps, context, null, ref transactionMeter);
UpdateGlobalReplicationInfoBeforeCommit(context);
}
finally
{
transactionMeter.Dispose();
}
}
catch (Exception e)
{
// need to dispose here since we are going to open new tx for each operation
if (tx != null)
{
_recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
tx.Dispose();
}
if (e is HighDirtyMemoryException highDirtyMemoryException)
{
if (_log.IsInfoEnabled)
{
var errorMessage = $"{pendingOps.Count:#,#0} operations were cancelled because of high dirty memory, details: {highDirtyMemoryException.Message}";
_log.Info(errorMessage, highDirtyMemoryException);
}
NotifyHighDirtyMemoryFailure(pendingOps, highDirtyMemoryException);
}
else
{
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to run merged transaction with {pendingOps.Count:#,#0}, will retry independently", e);
}
NotifyTransactionFailureAndRerunIndependently(pendingOps, e);
}
return;
}
switch (result)
{
case PendingOperations.CompletedAll:
try
{
tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
_recording.State?.Record(context, TxInstruction.Commit);
tx.Commit();
SlowWriteNotification.Notify(stats, _parent);
_recording.State?.Record(context, TxInstruction.DisposeTx, tx.Disposed == false);
tx.Dispose();
}
catch (Exception e)
{
foreach (var op in pendingOps)
{
op.Exception = e;
}
}
finally
{
NotifyOnThreadPool(pendingOps);
}
return;
case PendingOperations.HasMore:
MergeTransactionsWithAsyncCommit(ref context, ref returnContext, pendingOps);
return;
default:
Debug.Assert(false, "Should never happen");
return;
}
}
}
finally
{
if (context?.Transaction != null)
{
using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
{
_recording.State?.Record(ctx, TxInstruction.DisposeTx, context.Transaction.Disposed == false);
}
context.Transaction.Dispose();
}
returnContext?.Dispose();
}
}
private void MergeTransactionsWithAsyncCommit(
DocumentsOperationContext context,
List <MergedTransactionCommand> previousPendingOps)
{
var previous = context.Transaction;
try
{
while (true)
{
if (_log.IsInfoEnabled)
{
_log.Info($"BeginAsyncCommit on {previous.InnerTransaction.LowLevelTransaction.Id} with {_operations.Count} additional operations pending");
}
CommitStats commitStats = null;
try
{
previous.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out commitStats);
context.Transaction = previous.BeginAsyncCommitAndStartNewTransaction();
}
catch (Exception e)
{
foreach (var op in previousPendingOps)
{
op.Exception = e;
}
NotifyOnThreadPool(previousPendingOps);
if (e is OutOfMemoryException)
{
try
{
//already throwing, attempt to complete previous tx
CompletePreviousTransaction(previous, commitStats, ref previousPendingOps, throwOnError: false);
}
finally
{
context.Transaction?.Dispose();
}
}
return;
}
try
{
var currentPendingOps = GetBufferForPendingOps();
PendingOperations result;
bool calledCompletePreviousTx = false;
try
{
var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
try
{
result = ExecutePendingOperationsInTransaction(
currentPendingOps, context,
previous.InnerTransaction.LowLevelTransaction.AsyncCommit, ref transactionMeter);
UpdateGlobalReplicationInfoBeforeCommit(context);
}
finally
{
transactionMeter.Dispose();
}
calledCompletePreviousTx = true;
CompletePreviousTransaction(previous, commitStats, ref previousPendingOps, throwOnError: true);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Failed to run merged transaction with {currentPendingOps.Count:#,#0} operations in async manner, will retry independently",
e);
}
using (previous)
using (context.Transaction)
{
if (calledCompletePreviousTx == false)
{
CompletePreviousTransaction(previous,
commitStats,
ref previousPendingOps,
// if this previous threw, it won't throw again
throwOnError: false);
}
else
{
throw;
}
}
NotifyTransactionFailureAndRerunIndependently(currentPendingOps, e);
return;
}
previous.Dispose();
switch (result)
{
case PendingOperations.CompletedAll:
try
{
context.Transaction.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
context.Transaction.Commit();
SlowWriteNotification.Notify(stats, _parent);
context.Transaction.Dispose();
}
catch (Exception e)
{
foreach (var op in currentPendingOps)
{
op.Exception = e;
}
}
NotifyOnThreadPool(currentPendingOps);
return;
case PendingOperations.HasMore:
previousPendingOps = currentPendingOps;
previous = context.Transaction;
context.Transaction = null;
break;
default:
Debug.Assert(false);
return;
}
}
finally
{
context.Transaction?.Dispose();
}
}
}
finally
{
previous.Dispose();
}
}
private void MergeTransactionsOnce()
{
DocumentsTransaction tx = null;
try
{
var pendingOps = GetBufferForPendingOps();
using (_parent.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
try
{
tx = context.OpenWriteTransaction();
}
catch (Exception e)
{
try
{
if (_operations.TryDequeue(out MergedTransactionCommand command))
{
command.Exception = e;
DoCommandNotification(command);
}
return;
}
finally
{
tx?.Dispose();
}
}
PendingOperations result;
try
{
var transactionMeter = TransactionPerformanceMetrics.MeterPerformanceRate();
try
{
result = ExecutePendingOperationsInTransaction(pendingOps, context, null, ref transactionMeter);
UpdateGlobalReplicationInfoBeforeCommit(context);
}
finally
{
transactionMeter.Dispose();
}
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info(
$"Failed to run merged transaction with {pendingOps.Count:#,#0}, will retry independently",
e);
}
// need to dispose here since we are going to open new tx for each operation
tx?.Dispose();
NotifyTransactionFailureAndRerunIndependently(pendingOps, e);
return;
}
switch (result)
{
case PendingOperations.CompletedAll:
try
{
tx.InnerTransaction.LowLevelTransaction.RetrieveCommitStats(out var stats);
tx.Commit();
SlowWriteNotification.Notify(stats, _parent);
tx.Dispose();
}
catch (Exception e)
{
foreach (var op in pendingOps)
{
op.Exception = e;
}
}
finally
{
NotifyOnThreadPool(pendingOps);
}
return;
case PendingOperations.HasMore:
MergeTransactionsWithAsyncCommit(context, pendingOps);
return;
default:
Debug.Assert(false, "Should never happen");
return;
}
}
}
finally
{
tx?.Dispose();
}
}