private PendingOperations ExecutePendingOperationsInTransaction( List <MergedTransactionCommand> pendingOps, DocumentsOperationContext context, Task previousOperation, ref PerformanceMetrics.DurationMeasurement meter) { _alreadyListeningToPreviousOperationEnd = false; var percentageFromPhysicalMem = _parent.Configuration.Memory.TemporaryDirtyMemoryAllowedPercentage; context.TransactionMarkerOffset = 1; // ensure that we are consistent here and don't use old values var sp = Stopwatch.StartNew(); do { // RavenDB-7732 - Even if we merged multiple separate operations into // a single transaction in Voron, we're still going to have a separate // tx marker for them for the purpose of replication, to avoid creating // overly large replication batches. context.TransactionMarkerOffset++; if (TryGetNextOperation(previousOperation, out MergedTransactionCommand op, ref meter) == false) { break; } pendingOps.Add(op); var llt = context.Transaction.InnerTransaction.LowLevelTransaction; if (_parent.Configuration.Memory.EnableHighTemporaryDirtyMemoryUse) { var now = _parent.Time.GetUtcNow(); if (now - _lastHighDirtyMemCheck > _timeToCheckHighDirtyMemory.AsTimeSpan) // we do not need to test scratch dirty mem every write { if (MemoryInformation.IsHighDirtyMemory(percentageFromPhysicalMem, out var details)) { var highDirtyMemory = new HighDirtyMemoryException( $"Operation was cancelled by the transaction merger for transaction #{llt.Id} due to high dirty memory in scratch files." + $" This might be caused by a slow IO storage. Current memory usage: {details}"); foreach (var pendingOp in pendingOps) { pendingOp.Exception = highDirtyMemory; } NotifyOnThreadPool(pendingOps); var rejectedBuffer = GetBufferForPendingOps(); while (_operations.TryDequeue(out var operationToReject)) { operationToReject.Exception = highDirtyMemory; rejectedBuffer.Add(operationToReject); } NotifyOnThreadPool(rejectedBuffer); break; } _lastHighDirtyMemCheck = now; // reset timer for next check only if no errors (otherwise check every single write until back to normal) } } meter.IncrementCounter(1); meter.IncrementCommands(op.Execute(context, _recording.State)); var modifiedSize = llt.NumberOfModifiedPages * Constants.Storage.PageSize; modifiedSize += llt.TotalEncryptionBufferSize.GetValue(SizeUnit.Bytes); var canCloseCurrentTx = previousOperation == null || previousOperation.IsCompleted; if (canCloseCurrentTx || _parent.Is32Bits) { if (_operations.IsEmpty) { break; // nothing remaining to do, let's us close this work } if (sp.ElapsedMilliseconds > _maxTimeToWaitForPreviousTxInMs) { break; // too much time } if (modifiedSize > _maxTxSizeInBytes) { break; // transaction is too big, let's clean it } // even though we can close the tx, we choose to keep it a bit longer // we want to keep processing operations until we clear the queue, time / size // limits are reached continue; } // if I can't close the tx, this means that the previous async operation is still in progress // and there are incoming requests coming in. We'll accept them, to a certain limit if (modifiedSize < _maxTxSizeInBytes) { continue; // we can still process requests at this time, so let's do that... } UnlikelyRejectOperations(previousOperation, sp, llt, modifiedSize); break; } while (true); var status = GetPendingOperationsStatus(context, pendingOps.Count == 0); if (_log.IsInfoEnabled) { var opType = previousOperation == null ? string.Empty : "(async) "; _log.Info($"Merged {pendingOps.Count:#,#;;0} operations in {sp.Elapsed} {opType}with {_operations.Count:#,#;;0} operations remaining. Status: {status}"); } return(status); }