public void Open(ReplicatedLogManager logManager, ITracer tracer, ulong checkpointSizeMB, ulong minLogSizeInMB, uint truncationThresholdFactor, uint throttlingThresholdFactor)
{
this.tracer = tracer;
this.logManager = logManager;
this.checkpointIntervalBytes = checkpointSizeMB * MBtoBytesMultiplier;
this.minLogSizeInBytes = minLogSizeInMB * MBtoBytesMultiplier;
this.truncationThresholdInBytes = truncationThresholdFactor * this.minLogSizeInBytes;
this.throttleAtLogUsageBytes = this.GetThrottleThresholdInBytes(throttlingThresholdFactor, this.checkpointIntervalBytes, this.minLogSizeInBytes);
this.minTruncationAmountInBytes = this.GetMinTruncationAmountInBytes(this.checkpointIntervalBytes);
this.indexIntervalBytes = this.checkpointIntervalBytes / NumberOfIndexRecordsPerCheckpoint;
this.txnAbortThresholdInBytes = this.checkpointIntervalBytes / AbortOldTxFactor;
Utility.Assert(this.minTruncationAmountInBytes > 0, "Min truncation amount in bytes cannot be less than 1");
Utility.Assert(this.indexIntervalBytes != 0, "Index interval in bytes cannot be 0");
Utility.Assert(this.checkpointIntervalBytes != 0, "Checkpoint Interval cannot be 0");
Utility.Assert(this.minLogSizeInBytes != 0, "Min Log Size in bytes cannot be 0");
Utility.Assert(this.truncationThresholdInBytes >= this.minLogSizeInBytes, "truncationThresholdInBytes {0} must be larger than minLogSizeInBytes {1}", this.truncationThresholdInBytes, this.minLogSizeInBytes);
Utility.Assert(this.throttleAtLogUsageBytes > this.truncationThresholdInBytes, "throttleAtLogUsageBytes {0} must be larger than truncationThresholdInBytes {1}", this.throttleAtLogUsageBytes, this.truncationThresholdInBytes);
Utility.Assert(this.throttleAtLogUsageBytes > this.checkpointIntervalBytes, "throttleAtLogUsageBytes {0} must be larger than checkpointIntervalBytes {1}", this.throttleAtLogUsageBytes, this.checkpointIntervalBytes);
var trace = string.Format(
CultureInfo.InvariantCulture,
"IndexIntervalBytes = {0}, CheckpointIntervalBytes = {1}, MinLogSizeInMB = {2}, TruncateIntervalBytes = {3}, ThrottleAtLogUsageBytes = {4}, MinTruncationThresholdInBytes = {5}",
this.indexIntervalBytes,
this.checkpointIntervalBytes,
this.minLogSizeInBytes,
this.truncationThresholdInBytes,
this.throttleAtLogUsageBytes,
this.minTruncationAmountInBytes);
LoggingReplicator.ReplicatorTraceOnApi(tracer, trace);
}
/// <summary>
/// Primary: Process the logical log record.
/// </summary>
/// <param name="record"></param>
/// <returns></returns>
/// <remarks>
/// Wait for the logical record to be flushed and replicated.
/// If it is a barrier, increase stable lsn.
/// If it is a operation record, unlock it.
/// </remarks>
private async Task ProcessLogicalRecordOnPrimaryAsync(TransactionLogRecord record)
{
Exception exception = null;
this.ReplicateAndLogLogicalRecord(record);
try
{
// Wait for the record to be flushed and replicated
await Task.WhenAll(record.AwaitApply(), LoggingReplicator.AwaitReplication(record, this.replicatedLogManager)).ConfigureAwait(false);
}
catch (Exception e)
{
exception = e;
}
finally
{
this.InvokeUnlockCallback(record);
}
// Throw TransactionFaultedException if there has been any failure during replication and apply awaits.
if (exception != null)
{
throw new TransactionFaultedException(exception.Message);
}
return;
}
private void ProcessLogicalRecordOnPrimary(TransactionLogRecord record)
{
this.ReplicateAndLogLogicalRecord(record);
Task.WhenAll(record.AwaitApply(), LoggingReplicator.AwaitReplication(record, this.replicatedLogManager)).IgnoreException().ContinueWith(
task =>
{
// Simply retrieve task exception to mark it as handled
if (task.Exception != null)
{
Utility.Assert(task.IsFaulted == true, "task.IsFaulted == true");
}
this.InvokeUnlockCallback(record);
}).IgnoreExceptionVoid();
}
public async Task PerformRecoveryAsync(
RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
OperationProcessor recordsProcessor,
CheckpointManager checkpointManager,
TransactionManager transactionManager,
LogRecordsDispatcher logRecordsDispatcher,
ReplicatedLogManager replicatedLogManager)
{
var lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord;
LogicalSequenceNumber lsn;
var recoveredRecords = new List <LogRecord>();
var recoveredLastCompletedBeginCheckpointRecord = await this.RecoveryLogsReader.GetLastCompletedBeginCheckpointRecord(this.RecoveredLastEndCheckpointRecord).ConfigureAwait(false);
var recoveredLastCompletedBeginCheckpointRecordPosition = recoveredLastCompletedBeginCheckpointRecord.RecordPosition;
var recoveryStartingPosition = recoveredLastCompletedBeginCheckpointRecordPosition - recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset;
// Set the recovered lsn before performing recovery as it impacts unlock callback logic during recovery
recoveredOrCopiedCheckpointLsn.Update(recoveredLastCompletedBeginCheckpointRecord.Lsn);
var trace =
string.Format(
CultureInfo.InvariantCulture,
"PerformRecoveryAsync: " + Environment.NewLine
+ " Recovered last completed begin checkpoint record." + Environment.NewLine
+ " Type: {0} LSN: {1} PSN:{2} Position: {3}" + Environment.NewLine
+ " Recovery Starting Position: {4} Recovery Starting Epoch: {5},{6}" + Environment.NewLine
+ " LogCompleteCheckpointAfterRecovery: {7}" + " Recovered ProgressVector: {8}"
+ Environment.NewLine,
recoveredLastCompletedBeginCheckpointRecord.RecordType,
recoveredLastCompletedBeginCheckpointRecord.Lsn.LSN,
recoveredLastCompletedBeginCheckpointRecord.Psn.PSN,
recoveredLastCompletedBeginCheckpointRecord.RecordPosition,
recoveryStartingPosition,
recoveredLastCompletedBeginCheckpointRecord.Epoch.DataLossNumber,
recoveredLastCompletedBeginCheckpointRecord.Epoch.ConfigurationNumber,
this.logCompleteCheckpointAfterRecovery,
recoveredLastCompletedBeginCheckpointRecord.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb));
FabricEvents.Events.Api(tracer.Type, trace);
LogRecord lastRecoverableRecord = null;
replicatedLogManager.LogManager.SetSequentialAccessReadSize(
this.RecoveryLogsReader.ReadStream,
LogManager.ReadAheadBytesForSequentialStream);
// Updated to the recovered value in LogRecordsMap during successful recovery
long lastPeriodicTruncationTimeTicks = 0;
using (var records = new LogRecords(this.RecoveryLogsReader.ReadStream, recoveryStartingPosition, this.tailRecordAtStart.RecordPosition))
{
var hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
Utility.Assert(hasMoved == true, "hasMoved == true");
lsn = records.Current.Lsn;
if (recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset != 0)
{
lsn -= 1;
}
var logRecordsMap = new LogRecordsMap(
lsn,
transactionManager.TransactionsMap,
recoveredLastCompletedBeginCheckpointRecord.Epoch,
checkpointManager.LastStableLsn,
recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
recoveredLastCompletedBeginCheckpointRecord,
this.tracer,
this.RecoveredLastEndCheckpointRecord);
do
{
var isRecoverableRecord = true;
var record = records.Current;
record.CompletedFlush(null);
Utility.Assert(
record.PreviousPhysicalRecord == logRecordsMap.LastPhysicalRecord,
"record.PreviousPhysicalRecord == lastPhysicalRecord");
logRecordsMap.ProcessLogRecord(record, out isRecoverableRecord);
if (isRecoverableRecord == true)
{
recoveredRecords.Add(record);
lastRecoverableRecord = record;
if (LoggingReplicator.IsBarrierRecord(record) &&
recoveredRecords.Count >= ReplicatorDynamicConstants.ParallelRecoveryBatchSize)
{
logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
recoveredRecords = new List <LogRecord>();
this.recoveryException = recordsProcessor.ServiceException;
if (this.recoveryException == null)
{
this.recoveryException = recordsProcessor.LogException;
}
if (this.recoveryException != null)
{
// If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
// The only solution here is to throw during OpenAsync
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: RecoveryFailed");
await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);
await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);
throw this.recoveryException;
}
}
}
else
{
Utility.Assert(LoggingReplicator.IsBarrierRecord(record) == false, "IsBarrierRecord(record) == false");
record.CompletedApply(null);
record.CompletedProcessing(null);
}
hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
} while (hasMoved == true);
this.LastRecoveredAtomicRedoOperationLsn = logRecordsMap.LastRecoveredAtomicRedoOperationLsn;
checkpointManager.ResetStableLsn(logRecordsMap.LastStableLsn);
lastPhysicalRecord = logRecordsMap.LastPhysicalRecord;
lsn = logRecordsMap.LastLogicalSequenceNumber;
if (recoveredRecords.Count > 0)
{
logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
}
var tailRecord = records.Current;
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: Current tail record Type: " + tailRecord.RecordType + " LSN: "
+ tailRecord.Lsn.LSN + " PSN: " + tailRecord.Psn.PSN + " Position: " + tailRecord.RecordPosition);
Utility.Assert(
lastPhysicalRecord == records.LastPhysicalRecord,
"lastPhysicalRecord == records.LastPhysicalRecord");
Utility.Assert(
(tailRecord == lastPhysicalRecord) || (tailRecord.PreviousPhysicalRecord == lastPhysicalRecord),
"(tailRecord == lastPhysicalRecord) || "
+ "(tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), " + "LastPhysicalRecord PSN: {0}",
lastPhysicalRecord.Psn.PSN);
Utility.Assert(
logRecordsMap.LastCompletedEndCheckpointRecord != null,
"this.lastCompletedEndCheckpointRecord != null");
Utility.Assert(
logRecordsMap.LastInProgressCheckpointRecord == null,
"this.lastInProgressCheckpointRecord == null");
Utility.Assert(logRecordsMap.LastLinkedPhysicalRecord != null, "this.lastLinkedPhysicalRecord != nul");
Utility.Assert(lsn == tailRecord.Lsn, "lsn == tailRecord.LastLogicalSequenceNumber");
// Disable read ahead as indexing physical records will read the log backwards
replicatedLogManager.LogManager.SetSequentialAccessReadSize(this.RecoveryLogsReader.ReadStream, 0);
await this.RecoveryLogsReader.IndexPhysicalRecords(lastPhysicalRecord).ConfigureAwait(false);
var callbackManager = new PhysicalLogWriterCallbackManager(this.flushedRecordsCallback);
callbackManager.FlushedPsn = tailRecord.Psn + 1;
replicatedLogManager.LogManager.PrepareToLog(tailRecord, callbackManager);
replicatedLogManager.Reuse(
recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
logRecordsMap.LastCompletedEndCheckpointRecord,
logRecordsMap.LastInProgressCheckpointRecord,
logRecordsMap.LastLinkedPhysicalRecord,
replicatedLogManager.LastInformationRecord,
(IndexingLogRecord)this.RecoveryLogsReader.StartingRecord,
logRecordsMap.CurrentLogTailEpoch,
lsn);
this.RecoveredLsn = lsn;
// GopalK: The order of the following statements is significant
if (this.logCompleteCheckpointAfterRecovery)
{
replicatedLogManager.CompleteCheckpoint();
}
replicatedLogManager.Information(InformationEvent.Recovered);
await replicatedLogManager.LogManager.FlushAsync("PerformRecovery").ConfigureAwait(false);
await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);
await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);
lastPeriodicTruncationTimeTicks = logRecordsMap.LastPeriodicTruncationTimeTicks;
}
this.recoveryException = recordsProcessor.ServiceException;
if (this.recoveryException == null)
{
this.recoveryException = recordsProcessor.LogException;
}
if (this.recoveryException != null)
{
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: RecoveryFailed");
// If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
// The only solution here is to throw during OpenAsync
throw this.recoveryException;
}
checkpointManager.Recover(
recoveredLastCompletedBeginCheckpointRecord,
lastPeriodicTruncationTimeTicks);
}
public RecordProcessingMode IdentifyProcessingModeForRecord(LogRecord record)
{
var stream = this.roleContextDrainState.DrainingStream;
Utility.Assert(
stream != DrainingStream.Invalid || record is UpdateEpochLogRecord || record is PhysicalLogRecord,
@"Draining stream should not be invalid or record should be of type epoch or physical log record.
DrainingStream: {0} RecordType: {1}",
stream, record.RecordType);
if (LoggingReplicator.IsBarrierRecord(record))
{
this.LastAppliedBarrierRecord = record;
}
RecordProcessingMode processingMode;
switch (record.RecordType)
{
case LogRecordType.EndCheckpoint:
case LogRecordType.TruncateTail:
case LogRecordType.Indexing:
case LogRecordType.UpdateEpoch:
case LogRecordType.Information:
case LogRecordType.CompleteCheckpoint:
processingMode = RecordProcessingMode.ProcessImmediately;
break;
case LogRecordType.Barrier:
processingMode = (this.roleContextDrainState.ReplicaRole == ReplicaRole.Unknown)
? RecordProcessingMode.ProcessImmediately
: RecordProcessingMode.ApplyImmediately;
break;
case LogRecordType.TruncateHead:
processingMode = RecordProcessingMode.ApplyImmediately;
break;
case LogRecordType.BeginCheckpoint:
Utility.Assert(
(record.Lsn >= this.recoveredOrCopiedCheckpointLsn.Value) ||
(this.roleContextDrainState.ReplicaRole == ReplicaRole.Unknown),
"record.LastLogicalSequenceNumber >= this.recoveredOrCopiedCheckpointLsn || "
+ "(this.replicaRole == ReplicaRole.Unknown)");
processingMode = (this.roleContextDrainState.ReplicaRole == ReplicaRole.Unknown)
? RecordProcessingMode.ProcessImmediately
: RecordProcessingMode.ApplyImmediately;
break;
case LogRecordType.BeginTransaction:
case LogRecordType.Operation:
case LogRecordType.EndTransaction:
case LogRecordType.Backup:
processingMode = RecordProcessingMode.Normal;
break;
default:
Utility.CodingError("Unexpected record type {0}", record.RecordType);
processingMode = RecordProcessingMode.Normal;
break;
}
return(processingMode);
}
private async Task <bool> ProcessLoggedRecords()
{
while (this.processingRecords.Count > 0)
{
var isBarrierRecord = false;
do
{
var exception = this.processingRecords[0].Exception;
var records = this.processingRecords[0].Records;
if (exception == null)
{
// Copy from processingRecords to concurrentRecords until barrierRecord
while (this.processingIndex < records.Count)
{
var record = records[this.processingIndex];
// TODO: GopalK, currently all barrier operations are full barriers.
// Add support for weaker forms of barriers as well.
isBarrierRecord = LoggingReplicator.IsBarrierRecord(record);
// If barrier is hit, process records we currently have before barrier
if (isBarrierRecord == true)
{
if (this.concurrentRecords.Count > 0)
{
this.processor.UpdateDispatchingBarrierTask(record.AppliedTask);
break;
}
}
++this.processingIndex;
this.concurrentRecords.Add(record);
if (isBarrierRecord == true)
{
break;
}
}
}
else if (this.concurrentRecords.Count > 0)
{
// If an exception happened, process what we have so far
isBarrierRecord = true;
}
else
{
// If first record is an exception, process everything with exception
while (this.processingIndex < records.Count)
{
var record = records[this.processingIndex];
++this.processingIndex;
await
this.processor.ImmediatelyProcessRecord(
record,
exception,
RecordProcessingMode.ProcessImmediately).ConfigureAwait(false);
}
}
if (this.processingIndex == records.Count)
{
this.processingRecords.RemoveAt(0);
this.processingIndex = 0;
}
if (isBarrierRecord == true)
{
break;
}
} while (this.processingRecords.Count > 0);
// If we do have records to process, but didn't hit barrier, we do not process yet
if (isBarrierRecord == false)
{
if (this.concurrentRecords.Count > 0)
{
var lastIndex = this.concurrentRecords.Count - 1;
FabricEvents.Events.ProcessLoggedRecords(
this.tracer.Type,
this.concurrentRecords.Count,
this.concurrentRecords[0].Lsn.LSN,
this.concurrentRecords[lastIndex].Lsn.LSN,
this.concurrentRecords[0].Psn.PSN,
this.concurrentRecords[lastIndex].Psn.PSN);
}
break;
}
for (var i = 0; i < this.concurrentRecords.Count; i++)
{
var record = this.concurrentRecords[i];
var processingMode = this.processor.IdentifyProcessingModeForRecord(record);
// processingMode == Normal, ApplyImmediately
if (processingMode < RecordProcessingMode.ProcessImmediately)
{
this.PrepareToProcessRecord(record, processingMode);
}
// processingMode == ApplyImmediately, ProcessImmediately
if (processingMode > RecordProcessingMode.Normal)
{
await this.processor.ImmediatelyProcessRecord(record, null, processingMode).ConfigureAwait(false);
this.concurrentRecords.RemoveAt(i);
i--;
}
}
if (this.concurrentRecords.Count == 0)
{
continue;
}
else if (this.concurrentRecords.Count == 1)
{
await this.processor.ProcessLoggedRecordAsync(this.concurrentRecords[0]).ConfigureAwait(false);
}
else
{
var numberOfTransactions = await this.SeparateTransactions().ConfigureAwait(false);
if (numberOfTransactions > 1)
{
return(false);
}
}
this.concurrentRecords.Clear();
}
return(true);
}
