internal PhysicalLogWriter(
ILogicalLog logicalLogStream,
LogRecord tailRecord,
PhysicalLogWriterCallbackManager callbackManager,
Exception closedException,
ITracer tracer,
int maxWriteCacheSizeInMB,
IncomingBytesRateCounterWriter incomingBytesRateCounterWriter,
LogFlushBytesRateCounterWriter logFlushBytesRateCounterWriter,
AvgBytesPerFlushCounterWriter bytesPerFlushCounterWriter,
AvgFlushLatencyCounterWriter avgFlushLatencyCounterWriter,
AvgSerializationLatencyCounterWriter avgSerializationLatencyCounterWriter)
{
Utility.Assert(callbackManager != null, "PhysicalLogWriter cannot accept null callback managers");
this.incomingBytesRateCounterWriter = incomingBytesRateCounterWriter;
this.logFlushBytesRateCounterWriter = logFlushBytesRateCounterWriter;
this.bytesPerFlushCounterWriter = bytesPerFlushCounterWriter;
this.avgFlushLatencyCounterWriter = avgFlushLatencyCounterWriter;
this.avgSerializationLatencyCounterWriter = avgSerializationLatencyCounterWriter;
this.maxWriteCacheSizeInBytes = maxWriteCacheSizeInMB * 1024 * 1024;
this.Init(tracer, closedException);
this.logicalLogStream = logicalLogStream;
this.callbackManager = callbackManager;
this.SetTailRecord(tailRecord);
Utility.Assert(
this.currentLogTailPosition == (ulong)logicalLogStream.WritePosition,
"this.currentLogTailPosition:{0} should be equal to logicalLogStream.WritePosition:{1}",
this.currentLogTailPosition, logicalLogStream.WritePosition);
this.recordWriter = new BinaryWriter(new MemoryStream());
this.recomputeRecordOffsets = false;
}
internal PhysicalLogWriter(
ILogicalLog logicalLogStream,
PhysicalLogWriterCallbackManager callbackManager,
ITracer tracer,
int maxWriteCacheSizeInMB,
IncomingBytesRateCounterWriter incomingBytesRateCounterWriter,
LogFlushBytesRateCounterWriter logFlushBytesRateCounterWriter,
AvgBytesPerFlushCounterWriter bytesPerFlushCounterWriter,
AvgFlushLatencyCounterWriter avgFlushLatencyCounterWriter,
AvgSerializationLatencyCounterWriter avgSerializationLatencyCounterWriter,
bool recomputeRecordOffsets)
{
Utility.Assert(callbackManager != null, "PhysicalLogWriter cannot accept null callback managers");
this.maxWriteCacheSizeInBytes = maxWriteCacheSizeInMB * 1024 * 1024;
this.Init(tracer, null);
this.logicalLogStream = logicalLogStream;
this.callbackManager = callbackManager;
this.currentLogTailPosition = (ulong)logicalLogStream.WritePosition;
this.currentLogTailRecord = LogRecord.InvalidLogRecord;
this.currentLogTailPsn = new PhysicalSequenceNumber(-1);
this.lastPhysicalRecord = null;
this.incomingBytesRateCounterWriter = incomingBytesRateCounterWriter;
this.logFlushBytesRateCounterWriter = logFlushBytesRateCounterWriter;
this.bytesPerFlushCounterWriter = bytesPerFlushCounterWriter;
this.avgFlushLatencyCounterWriter = avgFlushLatencyCounterWriter;
this.avgSerializationLatencyCounterWriter = avgSerializationLatencyCounterWriter;
this.recordWriter = new BinaryWriter(new MemoryStream());
this.recomputeRecordOffsets = recomputeRecordOffsets;
}
internal virtual Task InitializeAsync(ITracer tracer, LogManagerInitialization init, TransactionalReplicatorSettings settings)
{
this.Tracer = tracer;
this.LogFileDirectoryPath = init.dedicatedLogPath;
this.MaxWriteCacheSizeInMB = DefaultWriteCacheSizeMb;
this.emptyCallbackManager = new PhysicalLogWriterCallbackManager(this.EmptyFlushedRecordsCallback, tracer);
return(Task.FromResult(0));
}
internal override async Task <IndexingLogRecord> CreateCopyLogAsync(
Epoch startingEpoch,
LogicalSequenceNumber startingLsn)
{
var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback;
await this.CloseCurrentLogAsync().ConfigureAwait(false);
this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix;
try
{
var aliasGuid =
await this.physicalLog.ResolveAliasAsync(this.CurrentLogFileAlias, CancellationToken.None).ConfigureAwait(false);
FabricEvents.Events.LogManager(
this.Tracer.Type,
"CreateCopyLog: Attempt to delete logical log " + this.CurrentLogFileAlias + " guid: " + aliasGuid);
await this.physicalLog.DeleteLogicalLogAsync(aliasGuid, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
FabricEvents.Events.LogManager(
this.Tracer.Type,
"CreateCopyLog: Delete logical log: " + this.CurrentLogFileAlias + " failed: " + ex);
}
this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false);
var callbackManager = new PhysicalLogWriterCallbackManager(
flushCallback,
this.Tracer);
this.PhysicalLogWriter = new PhysicalLogWriter(
this.LogicalLog,
callbackManager,
this.Tracer,
this.MaxWriteCacheSizeInMB,
this.IncomingBytesRateCounterWriter,
this.LogFlushBytesRateCounterWriter,
this.BytesPerFlushCounterWriter,
this.AvgFlushLatencyCounterWriter,
this.AvgSerializationLatencyCounterWriter,
false);
var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null);
this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord);
await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false);
this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition;
return(firstIndexingRecord);
}
internal void PrepareToLog(LogRecord tailRecord, PhysicalLogWriterCallbackManager callbackManager)
{
this.PhysicalLogWriter = new PhysicalLogWriter(
this.LogicalLog,
tailRecord,
callbackManager,
null,
this.Tracer,
this.MaxWriteCacheSizeInMB,
this.IncomingBytesRateCounterWriter,
this.LogFlushBytesRateCounterWriter,
this.BytesPerFlushCounterWriter,
this.AvgFlushLatencyCounterWriter,
this.AvgSerializationLatencyCounterWriter);
}
internal override async Task <IndexingLogRecord> CreateCopyLogAsync(
Epoch startingEpoch,
LogicalSequenceNumber startingLsn)
{
var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback;
await this.CloseCurrentLogAsync().ConfigureAwait(false);
this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix;
this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false);
var callbackManager = new PhysicalLogWriterCallbackManager(
flushCallback,
this.Tracer);
this.PhysicalLogWriter = new PhysicalLogWriter(
this.LogicalLog,
callbackManager,
this.Tracer,
this.MaxWriteCacheSizeInMB,
this.IncomingBytesRateCounterWriter,
this.LogFlushBytesRateCounterWriter,
this.BytesPerFlushCounterWriter,
this.AvgFlushLatencyCounterWriter,
this.AvgSerializationLatencyCounterWriter,
false);
var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null);
this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord);
await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false);
this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition;
return(firstIndexingRecord);
}
public async Task PerformRecoveryAsync(
RecoveredOrCopiedCheckpointLsn recoveredOrCopiedCheckpointLsn,
OperationProcessor recordsProcessor,
CheckpointManager checkpointManager,
TransactionManager transactionManager,
LogRecordsDispatcher logRecordsDispatcher,
ReplicatedLogManager replicatedLogManager)
{
var lastPhysicalRecord = PhysicalLogRecord.InvalidPhysicalLogRecord;
LogicalSequenceNumber lsn;
var recoveredRecords = new List <LogRecord>();
var recoveredLastCompletedBeginCheckpointRecord = await this.RecoveryLogsReader.GetLastCompletedBeginCheckpointRecord(this.RecoveredLastEndCheckpointRecord).ConfigureAwait(false);
var recoveredLastCompletedBeginCheckpointRecordPosition = recoveredLastCompletedBeginCheckpointRecord.RecordPosition;
var recoveryStartingPosition = recoveredLastCompletedBeginCheckpointRecordPosition - recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset;
// Set the recovered lsn before performing recovery as it impacts unlock callback logic during recovery
recoveredOrCopiedCheckpointLsn.Update(recoveredLastCompletedBeginCheckpointRecord.Lsn);
var trace =
string.Format(
CultureInfo.InvariantCulture,
"PerformRecoveryAsync: " + Environment.NewLine
+ " Recovered last completed begin checkpoint record." + Environment.NewLine
+ " Type: {0} LSN: {1} PSN:{2} Position: {3}" + Environment.NewLine
+ " Recovery Starting Position: {4} Recovery Starting Epoch: {5},{6}" + Environment.NewLine
+ " LogCompleteCheckpointAfterRecovery: {7}" + " Recovered ProgressVector: {8}"
+ Environment.NewLine,
recoveredLastCompletedBeginCheckpointRecord.RecordType,
recoveredLastCompletedBeginCheckpointRecord.Lsn.LSN,
recoveredLastCompletedBeginCheckpointRecord.Psn.PSN,
recoveredLastCompletedBeginCheckpointRecord.RecordPosition,
recoveryStartingPosition,
recoveredLastCompletedBeginCheckpointRecord.Epoch.DataLossNumber,
recoveredLastCompletedBeginCheckpointRecord.Epoch.ConfigurationNumber,
this.logCompleteCheckpointAfterRecovery,
recoveredLastCompletedBeginCheckpointRecord.ProgressVector.ToString(Constants.ProgressVectorMaxStringSizeInKb));
FabricEvents.Events.Api(tracer.Type, trace);
LogRecord lastRecoverableRecord = null;
replicatedLogManager.LogManager.SetSequentialAccessReadSize(
this.RecoveryLogsReader.ReadStream,
LogManager.ReadAheadBytesForSequentialStream);
// Updated to the recovered value in LogRecordsMap during successful recovery
long lastPeriodicTruncationTimeTicks = 0;
using (var records = new LogRecords(this.RecoveryLogsReader.ReadStream, recoveryStartingPosition, this.tailRecordAtStart.RecordPosition))
{
var hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
Utility.Assert(hasMoved == true, "hasMoved == true");
lsn = records.Current.Lsn;
if (recoveredLastCompletedBeginCheckpointRecord.EarliestPendingTransactionOffset != 0)
{
lsn -= 1;
}
var logRecordsMap = new LogRecordsMap(
lsn,
transactionManager.TransactionsMap,
recoveredLastCompletedBeginCheckpointRecord.Epoch,
checkpointManager.LastStableLsn,
recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
recoveredLastCompletedBeginCheckpointRecord,
this.tracer,
this.RecoveredLastEndCheckpointRecord);
do
{
var isRecoverableRecord = true;
var record = records.Current;
record.CompletedFlush(null);
Utility.Assert(
record.PreviousPhysicalRecord == logRecordsMap.LastPhysicalRecord,
"record.PreviousPhysicalRecord == lastPhysicalRecord");
logRecordsMap.ProcessLogRecord(record, out isRecoverableRecord);
if (isRecoverableRecord == true)
{
recoveredRecords.Add(record);
lastRecoverableRecord = record;
if (LoggingReplicator.IsBarrierRecord(record) &&
recoveredRecords.Count >= ReplicatorDynamicConstants.ParallelRecoveryBatchSize)
{
logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
recoveredRecords = new List <LogRecord>();
this.recoveryException = recordsProcessor.ServiceException;
if (this.recoveryException == null)
{
this.recoveryException = recordsProcessor.LogException;
}
if (this.recoveryException != null)
{
// If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
// The only solution here is to throw during OpenAsync
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: RecoveryFailed");
await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);
await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);
throw this.recoveryException;
}
}
}
else
{
Utility.Assert(LoggingReplicator.IsBarrierRecord(record) == false, "IsBarrierRecord(record) == false");
record.CompletedApply(null);
record.CompletedProcessing(null);
}
hasMoved = await records.MoveNextAsync(CancellationToken.None).ConfigureAwait(false);
} while (hasMoved == true);
this.LastRecoveredAtomicRedoOperationLsn = logRecordsMap.LastRecoveredAtomicRedoOperationLsn;
checkpointManager.ResetStableLsn(logRecordsMap.LastStableLsn);
lastPhysicalRecord = logRecordsMap.LastPhysicalRecord;
lsn = logRecordsMap.LastLogicalSequenceNumber;
if (recoveredRecords.Count > 0)
{
logRecordsDispatcher.DispatchLoggedRecords(new LoggedRecords(recoveredRecords, null));
}
var tailRecord = records.Current;
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: Current tail record Type: " + tailRecord.RecordType + " LSN: "
+ tailRecord.Lsn.LSN + " PSN: " + tailRecord.Psn.PSN + " Position: " + tailRecord.RecordPosition);
Utility.Assert(
lastPhysicalRecord == records.LastPhysicalRecord,
"lastPhysicalRecord == records.LastPhysicalRecord");
Utility.Assert(
(tailRecord == lastPhysicalRecord) || (tailRecord.PreviousPhysicalRecord == lastPhysicalRecord),
"(tailRecord == lastPhysicalRecord) || "
+ "(tailRecord.PreviousPhysicalRecord == lastPhysicalRecord), " + "LastPhysicalRecord PSN: {0}",
lastPhysicalRecord.Psn.PSN);
Utility.Assert(
logRecordsMap.LastCompletedEndCheckpointRecord != null,
"this.lastCompletedEndCheckpointRecord != null");
Utility.Assert(
logRecordsMap.LastInProgressCheckpointRecord == null,
"this.lastInProgressCheckpointRecord == null");
Utility.Assert(logRecordsMap.LastLinkedPhysicalRecord != null, "this.lastLinkedPhysicalRecord != nul");
Utility.Assert(lsn == tailRecord.Lsn, "lsn == tailRecord.LastLogicalSequenceNumber");
// Disable read ahead as indexing physical records will read the log backwards
replicatedLogManager.LogManager.SetSequentialAccessReadSize(this.RecoveryLogsReader.ReadStream, 0);
await this.RecoveryLogsReader.IndexPhysicalRecords(lastPhysicalRecord).ConfigureAwait(false);
var callbackManager = new PhysicalLogWriterCallbackManager(this.flushedRecordsCallback);
callbackManager.FlushedPsn = tailRecord.Psn + 1;
replicatedLogManager.LogManager.PrepareToLog(tailRecord, callbackManager);
replicatedLogManager.Reuse(
recoveredLastCompletedBeginCheckpointRecord.ProgressVector,
logRecordsMap.LastCompletedEndCheckpointRecord,
logRecordsMap.LastInProgressCheckpointRecord,
logRecordsMap.LastLinkedPhysicalRecord,
replicatedLogManager.LastInformationRecord,
(IndexingLogRecord)this.RecoveryLogsReader.StartingRecord,
logRecordsMap.CurrentLogTailEpoch,
lsn);
this.RecoveredLsn = lsn;
// GopalK: The order of the following statements is significant
if (this.logCompleteCheckpointAfterRecovery)
{
replicatedLogManager.CompleteCheckpoint();
}
replicatedLogManager.Information(InformationEvent.Recovered);
await replicatedLogManager.LogManager.FlushAsync("PerformRecovery").ConfigureAwait(false);
await lastRecoverableRecord.AwaitProcessing().ConfigureAwait(false);
await recordsProcessor.WaitForAllRecordsProcessingAsync().ConfigureAwait(false);
lastPeriodicTruncationTimeTicks = logRecordsMap.LastPeriodicTruncationTimeTicks;
}
this.recoveryException = recordsProcessor.ServiceException;
if (this.recoveryException == null)
{
this.recoveryException = recordsProcessor.LogException;
}
if (this.recoveryException != null)
{
FabricEvents.Events.Api(
tracer.Type,
"PerformRecoveryAsync: RecoveryFailed");
// If there was an apply or unlock failure, report fault does not help during recovery because the replica is not opened yet.
// The only solution here is to throw during OpenAsync
throw this.recoveryException;
}
checkpointManager.Recover(
recoveredLastCompletedBeginCheckpointRecord,
lastPeriodicTruncationTimeTicks);
}
