// Derived class exposed API
internal void RecoverFuzzyIndex(IndexCheckpointInfo info)
{
var token = info.info.token;
var ht_version = resizeInfo.version;
if (state[ht_version].size != info.info.table_size)
{
throw new FasterException($"Incompatible hash table size during recovery; allocated {state[ht_version].size} buckets, recovering {info.info.table_size} buckets");
}
// Create devices to read from using Async API
info.main_ht_device = checkpointManager.GetIndexDevice(token);
BeginMainIndexRecovery(ht_version,
info.main_ht_device,
info.info.num_ht_bytes);
var sectorSize = info.main_ht_device.SectorSize;
var alignedIndexSize = (uint)((info.info.num_ht_bytes + (sectorSize - 1)) & ~(sectorSize - 1));
overflowBucketsAllocator.Recover(info.main_ht_device, alignedIndexSize, info.info.num_buckets, info.info.num_ofb_bytes);
// Wait until reading is complete
IsFuzzyIndexRecoveryComplete(true);
// close index checkpoint files appropriately
info.main_ht_device.Dispose();
// Delete all tentative entries!
DeleteTentativeEntries();
}
// Derived class exposed API
internal void RecoverFuzzyIndex(IndexCheckpointInfo info)
{
var token = info.info.token;
var ht_version = resizeInfo.version;
Debug.Assert(state[ht_version].size == info.info.table_size);
// Create devices to read from using Async API
info.main_ht_device = Devices.CreateLogDevice(directoryConfiguration.GetPrimaryHashTableFileName(token), false);
info.ofb_device = Devices.CreateLogDevice(directoryConfiguration.GetOverflowBucketsFileName(token), false);
BeginMainIndexRecovery(ht_version,
info.main_ht_device,
info.info.num_ht_bytes);
overflowBucketsAllocator.Recover(
info.ofb_device,
info.info.num_buckets,
info.info.num_ofb_bytes);
// Wait until reading is complete
IsFuzzyIndexRecoveryComplete(true);
// close index checkpoint files appropriately
info.main_ht_device.Close();
info.ofb_device.Close();
// Delete all tentative entries!
DeleteTentativeEntries();
}
private void FinalizeMainIndexRecovery(IndexCheckpointInfo info)
{
// close index checkpoint files appropriately
info.main_ht_device.Dispose();
// Delete all tentative entries!
DeleteTentativeEntries();
}
private void GetRecoveryInfoFromLatestCheckpoints(out HybridLogCheckpointInfo recoveredHLCInfo, out IndexCheckpointInfo recoveredICInfo)
{
Debug.WriteLine("********* Primary Recovery Information ********");
recoveredHLCInfo = default;
foreach (var hybridLogToken in checkpointManager.GetLogCheckpointTokens())
{
try
{
recoveredHLCInfo = new HybridLogCheckpointInfo();
recoveredHLCInfo.Recover(hybridLogToken, checkpointManager, hlog.LogPageSizeBits);
}
catch
{
continue;
}
Debug.WriteLine("HybridLog Checkpoint: {0}", hybridLogToken);
break;
}
if (recoveredHLCInfo.IsDefault())
{
throw new FasterException("Unable to find valid index token");
}
recoveredHLCInfo.info.DebugPrint();
recoveredICInfo = default;
foreach (var indexToken in checkpointManager.GetIndexCheckpointTokens())
{
try
{
// Recovery appropriate context information
recoveredICInfo = new IndexCheckpointInfo();
recoveredICInfo.Recover(indexToken, checkpointManager);
}
catch
{
continue;
}
if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info))
{
recoveredICInfo = default;
continue;
}
Debug.WriteLine("Index Checkpoint: {0}", indexToken);
recoveredICInfo.info.DebugPrint();
break;
}
if (recoveredICInfo.IsDefault())
{
Debug.WriteLine("No index checkpoint found, recovering from beginning of log");
}
}
private void InternalRecover(Guid indexToken, Guid hybridLogToken)
{
Debug.WriteLine("********* Primary Recovery Information ********");
Debug.WriteLine("Index Checkpoint: {0}", indexToken);
Debug.WriteLine("HybridLog Checkpoint: {0}", hybridLogToken);
// Recovery appropriate context information
var recoveredICInfo = new IndexCheckpointInfo();
recoveredICInfo.Recover(indexToken, checkpointManager);
recoveredICInfo.info.DebugPrint();
var recoveredHLCInfo = new HybridLogCheckpointInfo();
recoveredHLCInfo.Recover(hybridLogToken, checkpointManager);
recoveredHLCInfo.info.DebugPrint();
// Check if the two checkpoints are compatible for recovery
if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info))
{
throw new Exception("Cannot recover from (" + indexToken.ToString() + "," + hybridLogToken.ToString() + ") checkpoint pair!\n");
}
// Set new system state after recovery
var v = recoveredHLCInfo.info.version;
_systemState.phase = Phase.REST;
_systemState.version = (v + 1);
// Recover fuzzy index from checkpoint
RecoverFuzzyIndex(recoveredICInfo);
// Recover segment offsets for object log
if (recoveredHLCInfo.info.objectLogSegmentOffsets != null)
{
Array.Copy(recoveredHLCInfo.info.objectLogSegmentOffsets,
hlog.GetSegmentOffsets(),
recoveredHLCInfo.info.objectLogSegmentOffsets.Length);
}
// Make index consistent for version v
if (FoldOverSnapshot)
{
RecoverHybridLog(recoveredICInfo.info, recoveredHLCInfo.info);
}
else
{
RecoverHybridLogFromSnapshotFile(recoveredICInfo.info, recoveredHLCInfo.info);
}
// Read appropriate hybrid log pages into memory
RestoreHybridLog(recoveredHLCInfo.info.finalLogicalAddress, recoveredHLCInfo.info.headAddress, recoveredHLCInfo.info.beginAddress);
// Recover session information
_recoveredSessions = recoveredHLCInfo.info.continueTokens;
}
internal async ValueTask RecoverFuzzyIndexAsync(IndexCheckpointInfo info, CancellationToken cancellationToken)
{
ulong alignedIndexSize = InitializeMainIndexRecovery(ref info, isAsync: true);
await this.recoveryCountdown.WaitAsync(cancellationToken).ConfigureAwait(false);
await overflowBucketsAllocator.RecoverAsync(info.main_ht_device, alignedIndexSize, info.info.num_buckets, info.info.num_ofb_bytes, cancellationToken).ConfigureAwait(false);
FinalizeMainIndexRecovery(info);
}
// Derived class exposed API
internal void RecoverFuzzyIndex(IndexCheckpointInfo info)
{
ulong alignedIndexSize = InitializeMainIndexRecovery(ref info, isAsync: false);
overflowBucketsAllocator.Recover(info.main_ht_device, alignedIndexSize, info.info.num_buckets, info.info.num_ofb_bytes);
// Wait until reading is complete
IsFuzzyIndexRecoveryComplete(true);
FinalizeMainIndexRecovery(info);
}
// Derived class exposed API
internal void RecoverFuzzyIndex(IndexCheckpointInfo info)
{
var ht_version = resizeInfo.version;
var token = info.info.token;
Debug.Assert(state[ht_version].size == info.info.table_size);
BeginMainIndexRecovery(ht_version,
info.main_ht_device,
info.info.num_ht_bytes);
overflowBucketsAllocator.Recover(
info.ofb_device,
info.info.num_buckets,
info.info.num_ofb_bytes);
}
private uint InitializeMainIndexRecovery(ref IndexCheckpointInfo info, bool isAsync)
{
var token = info.info.token;
var ht_version = resizeInfo.version;
if (state[ht_version].size != info.info.table_size)
{
throw new FasterException($"Incompatible hash table size during recovery; allocated {state[ht_version].size} buckets, recovering {info.info.table_size} buckets");
}
// Create devices to read from using Async API
info.main_ht_device = checkpointManager.GetIndexDevice(token);
BeginMainIndexRecovery(ht_version, info.main_ht_device, info.info.num_ht_bytes, isAsync);
var sectorSize = info.main_ht_device.SectorSize;
var alignedIndexSize = (uint)((info.info.num_ht_bytes + (sectorSize - 1)) & ~(sectorSize - 1));
return(alignedIndexSize);
}
private ulong InitializeMainIndexRecovery(ref IndexCheckpointInfo info, bool isAsync)
{
var token = info.info.token;
var ht_version = resizeInfo.version;
// Create devices to read from using Async API
info.main_ht_device = checkpointManager.GetIndexDevice(token);
var sectorSize = info.main_ht_device.SectorSize;
if (state[ht_version].size != info.info.table_size)
{
Free(ht_version);
Initialize(info.info.table_size, (int)sectorSize);
}
BeginMainIndexRecovery(ht_version, info.main_ht_device, info.info.num_ht_bytes, isAsync);
var alignedIndexSize = (info.info.num_ht_bytes + (sectorSize - 1)) & ~((ulong)sectorSize - 1);
return(alignedIndexSize);
}
private void InternalRecover(Guid indexToken, Guid hybridLogToken)
{
Debug.WriteLine("********* Primary Recovery Information ********");
Debug.WriteLine("Index Checkpoint: {0}", indexToken);
Debug.WriteLine("HybridLog Checkpoint: {0}", hybridLogToken);
// Assert corresponding checkpoints are safe to recover from
Debug.Assert(IsCheckpointSafe(indexToken, CheckpointType.INDEX_ONLY),
"Cannot recover from incomplete index checkpoint " + indexToken.ToString());
Debug.Assert(IsCheckpointSafe(hybridLogToken, CheckpointType.HYBRID_LOG_ONLY),
"Cannot recover from incomplete hybrid log checkpoint " + hybridLogToken.ToString());
// Recovery appropriate context information
var recoveredICInfo = new IndexCheckpointInfo();
recoveredICInfo.Recover(indexToken, directoryConfiguration);
recoveredICInfo.info.DebugPrint();
var recoveredHLCInfo = new HybridLogCheckpointInfo();
recoveredHLCInfo.Recover(hybridLogToken, directoryConfiguration);
recoveredHLCInfo.info.DebugPrint();
// Check if the two checkpoints are compatible for recovery
if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info))
{
throw new Exception("Cannot recover from (" + indexToken.ToString() + "," + hybridLogToken.ToString() + ") checkpoint pair!\n");
}
// Set new system state after recovery
var v = recoveredHLCInfo.info.version;
_systemState.phase = Phase.REST;
_systemState.version = (v + 1);
// Recover fuzzy index from checkpoint
RecoverFuzzyIndex(recoveredICInfo);
// Recover segment offsets for object log
if (recoveredHLCInfo.info.objectLogSegmentOffsets != null)
{
Array.Copy(recoveredHLCInfo.info.objectLogSegmentOffsets,
hlog.GetSegmentOffsets(),
recoveredHLCInfo.info.objectLogSegmentOffsets.Length);
}
// Make index consistent for version v
if (FoldOverSnapshot)
{
RecoverHybridLog(recoveredICInfo.info, recoveredHLCInfo.info);
}
else
{
RecoverHybridLogFromSnapshotFile(recoveredICInfo.info, recoveredHLCInfo.info);
}
// Read appropriate hybrid log pages into memory
RestoreHybridLog(recoveredHLCInfo.info.finalLogicalAddress);
// Recover session information
_recoveredSessions = new SafeConcurrentDictionary <Guid, long>();
foreach (var sessionInfo in recoveredHLCInfo.info.continueTokens)
{
_recoveredSessions.GetOrAdd(sessionInfo.Key, sessionInfo.Value);
}
}
private void InternalRecoverFromLatestCheckpoints(int numPagesToPreload, bool undoFutureVersions)
{
Debug.WriteLine("********* Primary Recovery Information ********");
HybridLogCheckpointInfo recoveredHLCInfo = default;
foreach (var hybridLogToken in checkpointManager.GetLogCheckpointTokens())
{
try
{
recoveredHLCInfo = new HybridLogCheckpointInfo();
recoveredHLCInfo.Recover(hybridLogToken, checkpointManager);
}
catch
{
continue;
}
Debug.WriteLine("HybridLog Checkpoint: {0}", hybridLogToken);
break;
}
if (recoveredHLCInfo.IsDefault())
{
throw new FasterException("Unable to find valid index token");
}
recoveredHLCInfo.info.DebugPrint();
IndexCheckpointInfo recoveredICInfo = default;
foreach (var indexToken in checkpointManager.GetIndexCheckpointTokens())
{
try
{
// Recovery appropriate context information
recoveredICInfo = new IndexCheckpointInfo();
recoveredICInfo.Recover(indexToken, checkpointManager);
}
catch
{
continue;
}
if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info))
{
recoveredICInfo = default;
continue;
}
Debug.WriteLine("Index Checkpoint: {0}", indexToken);
recoveredICInfo.info.DebugPrint();
break;
}
if (recoveredICInfo.IsDefault())
{
Debug.WriteLine("No index checkpoint found, recovering from beginning of log");
}
InternalRecover(recoveredICInfo, recoveredHLCInfo, numPagesToPreload, undoFutureVersions);
}