public void Close(bool removeFiles = true)
{
if (!_backgroundRunningEvent.Wait(7000))
{
throw new TimeoutException("Could not finish background thread in reasonable time.");
}
if (_inMem)
{
return;
}
if (_indexMap == null)
{
return;
}
if (removeFiles)
{
_indexMap.InOrder().ToList().ForEach(x => x.MarkForDestruction());
var fileName = Path.Combine(_directory, IndexMapFilename);
if (File.Exists(fileName))
{
File.SetAttributes(fileName, FileAttributes.Normal);
File.Delete(fileName);
}
}
else
{
_indexMap.InOrder().ToList().ForEach(x => x.Dispose());
}
_indexMap.InOrder().ToList().ForEach(x => x.WaitForDisposal(TimeSpan.FromMilliseconds(5000)));
}
private void ScavengeInternal(IIndexScavengerLog log, CancellationToken ct)
{
var toScavenge = _indexMap.InOrder().ToList();
foreach (var pTable in toScavenge)
{
var startNew = Stopwatch.StartNew();
try
{
ct.ThrowIfCancellationRequested();
using (var reader = _tfReaderFactory())
{
var indexmapFile = Path.Combine(_directory, IndexMapFilename);
var scavengeResult = _indexMap.Scavenge(pTable.Id, ct,
(streamId, currentHash) => UpgradeHash(streamId, currentHash),
entry => reader.ExistsAt(entry.Position),
entry => ReadEntry(reader, entry.Position), _fileNameProvider, _ptableVersion,
_indexCacheDepth, _skipIndexVerify);
if (scavengeResult.IsSuccess)
{
_indexMap = scavengeResult.ScavengedMap;
_indexMap.SaveToFile(indexmapFile);
scavengeResult.OldTable.MarkForDestruction();
var entriesDeleted = scavengeResult.OldTable.Count - scavengeResult.NewTable.Count;
log.IndexTableScavenged(scavengeResult.Level, scavengeResult.Index, startNew.Elapsed, entriesDeleted, scavengeResult.NewTable.Count, scavengeResult.SpaceSaved);
}
else
{
log.IndexTableNotScavenged(scavengeResult.Level, scavengeResult.Index, startNew.Elapsed, pTable.Count, "");
}
}
}
catch (OperationCanceledException)
{
log.IndexTableNotScavenged(-1, -1, startNew.Elapsed, pTable.Count, "Scavenge cancelled");
throw;
}
catch (Exception ex)
{
log.IndexTableNotScavenged(-1, -1, startNew.Elapsed, pTable.Count, ex.Message);
throw;
}
}
}
public void Initialize()
{
//NOT THREAD SAFE (assumes one thread)
if (_initialized)
{
throw new IOException("TableIndex is already initialized.");
}
_initialized = true;
CreateIfDoesNotExist(_directory);
try
{
_indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename),
IsHashCollision,
_maxTablesPerLevel);
if (_indexMap.IsCorrupt(_directory))
{
foreach (var ptable in _indexMap.InOrder())
{
ptable.MarkForDestruction();
}
foreach (var ptable in _indexMap.InOrder())
{
ptable.WaitForDestroy(5000);
}
throw new CorruptIndexException("IndexMap is in unsafe state.");
}
}
catch (CorruptIndexException exc)
{
Log.ErrorException(exc, "ReadIndex was corrupted. Rebuilding from scratch...");
foreach (var filePath in Directory.EnumerateFiles(_directory))
{
File.Delete(filePath);
}
_indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename),
IsHashCollision,
_maxTablesPerLevel);
}
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
}
public void Initialize(long chaserCheckpoint)
{
Ensure.Nonnegative(chaserCheckpoint, "chaserCheckpoint");
//NOT THREAD SAFE (assumes one thread)
if (_initialized)
{
throw new IOException("TableIndex is already initialized.");
}
_initialized = true;
if (_inMem)
{
_indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel);
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
return;
}
CreateIfDoesNotExist(_directory);
var indexmapFile = Path.Combine(_directory, IndexMapFilename);
// if TableIndex's CommitCheckpoint is >= amount of written TFChunk data,
// we'll have to remove some of PTables as they point to non-existent data
// this can happen (very unlikely, though) on master crash
try
{
_indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth);
if (_indexMap.CommitCheckpoint >= chaserCheckpoint)
{
_indexMap.Dispose(TimeSpan.FromMilliseconds(5000));
throw new CorruptIndexException(String.Format("IndexMap's CommitCheckpoint ({0}) is greater than ChaserCheckpoint ({1}).", _indexMap.CommitCheckpoint, chaserCheckpoint));
}
}
catch (CorruptIndexException exc)
{
Log.ErrorException(exc, "ReadIndex is corrupted...");
LogIndexMapContent(indexmapFile);
DumpAndCopyIndex();
File.Delete(indexmapFile);
_indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth);
}
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
// clean up all other remaining files
var indexFiles = _indexMap.InOrder().Select(x => Path.GetFileName(x.Filename))
.Union(new[] { IndexMapFilename });
var toDeleteFiles = Directory.EnumerateFiles(_directory).Select(Path.GetFileName)
.Except(indexFiles, StringComparer.OrdinalIgnoreCase);
foreach (var filePath in toDeleteFiles)
{
var file = Path.Combine(_directory, filePath);
File.SetAttributes(file, FileAttributes.Normal);
File.Delete(file);
}
}
public void Initialize(long chaserCheckpoint)
{
Ensure.Nonnegative(chaserCheckpoint, "chaserCheckpoint");
//NOT THREAD SAFE (assumes one thread)
if (_initialized)
throw new IOException("TableIndex is already initialized.");
_initialized = true;
if (_inMem)
{
_indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel);
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
return;
}
CreateIfDoesNotExist(_directory);
var indexmapFile = Path.Combine(_directory, IndexMapFilename);
// if TableIndex's CommitCheckpoint is >= amount of written TFChunk data,
// we'll have to remove some of PTables as they point to non-existent data
// this can happen (very unlikely, though) on master crash
try
{
_indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth);
if (_indexMap.CommitCheckpoint >= chaserCheckpoint)
{
_indexMap.Dispose(TimeSpan.FromMilliseconds(5000));
throw new CorruptIndexException("IndexMap's CommitCheckpoint is greater than WriterCheckpoint.");
}
}
catch (CorruptIndexException exc)
{
Log.ErrorException(exc, "ReadIndex is corrupted...");
LogIndexMapContent(indexmapFile);
DumpAndCopyIndex();
File.Delete(indexmapFile);
_indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth);
}
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
// clean up all other remaining files
var indexFiles = _indexMap.InOrder().Select(x => Path.GetFileName(x.Filename))
.Union(new[] { IndexMapFilename });
var toDeleteFiles = Directory.EnumerateFiles(_directory).Select(Path.GetFileName)
.Except(indexFiles, StringComparer.OrdinalIgnoreCase);
foreach (var filePath in toDeleteFiles)
{
var file = Path.Combine(_directory, filePath);
File.SetAttributes(file, FileAttributes.Normal);
File.Delete(file);
}
}
public void Initialize()
{
//NOT THREAD SAFE (assumes one thread)
if (_initialized)
throw new IOException("TableIndex is already initialized.");
_initialized = true;
CreateIfDoesNotExist(_directory);
try
{
_indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename), IsHashCollision, _maxTablesPerLevel);
if (_indexMap.IsCorrupt(_directory))
{
foreach (var ptable in _indexMap.InOrder())
{
ptable.MarkForDestruction();
}
foreach (var ptable in _indexMap.InOrder())
{
ptable.WaitForDestroy(5000);
}
throw new CorruptIndexException("IndexMap is in unsafe state.");
}
}
catch (CorruptIndexException exc)
{
Log.ErrorException(exc, "ReadIndex was corrupted. Rebuilding from scratch...");
foreach (var filePath in Directory.EnumerateFiles(_directory))
{
File.Delete(filePath);
}
_indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename),
IsHashCollision,
_maxTablesPerLevel);
}
_prepareCheckpoint = _indexMap.PrepareCheckpoint;
_commitCheckpoint = _indexMap.CommitCheckpoint;
}
