public void Close(bool removeFiles = true) { if (!_backgroundRunningEvent.Wait(7000)) { throw new TimeoutException("Could not finish background thread in reasonable time."); } if (_inMem) { return; } if (_indexMap == null) { return; } if (removeFiles) { _indexMap.InOrder().ToList().ForEach(x => x.MarkForDestruction()); var fileName = Path.Combine(_directory, IndexMapFilename); if (File.Exists(fileName)) { File.SetAttributes(fileName, FileAttributes.Normal); File.Delete(fileName); } } else { _indexMap.InOrder().ToList().ForEach(x => x.Dispose()); } _indexMap.InOrder().ToList().ForEach(x => x.WaitForDisposal(TimeSpan.FromMilliseconds(5000))); }
private void ScavengeInternal(IIndexScavengerLog log, CancellationToken ct) { var toScavenge = _indexMap.InOrder().ToList(); foreach (var pTable in toScavenge) { var startNew = Stopwatch.StartNew(); try { ct.ThrowIfCancellationRequested(); using (var reader = _tfReaderFactory()) { var indexmapFile = Path.Combine(_directory, IndexMapFilename); var scavengeResult = _indexMap.Scavenge(pTable.Id, ct, (streamId, currentHash) => UpgradeHash(streamId, currentHash), entry => reader.ExistsAt(entry.Position), entry => ReadEntry(reader, entry.Position), _fileNameProvider, _ptableVersion, _indexCacheDepth, _skipIndexVerify); if (scavengeResult.IsSuccess) { _indexMap = scavengeResult.ScavengedMap; _indexMap.SaveToFile(indexmapFile); scavengeResult.OldTable.MarkForDestruction(); var entriesDeleted = scavengeResult.OldTable.Count - scavengeResult.NewTable.Count; log.IndexTableScavenged(scavengeResult.Level, scavengeResult.Index, startNew.Elapsed, entriesDeleted, scavengeResult.NewTable.Count, scavengeResult.SpaceSaved); } else { log.IndexTableNotScavenged(scavengeResult.Level, scavengeResult.Index, startNew.Elapsed, pTable.Count, ""); } } } catch (OperationCanceledException) { log.IndexTableNotScavenged(-1, -1, startNew.Elapsed, pTable.Count, "Scavenge cancelled"); throw; } catch (Exception ex) { log.IndexTableNotScavenged(-1, -1, startNew.Elapsed, pTable.Count, ex.Message); throw; } } }
public void Initialize() { //NOT THREAD SAFE (assumes one thread) if (_initialized) { throw new IOException("TableIndex is already initialized."); } _initialized = true; CreateIfDoesNotExist(_directory); try { _indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename), IsHashCollision, _maxTablesPerLevel); if (_indexMap.IsCorrupt(_directory)) { foreach (var ptable in _indexMap.InOrder()) { ptable.MarkForDestruction(); } foreach (var ptable in _indexMap.InOrder()) { ptable.WaitForDestroy(5000); } throw new CorruptIndexException("IndexMap is in unsafe state."); } } catch (CorruptIndexException exc) { Log.ErrorException(exc, "ReadIndex was corrupted. Rebuilding from scratch..."); foreach (var filePath in Directory.EnumerateFiles(_directory)) { File.Delete(filePath); } _indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename), IsHashCollision, _maxTablesPerLevel); } _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; }
public void Initialize(long chaserCheckpoint) { Ensure.Nonnegative(chaserCheckpoint, "chaserCheckpoint"); //NOT THREAD SAFE (assumes one thread) if (_initialized) { throw new IOException("TableIndex is already initialized."); } _initialized = true; if (_inMem) { _indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel); _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; return; } CreateIfDoesNotExist(_directory); var indexmapFile = Path.Combine(_directory, IndexMapFilename); // if TableIndex's CommitCheckpoint is >= amount of written TFChunk data, // we'll have to remove some of PTables as they point to non-existent data // this can happen (very unlikely, though) on master crash try { _indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth); if (_indexMap.CommitCheckpoint >= chaserCheckpoint) { _indexMap.Dispose(TimeSpan.FromMilliseconds(5000)); throw new CorruptIndexException(String.Format("IndexMap's CommitCheckpoint ({0}) is greater than ChaserCheckpoint ({1}).", _indexMap.CommitCheckpoint, chaserCheckpoint)); } } catch (CorruptIndexException exc) { Log.ErrorException(exc, "ReadIndex is corrupted..."); LogIndexMapContent(indexmapFile); DumpAndCopyIndex(); File.Delete(indexmapFile); _indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth); } _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; // clean up all other remaining files var indexFiles = _indexMap.InOrder().Select(x => Path.GetFileName(x.Filename)) .Union(new[] { IndexMapFilename }); var toDeleteFiles = Directory.EnumerateFiles(_directory).Select(Path.GetFileName) .Except(indexFiles, StringComparer.OrdinalIgnoreCase); foreach (var filePath in toDeleteFiles) { var file = Path.Combine(_directory, filePath); File.SetAttributes(file, FileAttributes.Normal); File.Delete(file); } }
public void Initialize(long chaserCheckpoint) { Ensure.Nonnegative(chaserCheckpoint, "chaserCheckpoint"); //NOT THREAD SAFE (assumes one thread) if (_initialized) throw new IOException("TableIndex is already initialized."); _initialized = true; if (_inMem) { _indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel); _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; return; } CreateIfDoesNotExist(_directory); var indexmapFile = Path.Combine(_directory, IndexMapFilename); // if TableIndex's CommitCheckpoint is >= amount of written TFChunk data, // we'll have to remove some of PTables as they point to non-existent data // this can happen (very unlikely, though) on master crash try { _indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth); if (_indexMap.CommitCheckpoint >= chaserCheckpoint) { _indexMap.Dispose(TimeSpan.FromMilliseconds(5000)); throw new CorruptIndexException("IndexMap's CommitCheckpoint is greater than WriterCheckpoint."); } } catch (CorruptIndexException exc) { Log.ErrorException(exc, "ReadIndex is corrupted..."); LogIndexMapContent(indexmapFile); DumpAndCopyIndex(); File.Delete(indexmapFile); _indexMap = IndexMap.FromFile(indexmapFile, maxTablesPerLevel: _maxTablesPerLevel, cacheDepth: _indexCacheDepth); } _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; // clean up all other remaining files var indexFiles = _indexMap.InOrder().Select(x => Path.GetFileName(x.Filename)) .Union(new[] { IndexMapFilename }); var toDeleteFiles = Directory.EnumerateFiles(_directory).Select(Path.GetFileName) .Except(indexFiles, StringComparer.OrdinalIgnoreCase); foreach (var filePath in toDeleteFiles) { var file = Path.Combine(_directory, filePath); File.SetAttributes(file, FileAttributes.Normal); File.Delete(file); } }
public void Initialize() { //NOT THREAD SAFE (assumes one thread) if (_initialized) throw new IOException("TableIndex is already initialized."); _initialized = true; CreateIfDoesNotExist(_directory); try { _indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename), IsHashCollision, _maxTablesPerLevel); if (_indexMap.IsCorrupt(_directory)) { foreach (var ptable in _indexMap.InOrder()) { ptable.MarkForDestruction(); } foreach (var ptable in _indexMap.InOrder()) { ptable.WaitForDestroy(5000); } throw new CorruptIndexException("IndexMap is in unsafe state."); } } catch (CorruptIndexException exc) { Log.ErrorException(exc, "ReadIndex was corrupted. Rebuilding from scratch..."); foreach (var filePath in Directory.EnumerateFiles(_directory)) { File.Delete(filePath); } _indexMap = IndexMap.FromFile(Path.Combine(_directory, IndexMapFilename), IsHashCollision, _maxTablesPerLevel); } _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; }