private void ReadOffQueue() { try { while (true) { var indexmapFile = Path.Combine(_directory, IndexMapFilename); if (_isManualMergePending) { Log.Debug("Performing manual index merge."); _isManualMergePending = false; using (var reader = _tfReaderFactory()) { var manualMergeResult = _indexMap.TryManualMerge( (streamId, currentHash) => UpgradeHash(streamId, currentHash), entry => reader.ExistsAt(entry.Position), entry => ReadEntry(reader, entry.Position), _fileNameProvider, _ptableVersion, _indexCacheDepth, _skipIndexVerify); if (manualMergeResult.HasMergedAny) { _indexMap = manualMergeResult.MergedMap; _indexMap.SaveToFile(indexmapFile); manualMergeResult.ToDelete.ForEach(x => x.MarkForDestruction()); } Log.Debug("Manual index merge completed: {numMergedPTables} PTable(s) merged.", manualMergeResult.ToDelete.Count); } } TableItem tableItem; //ISearchTable table; lock (_awaitingTablesLock) { Log.Debug("Awaiting tables queue size is: {awaitingMemTables}.", _awaitingMemTables.Count); if (_awaitingMemTables.Count == 1) { return; } tableItem = _awaitingMemTables[_awaitingMemTables.Count - 1]; } PTable ptable; var memtable = tableItem.Table as IMemTable; if (memtable != null) { memtable.MarkForConversion(); ptable = PTable.FromMemtable(memtable, _fileNameProvider.GetFilenameNewTable(), ESConsts.PTableInitialReaderCount, _pTableMaxReaderCount, _indexCacheDepth, _skipIndexVerify); } else { ptable = (PTable)tableItem.Table; } var addResult = _indexMap.AddPTable(ptable, tableItem.PrepareCheckpoint, tableItem.CommitCheckpoint); _indexMap = addResult.NewMap; _indexMap.SaveToFile(indexmapFile); if (addResult.CanMergeAny) { using (var reader = _tfReaderFactory()) { MergeResult mergeResult; do { mergeResult = _indexMap.TryMergeOneLevel( (streamId, currentHash) => UpgradeHash(streamId, currentHash), entry => reader.ExistsAt(entry.Position), entry => ReadEntry(reader, entry.Position), _fileNameProvider, _ptableVersion, _indexCacheDepth, _skipIndexVerify); if (mergeResult.HasMergedAny) { _indexMap = mergeResult.MergedMap; _indexMap.SaveToFile(indexmapFile); mergeResult.ToDelete.ForEach(x => x.MarkForDestruction()); } } while (mergeResult.CanMergeAny); } } lock (_awaitingTablesLock) { var memTables = _awaitingMemTables.ToList(); var corrTable = memTables.First(x => x.Table.Id == ptable.Id); memTables.Remove(corrTable); // parallel thread could already switch table, // so if we have another PTable instance with same ID, // we need to kill that instance as we added ours already if (!ReferenceEquals(corrTable.Table, ptable) && corrTable.Table is PTable) { ((PTable)corrTable.Table).MarkForDestruction(); } Log.Debug("There are now {awaitingMemTables} awaiting tables.", memTables.Count); _awaitingMemTables = memTables; } } } catch (FileBeingDeletedException exc) { Log.Error(exc, "Could not acquire chunk in TableIndex.ReadOffQueue. It is OK if node is shutting down."); } catch (Exception exc) { Log.Error(exc, "Error in TableIndex.ReadOffQueue"); throw; } finally { lock (_awaitingTablesLock) { _backgroundRunning = false; _backgroundRunningEvent.Set(); } } }