private void ReadOffQueue()
{
try {
while (true)
{
var indexmapFile = Path.Combine(_directory, IndexMapFilename);
if (_isManualMergePending)
{
Log.Debug("Performing manual index merge.");
_isManualMergePending = false;
using (var reader = _tfReaderFactory()) {
var manualMergeResult = _indexMap.TryManualMerge(
(streamId, currentHash) => UpgradeHash(streamId, currentHash),
entry => reader.ExistsAt(entry.Position),
entry => ReadEntry(reader, entry.Position),
_fileNameProvider,
_ptableVersion,
_indexCacheDepth,
_skipIndexVerify);
if (manualMergeResult.HasMergedAny)
{
_indexMap = manualMergeResult.MergedMap;
_indexMap.SaveToFile(indexmapFile);
manualMergeResult.ToDelete.ForEach(x => x.MarkForDestruction());
}
Log.Debug("Manual index merge completed: {numMergedPTables} PTable(s) merged.", manualMergeResult.ToDelete.Count);
}
}
TableItem tableItem;
//ISearchTable table;
lock (_awaitingTablesLock) {
Log.Debug("Awaiting tables queue size is: {awaitingMemTables}.", _awaitingMemTables.Count);
if (_awaitingMemTables.Count == 1)
{
return;
}
tableItem = _awaitingMemTables[_awaitingMemTables.Count - 1];
}
PTable ptable;
var memtable = tableItem.Table as IMemTable;
if (memtable != null)
{
memtable.MarkForConversion();
ptable = PTable.FromMemtable(memtable, _fileNameProvider.GetFilenameNewTable(),
ESConsts.PTableInitialReaderCount,
_pTableMaxReaderCount,
_indexCacheDepth, _skipIndexVerify);
}
else
{
ptable = (PTable)tableItem.Table;
}
var addResult = _indexMap.AddPTable(ptable, tableItem.PrepareCheckpoint, tableItem.CommitCheckpoint);
_indexMap = addResult.NewMap;
_indexMap.SaveToFile(indexmapFile);
if (addResult.CanMergeAny)
{
using (var reader = _tfReaderFactory()) {
MergeResult mergeResult;
do
{
mergeResult = _indexMap.TryMergeOneLevel(
(streamId, currentHash) => UpgradeHash(streamId, currentHash),
entry => reader.ExistsAt(entry.Position),
entry => ReadEntry(reader, entry.Position),
_fileNameProvider,
_ptableVersion,
_indexCacheDepth,
_skipIndexVerify);
if (mergeResult.HasMergedAny)
{
_indexMap = mergeResult.MergedMap;
_indexMap.SaveToFile(indexmapFile);
mergeResult.ToDelete.ForEach(x => x.MarkForDestruction());
}
} while (mergeResult.CanMergeAny);
}
}
lock (_awaitingTablesLock) {
var memTables = _awaitingMemTables.ToList();
var corrTable = memTables.First(x => x.Table.Id == ptable.Id);
memTables.Remove(corrTable);
// parallel thread could already switch table,
// so if we have another PTable instance with same ID,
// we need to kill that instance as we added ours already
if (!ReferenceEquals(corrTable.Table, ptable) && corrTable.Table is PTable)
{
((PTable)corrTable.Table).MarkForDestruction();
}
Log.Debug("There are now {awaitingMemTables} awaiting tables.", memTables.Count);
_awaitingMemTables = memTables;
}
}
} catch (FileBeingDeletedException exc) {
Log.Error(exc,
"Could not acquire chunk in TableIndex.ReadOffQueue. It is OK if node is shutting down.");
} catch (Exception exc) {
Log.Error(exc, "Error in TableIndex.ReadOffQueue");
throw;
} finally {
lock (_awaitingTablesLock) {
_backgroundRunning = false;
_backgroundRunningEvent.Set();
}
}
}