private static void FlushLowRepMersInPartition(object threadParams)
{
FlushingThreadParams flushingParams = (FlushingThreadParams)threadParams;
MerTables merTable = flushingParams.merTable;
int partitionNo = flushingParams.partitionNo;
int culledMers = merTable.repeatedMers[partitionNo].Reduce(minKeepDepth, merTable);
// and it can't possibly be full if we've just culled some entries for it
if (culledMers > 0)
{
merTable.repeatedMersFull[partitionNo] = false;
}
//Console.WriteLine("flushed " + culledMers + " from partition " + partitionNo + ". " + merTable.repeatedMers[partitionNo].Count + " left");
}
// flush the low-rep mers from the repeat tables, condense the remaining repeated mers and fold in the per-thread repeats. Can only be called after all the
// threads have finished for a seq data file. This code is *not* thread-safe.
public void FlushLowRepMers(MerTables merTable, int fileNo)
{
// allocate a buffer to hold the flushed low-rep mers
//int initialBufferLength = 500000;
int initialBufferLength = this.repeatedMers[0].Capacity;
culledBuffer = new LowRepMerBuffer();
culledBuffer.keys = new ulong[initialBufferLength + noOfPartitions];
culledBuffer.values = new long[initialBufferLength + noOfPartitions];
culledBuffer.idx = 0;
culledBuffer.bufferActive = true;
culledBuffer.bufferNo = 1;
culledBuffer.limit = initialBufferLength;
culledLock = new object();
FlushingThreadParams[] flushingParams = new FlushingThreadParams[noOfPartitions];
Thread[] flushingThreads = new Thread[noOfPartitions];
for (int p = 0; p < noOfPartitions; p++)
{
flushingParams[p] = new FlushingThreadParams();
flushingParams[p].merTable = merTable;
flushingParams[p].partitionNo = p;
flushingThreads[p] = new Thread(new ParameterizedThreadStart(MerTables.FlushLowRepMersInPartition));
flushingThreads[p].Priority = ThreadPriority.BelowNormal;
flushingThreads[p].Start(flushingParams[p]);
}
for (int p = 0; p < noOfPartitions; p++)
{
flushingThreads[p].Join();
flushingThreads[p] = null;
}
// write out any filled culled buffers
int bufferNo = 0;
if (filledCulledBuffers != null)
{
for (int i = 0; i < filledCulledBuffers.Count; i++)
{
WriteLowRepMers(fileNo, bufferNo, filledCulledBuffers[i], filledCulledBuffers[i].keys.Length);
bufferNo++;
filledCulledBuffers[i] = null;
}
filledCulledBuffers = null;
}
// finally write out the remaining culled low-rep mers
WriteLowRepMers(fileNo, bufferNo, culledBuffer, culledBuffer.idx);
// return the temporary buffers
culledBuffer = null;
// finally push the per-thread dictionaries to the shared dictionary
for (int t = 0; t < overflowMers.Length; t++)
{
if (overflowMers[t] == null)
{
continue;
}
MerDictionary currentOverflow = overflowMers[t];
MerDictionary replacementOverflow = new MerDictionary(currentOverflow.Capacity, fullMerMask);
foreach (KeyValuePair <ulong, long> kvp in currentOverflow)
{
int absMerHashCode = kvp.Key.GetHashCode() & int31Mask;
int partitionNo = absMerHashCode % noOfPartitions;
if (repeatedMersFull[partitionNo])
{
replacementOverflow.Add(kvp.Key, kvp.Value);
}
else
{
bool OK = repeatedMers[partitionNo].Add(kvp.Key, kvp.Value);
if (!OK)
{
repeatedMersFull[partitionNo] = true;
}
}
}
overflowMers[t] = replacementOverflow;
}
}
