private static bool FillBuffer(MerSource[] merSources, ref ulong[] bufferMers, ref ulong[] bufferValues, out int bufferCount, ulong startingMer, out ulong endingMer)
{
bufferCount = 0;
bool foundSomeMers = false;
ulong highestFromFirstSource = ulong.MaxValue;
// initial buffer loading from the first repeated mer table (it's as good a one as any)
MerSource repeats = merSources[0];
if (repeats.valid)
{
foundSomeMers = true;
// get a buffer load of mers from this source
for (int i = 0; i < bufferSize; i++)
{
bufferMers[bufferCount] = repeats.lowestMer;
bufferValues[bufferCount] = repeats.value;
highestFromFirstSource = repeats.lowestMer;
bufferCount++;
repeats.MoveToNextDistinctMer();
if (!repeats.valid)
{
highestFromFirstSource = ulong.MaxValue;
break;
}
}
}
endingMer = highestFromFirstSource;
//Console.WriteLine("buffer: " + startingMer.ToString("X16") + " " + endingMer.ToString("X16"));
// now add the matching mers/counts from all the other sources
for (int s = 1; s < merSources.Length; s++)
{
MerSource ms = merSources[s];
// if this source has not yet been opened but overlaps the current buffer range, then open it
if (!ms.opened)
{
if (startingMer <= ms.firstMer && ms.firstMer <= endingMer)
{
ms.Open();
//Console.WriteLine("opened " + ms.sourceType + "-" + s + ": " + startingMer.ToString("X16") + " " + endingMer.ToString("X16") + " " + ms.firstMer.ToString("X16") + " " + ms.lastMer.ToString("X16"));
}
}
if (!ms.valid)
{
continue;
}
foundSomeMers = true;
while (highestFromFirstSource >= ms.lowestMer)
{
if (bufferCount == bufferMers.Length)
{
Array.Resize <ulong>(ref bufferMers, (bufferMers.Length + bufferMers.Length / 4));
Array.Resize <ulong>(ref bufferValues, (bufferValues.Length + bufferValues.Length / 4));
}
bufferMers[bufferCount] = ms.lowestMer;
bufferValues[bufferCount] = ms.value;
bufferCount++;
ms.MoveToNextDistinctMer();
if (!ms.valid)
{
break;
}
}
}
if (foundSomeMers)
{
Array.Sort <ulong, ulong>(bufferMers, bufferValues, 0, bufferCount);
}
return(foundSomeMers);
}
private static void MergeAndWrite(BinaryWriter pairsFile, MerCollections.MerDictionary[] repeatedMers, MerCollections.MerDictionary[] overflowMers)
{
mergingPhase = true;
int noOfOverflows = 0;
for (int p = 0; p < overflowMers.Length; p++)
{
if (overflowMers[p] != null)
{
noOfOverflows++;
}
}
// shared mers overflow
int noMerSources = repeatedMers.Length + noOfOverflows;
MerSource[] merSources = new MerSource[noMerSources];
int sourceCounts = 0;
int nextSource = 0;
// shared repeated mers partitions
for (int i = 0; i < repeatedMers.Length; i++)
{
merSources[nextSource] = new MerDictionarySource(repeatedMers[i]);
nextSource++;
sourceCounts += repeatedMers[i].Count;
//Console.WriteLine("repeatedMers[" + i + "]=" + repeatedMers[i].Count);
}
// all the overflow mer tables
for (int i = 0; i < overflowMers.Length; i++)
{
if (overflowMers[i] != null)
{
merSources[nextSource] = new MerDictionarySource(overflowMers[i]);
nextSource++;
sourceCounts += overflowMers[i].Count;
//Console.WriteLine("overflowMers[" + i + "]=" + overflowMers[i].Count);
}
}
//Console.WriteLine("Total mers=" + sourceCounts);
//Console.WriteLine("Dictionary=" + pairDictionary.Count);
WriteBufferDelegate wbd = new WriteBufferDelegate(WriteBuffer);
// now just merge and write until all mers have been written
bool mersLeft = true;
ulong[][] bufferMers = new ulong[2][];
bufferMers[0] = new ulong[noMerSources * bufferSize];
bufferMers[1] = new ulong[noMerSources * bufferSize];
ulong[][] bufferValues = new ulong[2][];
bufferValues[0] = new ulong[noMerSources * bufferSize];
bufferValues[1] = new ulong[noMerSources * bufferSize];
int[] bufferCount = new int[2];
IAsyncResult[] iarWriteBuffer = new IAsyncResult[2];
int currentBuffer = 0;
int previousBuffer = 1;
ulong highestMerInBuffer = 0;
merSources[0].Open(); // just being polite
while (mersLeft)
{
if (iarWriteBuffer[currentBuffer] != null)
{
wbd.EndInvoke(iarWriteBuffer[currentBuffer]);
}
mersLeft = FillBuffer(merSources, ref bufferMers[currentBuffer], ref bufferValues[currentBuffer], out bufferCount[currentBuffer], highestMerInBuffer, out highestMerInBuffer);
if (!mersLeft)
{
break;
}
if (iarWriteBuffer[previousBuffer] != null && !iarWriteBuffer[previousBuffer].IsCompleted)
{
iarWriteBuffer[previousBuffer].AsyncWaitHandle.WaitOne();
}
iarWriteBuffer[currentBuffer] = wbd.BeginInvoke(pairsFile, bufferMers[currentBuffer], bufferValues[currentBuffer], bufferCount[currentBuffer], null, null);
previousBuffer = currentBuffer;
if (currentBuffer == 0)
{
currentBuffer = 1;
}
else
{
currentBuffer = 0;
}
}
for (int i = 0; i < 2; i++)
{
if (iarWriteBuffer[i] != null && !iarWriteBuffer[i].IsCompleted)
{
wbd.EndInvoke(iarWriteBuffer[i]);
}
}
//for (int s = 0; s < merSources.Length; s++)
//{
// Console.WriteLine("skipped[" + s + "]=" + merSources[s].repeatsSkipped);
//}
}