private static bool FillBuffer(MerSource[] merSources, ref ulong[] bufferMers, ref ulong[] bufferValues, out int bufferCount, ulong startingMer, out ulong endingMer) { bufferCount = 0; bool foundSomeMers = false; ulong highestFromFirstSource = ulong.MaxValue; // initial buffer loading from the first repeated mer table (it's as good a one as any) MerSource repeats = merSources[0]; if (repeats.valid) { foundSomeMers = true; // get a buffer load of mers from this source for (int i = 0; i < bufferSize; i++) { bufferMers[bufferCount] = repeats.lowestMer; bufferValues[bufferCount] = repeats.value; highestFromFirstSource = repeats.lowestMer; bufferCount++; repeats.MoveToNextDistinctMer(); if (!repeats.valid) { highestFromFirstSource = ulong.MaxValue; break; } } } endingMer = highestFromFirstSource; //Console.WriteLine("buffer: " + startingMer.ToString("X16") + " " + endingMer.ToString("X16")); // now add the matching mers/counts from all the other sources for (int s = 1; s < merSources.Length; s++) { MerSource ms = merSources[s]; // if this source has not yet been opened but overlaps the current buffer range, then open it if (!ms.opened) { if (startingMer <= ms.firstMer && ms.firstMer <= endingMer) { ms.Open(); //Console.WriteLine("opened " + ms.sourceType + "-" + s + ": " + startingMer.ToString("X16") + " " + endingMer.ToString("X16") + " " + ms.firstMer.ToString("X16") + " " + ms.lastMer.ToString("X16")); } } if (!ms.valid) { continue; } foundSomeMers = true; while (highestFromFirstSource >= ms.lowestMer) { if (bufferCount == bufferMers.Length) { Array.Resize <ulong>(ref bufferMers, (bufferMers.Length + bufferMers.Length / 4)); Array.Resize <ulong>(ref bufferValues, (bufferValues.Length + bufferValues.Length / 4)); } bufferMers[bufferCount] = ms.lowestMer; bufferValues[bufferCount] = ms.value; bufferCount++; ms.MoveToNextDistinctMer(); if (!ms.valid) { break; } } } if (foundSomeMers) { Array.Sort <ulong, ulong>(bufferMers, bufferValues, 0, bufferCount); } return(foundSomeMers); }
private static void MergeAndWrite(BinaryWriter pairsFile, MerCollections.MerDictionary[] repeatedMers, MerCollections.MerDictionary[] overflowMers) { mergingPhase = true; int noOfOverflows = 0; for (int p = 0; p < overflowMers.Length; p++) { if (overflowMers[p] != null) { noOfOverflows++; } } // shared mers overflow int noMerSources = repeatedMers.Length + noOfOverflows; MerSource[] merSources = new MerSource[noMerSources]; int sourceCounts = 0; int nextSource = 0; // shared repeated mers partitions for (int i = 0; i < repeatedMers.Length; i++) { merSources[nextSource] = new MerDictionarySource(repeatedMers[i]); nextSource++; sourceCounts += repeatedMers[i].Count; //Console.WriteLine("repeatedMers[" + i + "]=" + repeatedMers[i].Count); } // all the overflow mer tables for (int i = 0; i < overflowMers.Length; i++) { if (overflowMers[i] != null) { merSources[nextSource] = new MerDictionarySource(overflowMers[i]); nextSource++; sourceCounts += overflowMers[i].Count; //Console.WriteLine("overflowMers[" + i + "]=" + overflowMers[i].Count); } } //Console.WriteLine("Total mers=" + sourceCounts); //Console.WriteLine("Dictionary=" + pairDictionary.Count); WriteBufferDelegate wbd = new WriteBufferDelegate(WriteBuffer); // now just merge and write until all mers have been written bool mersLeft = true; ulong[][] bufferMers = new ulong[2][]; bufferMers[0] = new ulong[noMerSources * bufferSize]; bufferMers[1] = new ulong[noMerSources * bufferSize]; ulong[][] bufferValues = new ulong[2][]; bufferValues[0] = new ulong[noMerSources * bufferSize]; bufferValues[1] = new ulong[noMerSources * bufferSize]; int[] bufferCount = new int[2]; IAsyncResult[] iarWriteBuffer = new IAsyncResult[2]; int currentBuffer = 0; int previousBuffer = 1; ulong highestMerInBuffer = 0; merSources[0].Open(); // just being polite while (mersLeft) { if (iarWriteBuffer[currentBuffer] != null) { wbd.EndInvoke(iarWriteBuffer[currentBuffer]); } mersLeft = FillBuffer(merSources, ref bufferMers[currentBuffer], ref bufferValues[currentBuffer], out bufferCount[currentBuffer], highestMerInBuffer, out highestMerInBuffer); if (!mersLeft) { break; } if (iarWriteBuffer[previousBuffer] != null && !iarWriteBuffer[previousBuffer].IsCompleted) { iarWriteBuffer[previousBuffer].AsyncWaitHandle.WaitOne(); } iarWriteBuffer[currentBuffer] = wbd.BeginInvoke(pairsFile, bufferMers[currentBuffer], bufferValues[currentBuffer], bufferCount[currentBuffer], null, null); previousBuffer = currentBuffer; if (currentBuffer == 0) { currentBuffer = 1; } else { currentBuffer = 0; } } for (int i = 0; i < 2; i++) { if (iarWriteBuffer[i] != null && !iarWriteBuffer[i].IsCompleted) { wbd.EndInvoke(iarWriteBuffer[i]); } } //for (int s = 0; s < merSources.Length; s++) //{ // Console.WriteLine("skipped[" + s + "]=" + merSources[s].repeatsSkipped); //} }