// static void WriteBatchHpcSort(KeyData[] keyDataArray, string outFile, int count)
// {
// var data = keyDataArray.SortMergePar(0, count, new KeyDataComparer());
// //write to batch file
//
// using var stream = new FileStream(outFile, FileMode.Create);
// var writer = new BinaryWriter(stream);
// for (int i = 0; i < count; i++)
// {
// stream.Write(data[i].Key.Hash, 0, data[i].Key.Length);
// writer.Write(data[i].DataAddr.addr);
// writer.Write(data[i].DataAddr.len);
// }
// }
static IEnumerable <CurrentBatchInfoItem> SortAndDedupe(List <CurrentBatchInfo> batchInfos)
{
CurrentBatchInfoItem last = null;
CurrentBatchInfoItem next = null;
bool skip = false;
while (batchInfos.Count > 0)
{
skip = false;
var lowestBatch = batchInfos.OrderBy(i => i.CurrentBatchInfoItem.CurrentHashBin).First();
next = lowestBatch.CurrentBatchInfoItem;
if (last == null)
{
skip = true;
last = lowestBatch.CurrentBatchInfoItem;
}
if (!skip)
{
#if DEBUG
//Console.WriteLine($"last: {last.CurrentHashBin.Hash.ToHexString()}. next: {next.CurrentHashBin.Hash.ToHexString()}");
#endif
if (next.CurrentHashBin != last.CurrentHashBin)
{
#if DEBUG
//Console.WriteLine($"writing out: {last.CurrentHashBin.Hash.ToHexString()}");
#endif
yield return(last);
last = next;
}
else
{
#if DEBUG
//Console.WriteLine($"skipping: {last.CurrentHashBin}");
#endif
if (last.CurrentAddr.addr < next.CurrentAddr.addr)
{
last = next;
}
}
}
if (!lowestBatch.Next())
{
batchInfos.Remove(lowestBatch);
}
}
yield return(last);
}
public bool Next()
{
if (CheckComplete())
{
CurrentBatchInfoItem = null;
return(false);
}
CurrentBatchInfoItem = new CurrentBatchInfoItem();
CurrentBatchInfoItem.CurrentHashBin = new HashBin(Stream, _keyLength);
_bytesRead += _keyLength;
CurrentBatchInfoItem.CurrentAddr.addr = Reader.ReadUInt64();
CurrentBatchInfoItem.CurrentAddr.len = Reader.ReadUInt64();
_bytesRead += 16;
return(true);
}
