///<summary>ファイル2個をマージするやつ</summary>
static void MargeSortUnit(long SortMask, string InputPathA, string InputPathB, string OutputPath)
{
using (BufferedLongWriter writer = new BufferedLongWriter(OutputPath))
using (BufferedLongReader readerA = new BufferedLongReader(InputPathA))
using (BufferedLongReader readerB = new BufferedLongReader(InputPathB))
{
//2つのファイルを並行して読んでいく
//残りの書込回数をこれで数える(境界条件対策
long restA = readerA.Length / sizeof(long), restB = readerB.Length / sizeof(long);
long lastA = 0, lastB = 0, maskedA = 0, maskedB = 0;
void ReadA()
{
if (readerA.Readable())
{
lastA = readerA.Read();
maskedA = lastA & SortMask;
}
else
{
maskedA = long.MaxValue;
} //maskされてないMax→もう読めない
}
void ReadB()
{
if (readerB.Readable())
{
lastB = readerB.Read();
maskedB = lastB & SortMask;
}
else
{
maskedB = long.MaxValue;
} //maskされてないMax→もう読めない
}
ReadA(); ReadB();
do
{
if (maskedA < maskedB)
{
writer.Write(lastA);
ReadA();
restA--;
}
else
{
writer.Write(lastB);
ReadB();
restB--;
}
} while (restA > 0 || restB > 0);
}
//終わったら古いやつは消す
File.Delete(InputPathA);
File.Delete(InputPathB);
}
///<summary>ブロックソートで一致する範囲だけ読み出す
///長さ2以上になるやつだけ返す</summary>
public long[] ReadBlock()
{
long TempHash;
do
{
TempList.Clear();
if (HasExtraHash)
{
TempHash = ExtraReadHash; HasExtraHash = false;
}
else if (reader.Readable())
{
TempHash = reader.Read();
}
else
{
return(null);
}
TempList.Add(TempHash);
//MaskしたやつがMaskedKeyと一致しなかったら終了
long MaskedKey = TempHash & FullMask;
while (reader.Readable())
{
TempHash = reader.Read();
if ((TempHash & FullMask) == MaskedKey)
{
TempList.Add(TempHash);
}
//1個余計に読んだので記録しておく
else
{
ExtraReadHash = TempHash; HasExtraHash = true; break;
}
}
} while (TempList.Count < 2);
return(TempList.ToArray());
}
///<summary>全ハッシュをファイルに書き出しながらソートしていくやつ</summary>
public static async ValueTask <string> MergeSortAll(long SortMask)
{
long FileCount = 0;
//最初はAllHashから読み出しながら個別のファイルを作る
using (BufferedLongReader reader = new BufferedLongReader(AllHashFilePath))
{
BlockSortComparer SortComp = new BlockSortComparer(SortMask);
var FirstSortBlock = new ActionBlock <(string FilePath, long[] ToSort)>((t) =>
{
Array.Sort(t.ToSort, SortComp);
using (BufferedLongWriter w = new BufferedLongWriter(t.FilePath))
{
foreach (long h in t.ToSort)
{
w.Write(h);
}
}
}, new ExecutionDataflowBlockOptions()
{
SingleProducerConstrained = true,
MaxDegreeOfParallelism = config.hash.FileSortThreads,
BoundedCapacity = config.hash.FileSortThreads << 1
});
int InitialSortUnit = config.hash.InitialSortFileSize / sizeof(long);
for (; reader.Length - reader.Position >= config.hash.InitialSortFileSize; FileCount++)
{
long[] ToSort = new long[InitialSortUnit];
for (int i = 0; i < InitialSortUnit; i++)
{
ToSort[i] = reader.Read();
}
await FirstSortBlock.SendAsync((SortingFilePath(0, FileCount), ToSort)).ConfigureAwait(false);
}
int SortLastCount = (int)(reader.Length - reader.Position) / sizeof(long);
if (SortLastCount > 0)
{
long[] ToSortLast = new long[SortLastCount];
for (int i = 0; i < SortLastCount; i++)
{
ToSortLast[i] = reader.Read();
}
await FirstSortBlock.SendAsync((SortingFilePath(0, FileCount), ToSortLast)).ConfigureAwait(false);
FileCount++; //最後に作ったから足す
}
FirstSortBlock.Complete(); FirstSortBlock.Completion.Wait();
}
GC.Collect();
Random rand = new Random();
int step = 0;
//ファイル単位でマージソートしていく
for (; FileCount > 1; step++)
{
ActionBlock <int> MergeSortBlock = new ActionBlock <int>((i) =>
{
MargeSortUnit(SortMask,
SortingFilePath(step, i << 1),
SortingFilePath(step, (i << 1) | 1),
SortingFilePath(step + 1, i));
//余りファイルは最初のファイルにくっつけてしまう
if (i == 0 && (FileCount & 1) != 0)
{
string NextFirstPath = SortingFilePath(step + 1, 0);
File.Delete(NextFirstPath + "_");
File.Move(NextFirstPath, NextFirstPath + "_");
MargeSortUnit(SortMask, SortingFilePath(step, FileCount - 1),
NextFirstPath + "_", NextFirstPath);
}
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = config.hash.FileSortThreads
});
for (int i = 0; i < FileCount >> 1; i++)
{
MergeSortBlock.Post(i);
}
MergeSortBlock.Complete();
await MergeSortBlock.Completion.ConfigureAwait(false);
FileCount >>= 1; //ソート後のファイル数はこうなる
}
return(SortingFilePath(step, 0));
}