/// <summary>
///dcthashpairに追加する必要があるハッシュを取得するやつ
///これが始まった後に追加されたハッシュは無視されるが
///次回の実行で拾われるから問題ない
/// </summary>
///<param name="SaveTime">保存するファイル名に付けるUNIX時刻</param>
///<param name="BeginTime">downloaded_atがこれ以降のハッシュを取得する</param>
public async Task <HashSet <long> > NewerMediaHash(long SaveTime, long BeginTime)
{
string FilePath = HashFile.NewerHashFilePathBase(SaveTime.ToString());
try
{
var ret = new HashSet <long>();
const int QueryRangeSeconds = 600;
var LoadHashBlock = new ActionBlock <long>(async(i) =>
{
var Table = new List <long>();
while (true)
{
using (MySqlCommand cmd = new MySqlCommand(@"SELECT dcthash
FROM media_downloaded_at
NATURAL JOIN media
WHERE downloaded_at BETWEEN @begin AND @end;"))
{
cmd.Parameters.Add("@begin", MySqlDbType.Int64).Value = BeginTime + QueryRangeSeconds * i;
cmd.Parameters.Add("@end", MySqlDbType.Int64).Value = BeginTime + QueryRangeSeconds * (i + 1) - 1;
if (await ExecuteReader(cmd, (r) => Table.Add(r.GetInt64(0)), IsolationLevel.ReadUncommitted).ConfigureAwait(false))
{
break;
}
else
{
Table.Clear();
}
}
}
lock (ret) { foreach (long h in Table)
{
ret.Add(h);
}
}
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
for (long i = 0; i < Math.Max(0, DateTimeOffset.UtcNow.ToUnixTimeSeconds() - BeginTime) / QueryRangeSeconds + 1; i++)
{
LoadHashBlock.Post(i);
}
LoadHashBlock.Complete();
await LoadHashBlock.Completion.ConfigureAwait(false);
using (var writer = new UnbufferedLongWriter(HashFile.TempFilePath(FilePath)))
{
writer.WriteDestructive(ret.ToArray(), ret.Count);
}
File.Move(HashFile.TempFilePath(FilePath), FilePath);
return(ret);
}catch (Exception e) { Console.WriteLine(e); return(null); }
}
///<summary>全ハッシュを一定サイズに分割してソートする
///分割されたファイルをマージソートすると完全なソート済み列が得られる</summary>
public static async Task <int> QuickSortAll(int Index, long SortMask)
{
int FileCount = 0;
var SortComp = new BlockSortComparer(SortMask);
//これにソート用の配列を入れてメモリ割り当てを減らしてみる
var LongPool = new ConcurrentBag <long[]>();
bool LongPoolReturn = true;
//ソート用の配列の個数に制限を設ける
var FirstSortSemaphore = new SemaphoreSlim(config.hash.InitialSortConcurrency);
//ソートは並列 書き込みは並列させない
var FirstSortBlock = new TransformBlock <FirstSort, FirstSort>(async(t) =>
{
await QuickSortParllel(SortMask, t.ToSort, t.Length, SortComp).ConfigureAwait(false);
return(t);
}, new ExecutionDataflowBlockOptions()
{
SingleProducerConstrained = true,
MaxDegreeOfParallelism = config.hash.InitialSortConcurrency,
});
var WriterBlock = new ActionBlock <FirstSort>((t) =>
{
using (var writer = new UnbufferedLongWriter(t.WriteFilePath))
{
writer.WriteDestructive(t.ToSort, t.Length);
}
//ここでLongPoolに配列を返却する
if (LongPoolReturn)
{
LongPool.Add(t.ToSort);
}
FirstSortSemaphore.Release();
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
FirstSortBlock.LinkTo(WriterBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
//まずはAllHashを読む
using (var reader = new UnbufferedLongReader(AllHashFilePath))
{
for (; reader.Readable; FileCount++)
{
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out var ToSort))
{
ToSort = new long[InitialSortUnit];
}
int ToSortLength = reader.Read(ToSort);
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSort, ToSortLength));
}
}
//NewerHashを読む
int ToSortNewerCursor = 0;
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out var ToSortNewer))
{
ToSortNewer = new long[InitialSortUnit];
}
foreach (var filePath in Directory.EnumerateFiles(config.hash.TempDir, Path.GetFileName(NewerHashFilePathBase("*"))))
{
using (var reader = new BufferedLongReader(filePath))
{
while (reader.Readable)
{
for (; ToSortNewerCursor < ToSortNewer.Length; ToSortNewerCursor++)
{
if (!reader.MoveNext(out var next))
{
break;
}
ToSortNewer[ToSortNewerCursor] = next;
}
if (InitialSortUnit <= ToSortNewerCursor)
{
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewer.Length));
FileCount++;
ToSortNewerCursor = 0;
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out ToSortNewer))
{
ToSortNewer = new long[InitialSortUnit];
}
}
}
}
}
//余った要素もソートさせる FirstSortingCountはもう使わないので放置
if (0 < ToSortNewerCursor)
{
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewerCursor));
FileCount++;
}
FirstSortBlock.Complete();
//ソート用配列は作り終わったので用が済んだ配列は解放させる
LongPoolReturn = false;
LongPool.Clear();
await WriterBlock.Completion.ConfigureAwait(false);
return(FileCount);
}
