/// <summary>
///dcthashpairに追加する必要があるハッシュを取得するやつ
///これが始まった後に追加されたハッシュは無視されるが
///次回の実行で拾われるから問題ない
/// </summary>
///<param name="SaveTime">保存するファイル名に付けるUNIX時刻</param>
///<param name="BeginTime">downloaded_atがこれ以降のハッシュを取得する</param>
public async Task <HashSet <long> > NewerMediaHash(long SaveTime, long BeginTime)
{
string FilePath = HashFile.NewerHashFilePathBase(SaveTime.ToString());
try
{
var ret = new HashSet <long>();
const int QueryRangeSeconds = 600;
var LoadHashBlock = new ActionBlock <long>(async(i) =>
{
var Table = new List <long>();
while (true)
{
using (MySqlCommand cmd = new MySqlCommand(@"SELECT dcthash
FROM media_downloaded_at
NATURAL JOIN media
WHERE downloaded_at BETWEEN @begin AND @end;"))
{
cmd.Parameters.Add("@begin", MySqlDbType.Int64).Value = BeginTime + QueryRangeSeconds * i;
cmd.Parameters.Add("@end", MySqlDbType.Int64).Value = BeginTime + QueryRangeSeconds * (i + 1) - 1;
if (await ExecuteReader(cmd, (r) => Table.Add(r.GetInt64(0)), IsolationLevel.ReadUncommitted).ConfigureAwait(false))
{
break;
}
else
{
Table.Clear();
}
}
}
lock (ret) { foreach (long h in Table)
{
ret.Add(h);
}
}
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
for (long i = 0; i < Math.Max(0, DateTimeOffset.UtcNow.ToUnixTimeSeconds() - BeginTime) / QueryRangeSeconds + 1; i++)
{
LoadHashBlock.Post(i);
}
LoadHashBlock.Complete();
await LoadHashBlock.Completion.ConfigureAwait(false);
using (var writer = new UnbufferedLongWriter(HashFile.TempFilePath(FilePath)))
{
writer.WriteDestructive(ret.ToArray(), ret.Count);
}
File.Move(HashFile.TempFilePath(FilePath), FilePath);
return(ret);
}catch (Exception e) { Console.WriteLine(e); return(null); }
}
///<summary>DBから読み込んだハッシュをそのままファイルに書き出す</summary>
///<param name="SaveTime">保存するファイル名に付けるUNIX時刻</param>
public async Task <long> AllMediaHash(long SaveTime)
{
try
{
long TotalHashCount = 0;
string HashFilePath = HashFile.AllHashFilePathBase(SaveTime.ToString());
using (var writer = new BufferedLongWriter(HashFile.TempFilePath(HashFilePath)))
{
var LoadHashBlock = new TransformBlock <long, AddOnlyList <long> >(async(i) =>
{
var table = new AddOnlyList <long>(TableListSize);
while (true)
{
using var cmd = new MySqlCommand(@"SELECT DISTINCT dcthash
FROM media
WHERE dcthash BETWEEN @begin AND @end
GROUP BY dcthash;");
cmd.Parameters.Add("@begin", MySqlDbType.Int64).Value = i << HashUnitBits;
cmd.Parameters.Add("@end", MySqlDbType.Int64).Value = ((i + 1) << HashUnitBits) - 1;
if (await ExecuteReader(cmd, (r) => table.Add(r.GetInt64(0)), IsolationLevel.ReadUncommitted).ConfigureAwait(false))
{
break;
}
else
{
table.Clear();
}
}
return(table);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
BoundedCapacity = Environment.ProcessorCount << 1,
SingleProducerConstrained = true
});
var WriterBlock = new ActionBlock <AddOnlyList <long> >(async(table) =>
{
await writer.Write(table.InnerArray, table.Count).ConfigureAwait(false);
TotalHashCount += table.Count;
table.Dispose();
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
LoadHashBlock.LinkTo(WriterBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
for (int i = 0; i < 1 << (64 - HashUnitBits); i++)
{
await LoadHashBlock.SendAsync(i).ConfigureAwait(false);
}
LoadHashBlock.Complete();
await WriterBlock.Completion.ConfigureAwait(false);
}
File.Move(HashFile.TempFilePath(HashFilePath), HashFilePath);
return(TotalHashCount);
}
catch (Exception e) { Console.WriteLine(e); return(-1); }
}