Пример #1
0
        /// <summary>
        ///dcthashpairに追加する必要があるハッシュを取得するやつ
        ///これが始まった後に追加されたハッシュは無視されるが
        ///次回の実行で拾われるから問題ない
        /// </summary>
        ///<param name="SaveTime">保存するファイル名に付けるUNIX時刻</param>
        ///<param name="BeginTime">downloaded_atがこれ以降のハッシュを取得する</param>
        public async Task <HashSet <long> > NewerMediaHash(long SaveTime, long BeginTime)
        {
            string FilePath = HashFile.NewerHashFilePathBase(SaveTime.ToString());

            try
            {
                var       ret = new HashSet <long>();
                const int QueryRangeSeconds = 600;
                var       LoadHashBlock     = new ActionBlock <long>(async(i) =>
                {
                    var Table = new List <long>();
                    while (true)
                    {
                        using (MySqlCommand cmd = new MySqlCommand(@"SELECT dcthash
FROM media_downloaded_at
NATURAL JOIN media
WHERE downloaded_at BETWEEN @begin AND @end;"))
                        {
                            cmd.Parameters.Add("@begin", MySqlDbType.Int64).Value = BeginTime + QueryRangeSeconds * i;
                            cmd.Parameters.Add("@end", MySqlDbType.Int64).Value   = BeginTime + QueryRangeSeconds * (i + 1) - 1;
                            if (await ExecuteReader(cmd, (r) => Table.Add(r.GetInt64(0)), IsolationLevel.ReadUncommitted).ConfigureAwait(false))
                            {
                                break;
                            }
                            else
                            {
                                Table.Clear();
                            }
                        }
                    }
                    lock (ret) { foreach (long h in Table)
                                 {
                                     ret.Add(h);
                                 }
                    }
                }, new ExecutionDataflowBlockOptions()
                {
                    MaxDegreeOfParallelism = Environment.ProcessorCount
                });
                for (long i = 0; i < Math.Max(0, DateTimeOffset.UtcNow.ToUnixTimeSeconds() - BeginTime) / QueryRangeSeconds + 1; i++)
                {
                    LoadHashBlock.Post(i);
                }
                LoadHashBlock.Complete();
                await LoadHashBlock.Completion.ConfigureAwait(false);

                using (var writer = new UnbufferedLongWriter(HashFile.TempFilePath(FilePath)))
                {
                    writer.WriteDestructive(ret.ToArray(), ret.Count);
                }
                File.Move(HashFile.TempFilePath(FilePath), FilePath);
                return(ret);
            }catch (Exception e) { Console.WriteLine(e); return(null); }
        }
Пример #2
0
        ///<summary>全ハッシュを一定サイズに分割してソートする
        ///分割されたファイルをマージソートすると完全なソート済み列が得られる</summary>
        public static async Task <int> QuickSortAll(int Index, long SortMask)
        {
            int FileCount = 0;
            var SortComp  = new BlockSortComparer(SortMask);

            //これにソート用の配列を入れてメモリ割り当てを減らしてみる
            var  LongPool       = new ConcurrentBag <long[]>();
            bool LongPoolReturn = true;
            //ソート用の配列の個数に制限を設ける
            var FirstSortSemaphore = new SemaphoreSlim(config.hash.InitialSortConcurrency);

            //ソートは並列 書き込みは並列させない
            var FirstSortBlock = new TransformBlock <FirstSort, FirstSort>(async(t) =>
            {
                await QuickSortParllel(SortMask, t.ToSort, t.Length, SortComp).ConfigureAwait(false);
                return(t);
            }, new ExecutionDataflowBlockOptions()
            {
                SingleProducerConstrained = true,
                MaxDegreeOfParallelism    = config.hash.InitialSortConcurrency,
            });
            var WriterBlock = new ActionBlock <FirstSort>((t) =>
            {
                using (var writer = new UnbufferedLongWriter(t.WriteFilePath))
                {
                    writer.WriteDestructive(t.ToSort, t.Length);
                }
                //ここでLongPoolに配列を返却する
                if (LongPoolReturn)
                {
                    LongPool.Add(t.ToSort);
                }
                FirstSortSemaphore.Release();
            }, new ExecutionDataflowBlockOptions()
            {
                MaxDegreeOfParallelism = 1
            });

            FirstSortBlock.LinkTo(WriterBlock, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });

            //まずはAllHashを読む
            using (var reader = new UnbufferedLongReader(AllHashFilePath))
            {
                for (; reader.Readable; FileCount++)
                {
                    await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);

                    if (!LongPool.TryTake(out var ToSort))
                    {
                        ToSort = new long[InitialSortUnit];
                    }
                    int ToSortLength = reader.Read(ToSort);
                    FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSort, ToSortLength));
                }
            }

            //NewerHashを読む
            int ToSortNewerCursor = 0;
            await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);

            if (!LongPool.TryTake(out var ToSortNewer))
            {
                ToSortNewer = new long[InitialSortUnit];
            }
            foreach (var filePath in Directory.EnumerateFiles(config.hash.TempDir, Path.GetFileName(NewerHashFilePathBase("*"))))
            {
                using (var reader = new BufferedLongReader(filePath))
                {
                    while (reader.Readable)
                    {
                        for (; ToSortNewerCursor < ToSortNewer.Length; ToSortNewerCursor++)
                        {
                            if (!reader.MoveNext(out var next))
                            {
                                break;
                            }
                            ToSortNewer[ToSortNewerCursor] = next;
                        }
                        if (InitialSortUnit <= ToSortNewerCursor)
                        {
                            FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewer.Length));
                            FileCount++;
                            ToSortNewerCursor = 0;
                            await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);

                            if (!LongPool.TryTake(out ToSortNewer))
                            {
                                ToSortNewer = new long[InitialSortUnit];
                            }
                        }
                    }
                }
            }
            //余った要素もソートさせる FirstSortingCountはもう使わないので放置
            if (0 < ToSortNewerCursor)
            {
                FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewerCursor));
                FileCount++;
            }
            FirstSortBlock.Complete();
            //ソート用配列は作り終わったので用が済んだ配列は解放させる
            LongPoolReturn = false;
            LongPool.Clear();

            await WriterBlock.Completion.ConfigureAwait(false);

            return(FileCount);
        }