public static async Task Marathon() { int mismatch = 0; int failure = 0; long[] mismatchBits = new long[sizeof(long) * 8]; var config = Config.Instance; var db = new DBHandler(); using var client = new TcpClient("localhost", 12306); client.NoDelay = true; using var tcp = client.GetStream(); using var reader = new MessagePackStreamReader(tcp); long downloaded_at = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); int mediaCount = 0; while (true) { var result = await db.GetMediaPath(downloaded_at).ConfigureAwait(false); downloaded_at = result.MinDownloadedAt - 1; var compareHashBlock = new ActionBlock <string>(async(p) => { byte[] mediabytes; try { using (var file = File.OpenRead(Path.Combine(config.crawl.PictPaththumb, p))) using (var mem = new MemoryStream()) { await file.CopyToAsync(mem).ConfigureAwait(false); mediabytes = mem.ToArray(); } } catch (Exception e) { Console.WriteLine(e.Message); return; } var a = PictHashClient.DCTHash(mediabytes, 0, "192.168.238.126"); var b = PictHashClient.DCTHash(mediabytes, 0, "localhost"); await Task.WhenAll(a, b).ConfigureAwait(false); if (!a.Result.HasValue || !b.Result.HasValue) { Interlocked.Increment(ref failure); Console.WriteLine("\t\t\tfailure"); } else if (a.Result.Value != b.Result.Value) { Interlocked.Increment(ref mismatch); ulong bits = (ulong)(a.Result.Value ^ b.Result.Value); Interlocked.Increment(ref mismatchBits[Popcnt.X64.PopCount(bits)]); //Console.WriteLine("{0:X16}", bits); } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }); foreach (string p in result.MediaPath) { compareHashBlock.Post(p); } mediaCount += result.MediaPath.Length; compareHashBlock.Complete(); await compareHashBlock.Completion; for (int i = 0; i < mismatchBits.Length; i++) { if (0 < mismatchBits[i]) { Console.WriteLine("{0}: {1}", i, mismatchBits[i]); } } Console.WriteLine("{0} / {1} mismatches.", mismatch, mediaCount); } }
public static async Task Proceed() { int mismatch = 0; int failure = 0; long[] mismatchBits = new long[sizeof(long) * 8]; var config = Config.Instance; var db = new DBHandler(); int mediaCount = 0; foreach (string p in (await db.GetMediaPath(DateTimeOffset.UtcNow.ToUnixTimeSeconds()).ConfigureAwait(false)).MediaPath) { byte[] mediabytes; try { using (var file = File.OpenRead(Path.Combine(config.crawl.PictPaththumb, p))) using (var mem = new MemoryStream()) { await file.CopyToAsync(mem).ConfigureAwait(false); mediabytes = mem.ToArray(); } } catch (Exception e) { Console.WriteLine(e.Message); continue; } mediaCount++; var a = PictHashClient.DCTHash(mediabytes, 0, "192.168.238.126"); var b = PictHashClient.DCTHash(mediabytes, 0, "localhost"); await Task.WhenAll(a, b).ConfigureAwait(false); if (!a.Result.HasValue || !b.Result.HasValue) { failure++; Console.Write(mediaCount); if (!a.Result.HasValue) { Console.Write(" a "); } if (!b.Result.HasValue) { Console.Write(" b "); } Console.WriteLine("\tfailure"); } else if (a.Result.Value != b.Result.Value) { mismatch++; ulong bits = (ulong)(a.Result.Value ^ b.Result.Value); mismatchBits[Popcnt.X64.PopCount(bits)]++; Console.WriteLine("{0:X16}", bits); } } for (int i = 0; i < mismatchBits.Length; i++) { if (0 < mismatchBits[i]) { Console.WriteLine("{0}: {1}", i, mismatchBits[i]); } } Console.WriteLine("{0} / {1} mismatches.", mismatch, mediaCount); }