public async Task <byte[]?> GetImageCached(string path, bool forced = false) { if (string.IsNullOrEmpty(path)) { return(null); } using (var key = await KeyedSemaphore.LockAsync(path)) { byte[]? image; if (imagesCache.TryGet(path, out image) && !forced) { return(image); } else { image = await ArchivesProvider.GetImage(path); if (image == null) { return(null); } imagesCache.AddReplace(path, image); return(image); } } }
public string get_archives_html() { StringBuilder sb = new StringBuilder(); int c = 0; foreach (var a in ArchivesProvider.GetAllArchives()) { sb.AppendFormat("<option value='{0}'>{1}</option>", c, System.Web.HttpUtility.HtmlEncode(a.Name)); c++; } return(sb.ToString()); }
private async Task <byte[]?> GetThumbnailRaw(string id, int page = 0) { var data = await ArchivesProvider.GetThumbnail(id, true, page); if (data == null) { return(null); } if (data.Thumbnail != null) { return(data.Thumbnail); } if (data.Job != null && await data.Job.WaitForMinionJob()) { return(await GetThumbnailRaw(id, page)); } return(null); }
public Task <bool> Validate() { #if false #if WINDOWS_UWP await Crashes.SetEnabledAsync(false); // Disable to prevent false-positive errors #endif var archives = await ArchivesProvider.Validate(); var categories = await CategoriesProvider.Validate(); var database = await DatabaseProvider.Validate(); if (archives && categories && database) { #if WINDOWS_UWP await Crashes.SetEnabledAsync(Settings.CrashReporting); #endif Settings.Profile.AcceptedDisclaimer = false; Settings.SaveProfiles(); return(true); } if (Settings.Profile.AcceptedDisclaimer) { #if WINDOWS_UWP await Crashes.SetEnabledAsync(false); #endif return(true); } var result = (await Platform.OpenDialog(Dialog.ValidateApi, archives, categories, database)) == IDialogResult.Primary; Settings.Profile.AcceptedDisclaimer = result; if (result) { #if WINDOWS_UWP await Crashes.SetEnabledAsync(false); #endif } Settings.SaveProfiles(); return(result); #else return(Task.Run(() => true)); #endif }
private string get_ai_items() { StringBuilder sb = new StringBuilder(); /* * * { * supported_boards: [""], * supported_files: [""], * name: "", * ishttp: true, * ishttps: false * } */ int a_c = ArchivesProvider.GetAllArchives().Count(); int w_ac = 0; foreach (ArchiveInfo w in ArchivesProvider.GetAllArchives()) { sb.Append("{"); sb.Append("supported_boards:["); int c = w.GetSupportedBoards().Count(); int c_w = 0; foreach (string b in w.GetSupportedBoards()) { sb.Append("\""); sb.Append(b); sb.Append("\""); if (c_w < c - 1) { sb.Append(","); } c_w++; } sb.Append("],"); sb.Append("supported_files:["); c = w.GetSupportedFiles().Count(); c_w = 0; foreach (string b in w.GetSupportedFiles()) { sb.Append("\""); sb.Append(b); sb.Append("\""); if (c_w < c - 1) { sb.Append(","); } c_w++; } sb.Append("],"); sb.Append("name: \""); sb.Append(w.Name.Replace("\"", @"\""")); sb.Append("\","); sb.Append("ishttp: "); sb.Append(w.SupportHttp ? "true" : "false"); sb.Append(","); sb.Append("ishttps: "); sb.Append(w.SupportHttps ? "true" : "false"); sb.Append("}"); if (w_ac < a_c - 1) { sb.Append(","); } } return(sb.ToString()); }
protected override async Task <ToolResult <List <ArchiveHit>, List <string> > > Process(DeduplicatorParams @params, int threads) { var factory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(threads)); int pixelThreshold = @params.PixelThreshold; float percentDifference = @params.PercentDifference; int width = @params.Width; float aspectRatioLimit = @params.AspectRatioLimit; int delay = @params.Delay; bool skipMissing = @params.SkipMissing; var archives = Archives.Archives; var thumbnailJob = await ArchivesProvider.RegenerateThumbnails(); if (thumbnailJob is null) { return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Title"), Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Message"))); } UpdateProgress(DeduplicatorStatus.GenerateThumbnails, archives.Count, -2, 3, 0); while (true) { await Task.Delay(1000); if ((await ServerProvider.GetMinionStatus(thumbnailJob.job))?.state?.Equals("finished") ?? true) { break; } } UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, 0, 3, 1); await Task.Delay(1000); int count = 0; var tmp = (await Task.WhenAll(archives.Select(pair => factory.StartNew(() => { int tries = 5; Image <Rgba32>?image = null; while (tries > 0) { Thread.Sleep(delay * (6 - tries)); // TODO Good ol' Thread.Sleep var bytes = Task.Run(async() => await Images.GetThumbnailCached(pair.Key)).GetAwaiter().GetResult(); if (bytes != null) { image = Image.Load(bytes); image.Mutate(i => i.Resize(width, 0)); break; } else { tries--; } } int itemCount = Interlocked.Increment(ref count); UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, itemCount); return(new Tuple <string, Image <Rgba32>?>(pair.Key, image)); })))).AsEnumerable().ToList(); List <string> removed = new List <string>(); tmp.RemoveAll(pair => { if (pair.Item2 == null) { removed.Add(pair.Item1); } return(pair.Item2 == null); }); if (removed.Count > 0 && !skipMissing) { return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Title"), Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Message"), removed)); } var decodedThumbnails = tmp.ToDictionary(pair => pair.Item1, pair => pair.Item2); tmp.Clear(); UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, count); await Task.Delay(1000); count = 0; UpdateProgress(DeduplicatorStatus.Comparing, decodedThumbnails.Count, 0, 3, 2); await Task.Delay(1000); var markedNonDuplicated = Settings.Profile.MarkedAsNonDuplicated; var hits = new ConcurrentBag <ArchiveHit>(); int maxItems = decodedThumbnails.Count; await Task.Run(async() => { while (decodedThumbnails.Count != 0) { var start = DateTime.Now; var sourcePair = decodedThumbnails.First(); decodedThumbnails.Remove(sourcePair.Key); using (var source = sourcePair.Value) { await Task.WhenAll(decodedThumbnails.Select(targetPair => factory.StartNew(() => { var target = targetPair.Value; if (Math.Abs((float)source !.Height / source.Width - (float)target !.Height / target.Width) > aspectRatioLimit) { return; } var hit = new ArchiveHit { Left = sourcePair.Key, Right = targetPair.Key }; if (markedNonDuplicated.Contains(hit)) { return; } int differences = 0; for (int y = 0; y < Math.Min(source.Height, target.Height); y++) { Span <Rgba32> sourcePixelRow = source.GetPixelRowSpan(y); Span <Rgba32> targetPixelRow = target.GetPixelRowSpan(y); for (int x = 0; x < source.Width; x++) { float diff = GetManhattanDistanceInRgbSpace(ref sourcePixelRow[x], ref targetPixelRow[x]) / 765f; //255+255+255 if (diff > pixelThreshold / 765f) { differences++; } } } float diffPixels = differences; diffPixels /= source.Width *source.Height; if (diffPixels < percentDifference) { hits.Add(hit); } }))); } int itemCount = Interlocked.Increment(ref count); var delta = DateTime.Now.Subtract(start).Ticks; long time = (maxItems - itemCount) * delta; UpdateProgress(DeduplicatorStatus.Comparing, maxItems, itemCount, time: time); } }); UpdateProgress(DeduplicatorStatus.Comparing, maxItems, count, time: 0); await Task.Delay(1000); UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 3, 3); await Task.Delay(1000); UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 0, 0); return(new ToolResult <List <ArchiveHit>, List <string> > { Data = hits.ToList(), Ok = true, Error = removed.Count > 0 ? removed : null }); }