public async Task <byte[]?> GetImageCached(string path, bool forced = false)
{
if (string.IsNullOrEmpty(path))
{
return(null);
}
using (var key = await KeyedSemaphore.LockAsync(path))
{
byte[]? image;
if (imagesCache.TryGet(path, out image) && !forced)
{
return(image);
}
else
{
image = await ArchivesProvider.GetImage(path);
if (image == null)
{
return(null);
}
imagesCache.AddReplace(path, image);
return(image);
}
}
}
public string get_archives_html()
{
StringBuilder sb = new StringBuilder();
int c = 0;
foreach (var a in ArchivesProvider.GetAllArchives())
{
sb.AppendFormat("<option value='{0}'>{1}</option>", c, System.Web.HttpUtility.HtmlEncode(a.Name));
c++;
}
return(sb.ToString());
}
private async Task <byte[]?> GetThumbnailRaw(string id, int page = 0)
{
var data = await ArchivesProvider.GetThumbnail(id, true, page);
if (data == null)
{
return(null);
}
if (data.Thumbnail != null)
{
return(data.Thumbnail);
}
if (data.Job != null && await data.Job.WaitForMinionJob())
{
return(await GetThumbnailRaw(id, page));
}
return(null);
}
public Task <bool> Validate()
{
#if false
#if WINDOWS_UWP
await Crashes.SetEnabledAsync(false); // Disable to prevent false-positive errors
#endif
var archives = await ArchivesProvider.Validate();
var categories = await CategoriesProvider.Validate();
var database = await DatabaseProvider.Validate();
if (archives && categories && database)
{
#if WINDOWS_UWP
await Crashes.SetEnabledAsync(Settings.CrashReporting);
#endif
Settings.Profile.AcceptedDisclaimer = false;
Settings.SaveProfiles();
return(true);
}
if (Settings.Profile.AcceptedDisclaimer)
{
#if WINDOWS_UWP
await Crashes.SetEnabledAsync(false);
#endif
return(true);
}
var result = (await Platform.OpenDialog(Dialog.ValidateApi, archives, categories, database)) == IDialogResult.Primary;
Settings.Profile.AcceptedDisclaimer = result;
if (result)
{
#if WINDOWS_UWP
await Crashes.SetEnabledAsync(false);
#endif
}
Settings.SaveProfiles();
return(result);
#else
return(Task.Run(() => true));
#endif
}
private string get_ai_items()
{
StringBuilder sb = new StringBuilder();
/*
*
* {
* supported_boards: [""],
* supported_files: [""],
* name: "",
* ishttp: true,
* ishttps: false
* }
*/
int a_c = ArchivesProvider.GetAllArchives().Count();
int w_ac = 0;
foreach (ArchiveInfo w in ArchivesProvider.GetAllArchives())
{
sb.Append("{");
sb.Append("supported_boards:[");
int c = w.GetSupportedBoards().Count();
int c_w = 0;
foreach (string b in w.GetSupportedBoards())
{
sb.Append("\"");
sb.Append(b);
sb.Append("\"");
if (c_w < c - 1)
{
sb.Append(",");
}
c_w++;
}
sb.Append("],");
sb.Append("supported_files:[");
c = w.GetSupportedFiles().Count();
c_w = 0;
foreach (string b in w.GetSupportedFiles())
{
sb.Append("\"");
sb.Append(b);
sb.Append("\"");
if (c_w < c - 1)
{
sb.Append(",");
}
c_w++;
}
sb.Append("],");
sb.Append("name: \"");
sb.Append(w.Name.Replace("\"", @"\"""));
sb.Append("\",");
sb.Append("ishttp: ");
sb.Append(w.SupportHttp ? "true" : "false");
sb.Append(",");
sb.Append("ishttps: ");
sb.Append(w.SupportHttps ? "true" : "false");
sb.Append("}");
if (w_ac < a_c - 1)
{
sb.Append(",");
}
}
return(sb.ToString());
}
protected override async Task <ToolResult <List <ArchiveHit>, List <string> > > Process(DeduplicatorParams @params, int threads)
{
var factory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(threads));
int pixelThreshold = @params.PixelThreshold;
float percentDifference = @params.PercentDifference;
int width = @params.Width;
float aspectRatioLimit = @params.AspectRatioLimit;
int delay = @params.Delay;
bool skipMissing = @params.SkipMissing;
var archives = Archives.Archives;
var thumbnailJob = await ArchivesProvider.RegenerateThumbnails();
if (thumbnailJob is null)
{
return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Title"), Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Message")));
}
UpdateProgress(DeduplicatorStatus.GenerateThumbnails, archives.Count, -2, 3, 0);
while (true)
{
await Task.Delay(1000);
if ((await ServerProvider.GetMinionStatus(thumbnailJob.job))?.state?.Equals("finished") ?? true)
{
break;
}
}
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, 0, 3, 1);
await Task.Delay(1000);
int count = 0;
var tmp = (await Task.WhenAll(archives.Select(pair => factory.StartNew(() =>
{
int tries = 5;
Image <Rgba32>?image = null;
while (tries > 0)
{
Thread.Sleep(delay * (6 - tries)); // TODO Good ol' Thread.Sleep
var bytes = Task.Run(async() => await Images.GetThumbnailCached(pair.Key)).GetAwaiter().GetResult();
if (bytes != null)
{
image = Image.Load(bytes);
image.Mutate(i => i.Resize(width, 0));
break;
}
else
{
tries--;
}
}
int itemCount = Interlocked.Increment(ref count);
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, itemCount);
return(new Tuple <string, Image <Rgba32>?>(pair.Key, image));
})))).AsEnumerable().ToList();
List <string> removed = new List <string>();
tmp.RemoveAll(pair =>
{
if (pair.Item2 == null)
{
removed.Add(pair.Item1);
}
return(pair.Item2 == null);
});
if (removed.Count > 0 && !skipMissing)
{
return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Title"), Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Message"), removed));
}
var decodedThumbnails = tmp.ToDictionary(pair => pair.Item1, pair => pair.Item2);
tmp.Clear();
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, count);
await Task.Delay(1000);
count = 0;
UpdateProgress(DeduplicatorStatus.Comparing, decodedThumbnails.Count, 0, 3, 2);
await Task.Delay(1000);
var markedNonDuplicated = Settings.Profile.MarkedAsNonDuplicated;
var hits = new ConcurrentBag <ArchiveHit>();
int maxItems = decodedThumbnails.Count;
await Task.Run(async() =>
{
while (decodedThumbnails.Count != 0)
{
var start = DateTime.Now;
var sourcePair = decodedThumbnails.First();
decodedThumbnails.Remove(sourcePair.Key);
using (var source = sourcePair.Value)
{
await Task.WhenAll(decodedThumbnails.Select(targetPair => factory.StartNew(() =>
{
var target = targetPair.Value;
if (Math.Abs((float)source !.Height / source.Width - (float)target !.Height / target.Width) > aspectRatioLimit)
{
return;
}
var hit = new ArchiveHit {
Left = sourcePair.Key, Right = targetPair.Key
};
if (markedNonDuplicated.Contains(hit))
{
return;
}
int differences = 0;
for (int y = 0; y < Math.Min(source.Height, target.Height); y++)
{
Span <Rgba32> sourcePixelRow = source.GetPixelRowSpan(y);
Span <Rgba32> targetPixelRow = target.GetPixelRowSpan(y);
for (int x = 0; x < source.Width; x++)
{
float diff = GetManhattanDistanceInRgbSpace(ref sourcePixelRow[x], ref targetPixelRow[x]) / 765f; //255+255+255
if (diff > pixelThreshold / 765f)
{
differences++;
}
}
}
float diffPixels = differences;
diffPixels /= source.Width *source.Height;
if (diffPixels < percentDifference)
{
hits.Add(hit);
}
})));
}
int itemCount = Interlocked.Increment(ref count);
var delta = DateTime.Now.Subtract(start).Ticks;
long time = (maxItems - itemCount) * delta;
UpdateProgress(DeduplicatorStatus.Comparing, maxItems, itemCount, time: time);
}
});
UpdateProgress(DeduplicatorStatus.Comparing, maxItems, count, time: 0);
await Task.Delay(1000);
UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 3, 3);
await Task.Delay(1000);
UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 0, 0);
return(new ToolResult <List <ArchiveHit>, List <string> > {
Data = hits.ToList(), Ok = true, Error = removed.Count > 0 ? removed : null
});
}
