private static int ScanTargetDir(ImageScraper scraper, string destDir) { Directory.CreateDirectory(destDir); int count = 0; foreach (ImageEntry entry in scraper.Scan(SearchOption.TopDirectoryOnly)) { string ext = string.Empty; switch (entry.Kind) { case ImageKind.None: continue; case ImageKind.Jpg: ext = ".jpg"; break; case ImageKind.Png: ext = ".png"; break; } string destName = entry.FileName + ext; string destPath = Path.Combine(destDir, destName); Console.WriteLine($"Copying [{entry.FileName}] into [{destPath}]."); File.Copy(entry.FullPath, destPath, true); count += 1; } return(count); }
public static void RunOptions(ScrapeOptions opts) { string srcDir = opts.SourceDir ?? ImageScraper.DefaultSourceDir; ImageScraper scraper = new ImageScraper(NativeLibrary.MagicFile, srcDir); int count = ScanTargetDir(scraper, opts.DestDir); Console.WriteLine($"Scraped {count} images from [{srcDir}]."); }
public void Scrape_NotFoundCountExceded_Returns() { var testff = new TestFetchFunction(); testff.AddNotFound(3); var scraper = new ImageScraper(testff.CreateFunc(), new MockImageRepo().Object); Assert.DoesNotThrow(() => SuppressAbort(() => scraper.Scrape(3))); testff.AssertActionsExausted(); }
public void Scrape_ImageNotFound_DeletesImage() { var testff = new TestFetchFunction(); testff.AddNotFound(1); var mockImageRepo = new MockImageRepo(); var scraper = new ImageScraper(testff.CreateFunc(), mockImageRepo.Object); SuppressAbort(() => scraper.Scrape(3)); mockImageRepo.VerifyDeleteCalled(); }
private static void DownloadCarImages(AppSettings settings) { var cars = _dbHelper.GetCarsWithoutDownloadedImage(); var interval = TimeSpan.FromMilliseconds(settings.RequestIntervalMs); int progress = 0; foreach (var car in cars) { var scraper = new ImageScraper(car.PictureUrl, interval); var path = scraper.DownloadImage(settings.ImagesDirectory); _dbHelper.UpdateCarImagePath(car, path); Console.Title = $"Scraped {++progress} / {cars.Count} car images ({(double)progress / cars.Count:P2}). Remaining time:{TimeSpan.FromMilliseconds((cars.Count - progress) * 150).ToLongString()}."; } }