private static int ScanTargetDir(ImageScraper scraper, string destDir)
{
Directory.CreateDirectory(destDir);
int count = 0;
foreach (ImageEntry entry in scraper.Scan(SearchOption.TopDirectoryOnly))
{
string ext = string.Empty;
switch (entry.Kind)
{
case ImageKind.None:
continue;
case ImageKind.Jpg:
ext = ".jpg";
break;
case ImageKind.Png:
ext = ".png";
break;
}
string destName = entry.FileName + ext;
string destPath = Path.Combine(destDir, destName);
Console.WriteLine($"Copying [{entry.FileName}] into [{destPath}].");
File.Copy(entry.FullPath, destPath, true);
count += 1;
}
return(count);
}
public static void RunOptions(ScrapeOptions opts)
{
string srcDir = opts.SourceDir ?? ImageScraper.DefaultSourceDir;
ImageScraper scraper = new ImageScraper(NativeLibrary.MagicFile, srcDir);
int count = ScanTargetDir(scraper, opts.DestDir);
Console.WriteLine($"Scraped {count} images from [{srcDir}].");
}
public void Scrape_NotFoundCountExceded_Returns()
{
var testff = new TestFetchFunction();
testff.AddNotFound(3);
var scraper = new ImageScraper(testff.CreateFunc(), new MockImageRepo().Object);
Assert.DoesNotThrow(() => SuppressAbort(() => scraper.Scrape(3)));
testff.AssertActionsExausted();
}
public void Scrape_ImageNotFound_DeletesImage()
{
var testff = new TestFetchFunction();
testff.AddNotFound(1);
var mockImageRepo = new MockImageRepo();
var scraper = new ImageScraper(testff.CreateFunc(), mockImageRepo.Object);
SuppressAbort(() => scraper.Scrape(3));
mockImageRepo.VerifyDeleteCalled();
}
private static void DownloadCarImages(AppSettings settings)
{
var cars = _dbHelper.GetCarsWithoutDownloadedImage();
var interval = TimeSpan.FromMilliseconds(settings.RequestIntervalMs);
int progress = 0;
foreach (var car in cars)
{
var scraper = new ImageScraper(car.PictureUrl, interval);
var path = scraper.DownloadImage(settings.ImagesDirectory);
_dbHelper.UpdateCarImagePath(car, path);
Console.Title = $"Scraped {++progress} / {cars.Count} car images ({(double)progress / cars.Count:P2}). Remaining time:{TimeSpan.FromMilliseconds((cars.Count - progress) * 150).ToLongString()}.";
}
}
