public async void DownloadAsync_WhenRequestsFileAlreadyDownloaded_DownloadsIndependently()
{
// Arrange
int delay = 2000;
IFileDownloader downloader = CreateFileDownloader(delay);
FileDownloadManager manager = new FileDownloadManager(downloader);
string file = "a file";
Task <FileResult> request2 = manager.DownloadAsync(file);
// Act
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
FileResult result1 = await manager.DownloadAsync(file);
FileResult result2 = await manager.DownloadAsync(file);
stopwatch.Stop();
//Assert
int timeRatio = GetTimeRation(stopwatch, delay);
A.CallTo(() => downloader.DownloadAsync(A <string> .Ignored)).MustHaveHappenedTwiceExactly();
result1.FileDownloadName.Should().Be(file);
result2.FileDownloadName.Should().Be(file);
result1.Should().NotBeSameAs(result2);
timeRatio.Should().Be(2);
}
public void DownloadAsync_WhenRequestsTwoDifferentFilesSimultaneously_DownloadsConcurrently()
{
// Arrange
int delay = 2000;
IFileDownloader downloader = CreateFileDownloader(delay);
FileDownloadManager manager = new FileDownloadManager(downloader);
string file1 = "a file";
Task <FileResult> request1 = manager.DownloadAsync(file1);
string file2 = "another file";
Task <FileResult> request2 = manager.DownloadAsync(file2);
// Act
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Task.WaitAll(Task.Run(async() => await request1),
Task.Run(async() => await request2));
stopwatch.Stop();
//Assert
FileResult result1 = request1.Result;
FileResult result2 = request2.Result;
int timeRatio = GetTimeRation(stopwatch, delay);
A.CallTo(() => downloader.DownloadAsync(A <string> .Ignored)).MustHaveHappenedTwiceExactly();
result1.FileDownloadName.Should().Be(file1);
result2.FileDownloadName.Should().Be(file2);
result1.Should().NotBeSameAs(result2);
timeRatio.Should().Be(1);
}
public void DownloadAsync_WhenRequestsFileBeingDownloadedAtTheMoment_WaitsForExistingRequestToFinishAndReturnsItsResult()
{
// Arrange
int delay = 2000;
IFileDownloader downloader = CreateFileDownloader(delay);
FileDownloadManager manager = new FileDownloadManager(downloader);
string file = "a file";
Task <FileResult> request1 = manager.DownloadAsync(file);
Task <FileResult> request2 = manager.DownloadAsync(file);
// Act
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Task.WaitAll(Task.Run(async() => await request1),
Task.Run(async() => await request2));
stopwatch.Stop();
//Assert
FileResult result1 = request1.Result;
FileResult result2 = request2.Result;
int timeRatio = GetTimeRation(stopwatch, delay);
A.CallTo(() => downloader.DownloadAsync(A <string> .Ignored)).MustHaveHappenedOnceExactly();
result1.FileDownloadName.Should().Be(file);
result2.FileDownloadName.Should().Be(file);
result1.Should().BeSameAs(result2);
timeRatio.Should().Be(1);
}
protected virtual int ProcessAvatars(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
foreach (var row in batch)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
continue;
}
var image = CreateDownloadImage(context, urlOrPath, 1);
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
using (var stream = File.OpenRead(image.Path))
{
if ((stream?.Length ?? 0) > 0)
{
var currentFiles = new List <MediaFileInfo>();
var fileId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
var file = _mediaService.GetFileById(fileId, MediaLoadFlags.AsNoTracking);
if (file != null)
{
currentFiles.Add(file);
}
if (!_mediaService.FindEqualFile(stream, currentFiles.Select(x => x.File), true, out var _))
{
// Don't manage avatar files. Just overwrite existing file.
var path = _mediaService.CombinePaths(SystemAlbumProvider.Customers, image.FileName.ToValidFileName());
var newFile = _mediaService.SaveFile(path, stream, false, DuplicateFileHandling.Overwrite);
if ((newFile?.Id ?? 0) != 0)
{
SaveAttribute(row, SystemCustomerAttributeNames.AvatarPictureId, newFile.Id);
}
}
else
{
context.Result.AddInfo("Found equal image in data store. Skipping field.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
}
}
else
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
return(_services.DbContext.SaveChanges());
}
protected virtual int ProcessAvatars(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
foreach (var row in batch)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
continue;
}
var equalPictureId = 0;
var currentPictures = new List <Picture>();
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(urlOrPath, seoName, 1);
if (image == null)
{
continue;
}
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var pictureId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
if (pictureId != 0)
{
var picture = _pictureRepository.TableUntracked.Expand(x => x.MediaStorage).FirstOrDefault(x => x.Id == pictureId);
if (picture != null)
{
currentPictures.Add(picture);
}
}
var size = Size.Empty;
pictureBinary = _pictureService.ValidatePicture(pictureBinary, out size);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, size.Width, size.Height, false);
if (picture != null)
{
SaveAttribute(row, SystemCustomerAttributeNames.AvatarPictureId, picture.Id);
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
}
else
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
return(_services.DbContext.SaveChanges());
}
protected virtual int ProcessAvatars(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
foreach (var row in batch)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
continue;
}
var image = CreateDownloadImage(context, urlOrPath, 1);
if (image == null)
{
continue;
}
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
using (var stream = File.OpenRead(image.Path))
{
if (stream?.Length > 0)
{
MediaFile sourceFile = null;
var currentFiles = new List <MediaFileInfo>();
var fileId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
var file = _mediaService.GetFileById(fileId, MediaLoadFlags.AsNoTracking);
if (file != null)
{
currentFiles.Add(file);
}
if (_mediaService.FindEqualFile(stream, currentFiles.Select(x => x.File), true, out var _))
{
context.Result.AddInfo($"Found equal image in customer data for {image.FileName}. Skipping file.", row.GetRowInfo(), "AvatarPictureUrl");
}
else
{
// An avatar may not be assigned to several customers. A customer could otherwise delete the avatar of another.
// Overwriting is probably too dangerous here, because we could overwrite the avatar of another customer, so better rename.
var path = _mediaService.CombinePaths(SystemAlbumProvider.Customers, image.FileName);
sourceFile = _mediaService.SaveFile(path, stream, false, DuplicateFileHandling.Rename)?.File;
}
if (sourceFile?.Id > 0)
{
SaveAttribute(row, SystemCustomerAttributeNames.AvatarPictureId, sourceFile.Id);
}
}
}
}
else
{
context.Result.AddInfo("Download of avatar failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
return(_services.DbContext.SaveChanges());
}
private void ImportAvatar(IImportExecuteContext context, ImportRow <Customer> row)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
return;
}
Picture picture = null;
var equalPictureId = 0;
var currentPictures = new List <Picture>();
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(urlOrPath, seoName, 1);
if (image == null)
{
return;
}
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var currentPictureId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
if (currentPictureId != 0 && (picture = _pictureRepository.GetById(currentPictureId)) != null)
{
currentPictures.Add(picture);
}
pictureBinary = _pictureService.ValidatePicture(pictureBinary);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
if ((picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, false, false)) != null)
{
_pictureRepository.Context.SaveChanges();
_genericAttributeService.SaveAttribute(row.Entity.Id, SystemCustomerAttributeNames.AvatarPictureId, _attributeKeyGroup, picture.Id.ToString());
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
}
else
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
private int ProcessPictures(IImportExecuteContext context,
ImportRow <Category>[] batch,
Dictionary <int, ImportCategoryMapping> srcToDestId)
{
Picture picture = null;
var equalPictureId = 0;
foreach (var row in batch)
{
try
{
var srcId = row.GetDataValue <int>("Id");
var urlOrPath = row.GetDataValue <string>("ImageUrl");
if (srcId != 0 && srcToDestId.ContainsKey(srcId) && urlOrPath.HasValue())
{
var currentPictures = new List <Picture>();
var category = _categoryRepository.GetById(srcToDestId[srcId].DestinationId);
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(urlOrPath, seoName, 1);
if (category != null && image != null)
{
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
if (category.PictureId.HasValue && (picture = _pictureRepository.GetById(category.PictureId.Value)) != null)
{
currentPictures.Add(picture);
}
pictureBinary = _pictureService.ValidatePicture(pictureBinary);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
if ((picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, false, false)) != null)
{
category.PictureId = picture.Id;
_categoryRepository.Update(category);
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls");
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls");
}
}
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.ToAllMessages(), row.GetRowInfo(), "ImageUrls");
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
private void ProcessProductPictures(IImportExecuteContext context, ImportRow <Product>[] batch)
{
// true, cause pictures must be saved and assigned an id prior adding a mapping.
_productPictureRepository.AutoCommitEnabled = true;
ProductPicture lastInserted = null;
var equalPictureId = 0;
foreach (var row in batch)
{
var imageUrls = row.GetDataValue <List <string> >("ImageUrls");
if (imageUrls == null || !imageUrls.Any())
{
continue;
}
var imageNumber = 0;
var displayOrder = -1;
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var imageFiles = new List <FileDownloadManagerItem>();
// collect required image file infos
foreach (var urlOrPath in imageUrls)
{
var image = CreateDownloadImage(urlOrPath, seoName, ++imageNumber);
if (image != null)
{
imageFiles.Add(image);
}
}
// download images
if (imageFiles.Any(x => x.Url.HasValue()))
{
// async downloading in batch processing is inefficient cause only the image processing benefits from async,
// not the record processing itself. a per record processing may speed up the import.
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, imageFiles.Where(x => x.Url.HasValue() && !x.Success.HasValue)));
}
// import images
foreach (var image in imageFiles.OrderBy(x => x.DisplayOrder))
{
try
{
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var currentProductPictures = _productPictureRepository.TableUntracked.Expand(x => x.Picture)
.Where(x => x.ProductId == row.Entity.Id)
.ToList();
var currentPictures = currentProductPictures
.Select(x => x.Picture)
.ToList();
if (displayOrder == -1)
{
displayOrder = (currentProductPictures.Any() ? currentProductPictures.Select(x => x.DisplayOrder).Max() : 0);
}
pictureBinary = _pictureService.ValidatePicture(pictureBinary);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
// no equal picture found in sequence
var newPicture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, false, false);
if (newPicture != null)
{
var mapping = new ProductPicture
{
ProductId = row.Entity.Id,
PictureId = newPicture.Id,
DisplayOrder = ++displayOrder
};
_productPictureRepository.Insert(mapping);
lastInserted = mapping;
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.ToAllMessages(), row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
}
