protected virtual int ProcessCustomerRoles(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
Dictionary <string, CustomerRole> allCustomerRoles = null;
foreach (var row in batch)
{
var customer = row.Entity;
var importRoleSystemNames = row.GetDataValue <List <string> >("CustomerRoleSystemNames");
var assignedRoles = customer.CustomerRoleMappings
.Where(x => !x.IsSystemMapping)
.Select(x => x.CustomerRole)
.ToDictionarySafe(x => x.SystemName, StringComparer.OrdinalIgnoreCase);
// Roles to remove.
foreach (var customerRole in assignedRoles)
{
var systemName = customerRole.Key;
if (!systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.Administrators) &&
!systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.SuperAdministrators) &&
!importRoleSystemNames.Contains(systemName))
{
var mappings = customer.CustomerRoleMappings.Where(x => !x.IsSystemMapping && x.CustomerRoleId == customerRole.Value.Id).ToList();
mappings.Each(x => _customerService.DeleteCustomerRoleMapping(x));
}
}
// Roles to add.
foreach (var systemName in importRoleSystemNames)
{
if (systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.Administrators) ||
systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.SuperAdministrators))
{
context.Result.AddInfo("Security. Ignored administrator role.", row.GetRowInfo(), "CustomerRoleSystemNames");
}
else if (!assignedRoles.ContainsKey(systemName))
{
// Add role mapping, never insert roles.
// Be careful not to insert the roles several times!
if (allCustomerRoles == null)
{
allCustomerRoles = _customerRoleRepository.TableUntracked
.Where(x => !string.IsNullOrEmpty(x.SystemName))
.ToDictionarySafe(x => x.SystemName, StringComparer.OrdinalIgnoreCase);
}
if (allCustomerRoles.TryGetValue(systemName, out var role))
{
_customerService.InsertCustomerRoleMapping(new CustomerRoleMapping {
CustomerId = customer.Id, CustomerRoleId = role.Id
});
}
}
}
}
return(_services.DbContext.SaveChanges());
}
protected virtual int ProcessParentMappings(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch,
Dictionary <int, ImportCategoryMapping> srcToDestId)
{
foreach (var row in batch)
{
var id = row.GetDataValue <int>("Id");
var rawParentId = row.GetDataValue <string>("ParentCategoryId");
var parentId = rawParentId.ToInt(-1);
if (id != 0 && parentId != -1 && srcToDestId.ContainsKey(id) && srcToDestId.ContainsKey(parentId))
{
// only touch hierarchical data if child and parent were inserted
if (srcToDestId[id].Inserted && srcToDestId[parentId].Inserted && srcToDestId[id].DestinationId != 0)
{
var category = _categoryRepository.GetById(srcToDestId[id].DestinationId);
if (category != null)
{
category.ParentCategoryId = srcToDestId[parentId].DestinationId;
_categoryRepository.Update(category);
}
}
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessProductMappings(
ImportExecuteContext context,
IEnumerable <ImportRow <Product> > batch,
Dictionary <int, ImportProductMapping> srcToDestId)
{
_productRepository.AutoCommitEnabled = false;
foreach (var row in batch)
{
var id = row.GetDataValue <int>("Id");
var parentGroupedProductId = row.GetDataValue <int>("ParentGroupedProductId");
if (id != 0 && parentGroupedProductId != 0 && srcToDestId.ContainsKey(id) && srcToDestId.ContainsKey(parentGroupedProductId))
{
// only touch relationship if child and parent were inserted
if (srcToDestId[id].Inserted && srcToDestId[parentGroupedProductId].Inserted && srcToDestId[id].DestinationId != 0)
{
var product = _productRepository.GetById(srcToDestId[id].DestinationId);
if (product != null)
{
product.ParentGroupedProductId = srcToDestId[parentGroupedProductId].DestinationId;
_productRepository.Update(product);
}
}
}
}
var num = _productRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessAvatars(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
foreach (var row in batch)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
continue;
}
var image = CreateDownloadImage(context, urlOrPath, 1);
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
using (var stream = File.OpenRead(image.Path))
{
if ((stream?.Length ?? 0) > 0)
{
var currentFiles = new List <MediaFileInfo>();
var fileId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
var file = _mediaService.GetFileById(fileId, MediaLoadFlags.AsNoTracking);
if (file != null)
{
currentFiles.Add(file);
}
if (!_mediaService.FindEqualFile(stream, currentFiles.Select(x => x.File), true, out var _))
{
// Don't manage avatar files. Just overwrite existing file.
var path = _mediaService.CombinePaths(SystemAlbumProvider.Customers, image.FileName.ToValidFileName());
var newFile = _mediaService.SaveFile(path, stream, false, DuplicateFileHandling.Overwrite);
if ((newFile?.Id ?? 0) != 0)
{
SaveAttribute(row, SystemCustomerAttributeNames.AvatarPictureId, newFile.Id);
}
}
else
{
context.Result.AddInfo("Found equal image in data store. Skipping field.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
}
}
else
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
return(_services.DbContext.SaveChanges());
}
public ImportBatchExecutedEvent(ImportExecuteContext context, IEnumerable <ImportRow <TEntity> > batch)
{
Guard.NotNull(context, nameof(context));
Guard.NotNull(batch, nameof(batch));
Context = context;
Batch = batch;
}
public DataImporterContext(
DataImportRequest request,
CancellationToken cancellationToken,
string progressInfo)
{
Request = request;
CancellationToken = cancellationToken;
ExecuteContext = new ImportExecuteContext(CancellationToken, Request.ProgressValueSetter, progressInfo);
}
protected virtual int ProcessProductCategories(ImportExecuteContext context, IEnumerable <ImportRow <Product> > batch)
{
_productCategoryRepository.AutoCommitEnabled = false;
ProductCategory lastInserted = null;
foreach (var row in batch)
{
var categoryIds = row.GetDataValue <List <int> >("CategoryIds");
if (!categoryIds.IsNullOrEmpty())
{
try
{
foreach (var id in categoryIds)
{
if (_productCategoryRepository.TableUntracked.Where(x => x.ProductId == row.Entity.Id && x.CategoryId == id).FirstOrDefault() == null)
{
// ensure that category exists
var category = _categoryService.GetCategoryById(id);
if (category != null)
{
var productCategory = new ProductCategory
{
ProductId = row.Entity.Id,
CategoryId = category.Id,
IsFeaturedProduct = false,
DisplayOrder = 1
};
_productCategoryRepository.Insert(productCategory);
lastInserted = productCategory;
}
}
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.Message, row.GetRowInfo(), "CategoryIds");
}
}
}
// commit whole batch at once
var num = _productCategoryRepository.Context.SaveChanges();
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
return(num);
}
protected virtual int ProcessAddresses(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch,
Dictionary <string, int> allCountries,
Dictionary <Tuple <int, string>, int> allStateProvinces)
{
foreach (var row in batch)
{
ImportAddress("BillingAddress.", row, context, allCountries, allStateProvinces);
ImportAddress("ShippingAddress.", row, context, allCountries, allStateProvinces);
}
return(_services.DbContext.SaveChanges());
}
public DataImporterContext(
DataImportRequest request,
CancellationToken cancellationToken,
string progressInfo)
{
Request = request;
CancellationToken = cancellationToken;
ExecuteContext = new ImportExecuteContext(CancellationToken, Request.ProgressValueSetter, progressInfo)
{
Request = request
};
ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(Request.Profile.ColumnMapping) ?? new ColumnMap();
Results = new Dictionary <string, ImportResult>();
}
private void AddInfoForDeprecatedFields(ImportExecuteContext context)
{
if (context.DataSegmenter.HasColumn("IsGuest"))
{
context.Result.AddInfo("Deprecated field. Use CustomerRoleSystemNames instead.", null, "IsGuest");
}
if (context.DataSegmenter.HasColumn("IsRegistered"))
{
context.Result.AddInfo("Deprecated field. Use CustomerRoleSystemNames instead.", null, "IsRegistered");
}
if (context.DataSegmenter.HasColumn("IsAdministrator"))
{
context.Result.AddInfo("Deprecated field. Use CustomerRoleSystemNames instead.", null, "IsAdministrator");
}
if (context.DataSegmenter.HasColumn("IsForumModerator"))
{
context.Result.AddInfo("Deprecated field. Use CustomerRoleSystemNames instead.", null, "IsForumModerator");
}
}
protected virtual int ProcessProductManufacturers(ImportExecuteContext context, IEnumerable <ImportRow <Product> > batch)
{
_productManufacturerRepository.AutoCommitEnabled = false;
foreach (var row in batch)
{
var manufacturerIds = row.GetDataValue <List <int> >("ManufacturerIds");
if (!manufacturerIds.IsNullOrEmpty())
{
try
{
foreach (var id in manufacturerIds)
{
if (_productManufacturerRepository.TableUntracked.Where(x => x.ProductId == row.Entity.Id && x.ManufacturerId == id).FirstOrDefault() == null)
{
// ensure that manufacturer exists
var manufacturer = _manufacturerService.GetManufacturerById(id);
if (manufacturer != null)
{
var productManufacturer = new ProductManufacturer
{
ProductId = row.Entity.Id,
ManufacturerId = manufacturer.Id,
IsFeaturedProduct = false,
DisplayOrder = 1
};
_productManufacturerRepository.Insert(productManufacturer);
}
}
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.Message, row.GetRowInfo(), "ManufacturerIds");
}
}
}
// commit whole batch at once
var num = _productManufacturerRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessLocalizations(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch,
string[] localizedProperties)
{
if (localizedProperties.Length == 0)
{
return(0);
}
bool shouldSave = false;
foreach (var row in batch)
{
foreach (var prop in localizedProperties)
{
var lambda = _localizableProperties[prop];
foreach (var lang in context.Languages)
{
var code = lang.UniqueSeoCode;
string value;
if (row.TryGetDataValue(prop /* ColumnName */, code, out value))
{
_localizedEntityService.SaveLocalizedValue(row.Entity, lambda, value, lang.Id);
shouldSave = true;
}
}
}
}
if (shouldSave)
{
// commit whole batch at once
return(context.Services.DbContext.SaveChanges());
}
return(0);
}
protected virtual int ProcessProductMappings(
ImportExecuteContext context,
IEnumerable<ImportRow<Product>> batch,
Dictionary<int, ImportProductMapping> srcToDestId)
{
_productRepository.AutoCommitEnabled = false;
foreach (var row in batch)
{
var id = row.GetDataValue<int>("Id");
var parentGroupedProductId = row.GetDataValue<int>("ParentGroupedProductId");
if (id != 0 && parentGroupedProductId != 0 && srcToDestId.ContainsKey(id) && srcToDestId.ContainsKey(parentGroupedProductId))
{
// only touch relationship if child and parent were inserted
if (srcToDestId[id].Inserted && srcToDestId[parentGroupedProductId].Inserted && srcToDestId[id].DestinationId != 0)
{
var product = _productRepository.GetById(srcToDestId[id].DestinationId);
if (product != null)
{
product.ParentGroupedProductId = srcToDestId[parentGroupedProductId].DestinationId;
_productRepository.Update(product);
}
}
}
}
var num = _productRepository.Context.SaveChanges();
return num;
}
protected override void Import(ImportExecuteContext context)
{
var srcToDestId = new Dictionary <int, ImportProductMapping>();
var templateViewPaths = _productTemplateService.GetAllProductTemplates().ToDictionarySafe(x => x.ViewPath, x => x.Id);
using (var scope = new DbContextScope(ctx: _productRepository.Context, autoDetectChanges: false, proxyCreation: false, validateOnSave: false))
{
var segmenter = context.DataSegmenter;
Initialize(context);
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <Product>();
// Perf: detach all entities
_productRepository.Context.DetachAll(false);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
// ===========================================================================
// 1.) Import products
// ===========================================================================
try
{
ProcessProducts(context, batch, templateViewPaths, srcToDestId);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProducts");
}
// reduce batch to saved (valid) products.
// No need to perform import operations on errored products.
batch = batch.Where(x => x.Entity != null && !x.IsTransient).ToArray();
// update result object
context.Result.NewRecords += batch.Count(x => x.IsNew && !x.IsTransient);
context.Result.ModifiedRecords += batch.Count(x => !x.IsNew && !x.IsTransient);
// ===========================================================================
// 2.) Import SEO Slugs
// IMPORTANT: Unlike with Products AutoCommitEnabled must be TRUE,
// as Slugs are going to be validated against existing ones in DB.
// ===========================================================================
if (segmenter.HasColumn("SeName", true) || batch.Any(x => x.IsNew || x.NameChanged))
{
try
{
_productRepository.Context.AutoDetectChangesEnabled = true;
ProcessSlugs(context, batch, typeof(Product).Name);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessSlugs");
}
finally
{
_productRepository.Context.AutoDetectChangesEnabled = false;
}
}
// ===========================================================================
// 3.) Import StoreMappings
// ===========================================================================
if (segmenter.HasColumn("StoreIds"))
{
try
{
ProcessStoreMappings(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessStoreMappings");
}
}
// ===========================================================================
// 4.) Import Localizations
// ===========================================================================
try
{
ProcessLocalizations(context, batch, _localizableProperties);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessLocalizations");
}
// ===========================================================================
// 5.) Import product category mappings
// ===========================================================================
if (segmenter.HasColumn("CategoryIds"))
{
try
{
ProcessProductCategories(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductCategories");
}
}
// ===========================================================================
// 6.) Import product manufacturer mappings
// ===========================================================================
if (segmenter.HasColumn("ManufacturerIds"))
{
try
{
ProcessProductManufacturers(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductManufacturers");
}
}
// ===========================================================================
// 7.) Import product picture mappings
// ===========================================================================
if (segmenter.HasColumn("ImageUrls"))
{
try
{
ProcessProductPictures(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductPictures");
}
}
}
// ===========================================================================
// 8.) Map parent id of inserted products
// ===========================================================================
if (srcToDestId.Any() && segmenter.HasColumn("Id") && segmenter.HasColumn("ParentGroupedProductId") && !segmenter.IsIgnored("ParentGroupedProductId"))
{
segmenter.Reset();
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <Product>();
_productRepository.Context.DetachAll(false);
try
{
ProcessProductMappings(context, batch, srcToDestId);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessParentMappings");
}
}
}
}
}
protected virtual int ProcessPictures(ImportExecuteContext context, IEnumerable <ImportRow <Category> > batch)
{
var allFileIds = batch
.Where(row => row.HasDataValue("ImageUrl") && row.Entity.MediaFileId > 0)
.Select(row => row.Entity.MediaFileId.Value)
.Distinct()
.ToArray();
var allFiles = _mediaService.GetFilesByIds(allFileIds).ToDictionary(x => x.Id, x => x.File);
var catalogAlbumId = _folderService.GetNodeByPath(SystemAlbumProvider.Catalog).Value.Id;
foreach (var row in batch)
{
try
{
var imageUrl = row.GetDataValue <string>("ImageUrl");
if (imageUrl.IsEmpty())
{
continue;
}
var image = CreateDownloadImage(context, imageUrl, 1);
if (image == null)
{
continue;
}
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
using (var stream = File.OpenRead(image.Path))
{
if ((stream?.Length ?? 0) > 0)
{
var assignedFile = allFiles.Get(row.Entity.MediaFileId ?? 0);
MediaFile sourceFile = null;
if (assignedFile != null && _mediaService.FindEqualFile(stream, new[] { assignedFile }, true, out var _))
{
context.Result.AddInfo($"Found equal image in data store for '{image.FileName}'. Skipping file.", row.GetRowInfo(), "ImageUrl");
}
else if (_mediaService.FindEqualFile(stream, image.FileName, catalogAlbumId, true, out sourceFile))
{
context.Result.AddInfo($"Found equal image in catalog album for '{image.FileName}'. Assigning existing file instead.", row.GetRowInfo(), "ImageUrl");
}
else
{
var path = _mediaService.CombinePaths(SystemAlbumProvider.Catalog, image.FileName);
sourceFile = _mediaService.SaveFile(path, stream, false, DuplicateFileHandling.Rename)?.File;
}
if (sourceFile?.Id > 0)
{
row.Entity.MediaFileId = sourceFile.Id;
_categoryRepository.Update(row.Entity);
}
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls");
}
}
catch (Exception ex)
{
context.Result.AddWarning(ex.ToAllMessages(), row.GetRowInfo(), "ImageUrls");
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
protected virtual void ProcessProductPictures(ImportExecuteContext context, IEnumerable <ImportRow <Product> > batch)
{
// true, cause pictures must be saved and assigned an id prior adding a mapping.
_productPictureRepository.AutoCommitEnabled = true;
ProductPicture lastInserted = null;
var equalPictureId = 0;
var numberOfPictures = (context.ExtraData.NumberOfPictures ?? int.MaxValue);
foreach (var row in batch)
{
var imageUrls = row.GetDataValue <List <string> >("ImageUrls");
if (imageUrls.IsNullOrEmpty())
{
continue;
}
var imageNumber = 0;
var displayOrder = -1;
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var imageFiles = new List <FileDownloadManagerItem>();
// collect required image file infos
foreach (var urlOrPath in imageUrls)
{
var image = CreateDownloadImage(urlOrPath, seoName, ++imageNumber);
if (image != null)
{
imageFiles.Add(image);
}
if (imageFiles.Count >= numberOfPictures)
{
break;
}
}
// download images
if (imageFiles.Any(x => x.Url.HasValue()))
{
// async downloading in batch processing is inefficient cause only the image processing benefits from async,
// not the record processing itself. a per record processing may speed up the import.
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, imageFiles.Where(x => x.Url.HasValue() && !x.Success.HasValue)));
}
// import images
foreach (var image in imageFiles.OrderBy(x => x.DisplayOrder))
{
try
{
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var currentProductPictures = _productPictureRepository.TableUntracked.Expand(x => x.Picture)
.Where(x => x.ProductId == row.Entity.Id)
.ToList();
var currentPictures = currentProductPictures
.Select(x => x.Picture)
.ToList();
if (displayOrder == -1)
{
displayOrder = (currentProductPictures.Any() ? currentProductPictures.Select(x => x.DisplayOrder).Max() : 0);
}
var size = Size.Empty;
pictureBinary = _pictureService.ValidatePicture(pictureBinary, out size);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
// no equal picture found in sequence
var newPicture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, size.Width, size.Height, false);
if (newPicture != null)
{
var mapping = new ProductPicture
{
ProductId = row.Entity.Id,
PictureId = newPicture.Id,
DisplayOrder = ++displayOrder
};
_productPictureRepository.Insert(mapping);
lastInserted = mapping;
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.ToAllMessages(), row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
}
protected virtual void ProcessProductPictures(ImportExecuteContext context, IEnumerable<ImportRow<Product>> batch)
{
// true, cause pictures must be saved and assigned an id prior adding a mapping.
_productPictureRepository.AutoCommitEnabled = true;
ProductPicture lastInserted = null;
var equalPictureId = 0;
var numberOfPictures = (context.ExtraData.NumberOfPictures ?? int.MaxValue);
foreach (var row in batch)
{
var imageUrls = row.GetDataValue<List<string>>("ImageUrls");
if (imageUrls.IsNullOrEmpty())
continue;
var imageNumber = 0;
var displayOrder = -1;
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var imageFiles = new List<FileDownloadManagerItem>();
// collect required image file infos
foreach (var urlOrPath in imageUrls)
{
var image = CreateDownloadImage(urlOrPath, seoName, ++imageNumber);
if (image != null)
imageFiles.Add(image);
if (imageFiles.Count >= numberOfPictures)
break;
}
// download images
if (imageFiles.Any(x => x.Url.HasValue()))
{
// async downloading in batch processing is inefficient cause only the image processing benefits from async,
// not the record processing itself. a per record processing may speed up the import.
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, imageFiles.Where(x => x.Url.HasValue() && !x.Success.HasValue)));
}
// import images
foreach (var image in imageFiles.OrderBy(x => x.DisplayOrder))
{
try
{
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var currentProductPictures = _productPictureRepository.TableUntracked.Expand(x => x.Picture)
.Where(x => x.ProductId == row.Entity.Id)
.ToList();
var currentPictures = currentProductPictures
.Select(x => x.Picture)
.ToList();
if (displayOrder == -1)
{
displayOrder = (currentProductPictures.Any() ? currentProductPictures.Select(x => x.DisplayOrder).Max() : 0);
}
pictureBinary = _pictureService.ValidatePicture(pictureBinary);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
// no equal picture found in sequence
var newPicture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, false, false);
if (newPicture != null)
{
var mapping = new ProductPicture
{
ProductId = row.Entity.Id,
PictureId = newPicture.Id,
DisplayOrder = ++displayOrder
};
_productPictureRepository.Insert(mapping);
lastInserted = mapping;
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.ToAllMessages(), row.GetRowInfo(), "ImageUrls" + image.DisplayOrder.ToString());
}
}
}
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
}
protected virtual int ProcessCustomers(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch,
List <int> allAffiliateIds)
{
_customerRepository.AutoCommitEnabled = true;
Customer lastInserted = null;
Customer lastUpdated = null;
var currentCustomer = _services.WorkContext.CurrentCustomer;
var customerQuery = _customerRepository.Table.Expand(x => x.Addresses);
var hasCustomerRoleSystemNames = context.DataSegmenter.HasColumn("CustomerRoleSystemNames");
foreach (var row in batch)
{
Customer customer = null;
var id = row.GetDataValue <int>("Id");
var email = row.GetDataValue <string>("Email");
foreach (var keyName in context.KeyFieldNames)
{
switch (keyName)
{
case "Id":
if (id != 0)
{
customer = customerQuery.FirstOrDefault(x => x.Id == id);
}
break;
case "CustomerGuid":
var customerGuid = row.GetDataValue <string>("CustomerGuid");
if (customerGuid.HasValue())
{
var guid = new Guid(customerGuid);
customer = customerQuery.FirstOrDefault(x => x.CustomerGuid == guid);
}
break;
case "Email":
if (email.HasValue())
{
customer = customerQuery.FirstOrDefault(x => x.Email == email);
}
break;
case "Username":
var userName = row.GetDataValue <string>("Username");
if (userName.HasValue())
{
customer = customerQuery.FirstOrDefault(x => x.Username == userName);
}
break;
}
if (customer != null)
{
break;
}
}
if (customer == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
customer = new Customer
{
CustomerGuid = new Guid(),
AffiliateId = 0,
Active = true
};
}
else
{
_customerRepository.Context.LoadCollection(customer, (Customer x) => x.CustomerRoles);
}
var affiliateId = row.GetDataValue <int>("AffiliateId");
row.Initialize(customer, email ?? id.ToString());
row.SetProperty(context.Result, (x) => x.CustomerGuid);
row.SetProperty(context.Result, (x) => x.Username);
row.SetProperty(context.Result, (x) => x.Email);
if (email.HasValue() && currentCustomer.Email.IsCaseInsensitiveEqual(email))
{
context.Result.AddInfo("Security. Ignored password of current customer (who started this import).", row.GetRowInfo(), "Password");
}
else
{
row.SetProperty(context.Result, (x) => x.Password);
row.SetProperty(context.Result, (x) => x.PasswordFormatId);
row.SetProperty(context.Result, (x) => x.PasswordSalt);
}
row.SetProperty(context.Result, (x) => x.AdminComment);
row.SetProperty(context.Result, (x) => x.IsTaxExempt);
row.SetProperty(context.Result, (x) => x.Active);
row.SetProperty(context.Result, (x) => x.CreatedOnUtc, UtcNow);
row.SetProperty(context.Result, (x) => x.LastActivityDateUtc, UtcNow);
if (affiliateId > 0 && allAffiliateIds.Contains(affiliateId))
{
customer.AffiliateId = affiliateId;
}
if (row.IsTransient)
{
_customerRepository.Insert(customer);
lastInserted = customer;
}
else
{
_customerRepository.Update(customer);
lastUpdated = customer;
}
}
var num = _customerRepository.Context.SaveChanges();
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
if (lastUpdated != null)
{
_services.EventPublisher.EntityUpdated(lastUpdated);
}
return(num);
}
protected virtual int ProcessProducts(
ImportExecuteContext context,
IEnumerable<ImportRow<Product>> batch,
Dictionary<string, int> templateViewPaths,
Dictionary<int, ImportProductMapping> srcToDestId)
{
_productRepository.AutoCommitEnabled = false;
Product lastInserted = null;
Product lastUpdated = null;
var defaultTemplateId = templateViewPaths["ProductTemplate.Simple"];
foreach (var row in batch)
{
Product product = null;
var id = row.GetDataValue<int>("Id");
foreach (var keyName in context.KeyFieldNames)
{
var keyValue = row.GetDataValue<string>(keyName);
if (keyValue.HasValue() || id > 0)
{
switch (keyName)
{
case "Id":
product = _productService.GetProductById(id);
break;
case "Sku":
product = _productService.GetProductBySku(keyValue);
break;
case "Gtin":
product = _productService.GetProductByGtin(keyValue);
break;
case "ManufacturerPartNumber":
product = _productService.GetProductByManufacturerPartNumber(keyValue);
break;
case "Name":
product = _productService.GetProductByName(keyValue);
break;
}
}
if (product != null)
break;
}
if (product == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
// a Name is required for new products.
if (!row.HasDataValue("Name"))
{
++context.Result.SkippedRecords;
context.Result.AddError("The 'Name' field is required for new products. Skipping row.", row.GetRowInfo(), "Name");
continue;
}
product = new Product();
}
var name = row.GetDataValue<string>("Name");
row.Initialize(product, name ?? product.Name);
if (!row.IsNew)
{
if (!product.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
{
// Perf: use this later for SeName updates.
row.NameChanged = true;
}
}
row.SetProperty(context.Result, (x) => x.ProductTypeId, (int)ProductType.SimpleProduct);
row.SetProperty(context.Result, (x) => x.VisibleIndividually, true);
row.SetProperty(context.Result, (x) => x.Name);
row.SetProperty(context.Result, (x) => x.ShortDescription);
row.SetProperty(context.Result, (x) => x.FullDescription);
row.SetProperty(context.Result, (x) => x.AdminComment);
row.SetProperty(context.Result, (x) => x.ShowOnHomePage);
row.SetProperty(context.Result, (x) => x.HomePageDisplayOrder);
row.SetProperty(context.Result, (x) => x.MetaKeywords);
row.SetProperty(context.Result, (x) => x.MetaDescription);
row.SetProperty(context.Result, (x) => x.MetaTitle);
row.SetProperty(context.Result, (x) => x.AllowCustomerReviews, true);
row.SetProperty(context.Result, (x) => x.ApprovedRatingSum);
row.SetProperty(context.Result, (x) => x.NotApprovedRatingSum);
row.SetProperty(context.Result, (x) => x.ApprovedTotalReviews);
row.SetProperty(context.Result, (x) => x.NotApprovedTotalReviews);
row.SetProperty(context.Result, (x) => x.Published, true);
row.SetProperty(context.Result, (x) => x.Sku);
row.SetProperty(context.Result, (x) => x.ManufacturerPartNumber);
row.SetProperty(context.Result, (x) => x.Gtin);
row.SetProperty(context.Result, (x) => x.IsGiftCard);
row.SetProperty(context.Result, (x) => x.GiftCardTypeId);
row.SetProperty(context.Result, (x) => x.RequireOtherProducts);
row.SetProperty(context.Result, (x) => x.RequiredProductIds); // TODO: global scope
row.SetProperty(context.Result, (x) => x.AutomaticallyAddRequiredProducts);
row.SetProperty(context.Result, (x) => x.IsDownload);
row.SetProperty(context.Result, (x) => x.DownloadId);
row.SetProperty(context.Result, (x) => x.UnlimitedDownloads, true);
row.SetProperty(context.Result, (x) => x.MaxNumberOfDownloads, 10);
row.SetProperty(context.Result, (x) => x.DownloadExpirationDays);
row.SetProperty(context.Result, (x) => x.DownloadActivationTypeId, 1);
row.SetProperty(context.Result, (x) => x.HasSampleDownload);
row.SetProperty(context.Result, (x) => x.SampleDownloadId, (int?)null, ZeroToNull); // TODO: global scope
row.SetProperty(context.Result, (x) => x.HasUserAgreement);
row.SetProperty(context.Result, (x) => x.UserAgreementText);
row.SetProperty(context.Result, (x) => x.IsRecurring);
row.SetProperty(context.Result, (x) => x.RecurringCycleLength, 100);
row.SetProperty(context.Result, (x) => x.RecurringCyclePeriodId);
row.SetProperty(context.Result, (x) => x.RecurringTotalCycles, 10);
row.SetProperty(context.Result, (x) => x.IsShipEnabled, true);
row.SetProperty(context.Result, (x) => x.IsFreeShipping);
row.SetProperty(context.Result, (x) => x.AdditionalShippingCharge);
row.SetProperty(context.Result, (x) => x.IsEsd);
row.SetProperty(context.Result, (x) => x.IsTaxExempt);
row.SetProperty(context.Result, (x) => x.TaxCategoryId, 1); // TODO: global scope
row.SetProperty(context.Result, (x) => x.ManageInventoryMethodId);
row.SetProperty(context.Result, (x) => x.StockQuantity, 10000);
row.SetProperty(context.Result, (x) => x.DisplayStockAvailability);
row.SetProperty(context.Result, (x) => x.DisplayStockQuantity);
row.SetProperty(context.Result, (x) => x.MinStockQuantity);
row.SetProperty(context.Result, (x) => x.LowStockActivityId);
row.SetProperty(context.Result, (x) => x.NotifyAdminForQuantityBelow, 1);
row.SetProperty(context.Result, (x) => x.BackorderModeId);
row.SetProperty(context.Result, (x) => x.AllowBackInStockSubscriptions);
row.SetProperty(context.Result, (x) => x.OrderMinimumQuantity, 1);
row.SetProperty(context.Result, (x) => x.OrderMaximumQuantity, 10000);
row.SetProperty(context.Result, (x) => x.AllowedQuantities);
row.SetProperty(context.Result, (x) => x.DisableBuyButton);
row.SetProperty(context.Result, (x) => x.DisableWishlistButton);
row.SetProperty(context.Result, (x) => x.AvailableForPreOrder);
row.SetProperty(context.Result, (x) => x.CallForPrice);
row.SetProperty(context.Result, (x) => x.Price);
row.SetProperty(context.Result, (x) => x.OldPrice);
row.SetProperty(context.Result, (x) => x.ProductCost);
row.SetProperty(context.Result, (x) => x.SpecialPrice);
row.SetProperty(context.Result, (x) => x.SpecialPriceStartDateTimeUtc);
row.SetProperty(context.Result, (x) => x.SpecialPriceEndDateTimeUtc);
row.SetProperty(context.Result, (x) => x.CustomerEntersPrice);
row.SetProperty(context.Result, (x) => x.MinimumCustomerEnteredPrice);
row.SetProperty(context.Result, (x) => x.MaximumCustomerEnteredPrice, 1000);
// HasTierPrices... ignore as long as no tier prices are imported
// LowestAttributeCombinationPrice... ignore as long as no combinations are imported
row.SetProperty(context.Result, (x) => x.Weight);
row.SetProperty(context.Result, (x) => x.Length);
row.SetProperty(context.Result, (x) => x.Width);
row.SetProperty(context.Result, (x) => x.Height);
row.SetProperty(context.Result, (x) => x.DisplayOrder);
row.SetProperty(context.Result, (x) => x.DeliveryTimeId); // TODO: global scope
row.SetProperty(context.Result, (x) => x.QuantityUnitId); // TODO: global scope
row.SetProperty(context.Result, (x) => x.BasePriceEnabled);
row.SetProperty(context.Result, (x) => x.BasePriceMeasureUnit);
row.SetProperty(context.Result, (x) => x.BasePriceAmount);
row.SetProperty(context.Result, (x) => x.BasePriceBaseAmount);
row.SetProperty(context.Result, (x) => x.BundleTitleText);
row.SetProperty(context.Result, (x) => x.BundlePerItemShipping);
row.SetProperty(context.Result, (x) => x.BundlePerItemPricing);
row.SetProperty(context.Result, (x) => x.BundlePerItemShoppingCart);
row.SetProperty(context.Result, (x) => x.AvailableStartDateTimeUtc);
row.SetProperty(context.Result, (x) => x.AvailableEndDateTimeUtc);
// With new entities, "LimitedToStores" is an implicit field, meaning
// it has to be set to true by code if it's absent but "StoreIds" exists.
row.SetProperty(context.Result, (x) => x.LimitedToStores, !row.GetDataValue<List<int>>("StoreIds").IsNullOrEmpty());
string tvp;
if (row.TryGetDataValue("ProductTemplateViewPath", out tvp, row.IsTransient))
{
product.ProductTemplateId = (tvp.HasValue() && templateViewPaths.ContainsKey(tvp) ? templateViewPaths[tvp] : defaultTemplateId);
}
row.SetProperty(context.Result, (x) => x.CreatedOnUtc, UtcNow);
product.UpdatedOnUtc = UtcNow;
if (id != 0 && !srcToDestId.ContainsKey(id))
{
srcToDestId.Add(id, new ImportProductMapping { Inserted = row.IsTransient });
}
if (row.IsTransient)
{
_productRepository.Insert(product);
lastInserted = product;
}
else
{
_productRepository.Update(product);
lastUpdated = product;
}
}
// commit whole batch at once
var num = _productRepository.Context.SaveChanges();
// get new product ids
foreach (var row in batch)
{
var id = row.GetDataValue<int>("Id");
if (id != 0 && srcToDestId.ContainsKey(id))
srcToDestId[id].DestinationId = row.Entity.Id;
}
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
if (lastUpdated != null)
{
_services.EventPublisher.EntityUpdated(lastUpdated);
}
return num;
}
protected override void Import(ImportExecuteContext context)
{
var customer = _services.WorkContext.CurrentCustomer;
var allowManagingCustomerRoles = _services.Permissions.Authorize(StandardPermissionProvider.ManageCustomerRoles, customer);
var allAffiliateIds = _affiliateService.GetAllAffiliates(true)
.Select(x => x.Id)
.ToList();
var allCountries = new Dictionary <string, int>();
foreach (var country in _countryService.GetAllCountries(true))
{
if (!allCountries.ContainsKey(country.TwoLetterIsoCode))
{
allCountries.Add(country.TwoLetterIsoCode, country.Id);
}
if (!allCountries.ContainsKey(country.ThreeLetterIsoCode))
{
allCountries.Add(country.ThreeLetterIsoCode, country.Id);
}
}
var allStateProvinces = _stateProvinceService.GetAllStateProvinces(true)
.ToDictionarySafe(x => new Tuple <int, string>(x.CountryId, x.Abbreviation), x => x.Id);
var allCustomerNumbers = new HashSet <string>(
_genericAttributeService.GetAttributes(SystemCustomerAttributeNames.CustomerNumber, _attributeKeyGroup).Select(x => x.Value),
StringComparer.OrdinalIgnoreCase);
var allCustomerRoles = _customerRoleRepository.Table.ToDictionarySafe(x => x.SystemName, StringComparer.OrdinalIgnoreCase);
using (var scope = new DbContextScope(ctx: _services.DbContext, autoDetectChanges: false, proxyCreation: false, validateOnSave: false, autoCommit: false))
{
var segmenter = context.DataSegmenter;
Initialize(context);
AddInfoForDeprecatedFields(context);
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <Customer>();
_customerRepository.Context.DetachAll(true);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
// ===========================================================================
// Process customers
// ===========================================================================
try
{
ProcessCustomers(context, batch, allAffiliateIds);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessCustomers");
}
// reduce batch to saved (valid) records.
// No need to perform import operations on errored records.
batch = batch.Where(x => x.Entity != null && !x.IsTransient).ToArray();
// update result object
context.Result.NewRecords += batch.Count(x => x.IsNew && !x.IsTransient);
context.Result.ModifiedRecords += batch.Count(x => !x.IsNew && !x.IsTransient);
// ===========================================================================
// Process customer roles
// ===========================================================================
try
{
_customerRepository.Context.AutoDetectChangesEnabled = true;
ProcessCustomerRoles(context, batch, allCustomerRoles);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessCustomerRoles");
}
finally
{
_customerRepository.Context.AutoDetectChangesEnabled = false;
}
// ===========================================================================
// Process generic attributes
// ===========================================================================
try
{
ProcessGenericAttributes(context, batch, allCountries, allStateProvinces, allCustomerNumbers);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessGenericAttributes");
}
// ===========================================================================
// Process avatars
// ===========================================================================
if (_customerSettings.AllowCustomersToUploadAvatars)
{
try
{
ProcessAvatars(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessAvatars");
}
}
// ===========================================================================
// Process addresses
// ===========================================================================
try
{
_services.DbContext.AutoDetectChangesEnabled = true;
ProcessAddresses(context, batch, allCountries, allStateProvinces);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessAddresses");
}
finally
{
_services.DbContext.AutoDetectChangesEnabled = false;
}
}
}
}
public void Execute(ImportExecuteContext context)
{
var utcNow = DateTime.UtcNow;
var currentStoreId = _services.StoreContext.CurrentStore.Id;
using (var scope = new DbContextScope(ctx: _services.DbContext, autoDetectChanges: false, proxyCreation: false, validateOnSave: false, autoCommit: false))
{
var segmenter = context.DataSegmenter;
context.Result.TotalRecords = segmenter.TotalRows;
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch<NewsLetterSubscription>();
// Perf: detach all entities
_subscriptionRepository.Context.DetachEntities<NewsLetterSubscription>(false);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
foreach (var row in batch)
{
try
{
var active = true;
var email = row.GetDataValue<string>("Email");
var storeId = row.GetDataValue<int>("StoreId");
if (storeId == 0)
{
storeId = currentStoreId;
}
if (row.HasDataValue("Active") && row.TryGetDataValue("Active", out active))
{
}
else
{
active = true; // default
}
if (email.IsEmpty())
{
context.Result.AddWarning("Skipped empty email address", row.GetRowInfo(), "Email");
continue;
}
if (email.Length > 255)
{
context.Result.AddWarning("Skipped email address '{0}'. It exceeds the maximum allowed length of 255".FormatInvariant(email), row.GetRowInfo(), "Email");
continue;
}
if (!email.IsEmail())
{
context.Result.AddWarning("Skipped invalid email address '{0}'".FormatInvariant(email), row.GetRowInfo(), "Email");
continue;
}
NewsLetterSubscription subscription = null;
foreach (var keyName in context.KeyFieldNames)
{
switch (keyName)
{
case "Email":
subscription = _subscriptionRepository.Table
.OrderBy(x => x.Id)
.FirstOrDefault(x => x.Email == email && x.StoreId == storeId);
break;
}
if (subscription != null)
break;
}
if (subscription == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
subscription = new NewsLetterSubscription
{
Active = active,
CreatedOnUtc = utcNow,
Email = email,
NewsLetterSubscriptionGuid = Guid.NewGuid(),
StoreId = storeId
};
_subscriptionRepository.Insert(subscription);
context.Result.NewRecords++;
}
else
{
subscription.Active = active;
_subscriptionRepository.Update(subscription);
context.Result.ModifiedRecords++;
}
}
catch (Exception exception)
{
context.Result.AddError(exception.ToAllMessages(), row.GetRowInfo());
}
} // for
_subscriptionRepository.Context.SaveChanges();
} // while
}
}
protected override void Import(ImportExecuteContext context)
{
var srcToDestId = new Dictionary <int, ImportCategoryMapping>();
var templateViewPaths = _categoryTemplateService.GetAllCategoryTemplates().ToDictionarySafe(x => x.ViewPath, x => x.Id);
using (var scope = new DbContextScope(ctx: context.Services.DbContext, hooksEnabled: false, autoDetectChanges: false, proxyCreation: false, validateOnSave: false))
{
var segmenter = context.DataSegmenter;
Initialize(context);
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <Category>();
// Perf: detach all entities
_categoryRepository.Context.DetachAll(true);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
try
{
ProcessCategories(context, batch, templateViewPaths, srcToDestId);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessCategories");
}
// Reduce batch to saved (valid) products.
// No need to perform import operations on errored products.
batch = batch.Where(x => x.Entity != null && !x.IsTransient).ToArray();
// Update result object.
context.Result.NewRecords += batch.Count(x => x.IsNew && !x.IsTransient);
context.Result.ModifiedRecords += batch.Count(x => !x.IsNew && !x.IsTransient);
// Process slugs.
if (segmenter.HasColumn("SeName", true) || batch.Any(x => x.IsNew || x.NameChanged))
{
try
{
_categoryRepository.Context.AutoDetectChangesEnabled = true;
ProcessSlugs(context, batch, typeof(Category).Name);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessSlugs");
}
finally
{
_categoryRepository.Context.AutoDetectChangesEnabled = false;
}
}
// Process store mappings.
if (segmenter.HasColumn("StoreIds"))
{
try
{
ProcessStoreMappings(context, batch);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessStoreMappings");
}
}
// Localizations.
try
{
ProcessLocalizations(context, batch, _localizableProperties);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessLocalizedProperties");
}
// Process pictures.
if (segmenter.HasColumn("ImageUrl") && !segmenter.IsIgnored("PictureId"))
{
try
{
_categoryRepository.Context.AutoDetectChangesEnabled = true;
ProcessPictures(context, batch);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessPictures");
}
finally
{
_categoryRepository.Context.AutoDetectChangesEnabled = false;
}
}
context.Services.EventPublisher.Publish(new ImportBatchExecutedEvent <Category>(context, batch));
}
// Map parent id of inserted categories.
if (srcToDestId.Any() && segmenter.HasColumn("Id") && segmenter.HasColumn("ParentCategoryId") && !segmenter.IsIgnored("ParentCategoryId"))
{
segmenter.Reset();
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <Category>();
_categoryRepository.Context.DetachAll(unchangedEntitiesOnly: false);
try
{
ProcessParentMappings(context, batch, srcToDestId);
}
catch (Exception ex)
{
context.Result.AddError(ex, segmenter.CurrentSegment, "ProcessParentMappings");
}
}
}
}
}
protected virtual int ProcessProductManufacturers(ImportExecuteContext context, IEnumerable<ImportRow<Product>> batch)
{
_productManufacturerRepository.AutoCommitEnabled = false;
ProductManufacturer lastInserted = null;
foreach (var row in batch)
{
var manufacturerIds = row.GetDataValue<List<int>>("ManufacturerIds");
if (!manufacturerIds.IsNullOrEmpty())
{
try
{
foreach (var id in manufacturerIds)
{
if (_productManufacturerRepository.TableUntracked.Where(x => x.ProductId == row.Entity.Id && x.ManufacturerId == id).FirstOrDefault() == null)
{
// ensure that manufacturer exists
var manufacturer = _manufacturerService.GetManufacturerById(id);
if (manufacturer != null)
{
var productManufacturer = new ProductManufacturer
{
ProductId = row.Entity.Id,
ManufacturerId = manufacturer.Id,
IsFeaturedProduct = false,
DisplayOrder = 1
};
_productManufacturerRepository.Insert(productManufacturer);
lastInserted = productManufacturer;
}
}
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.Message, row.GetRowInfo(), "ManufacturerIds");
}
}
}
// commit whole batch at once
var num = _productManufacturerRepository.Context.SaveChanges();
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
_services.EventPublisher.EntityInserted(lastInserted);
return num;
}
protected virtual int ProcessPictures(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch)
{
foreach (var row in batch)
{
try
{
var srcId = row.GetDataValue <int>("Id");
var imageUrl = row.GetDataValue <string>("ImageUrl");
if (imageUrl.IsEmpty())
{
continue;
}
var image = CreateDownloadImage(context, imageUrl, 1);
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
using (var stream = File.OpenRead(image.Path))
{
if ((stream?.Length ?? 0) > 0)
{
var currentFiles = new List <MediaFileInfo>();
var file = _mediaService.GetFileById(row.Entity.MediaFileId ?? 0, MediaLoadFlags.AsNoTracking);
if (file != null)
{
currentFiles.Add(file);
}
if (!_mediaService.FindEqualFile(stream, currentFiles.Select(x => x.File), true, out var _))
{
var path = _mediaService.CombinePaths(SystemAlbumProvider.Catalog, image.FileName.ToValidFileName());
var newFile = _mediaService.SaveFile(path, stream, false, DuplicateFileHandling.Rename);
if ((newFile?.Id ?? 0) != 0)
{
row.Entity.MediaFileId = newFile.Id;
_categoryRepository.Update(row.Entity);
}
}
else
{
context.Result.AddInfo("Found equal image in data store. Skipping field.", row.GetRowInfo(), "ImageUrls");
}
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls");
}
}
catch (Exception ex)
{
context.Result.AddWarning(ex.ToAllMessages(), row.GetRowInfo(), "ImageUrls");
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessPictures(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch)
{
foreach (var row in batch)
{
try
{
var srcId = row.GetDataValue <int>("Id");
var imageUrl = row.GetDataValue <string>("ImageUrl");
if (imageUrl.IsEmpty())
{
continue;
}
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(context, imageUrl, seoName, 1);
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var currentPictures = new List <MediaFile>();
var pictureId = row.Entity.MediaFileId ?? 0;
if (pictureId != 0)
{
var picture = _pictureRepository.TableUntracked.Expand(x => x.MediaStorage).FirstOrDefault(x => x.Id == pictureId);
if (picture != null)
{
currentPictures.Add(picture);
}
}
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out var equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, false, false, "category");
if (picture != null)
{
row.Entity.MediaFileId = picture.Id;
_categoryRepository.Update(row.Entity);
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls");
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls");
}
}
catch (Exception ex)
{
context.Result.AddWarning(ex.ToAllMessages(), row.GetRowInfo(), "ImageUrls");
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessCustomerRoles(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch,
Dictionary <string, CustomerRole> allCustomerRoles)
{
CustomerRole role = null;
var hasCustomerRoleSystemNames = context.DataSegmenter.HasColumn("CustomerRoleSystemNames");
// Deprecated fields. Still processed for backward compatibility. Use "CustomerRoleSystemNames" instead.
var hasIsGuest = context.DataSegmenter.HasColumn("IsGuest");
var hasIsRegistered = context.DataSegmenter.HasColumn("IsRegistered");
var hasIsForumModerator = context.DataSegmenter.HasColumn("IsForumModerator");
foreach (var row in batch)
{
var customer = row.Entity;
// Deprecated customer role fields.
if (hasIsGuest)
{
UpsertRole(customer, row.GetDataValue <bool>("IsGuest"), allCustomerRoles, SystemCustomerRoleNames.Guests);
}
if (hasIsRegistered)
{
UpsertRole(customer, row.GetDataValue <bool>("IsRegistered"), allCustomerRoles, SystemCustomerRoleNames.Registered);
}
if (hasIsForumModerator)
{
UpsertRole(customer, row.GetDataValue <bool>("IsForumModerator"), allCustomerRoles, SystemCustomerRoleNames.ForumModerators);
}
// New customer role field.
if (hasCustomerRoleSystemNames)
{
var importRoleSystemNames = row.GetDataValue <List <string> >("CustomerRoleSystemNames");
var assignedRoles = customer.CustomerRoles.ToDictionarySafe(x => x.SystemName, StringComparer.OrdinalIgnoreCase);
// Roles to remove.
foreach (var customerRole in assignedRoles)
{
var systemName = customerRole.Key;
if (!systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.Administrators) &&
!systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.SuperAdministrators) &&
!importRoleSystemNames.Contains(systemName))
{
customer.CustomerRoles.Remove(customerRole.Value);
}
}
// Roles to add.
foreach (var systemName in importRoleSystemNames)
{
if (systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.Administrators) ||
systemName.IsCaseInsensitiveEqual(SystemCustomerRoleNames.SuperAdministrators))
{
context.Result.AddInfo("Security. Ignored administrator role.", row.GetRowInfo(), "CustomerRoleSystemNames");
}
else if (!assignedRoles.ContainsKey(systemName) && allCustomerRoles.TryGetValue(systemName, out role))
{
// Never insert roles!
context.Services.DbContext.SetToUnchanged(role);
customer.CustomerRoles.Add(role);
}
}
}
}
return(context.Services.DbContext.SaveChanges());
}
protected virtual int ProcessPictures(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch,
Dictionary <int, ImportCategoryMapping> srcToDestId)
{
Picture picture = null;
var equalPictureId = 0;
foreach (var row in batch)
{
try
{
var srcId = row.GetDataValue <int>("Id");
var urlOrPath = row.GetDataValue <string>("ImageUrl");
if (srcId != 0 && srcToDestId.ContainsKey(srcId) && urlOrPath.HasValue())
{
var currentPictures = new List <Picture>();
var category = _categoryRepository.GetById(srcToDestId[srcId].DestinationId);
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(urlOrPath, seoName, 1);
if (category != null && image != null)
{
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
if (category.PictureId.HasValue && (picture = _pictureRepository.GetById(category.PictureId.Value)) != null)
{
currentPictures.Add(picture);
}
var size = Size.Empty;
pictureBinary = _pictureService.ValidatePicture(pictureBinary, out size);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
if ((picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, size.Width, size.Height, false)) != null)
{
category.PictureId = picture.Id;
_categoryRepository.Update(category);
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "ImageUrls");
}
}
}
else if (image.Url.HasValue())
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "ImageUrls");
}
}
}
}
catch (Exception exception)
{
context.Result.AddWarning(exception.ToAllMessages(), row.GetRowInfo(), "ImageUrls");
}
}
var num = _categoryRepository.Context.SaveChanges();
return(num);
}
protected virtual int ProcessProducts(
ImportExecuteContext context,
IEnumerable <ImportRow <Product> > batch,
Dictionary <string, int> templateViewPaths,
Dictionary <int, ImportProductMapping> srcToDestId)
{
_productRepository.AutoCommitEnabled = false;
Product lastInserted = null;
Product lastUpdated = null;
var defaultTemplateId = templateViewPaths["Product"];
foreach (var row in batch)
{
Product product = null;
var id = row.GetDataValue <int>("Id");
foreach (var keyName in context.KeyFieldNames)
{
var keyValue = row.GetDataValue <string>(keyName);
if (keyValue.HasValue() || id > 0)
{
switch (keyName)
{
case "Id":
product = _productRepository.GetById(id); // get it uncached
break;
case "Sku":
product = _productService.GetProductBySku(keyValue);
break;
case "Gtin":
product = _productService.GetProductByGtin(keyValue);
break;
case "ManufacturerPartNumber":
product = _productService.GetProductByManufacturerPartNumber(keyValue);
break;
case "Name":
product = _productService.GetProductByName(keyValue);
break;
}
}
if (product != null)
{
break;
}
}
if (product == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
// a Name is required for new products.
if (!row.HasDataValue("Name"))
{
++context.Result.SkippedRecords;
context.Result.AddError("The 'Name' field is required for new products. Skipping row.", row.GetRowInfo(), "Name");
continue;
}
product = new Product();
}
var name = row.GetDataValue <string>("Name");
row.Initialize(product, name ?? product.Name);
if (!row.IsNew)
{
if (!product.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
{
// Perf: use this later for SeName updates.
row.NameChanged = true;
}
}
row.SetProperty(context.Result, (x) => x.ProductTypeId, (int)ProductType.SimpleProduct);
row.SetProperty(context.Result, (x) => x.VisibleIndividually, true);
row.SetProperty(context.Result, (x) => x.Name);
row.SetProperty(context.Result, (x) => x.ShortDescription);
row.SetProperty(context.Result, (x) => x.FullDescription);
row.SetProperty(context.Result, (x) => x.AdminComment);
row.SetProperty(context.Result, (x) => x.ShowOnHomePage);
row.SetProperty(context.Result, (x) => x.HomePageDisplayOrder);
row.SetProperty(context.Result, (x) => x.MetaKeywords);
row.SetProperty(context.Result, (x) => x.MetaDescription);
row.SetProperty(context.Result, (x) => x.MetaTitle);
row.SetProperty(context.Result, (x) => x.AllowCustomerReviews, true);
row.SetProperty(context.Result, (x) => x.ApprovedRatingSum);
row.SetProperty(context.Result, (x) => x.NotApprovedRatingSum);
row.SetProperty(context.Result, (x) => x.ApprovedTotalReviews);
row.SetProperty(context.Result, (x) => x.NotApprovedTotalReviews);
row.SetProperty(context.Result, (x) => x.Published, true);
row.SetProperty(context.Result, (x) => x.Sku);
row.SetProperty(context.Result, (x) => x.ManufacturerPartNumber);
row.SetProperty(context.Result, (x) => x.Gtin);
row.SetProperty(context.Result, (x) => x.IsGiftCard);
row.SetProperty(context.Result, (x) => x.GiftCardTypeId);
row.SetProperty(context.Result, (x) => x.RequireOtherProducts);
row.SetProperty(context.Result, (x) => x.RequiredProductIds); // TODO: global scope
row.SetProperty(context.Result, (x) => x.AutomaticallyAddRequiredProducts);
row.SetProperty(context.Result, (x) => x.IsDownload);
row.SetProperty(context.Result, (x) => x.DownloadId);
row.SetProperty(context.Result, (x) => x.UnlimitedDownloads, true);
row.SetProperty(context.Result, (x) => x.MaxNumberOfDownloads, 10);
row.SetProperty(context.Result, (x) => x.DownloadExpirationDays);
row.SetProperty(context.Result, (x) => x.DownloadActivationTypeId, 1);
row.SetProperty(context.Result, (x) => x.HasSampleDownload);
row.SetProperty(context.Result, (x) => x.SampleDownloadId, (int?)null, ZeroToNull); // TODO: global scope
row.SetProperty(context.Result, (x) => x.HasUserAgreement);
row.SetProperty(context.Result, (x) => x.UserAgreementText);
row.SetProperty(context.Result, (x) => x.IsRecurring);
row.SetProperty(context.Result, (x) => x.RecurringCycleLength, 100);
row.SetProperty(context.Result, (x) => x.RecurringCyclePeriodId);
row.SetProperty(context.Result, (x) => x.RecurringTotalCycles, 10);
row.SetProperty(context.Result, (x) => x.IsShipEnabled, true);
row.SetProperty(context.Result, (x) => x.IsFreeShipping);
row.SetProperty(context.Result, (x) => x.AdditionalShippingCharge);
row.SetProperty(context.Result, (x) => x.IsEsd);
row.SetProperty(context.Result, (x) => x.IsTaxExempt);
row.SetProperty(context.Result, (x) => x.TaxCategoryId, 1); // TODO: global scope
row.SetProperty(context.Result, (x) => x.ManageInventoryMethodId);
row.SetProperty(context.Result, (x) => x.StockQuantity, 10000);
row.SetProperty(context.Result, (x) => x.DisplayStockAvailability);
row.SetProperty(context.Result, (x) => x.DisplayStockQuantity);
row.SetProperty(context.Result, (x) => x.MinStockQuantity);
row.SetProperty(context.Result, (x) => x.LowStockActivityId);
row.SetProperty(context.Result, (x) => x.NotifyAdminForQuantityBelow, 1);
row.SetProperty(context.Result, (x) => x.BackorderModeId);
row.SetProperty(context.Result, (x) => x.AllowBackInStockSubscriptions);
row.SetProperty(context.Result, (x) => x.OrderMinimumQuantity, 1);
row.SetProperty(context.Result, (x) => x.OrderMaximumQuantity, 100);
row.SetProperty(context.Result, (x) => x.HideQuantityControl);
row.SetProperty(context.Result, (x) => x.AllowedQuantities);
row.SetProperty(context.Result, (x) => x.DisableBuyButton);
row.SetProperty(context.Result, (x) => x.DisableWishlistButton);
row.SetProperty(context.Result, (x) => x.AvailableForPreOrder);
row.SetProperty(context.Result, (x) => x.CallForPrice);
row.SetProperty(context.Result, (x) => x.Price);
row.SetProperty(context.Result, (x) => x.OldPrice);
row.SetProperty(context.Result, (x) => x.ProductCost);
row.SetProperty(context.Result, (x) => x.SpecialPrice);
row.SetProperty(context.Result, (x) => x.SpecialPriceStartDateTimeUtc);
row.SetProperty(context.Result, (x) => x.SpecialPriceEndDateTimeUtc);
row.SetProperty(context.Result, (x) => x.CustomerEntersPrice);
row.SetProperty(context.Result, (x) => x.MinimumCustomerEnteredPrice);
row.SetProperty(context.Result, (x) => x.MaximumCustomerEnteredPrice, 1000);
// HasTierPrices... ignore as long as no tier prices are imported
// LowestAttributeCombinationPrice... ignore as long as no combinations are imported
row.SetProperty(context.Result, (x) => x.Weight);
row.SetProperty(context.Result, (x) => x.Length);
row.SetProperty(context.Result, (x) => x.Width);
row.SetProperty(context.Result, (x) => x.Height);
row.SetProperty(context.Result, (x) => x.DisplayOrder);
row.SetProperty(context.Result, (x) => x.DeliveryTimeId); // TODO: global scope
row.SetProperty(context.Result, (x) => x.QuantityUnitId); // TODO: global scope
row.SetProperty(context.Result, (x) => x.BasePriceEnabled);
row.SetProperty(context.Result, (x) => x.BasePriceMeasureUnit);
row.SetProperty(context.Result, (x) => x.BasePriceAmount);
row.SetProperty(context.Result, (x) => x.BasePriceBaseAmount);
row.SetProperty(context.Result, (x) => x.BundleTitleText);
row.SetProperty(context.Result, (x) => x.BundlePerItemShipping);
row.SetProperty(context.Result, (x) => x.BundlePerItemPricing);
row.SetProperty(context.Result, (x) => x.BundlePerItemShoppingCart);
row.SetProperty(context.Result, (x) => x.AvailableStartDateTimeUtc);
row.SetProperty(context.Result, (x) => x.AvailableEndDateTimeUtc);
// With new entities, "LimitedToStores" is an implicit field, meaning
// it has to be set to true by code if it's absent but "StoreIds" exists.
row.SetProperty(context.Result, (x) => x.LimitedToStores, !row.GetDataValue <List <int> >("StoreIds").IsNullOrEmpty());
row.SetProperty(context.Result, (x) => x.CustomsTariffNumber);
row.SetProperty(context.Result, (x) => x.CountryOfOriginId);
string tvp;
if (row.TryGetDataValue("ProductTemplateViewPath", out tvp, row.IsTransient))
{
product.ProductTemplateId = (tvp.HasValue() && templateViewPaths.ContainsKey(tvp) ? templateViewPaths[tvp] : defaultTemplateId);
}
if (id != 0 && !srcToDestId.ContainsKey(id))
{
srcToDestId.Add(id, new ImportProductMapping {
Inserted = row.IsTransient
});
}
if (row.IsTransient)
{
_productRepository.Insert(product);
lastInserted = product;
}
else
{
_productRepository.Update(product);
lastUpdated = product;
}
}
// commit whole batch at once
var num = _productRepository.Context.SaveChanges();
// get new product ids
foreach (var row in batch)
{
var id = row.GetDataValue <int>("Id");
if (id != 0 && srcToDestId.ContainsKey(id))
{
srcToDestId[id].DestinationId = row.Entity.Id;
}
}
// Perf: notify only about LAST insertion and update
if (lastInserted != null)
{
_services.EventPublisher.EntityInserted(lastInserted);
}
if (lastUpdated != null)
{
_services.EventPublisher.EntityUpdated(lastUpdated);
}
return(num);
}
public void Execute(ImportExecuteContext context)
{
var utcNow = DateTime.UtcNow;
var currentStoreId = _services.StoreContext.CurrentStore.Id;
using (var scope = new DbContextScope(ctx: _services.DbContext, hooksEnabled: false, autoDetectChanges: false, proxyCreation: false, validateOnSave: false, autoCommit: false))
{
var segmenter = context.DataSegmenter;
context.Result.TotalRecords = segmenter.TotalRows;
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch <NewsLetterSubscription>();
// Perf: detach all entities
_subscriptionRepository.Context.DetachEntities <NewsLetterSubscription>(false);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
foreach (var row in batch)
{
try
{
var active = true;
var email = row.GetDataValue <string>("Email");
var storeId = row.GetDataValue <int>("StoreId");
if (storeId == 0)
{
storeId = currentStoreId;
}
if (row.HasDataValue("Active") && row.TryGetDataValue("Active", out active))
{
}
else
{
active = true; // default
}
if (email.IsEmpty())
{
context.Result.AddWarning("Skipped empty email address", row.GetRowInfo(), "Email");
continue;
}
if (email.Length > 255)
{
context.Result.AddWarning("Skipped email address '{0}'. It exceeds the maximum allowed length of 255".FormatInvariant(email), row.GetRowInfo(), "Email");
continue;
}
if (!email.IsEmail())
{
context.Result.AddWarning("Skipped invalid email address '{0}'".FormatInvariant(email), row.GetRowInfo(), "Email");
continue;
}
NewsLetterSubscription subscription = null;
foreach (var keyName in context.KeyFieldNames)
{
switch (keyName)
{
case "Email":
subscription = _subscriptionRepository.Table
.OrderBy(x => x.Id)
.FirstOrDefault(x => x.Email == email && x.StoreId == storeId);
break;
}
if (subscription != null)
{
break;
}
}
if (subscription == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
subscription = new NewsLetterSubscription
{
Active = active,
CreatedOnUtc = utcNow,
Email = email,
NewsLetterSubscriptionGuid = Guid.NewGuid(),
StoreId = storeId
};
_subscriptionRepository.Insert(subscription);
context.Result.NewRecords++;
}
else
{
subscription.Active = active;
_subscriptionRepository.Update(subscription);
context.Result.ModifiedRecords++;
}
}
catch (Exception exception)
{
context.Result.AddError(exception.ToAllMessages(), row.GetRowInfo());
}
} // for
_subscriptionRepository.Context.SaveChanges();
} // while
}
}
protected virtual int ProcessGenericAttributes(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch,
Dictionary <string, int> allCountries,
Dictionary <Tuple <int, string>, int> allStateProvinces)
{
foreach (var row in batch)
{
if (_dateTimeSettings.AllowCustomersToSetTimeZone)
{
SaveAttribute(row, SystemCustomerAttributeNames.TimeZoneId);
}
if (_customerSettings.GenderEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Gender);
}
if (_customerSettings.StreetAddressEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.StreetAddress);
}
if (_customerSettings.StreetAddress2Enabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.StreetAddress2);
}
if (_customerSettings.ZipPostalCodeEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.ZipPostalCode);
}
if (_customerSettings.CityEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.City);
}
if (_customerSettings.CountryEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.CountryId);
}
if (_customerSettings.CountryEnabled && _customerSettings.StateProvinceEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.StateProvinceId);
}
if (_customerSettings.PhoneEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Phone);
}
if (_customerSettings.FaxEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Fax);
}
if (_forumSettings.ForumsEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.ForumPostCount);
}
if (_forumSettings.SignaturesEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Signature);
}
var countryId = CountryCodeToId(allCountries, row.GetDataValue <string>("CountryCode"));
var stateId = StateAbbreviationToId(allStateProvinces, countryId, row.GetDataValue <string>("StateAbbreviation"));
if (countryId.HasValue)
{
SaveAttribute(row, SystemCustomerAttributeNames.CountryId, countryId.Value);
}
if (stateId.HasValue)
{
SaveAttribute(row, SystemCustomerAttributeNames.StateProvinceId, stateId.Value);
}
}
return(_services.DbContext.SaveChanges());
}
private void ImportAddress(
string fieldPrefix,
ImportRow <Customer> row,
ImportExecuteContext context,
Dictionary <string, int> allCountries,
Dictionary <Tuple <int, string>, int> allStateProvinces)
{
// last name is mandatory for an address to be imported or updated
if (!row.HasDataValue(fieldPrefix + "LastName"))
{
return;
}
var address = new Address
{
CreatedOnUtc = UtcNow
};
var childRow = new ImportRow <Address>(row.Segmenter, row.DataRow, row.Position);
childRow.Initialize(address, row.EntityDisplayName);
childRow.SetProperty(context.Result, fieldPrefix + "LastName", x => x.LastName);
childRow.SetProperty(context.Result, fieldPrefix + "FirstName", x => x.FirstName);
childRow.SetProperty(context.Result, fieldPrefix + "Email", x => x.Email);
childRow.SetProperty(context.Result, fieldPrefix + "Company", x => x.Company);
childRow.SetProperty(context.Result, fieldPrefix + "City", x => x.City);
childRow.SetProperty(context.Result, fieldPrefix + "Address1", x => x.Address1);
childRow.SetProperty(context.Result, fieldPrefix + "Address2", x => x.Address2);
childRow.SetProperty(context.Result, fieldPrefix + "ZipPostalCode", x => x.ZipPostalCode);
childRow.SetProperty(context.Result, fieldPrefix + "PhoneNumber", x => x.PhoneNumber);
childRow.SetProperty(context.Result, fieldPrefix + "FaxNumber", x => x.FaxNumber);
childRow.SetProperty(context.Result, fieldPrefix + "CountryId", x => x.CountryId);
if (childRow.Entity.CountryId == null)
{
// try with country code
childRow.SetProperty(context.Result, fieldPrefix + "CountryCode", x => x.CountryId, converter: (val, ci) => CountryCodeToId(allCountries, val.ToString()));
}
var countryId = childRow.Entity.CountryId;
if (countryId.HasValue)
{
childRow.SetProperty(context.Result, fieldPrefix + "StateProvinceId", x => x.StateProvinceId);
if (childRow.Entity.StateProvinceId == null)
{
// try with state abbreviation
childRow.SetProperty(context.Result, fieldPrefix + "StateAbbreviation", x => x.StateProvinceId, converter: (val, ci) => StateAbbreviationToId(allStateProvinces, countryId, val.ToString()));
}
}
if (!childRow.IsDirty)
{
// Not one single property could be set. Get out!
return;
}
var appliedAddress = row.Entity.Addresses.FindAddress(address);
if (appliedAddress == null)
{
appliedAddress = address;
row.Entity.Addresses.Add(appliedAddress);
}
if (fieldPrefix == "BillingAddress.")
{
row.Entity.BillingAddress = appliedAddress;
}
else if (fieldPrefix == "ShippingAddress.")
{
row.Entity.ShippingAddress = appliedAddress;
}
_customerRepository.Update(row.Entity);
}
protected virtual int ProcessGenericAttributes(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch,
Dictionary <string, int> allCountries,
Dictionary <Tuple <int, string>, int> allStateProvinces,
HashSet <string> allCustomerNumbers)
{
foreach (var row in batch)
{
SaveAttribute(row, SystemCustomerAttributeNames.FirstName);
SaveAttribute(row, SystemCustomerAttributeNames.LastName);
if (_dateTimeSettings.AllowCustomersToSetTimeZone)
{
SaveAttribute(row, SystemCustomerAttributeNames.TimeZoneId);
}
if (_customerSettings.GenderEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Gender);
}
if (_customerSettings.DateOfBirthEnabled)
{
SaveAttribute <DateTime?>(row, SystemCustomerAttributeNames.DateOfBirth);
}
if (_customerSettings.CompanyEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Company);
}
if (_customerSettings.StreetAddressEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.StreetAddress);
}
if (_customerSettings.StreetAddress2Enabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.StreetAddress2);
}
if (_customerSettings.ZipPostalCodeEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.ZipPostalCode);
}
if (_customerSettings.CityEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.City);
}
if (_customerSettings.CountryEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.CountryId);
}
if (_customerSettings.CountryEnabled && _customerSettings.StateProvinceEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.StateProvinceId);
}
if (_customerSettings.PhoneEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Phone);
}
if (_customerSettings.FaxEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Fax);
}
if (_forumSettings.ForumsEnabled)
{
SaveAttribute <int>(row, SystemCustomerAttributeNames.ForumPostCount);
}
if (_forumSettings.SignaturesEnabled)
{
SaveAttribute(row, SystemCustomerAttributeNames.Signature);
}
var countryId = CountryCodeToId(allCountries, row.GetDataValue <string>("CountryCode"));
var stateId = StateAbbreviationToId(allStateProvinces, countryId, row.GetDataValue <string>("StateAbbreviation"));
if (countryId.HasValue)
{
SaveAttribute(row, SystemCustomerAttributeNames.CountryId, countryId.Value);
}
if (stateId.HasValue)
{
SaveAttribute(row, SystemCustomerAttributeNames.StateProvinceId, stateId.Value);
}
string customerNumber = null;
if (_customerSettings.CustomerNumberMethod == CustomerNumberMethod.AutomaticallySet)
{
customerNumber = row.Entity.Id.ToString();
}
else
{
customerNumber = row.GetDataValue <string>("CustomerNumber");
}
if (customerNumber.IsEmpty() || !allCustomerNumbers.Contains(customerNumber))
{
SaveAttribute(row, SystemCustomerAttributeNames.CustomerNumber, customerNumber);
if (!customerNumber.IsEmpty())
{
allCustomerNumbers.Add(customerNumber);
}
}
}
return(_services.DbContext.SaveChanges());
}
protected virtual int ProcessCategories(
ImportExecuteContext context,
IEnumerable <ImportRow <Category> > batch,
Dictionary <string, int> templateViewPaths,
Dictionary <int, ImportCategoryMapping> srcToDestId)
{
_categoryRepository.AutoCommitEnabled = true;
var defaultTemplateId = templateViewPaths["CategoryTemplate.ProductsInGridOrLines"];
var hasNameColumn = context.DataSegmenter.HasColumn("Name");
foreach (var row in batch)
{
Category category = null;
var id = row.GetDataValue <int>("Id");
var name = row.GetDataValue <string>("Name");
foreach (var keyName in context.KeyFieldNames)
{
switch (keyName)
{
case "Id":
if (id != 0)
{
category = _categoryRepository.GetById(id);
}
break;
case "Name":
if (name.HasValue())
{
category = _categoryRepository.Table.FirstOrDefault(x => x.Name == name);
}
break;
}
if (category != null)
{
break;
}
}
if (category == null)
{
if (context.UpdateOnly)
{
++context.Result.SkippedRecords;
continue;
}
// A name is required for new categories.
if (!row.HasDataValue("Name"))
{
++context.Result.SkippedRecords;
context.Result.AddError("The 'Name' field is required for new categories. Skipping row.", row.GetRowInfo(), "Name");
continue;
}
category = new Category();
}
row.Initialize(category, name ?? category.Name);
if (!row.IsNew && hasNameColumn && !category.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
{
// Perf: use this later for SeName updates.
row.NameChanged = true;
}
row.SetProperty(context.Result, (x) => x.Name);
row.SetProperty(context.Result, (x) => x.FullName);
row.SetProperty(context.Result, (x) => x.Description);
row.SetProperty(context.Result, (x) => x.BottomDescription);
row.SetProperty(context.Result, (x) => x.MetaKeywords);
row.SetProperty(context.Result, (x) => x.MetaDescription);
row.SetProperty(context.Result, (x) => x.MetaTitle);
row.SetProperty(context.Result, (x) => x.PageSize);
row.SetProperty(context.Result, (x) => x.AllowCustomersToSelectPageSize);
row.SetProperty(context.Result, (x) => x.PageSizeOptions);
row.SetProperty(context.Result, (x) => x.ShowOnHomePage);
row.SetProperty(context.Result, (x) => x.HasDiscountsApplied);
row.SetProperty(context.Result, (x) => x.Published, true);
row.SetProperty(context.Result, (x) => x.DisplayOrder);
row.SetProperty(context.Result, (x) => x.Alias);
row.SetProperty(context.Result, (x) => x.DefaultViewMode);
// With new entities, "LimitedToStores" is an implicit field, meaning
// it has to be set to true by code if it's absent but "StoreIds" exists.
row.SetProperty(context.Result, (x) => x.LimitedToStores, !row.GetDataValue <List <int> >("StoreIds").IsNullOrEmpty());
string tvp;
if (row.TryGetDataValue("CategoryTemplateViewPath", out tvp, row.IsTransient))
{
category.CategoryTemplateId = (tvp.HasValue() && templateViewPaths.ContainsKey(tvp) ? templateViewPaths[tvp] : defaultTemplateId);
}
if (id != 0 && !srcToDestId.ContainsKey(id))
{
srcToDestId.Add(id, new ImportCategoryMapping {
Inserted = row.IsTransient
});
}
if (row.IsTransient)
{
_categoryRepository.Insert(category);
}
else
{
_categoryRepository.Update(category);
}
}
// Commit whole batch at once.
var num = _categoryRepository.Context.SaveChanges();
// Get new category ids.
foreach (var row in batch)
{
var id = row.GetDataValue <int>("Id");
if (id != 0 && srcToDestId.ContainsKey(id))
{
srcToDestId[id].DestinationId = row.Entity.Id;
}
}
return(num);
}
protected virtual int ProcessAvatars(
ImportExecuteContext context,
IEnumerable <ImportRow <Customer> > batch)
{
foreach (var row in batch)
{
var urlOrPath = row.GetDataValue <string>("AvatarPictureUrl");
if (urlOrPath.IsEmpty())
{
continue;
}
var equalPictureId = 0;
var currentPictures = new List <Picture>();
var seoName = _pictureService.GetPictureSeName(row.EntityDisplayName);
var image = CreateDownloadImage(urlOrPath, seoName, 1);
if (image == null)
{
continue;
}
if (image.Url.HasValue() && !image.Success.HasValue)
{
AsyncRunner.RunSync(() => _fileDownloadManager.DownloadAsync(DownloaderContext, new FileDownloadManagerItem[] { image }));
}
if ((image.Success ?? false) && File.Exists(image.Path))
{
Succeeded(image);
var pictureBinary = File.ReadAllBytes(image.Path);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var pictureId = row.Entity.GetAttribute <int>(SystemCustomerAttributeNames.AvatarPictureId);
if (pictureId != 0)
{
var picture = _pictureRepository.TableUntracked.Expand(x => x.MediaStorage).FirstOrDefault(x => x.Id == pictureId);
if (picture != null)
{
currentPictures.Add(picture);
}
}
var size = Size.Empty;
pictureBinary = _pictureService.ValidatePicture(pictureBinary, out size);
pictureBinary = _pictureService.FindEqualPicture(pictureBinary, currentPictures, out equalPictureId);
if (pictureBinary != null && pictureBinary.Length > 0)
{
var picture = _pictureService.InsertPicture(pictureBinary, image.MimeType, seoName, true, size.Width, size.Height, false);
if (picture != null)
{
SaveAttribute(row, SystemCustomerAttributeNames.AvatarPictureId, picture.Id);
}
}
else
{
context.Result.AddInfo("Found equal picture in data store. Skipping field.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
}
else
{
context.Result.AddInfo("Download of an image failed.", row.GetRowInfo(), "AvatarPictureUrl");
}
}
return(_services.DbContext.SaveChanges());
}
public ImportExecutedEvent(ImportExecuteContext context)
{
Guard.NotNull(context, nameof(context));
Context = context;
}
protected override void Import(ImportExecuteContext context)
{
var srcToDestId = new Dictionary<int, ImportProductMapping>();
var templateViewPaths = _productTemplateService.GetAllProductTemplates().ToDictionarySafe(x => x.ViewPath, x => x.Id);
using (var scope = new DbContextScope(ctx: _productRepository.Context, autoDetectChanges: false, proxyCreation: false, validateOnSave: false))
{
var segmenter = context.DataSegmenter;
Initialize(context);
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch<Product>();
// Perf: detach all entities
_productRepository.Context.DetachAll(false);
context.SetProgress(segmenter.CurrentSegmentFirstRowIndex - 1, segmenter.TotalRows);
// ===========================================================================
// 1.) Import products
// ===========================================================================
try
{
ProcessProducts(context, batch, templateViewPaths, srcToDestId);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProducts");
}
// reduce batch to saved (valid) products.
// No need to perform import operations on errored products.
batch = batch.Where(x => x.Entity != null && !x.IsTransient).ToArray();
// update result object
context.Result.NewRecords += batch.Count(x => x.IsNew && !x.IsTransient);
context.Result.ModifiedRecords += batch.Count(x => !x.IsNew && !x.IsTransient);
// ===========================================================================
// 2.) Import SEO Slugs
// IMPORTANT: Unlike with Products AutoCommitEnabled must be TRUE,
// as Slugs are going to be validated against existing ones in DB.
// ===========================================================================
if (segmenter.HasColumn("SeName", true) || batch.Any(x => x.IsNew || x.NameChanged))
{
try
{
_productRepository.Context.AutoDetectChangesEnabled = true;
ProcessSlugs(context, batch, typeof(Product).Name);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessSlugs");
}
finally
{
_productRepository.Context.AutoDetectChangesEnabled = false;
}
}
// ===========================================================================
// 3.) Import StoreMappings
// ===========================================================================
if (segmenter.HasColumn("StoreIds"))
{
try
{
ProcessStoreMappings(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessStoreMappings");
}
}
// ===========================================================================
// 4.) Import Localizations
// ===========================================================================
try
{
ProcessLocalizations(context, batch, _localizableProperties);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessLocalizations");
}
// ===========================================================================
// 5.) Import product category mappings
// ===========================================================================
if (segmenter.HasColumn("CategoryIds"))
{
try
{
ProcessProductCategories(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductCategories");
}
}
// ===========================================================================
// 6.) Import product manufacturer mappings
// ===========================================================================
if (segmenter.HasColumn("ManufacturerIds"))
{
try
{
ProcessProductManufacturers(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductManufacturers");
}
}
// ===========================================================================
// 7.) Import product picture mappings
// ===========================================================================
if (segmenter.HasColumn("ImageUrls"))
{
try
{
ProcessProductPictures(context, batch);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessProductPictures");
}
}
}
// ===========================================================================
// 8.) Map parent id of inserted products
// ===========================================================================
if (srcToDestId.Any() && segmenter.HasColumn("Id") && segmenter.HasColumn("ParentGroupedProductId") && !segmenter.IsIgnored("ParentGroupedProductId"))
{
segmenter.Reset();
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
var batch = segmenter.GetCurrentBatch<Product>();
_productRepository.Context.DetachAll(false);
try
{
ProcessProductMappings(context, batch, srcToDestId);
}
catch (Exception exception)
{
context.Result.AddError(exception, segmenter.CurrentSegment, "ProcessParentMappings");
}
}
}
}
}