private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
return;
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize<ImportExtraData>(ctx.Request.Profile.ExtraData);
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom<ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
throw new SmartException("No files to import.");
if (!HasPermission(ctx))
throw new SmartException("You do not have permission to perform the selected import.");
ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType);
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = (profile.Skip == 0 ? (int?)null : profile.Skip);
model.Take = (profile.Take == 0 ? (int?)null : profile.Take);
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(Services.Localization, Services.WorkContext);
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
model.ImportResult = XmlHelper.Deserialize<SerializableImportResult>(profile.ResultInfo);
if (!forEdit)
return;
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom<CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// common configuration
var extraData = XmlHelper.Deserialize<ImportExtraData>(profile.ExtraData);
model.ExtraData.NumberOfPictures = extraData.NumberOfPictures;
// column mapping
model.AvailableSourceColumns = new List<ColumnMappingItemModel>();
model.AvailableEntityProperties = new List<ColumnMappingItemModel>();
model.AvailableKeyFieldNames = new List<SelectListItem>();
model.ColumnMappings = new List<ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
string[] disabledDefaultFieldNames = GetDisabledDefaultFieldNames(profile);
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom<ColumnMap>(profile.ColumnMapping);
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Property = x.Key,
PropertyDescription = x.Value,
IsDefaultDisabled = IsDefaultValueDisabled(x.Key, x.Key, disabledDefaultFieldNames)
};
return mapping;
})
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem { Value = x, Text = x };
if (x == "Id")
item.Text = T("Admin.Common.Entity.Fields.Id");
else if (allProperties.ContainsKey(x))
item.Text = allProperties[x];
return item;
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = x.Value.MappedName,
Property = x.Key,
Default = x.Value.Default
};
if (x.Value.IgnoreProperty)
{
// explicitly ignore the property
mapping.Column = null;
mapping.Default = null;
}
mapping.PropertyDescription = GetPropertyDescription(allProperties, mapping.Property);
mapping.IsDefaultDisabled = IsDefaultValueDisabled(mapping.Column, mapping.Property, disabledDefaultFieldNames);
return mapping;
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
return;
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceName(column.Name, out columnWithoutIndex, out columnIndex);
model.AvailableSourceColumns.Add(new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
PropertyDescription = GetPropertyDescription(allProperties, column.Name)
});
// auto map where field equals property name
if (!model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.IsEmpty())
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
}
if (kvp.Key.HasValue() && !model.ColumnMappings.Any(x => x.Property == kvp.Key))
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
PropertyDescription = kvp.Value,
IsDefaultDisabled = IsDefaultValueDisabled(column.Name, kvp.Key, disabledDefaultFieldNames)
});
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.PropertyDescription)
.ToList();
model.AvailableEntityProperties = model.AvailableEntityProperties
.OrderBy(x => x.PropertyDescription)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.PropertyDescription)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = (profile.Skip == 0 ? (int?)null : profile.Skip);
model.Take = (profile.Take == 0 ? (int?)null : profile.Take);
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(Services.Localization, Services.WorkContext);
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
{
model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo);
}
if (!forEdit)
{
return;
}
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// common configuration
var extraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData);
model.ExtraData.NumberOfPictures = extraData.NumberOfPictures;
// column mapping
model.AvailableSourceColumns = new List <ColumnMappingItemModel>();
model.AvailableEntityProperties = new List <ColumnMappingItemModel>();
model.AvailableKeyFieldNames = new List <SelectListItem>();
model.ColumnMappings = new List <ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
string[] disabledDefaultFieldNames = GetDisabledDefaultFieldNames(profile);
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping);
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Property = x.Key,
PropertyDescription = x.Value,
IsDefaultDisabled = IsDefaultValueDisabled(x.Key, x.Key, disabledDefaultFieldNames)
};
return(mapping);
})
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem {
Value = x, Text = x
};
if (x == "Id")
{
item.Text = T("Admin.Common.Entity.Fields.Id");
}
else if (allProperties.ContainsKey(x))
{
item.Text = allProperties[x];
}
return(item);
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = x.Value.MappedName,
Property = x.Key,
Default = x.Value.Default
};
if (x.Value.IgnoreProperty)
{
// explicitly ignore the property
mapping.Column = null;
mapping.Default = null;
}
mapping.PropertyDescription = GetPropertyDescription(allProperties, mapping.Property);
mapping.IsDefaultDisabled = IsDefaultValueDisabled(mapping.Column, mapping.Property, disabledDefaultFieldNames);
return(mapping);
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
{
return;
}
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceName(column.Name, out columnWithoutIndex, out columnIndex);
model.AvailableSourceColumns.Add(new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
PropertyDescription = GetPropertyDescription(allProperties, column.Name)
});
// auto map where field equals property name
if (!model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.IsEmpty())
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
}
if (kvp.Key.HasValue() && !model.ColumnMappings.Any(x => x.Property == kvp.Key))
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
PropertyDescription = kvp.Value,
IsDefaultDisabled = IsDefaultValueDisabled(column.Name, kvp.Key, disabledDefaultFieldNames)
});
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.PropertyDescription)
.ToList();
model.AvailableEntityProperties = model.AvailableEntityProperties
.OrderBy(x => x.PropertyDescription)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.PropertyDescription)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
if (profile == null)
{
if (model.Name.IsEmpty())
{
var defaultNames = T("Admin.DataExchange.Import.DefaultProfileNames").Text.SplitSafe(";");
model.Name = defaultNames.SafeGet((int)model.EntityType);
}
model.ExistingFileNames = new List <string>();
return;
}
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = profile.Skip;
model.Take = profile.Take;
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(_services.Localization, _services.WorkContext);
model.UnspecifiedString = T("Common.Unspecified");
model.AddNewString = T("Common.AddNew");
model.DeleteString = T("Common.Delete");
model.IgnoreString = T("Admin.Common.Ignore");
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
{
model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo);
}
if (!forEdit)
{
return;
}
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// column mapping
model.AvailableSourceColumns = new List <ColumnMappingItemModel>();
model.AvailableEntityProperties = new List <SelectListItem>();
model.AvailableKeyFieldNames = new List <SelectListItem>();
model.ColumnMappings = new List <ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping);
var hasStoredMappings = (storedMap != null && storedMap.Mappings.Any());
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x => new SelectListItem {
Value = x.Key, Text = x.Value
})
.OrderBy(x => x.Text)
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem {
Value = x, Text = x
};
if (x == "Id")
{
item.Text = T("Admin.Common.Entity.Fields.Id");
}
else if (allProperties.ContainsKey(x))
{
item.Text = allProperties[x];
}
return(item);
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = (x.Value.Property.IsEmpty() ? null : x.Key),
Property = (x.Value.Property.IsEmpty() ? x.Key : x.Value.Property),
Default = x.Value.Default
};
// add localized to make mappings sortable
if (allProperties.ContainsKey(mapping.Property))
{
mapping.ColumnLocalized = allProperties[mapping.Property];
}
return(mapping);
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
{
return;
}
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceColumn(column.Name, out columnWithoutIndex, out columnIndex);
var mapModel = new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
ColumnLocalized = (allProperties.ContainsKey(column.Name) ? allProperties[column.Name] : column.Name)
};
model.AvailableSourceColumns.Add(mapModel);
// auto map where field equals property name
if (!hasStoredMappings && !model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.HasValue())
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
ColumnLocalized = kvp.Value
});
}
else
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
if (kvp.Key.HasValue())
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
ColumnLocalized = kvp.Value
});
}
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.ColumnLocalized)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.ColumnLocalized)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}