public ImportDataSegmenter(IDataTable table, ColumnMap map)
{
Guard.NotNull(table, nameof(table));
Guard.NotNull(map, nameof(map));
_table = table;
_columnMap = map;
_bof = true;
_pageable = new PagedList(0, BATCHSIZE, table.Rows.Count);
_culture = CultureInfo.InvariantCulture;
}
public override object ConvertFrom(CultureInfo culture, object value)
{
if (value is string)
{
var dict = JsonConvert.DeserializeObject <Dictionary <string, ColumnMappingValue> >((string)value);
var map = new ColumnMap();
foreach (var kvp in dict)
{
map.AddMapping(kvp.Key, null, kvp.Value.Property, kvp.Value.Default);
}
return(map);
}
return(base.ConvertFrom(culture, value));
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = (profile.Skip == 0 ? (int?)null : profile.Skip);
model.Take = (profile.Take == 0 ? (int?)null : profile.Take);
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(Services.Localization, Services.WorkContext);
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
model.ImportResult = XmlHelper.Deserialize<SerializableImportResult>(profile.ResultInfo);
if (!forEdit)
return;
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom<CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// common configuration
var extraData = XmlHelper.Deserialize<ImportExtraData>(profile.ExtraData);
model.ExtraData.NumberOfPictures = extraData.NumberOfPictures;
// column mapping
model.AvailableSourceColumns = new List<ColumnMappingItemModel>();
model.AvailableEntityProperties = new List<ColumnMappingItemModel>();
model.AvailableKeyFieldNames = new List<SelectListItem>();
model.ColumnMappings = new List<ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
string[] disabledDefaultFieldNames = GetDisabledDefaultFieldNames(profile);
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom<ColumnMap>(profile.ColumnMapping);
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Property = x.Key,
PropertyDescription = x.Value,
IsDefaultDisabled = IsDefaultValueDisabled(x.Key, x.Key, disabledDefaultFieldNames)
};
return mapping;
})
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem { Value = x, Text = x };
if (x == "Id")
item.Text = T("Admin.Common.Entity.Fields.Id");
else if (allProperties.ContainsKey(x))
item.Text = allProperties[x];
return item;
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = x.Value.MappedName,
Property = x.Key,
Default = x.Value.Default
};
if (x.Value.IgnoreProperty)
{
// explicitly ignore the property
mapping.Column = null;
mapping.Default = null;
}
mapping.PropertyDescription = GetPropertyDescription(allProperties, mapping.Property);
mapping.IsDefaultDisabled = IsDefaultValueDisabled(mapping.Column, mapping.Property, disabledDefaultFieldNames);
return mapping;
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
return;
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceName(column.Name, out columnWithoutIndex, out columnIndex);
model.AvailableSourceColumns.Add(new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
PropertyDescription = GetPropertyDescription(allProperties, column.Name)
});
// auto map where field equals property name
if (!model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.IsEmpty())
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
}
if (kvp.Key.HasValue() && !model.ColumnMappings.Any(x => x.Property == kvp.Key))
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
PropertyDescription = kvp.Value,
IsDefaultDisabled = IsDefaultValueDisabled(column.Name, kvp.Key, disabledDefaultFieldNames)
});
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.PropertyDescription)
.ToList();
model.AvailableEntityProperties = model.AvailableEntityProperties
.OrderBy(x => x.PropertyDescription)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.PropertyDescription)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}
public ActionResult Edit(ImportProfileModel model, bool continueEditing, FormCollection form)
{
if (!Services.Permissions.Authorize(StandardPermissionProvider.ManageImports))
return AccessDeniedView();
var profile = _importProfileService.GetImportProfileById(model.Id);
if (profile == null)
return RedirectToAction("List");
var map = new ColumnMap();
var hasErrors = false;
var resetMappings = false;
try
{
var propertyKeyPrefix = "ColumnMapping.Property.";
var allPropertyKeys = form.AllKeys.Where(x => x.HasValue() && x.StartsWith(propertyKeyPrefix));
if (allPropertyKeys.Any())
{
var entityProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType);
foreach (var key in allPropertyKeys)
{
var index = key.Substring(propertyKeyPrefix.Length);
var property = form[key];
var column = form["ColumnMapping.Column." + index];
var defaultValue = form["ColumnMapping.Default." + index];
if (column.IsEmpty())
{
// tell mapper to explicitly ignore the property
map.AddMapping(property, null, property, "[IGNOREPROPERTY]");
}
else if (!column.IsCaseInsensitiveEqual(property) || defaultValue.HasValue())
{
if (defaultValue.HasValue() && GetDisabledDefaultFieldNames(profile).Contains(property))
defaultValue = null;
map.AddMapping(property, null, column, defaultValue);
}
}
}
}
catch (Exception exception)
{
hasErrors = true;
NotifyError(exception, true, false);
}
if (!ModelState.IsValid || hasErrors)
{
PrepareProfileModel(model, profile, true, map);
return View(model);
}
profile.Name = model.Name;
profile.EntityType = model.EntityType;
profile.Enabled = model.Enabled;
profile.Skip = model.Skip ?? 0;
profile.Take = model.Take ?? 0;
profile.UpdateOnly = model.UpdateOnly;
profile.KeyFieldNames = (model.KeyFieldNames == null ? null : string.Join(",", model.KeyFieldNames));
try
{
if (profile.FileType == ImportFileType.CSV && model.CsvConfiguration != null)
{
var csvConverter = new CsvConfigurationConverter();
var oldCsvConfig = csvConverter.ConvertFrom<CsvConfiguration>(profile.FileTypeConfiguration);
var oldDelimiter = (oldCsvConfig != null ? oldCsvConfig.Delimiter.ToString() : null);
// auto reset mappings cause they are invalid. note: delimiter can be whitespaced, so no oldDelimter.HasValue() etc.
resetMappings = (oldDelimiter != model.CsvConfiguration.Delimiter && profile.ColumnMapping.HasValue());
profile.FileTypeConfiguration = csvConverter.ConvertTo(model.CsvConfiguration.Clone());
}
else
{
profile.FileTypeConfiguration = null;
}
if (resetMappings)
{
profile.ColumnMapping = null;
}
else
{
var mapConverter = new ColumnMapConverter();
profile.ColumnMapping = mapConverter.ConvertTo(map);
}
if (model.ExtraData != null)
{
var extraData = new ImportExtraData
{
NumberOfPictures = model.ExtraData.NumberOfPictures
};
profile.ExtraData = XmlHelper.Serialize(extraData);
}
}
catch (Exception exception)
{
hasErrors = true;
NotifyError(exception, true, false);
}
if (!hasErrors)
{
_importProfileService.UpdateImportProfile(profile);
NotifySuccess(T("Admin.Common.DataSuccessfullySaved"));
if (resetMappings)
{
NotifyWarning(T("Admin.DataExchange.ColumnMapping.Validate.MappingsReset"));
}
}
return (continueEditing ? RedirectToAction("Edit", new { id = profile.Id }) : RedirectToAction("List"));
}
