protected override void Page_Load(object sender, EventArgs e)
{
RequireClientAuthentication();
importProfile = ObjectFromQueryString<ImportProfile>();
ImportProfileFormLayout.DataSourceID = null;
if (importProfile != null && !IsPostBack)
{
ImportProfileFormLayout.DataSource = importProfile;
ImportProfileFormLayout.DataBind();
}
if (importProfile != null)
{
ImportProfileHeadline.HeaderText = "Edit";
DeleteImportProfileButton.ClientVisible = true;
}
else
{
ImportProfileHeadline.HeaderText = "Add new import profile";
}
SetupRibbon();
base.Page_Load(sender, e);
}
/// <summary>
/// Gets import files for an import profile.
/// </summary>
/// <param name="profile">Import profile.</param>
/// <param name="includeRelatedFiles">Whether to include related data files.</param>
/// <returns>List of import files.</returns>
public static List <ImportFile> GetImportFiles(this ImportProfile profile, bool includeRelatedFiles = true)
{
var result = new List <ImportFile>();
var importFolder = profile.GetImportFolder(true);
if (System.IO.Directory.Exists(importFolder))
{
var paths = System.IO.Directory.EnumerateFiles(importFolder, "*", SearchOption.TopDirectoryOnly);
foreach (var path in paths)
{
var file = new ImportFile(path);
if (!includeRelatedFiles && file.RelatedType.HasValue)
{
continue;
}
result.Add(file);
}
// Always main data files first.
result = result.OrderBy(x => x.RelatedType).ThenBy(x => x.Path).ToList();
return(result);
}
return(result);
}
///////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Internal execute.
/// </summary>
/// <param name="args">Commnad arguments.</param>
protected override void _Execute(params object[] args)
{
_ClearGridSelectionOnParentPage();
ImportProfilesKeeper importProfileKeeper = _Application.ImportProfilesKeeper;
ImportProfile profile = importProfileKeeper.GetDefaultProfile(Type);
if (null != profile)
{
// start import with default profile
_DoImport(profile);
}
else
{ // default profile absent - use on fly profile
// create page to on fly creating profile
Type type = typeof(FleetSetupWizardImportObjectsPage);
_onFlyCreatingProfilePage =
(FleetSetupWizardImportObjectsPage)Activator.CreateInstance(type);
_onFlyCreatingProfilePage.Loaded +=
new RoutedEventHandler(_onFlyCreatingProfilePage_Loaded);
// create empty profile
_doesNewProfile = (null == importProfileKeeper.GetOneTimeProfile(Type));
ImportProfile onFlyProfile = CommonHelpers.GetOneTimeProfile(Type);
// init page state
_onFlyCreatingProfilePage.PostInit(onFlyProfile);
// show page
App.Current.MainWindow.PageFrame.Navigate(_onFlyCreatingProfilePage);
}
}
public IHttpActionResult Post([FromBody] ImportProfile profile)
{
var principal = (User)User.Identity;
if (!principal.UserRights.Any(ur => ur.RoleId == (int)Roles.Administrator))
{
return(new StatusCodeResult(HttpStatusCode.Forbidden, Request));
}
if (Repository.FindBy <ImportProfile>(p => p.Name == profile.Name).Any())
{
var warningMessage = "The ProfileName \"" + profile.Name + "\" already exists";
// Log.MonitoringLogger.Warn(warningMessage);
ModelState.AddModelError("alreadyExists", warningMessage);
return(BadRequest(ModelState));
}
profile.CreatedBy = principal.Username;
profile.CreatedDate = DateTime.Now;
Repository.Add(profile);
Repository.Save();
var message = "The Profile \"" + profile.Name + "\" has been added";
Log.MonitoringLogger.Info(message);
return(Ok(message));
}
public virtual void UpdateImportProfile(ImportProfile profile)
{
if (profile == null)
{
throw new ArgumentNullException("profile");
}
_importProfileRepository.Update(profile);
}
public DataImportRequest(ImportProfile profile)
{
Guard.ArgumentNotNull(() => profile);
Profile = profile;
ProgressValueSetter = _voidProgressValueSetter;
EntitiesToImport = new List <int>();
CustomData = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
}
/// <summary>
/// Do import process.
/// </summary>
/// <param name="profile">Profile to importing.</param>
private void _DoImport(ImportProfile profile)
{
var manager = new ImportManager();
// Subscribe to order completed. If parent page is OptimizeAndEditPage -
// call Ungeocoded orders page.
manager.ImportCompleted += new ImportCompletedEventHandler(_ImportCompleted);
manager.ImportAsync(ParentPage, profile, App.Current.CurrentDate);
}
/// <summary>
/// Get folder for import files
/// </summary>
/// <param name="profile">Import profile</param>
/// <returns>Folder path</returns>
public static string GetImportFolder(this ImportProfile profile, bool content = false, bool create = false)
{
var path = CommonHelper.MapPath(string.Concat("~/App_Data/ImportProfiles/", profile.FolderName, content ? "/Content" : ""));
if (create && !System.IO.Directory.Exists(path))
{
System.IO.Directory.CreateDirectory(path);
}
return(path);
}
/// <summary>
/// Get folder for import files
/// </summary>
/// <param name="profile">Import profile</param>
/// <returns>Folder path</returns>
public static string GetImportFolder(this ImportProfile profile, bool content = false, bool create = false)
{
var basePath = DataSettings.Current.TenantPath + "/ImportProfiles/";
var path = CommonHelper.MapPath(string.Concat(basePath, profile.FolderName, content ? "/Content" : ""));
if (create && !System.IO.Directory.Exists(path))
{
System.IO.Directory.CreateDirectory(path);
}
return(path);
}
/// <summary>
/// Gets import files for an import profile
/// </summary>
/// <param name="profile">Import profile</param>
/// <returns>List of file paths</returns>
public static List <string> GetImportFiles(this ImportProfile profile)
{
var importFolder = profile.GetImportFolder(true);
if (System.IO.Directory.Exists(importFolder))
{
return(System.IO.Directory.EnumerateFiles(importFolder, "*", SearchOption.TopDirectoryOnly)
.OrderBy(x => x)
.ToList());
}
return(new List <string>());
}
protected void SaveButton_Click(object sender, EventArgs e)
{
if (importProfile == null)
{
importProfile = new ImportProfile();
SetImportProfilePropertiesFromFormLayout(ImportProfileFormLayout, importProfile);
SaveOrShowErrors(importProfile, true);
}
else
{
SetImportProfilePropertiesFromFormLayout(ImportProfileFormLayout, importProfile);
SaveOrShowErrors(importProfile, false);
}
}
/// <summary>
/// Special initializing of wizard pages.
/// </summary>
protected override void PostInit()
{
base.PostInit();
foreach (WizardPageBase page in Pages)
{
FleetSetupWizardImportObjectsPage importObjectsPage = page as FleetSetupWizardImportObjectsPage;
if (importObjectsPage != null)
{
ImportProfile profile = CommonHelpers.GetOneTimeProfile(ImportType.Orders);
importObjectsPage.PostInit(profile);
}
}
}
/// <summary>
/// Get or create "one-time" import profile.
/// </summary>
/// <param name="type">Import object type.</param>
/// <returns>Application "one-time" import profile.</returns>
public static ImportProfile GetOneTimeProfile(ImportType type)
{
ImportProfile onFlyProfile = App.Current.ImportProfilesKeeper.GetOneTimeProfile(type);
if (null == onFlyProfile)
{ // create new one time profile
onFlyProfile = new ImportProfile();
onFlyProfile.Type = type;
onFlyProfile.IsDefault = false;
onFlyProfile.IsOnTime = true;
string objectsName = GetImportObjectsName(onFlyProfile.Type);
onFlyProfile.Name = _GetUniqueName(onFlyProfile.Type, objectsName);
}
return(onFlyProfile);
}
private string[] GetDisabledDefaultFieldNames(ImportProfile profile)
{
switch (profile.EntityType)
{
case ImportEntityType.Product:
return new string[] { "Name", "Sku", "ManufacturerPartNumber", "Gtin", "SeName" };
case ImportEntityType.Category:
return new string[] { "Name", "SeName" };
case ImportEntityType.Customer:
return new string[] { "CustomerGuid", "Email" };
case ImportEntityType.NewsLetterSubscription:
return new string[] { "Email" };
default:
return new string[0];
}
}
public ImportProfile ConvertEnquiryToTableRow(DataRow dr)
{
ImportProfile importProfile = new ImportProfile();
importProfile.UserName = Convert.ToString(dr[0]);
importProfile.Password = Convert.ToString(dr[1]);
importProfile.ConsumerNo = Convert.ToString(dr[2]);
importProfile.RegionCode = Convert.ToString(dr[3]);
importProfile.Address = Convert.ToString(dr[4]);
importProfile.Email = Convert.ToString(dr[5]);
importProfile.City = Convert.ToString(dr[6]);
importProfile.State = Convert.ToString(dr[7]);
importProfile.ZipCode = Convert.ToString(dr[8]);
importProfile.IsAdmin = Convert.ToString(dr[9]);
return(importProfile);
}
public ActionResult Create()
{
var importProfile = new ImportProfile {
IsNew = true
};
_importProfileRepository.InsertAndCommit(importProfile);
//Create a new Directory with this profile
var rootPath = ConfigurationManager.AppSettings["ImportFolder"];
var importProfileFolder = string.Format("Profile{0}", importProfile.Id);
var importProfilePath = Path.Combine(rootPath, importProfileFolder);
if (!Directory.Exists(importProfilePath))
{
Directory.CreateDirectory(importProfilePath);
}
return(Json(new { Id = importProfile.Id }));
}
private static void LogResults(ImportProfile profile, DataImporterContext ctx)
{
using var psb = StringBuilderPool.Instance.Get(out var sb);
foreach (var item in ctx.Results)
{
var result = item.Value;
var entityName = item.Key.HasValue() ? item.Key : profile.EntityType.ToString();
sb.Clear();
sb.AppendLine();
sb.AppendLine(new string('-', 40));
sb.AppendLine("Object: " + entityName);
sb.AppendLine("Started: " + result.StartDateUtc.ToLocalTime());
sb.AppendLine("Finished: " + result.EndDateUtc.ToLocalTime());
sb.AppendLine("Duration: " + (result.EndDateUtc - result.StartDateUtc).ToString("g"));
sb.AppendLine("Rows total: " + result.TotalRecords);
sb.AppendLine("Rows processed: " + result.AffectedRecords);
sb.AppendLine("Rows imported: " + result.NewRecords);
sb.AppendLine("Rows updated: " + result.ModifiedRecords);
sb.AppendLine("Warnings: " + result.Warnings);
sb.Append("Errors: " + result.Errors);
ctx.Log.Info(sb.ToString());
foreach (var message in result.Messages)
{
if (message.MessageType == ImportMessageType.Error)
{
ctx.Log.Error(new Exception(message.FullMessage), message.ToString());
}
else if (message.MessageType == ImportMessageType.Warning)
{
ctx.Log.Warn(message.ToString());
}
else
{
ctx.Log.Info(message.ToString());
}
}
}
}
public virtual void DeleteImportProfile(ImportProfile profile)
{
if (profile == null)
{
throw new ArgumentNullException("profile");
}
var scheduleTaskId = profile.SchedulingTaskId;
var folder = profile.GetImportFolder();
_importProfileRepository.Delete(profile);
var scheduleTask = _scheduleTaskService.GetTaskById(scheduleTaskId);
_scheduleTaskService.DeleteTask(scheduleTask);
if (System.IO.Directory.Exists(folder))
{
FileSystemHelper.ClearDirectory(folder, true);
}
}
private string CreateLogHeader(ImportProfile profile, Multimap <string, ImportFile> files)
{
var executingCustomer = _services.WorkContext.CurrentCustomer;
using var psb = StringBuilderPool.Instance.Get(out var sb);
sb.AppendLine();
sb.AppendLine(new string('-', 40));
sb.AppendLine("Smartstore: v." + SmartstoreVersion.CurrentFullVersion);
sb.AppendLine("Import profile: " + profile.Name);
sb.AppendLine(profile.Id == 0 ? " (transient)" : $" (ID {profile.Id})");
foreach (var fileGroup in files)
{
var entityName = fileGroup.Key.NullEmpty() ?? profile.EntityType.ToString();
var fileNames = string.Join(", ", fileGroup.Value.Select(x => x.File.Name));
sb.AppendLine($"{entityName} files: {fileNames}");
}
sb.Append("Executed by: " + (executingCustomer.Email.HasValue() ? executingCustomer.Email : executingCustomer.SystemName));
return(sb.ToString());
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = (profile.Skip == 0 ? (int?)null : profile.Skip);
model.Take = (profile.Take == 0 ? (int?)null : profile.Take);
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(Services.Localization, Services.WorkContext);
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
{
model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo);
}
if (!forEdit)
{
return;
}
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// common configuration
var extraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData);
model.ExtraData.NumberOfPictures = extraData.NumberOfPictures;
// column mapping
model.AvailableSourceColumns = new List <ColumnMappingItemModel>();
model.AvailableEntityProperties = new List <ColumnMappingItemModel>();
model.AvailableKeyFieldNames = new List <SelectListItem>();
model.ColumnMappings = new List <ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
string[] disabledDefaultFieldNames = GetDisabledDefaultFieldNames(profile);
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping);
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Property = x.Key,
PropertyDescription = x.Value,
IsDefaultDisabled = IsDefaultValueDisabled(x.Key, x.Key, disabledDefaultFieldNames)
};
return(mapping);
})
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem {
Value = x, Text = x
};
if (x == "Id")
{
item.Text = T("Admin.Common.Entity.Fields.Id");
}
else if (allProperties.ContainsKey(x))
{
item.Text = allProperties[x];
}
return(item);
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = x.Value.MappedName,
Property = x.Key,
Default = x.Value.Default
};
if (x.Value.IgnoreProperty)
{
// explicitly ignore the property
mapping.Column = null;
mapping.Default = null;
}
mapping.PropertyDescription = GetPropertyDescription(allProperties, mapping.Property);
mapping.IsDefaultDisabled = IsDefaultValueDisabled(mapping.Column, mapping.Property, disabledDefaultFieldNames);
return(mapping);
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
{
return;
}
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceName(column.Name, out columnWithoutIndex, out columnIndex);
model.AvailableSourceColumns.Add(new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
PropertyDescription = GetPropertyDescription(allProperties, column.Name)
});
// auto map where field equals property name
if (!model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.IsEmpty())
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
}
if (kvp.Key.HasValue() && !model.ColumnMappings.Any(x => x.Property == kvp.Key))
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
PropertyDescription = kvp.Value,
IsDefaultDisabled = IsDefaultValueDisabled(column.Name, kvp.Key, disabledDefaultFieldNames)
});
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.PropertyDescription)
.ToList();
model.AvailableEntityProperties = model.AvailableEntityProperties
.OrderBy(x => x.PropertyDescription)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.PropertyDescription)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}
public async Task <IHttpActionResult> ImportFiles()
{
if (!Request.Content.IsMimeMultipartContent())
{
return(StatusCode(HttpStatusCode.UnsupportedMediaType));
}
ImportProfile profile = null;
string identifier = null;
var tempDir = FileSystemHelper.TempDirTenant(Guid.NewGuid().ToString());
var provider = new MultipartFormDataStreamProvider(tempDir);
try
{
await Request.Content.ReadAsMultipartAsync(provider);
}
catch (Exception ex)
{
FileSystemHelper.ClearDirectory(tempDir, true);
return(InternalServerError(ex));
}
// Find import profile.
if (provider.FormData.AllKeys.Contains("Id"))
{
identifier = provider.FormData.GetValues("Id").FirstOrDefault();
profile = _importProfileService.Value.GetImportProfileById(identifier.ToInt());
}
else if (provider.FormData.AllKeys.Contains("Name"))
{
identifier = provider.FormData.GetValues("Name").FirstOrDefault();
profile = _importProfileService.Value.GetImportProfileByName(identifier);
}
if (profile == null)
{
FileSystemHelper.ClearDirectory(tempDir, true);
throw Request.NotFoundException(WebApiGlobal.Error.EntityNotFound.FormatInvariant(identifier.NaIfEmpty()));
}
var startImport = false;
var deleteExisting = false;
var result = new List <UploadImportFile>();
var unzippedFiles = new List <MultipartFileData>();
var importFolder = profile.GetImportFolder(true, true);
var csvTypes = new string[] { ".csv", ".txt", ".tab" };
if (provider.FormData.AllKeys.Contains("deleteExisting"))
{
var strDeleteExisting = provider.FormData.GetValues("deleteExisting").FirstOrDefault();
deleteExisting = strDeleteExisting.HasValue() && strDeleteExisting.ToBool();
}
if (provider.FormData.AllKeys.Contains("startImport"))
{
var strStartImport = provider.FormData.GetValues("startImport").FirstOrDefault();
startImport = strStartImport.HasValue() && strStartImport.ToBool();
}
// Unzip files.
foreach (var file in provider.FileData)
{
var import = new UploadImportFile(file.Headers);
if (import.FileExtension.IsCaseInsensitiveEqual(".zip"))
{
var subDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
ZipFile.ExtractToDirectory(file.LocalFileName, subDir);
FileSystemHelper.DeleteFile(file.LocalFileName);
foreach (var unzippedFile in Directory.GetFiles(subDir, "*.*"))
{
var content = CloneHeaderContent(unzippedFile, file);
unzippedFiles.Add(new MultipartFileData(content.Headers, unzippedFile));
}
}
else
{
unzippedFiles.Add(new MultipartFileData(file.Headers, file.LocalFileName));
}
}
// Copy files to import folder.
if (unzippedFiles.Any())
{
using (_rwLock.GetWriteLock())
{
if (deleteExisting)
{
FileSystemHelper.ClearDirectory(importFolder, false);
}
foreach (var file in unzippedFiles)
{
var import = new UploadImportFile(file.Headers);
var destPath = Path.Combine(importFolder, import.FileName);
import.Exists = File.Exists(destPath);
switch (profile.FileType)
{
case ImportFileType.XLSX:
import.IsSupportedByProfile = import.FileExtension.IsCaseInsensitiveEqual(".xlsx");
break;
case ImportFileType.CSV:
import.IsSupportedByProfile = csvTypes.Contains(import.FileExtension, StringComparer.OrdinalIgnoreCase);
break;
}
import.Inserted = FileSystemHelper.CopyFile(file.LocalFileName, destPath);
result.Add(import);
}
}
}
FileSystemHelper.ClearDirectory(tempDir, true);
if (startImport)
{
var customer = _workContext.Value.CurrentCustomer;
if (_permissionService.Value.Authorize(Permissions.System.ScheduleTask.Execute, customer))
{
_taskScheduler.Value.RunSingleTask(profile.SchedulingTaskId, new Dictionary <string, string>
{
{ TaskExecutor.CurrentCustomerIdParamName, customer.Id.ToString() },
{ TaskExecutor.CurrentStoreIdParamName, _storeContext.Value.CurrentStore.Id.ToString() }
});
}
}
return(Ok(result.AsQueryable()));
}
private void SaveOrShowErrors(ImportProfile importProfile, bool isNew)
{
ValidationResult validation = new ImportProfileValidator().Validate(importProfile);
if (validation.IsValid == true)
{
if (isNew == true)
{
CustomerUtilities.CurrentCustomerFromXPODefaultSession().ImportProfiles.Add(importProfile);
}
importProfile.Save();
Response.Redirect(Request.RawUrl);
}
else
{
foreach (var error in validation.Errors)
{
ASPxTextEdit control = (ASPxTextEdit)ImportProfileFormLayout.FindNestedControlByFieldName(error.PropertyName);
ShowValidationErrorForControl(control, error.ErrorMessage);
}
}
}
private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken)
{
var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true);
var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken)
{
Request = request,
ProgressCallback = request.ProgressCallback,
UpdateOnly = profile.UpdateOnly,
KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(),
ImportDirectory = dir,
ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true),
ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData),
Languages = await _languageService.GetAllLanguagesAsync(true),
Stores = _services.StoreContext.GetAllStores().AsReadOnly(),
DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout))
};
// Relative paths for images always refer to the profile directory, not to its "Content" sub-directory.
executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue()
? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false)
: dir.Parent;
var context = new DataImporterContext
{
Request = request,
CancelToken = cancelToken,
ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(),
ExecuteContext = executeContext
};
var logFile = await dir.Parent.GetFileAsync("log.txt");
return(context, logFile);
}
private async Task SendCompletionEmail(ImportProfile profile, DataImporterContext ctx)
{
var emailAccount = _emailAccountService.GetDefaultEmailAccount();
var result = ctx.ExecuteContext.Result;
var store = _services.StoreContext.CurrentStore;
var storeInfo = $"{store.Name} ({store.Url})";
using var psb = StringBuilderPool.Instance.Get(out var body);
body.Append(T("Admin.DataExchange.Import.CompletedEmail.Body", storeInfo));
if (result.LastError.HasValue())
{
body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError);
}
body.Append("<p>");
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.TotalRows"), result.TotalRecords,
T("Admin.Common.Skipped"), result.SkippedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.NewRecords"), result.NewRecords,
T("Admin.Common.Updated"), result.ModifiedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.Errors"), result.Errors,
T("Admin.Common.Warnings"), result.Warnings);
body.Append("</p>");
var message = new MailMessage
{
From = new(emailAccount.Email, emailAccount.DisplayName),
Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Value.FormatInvariant(profile.Name),
Body = body.ToString()
};
if (_contactDataSettings.WebmasterEmailAddress.HasValue())
{
message.To.Add(new(_contactDataSettings.WebmasterEmailAddress));
}
if (!message.To.Any() && _contactDataSettings.CompanyEmailAddress.HasValue())
{
message.To.Add(new(_contactDataSettings.CompanyEmailAddress));
}
if (!message.To.Any())
{
message.To.Add(new(emailAccount.Email, emailAccount.DisplayName));
}
await using var client = await _mailService.ConnectAsync(emailAccount);
await client.SendAsync(message, ctx.CancelToken);
//_db.QueuedEmails.Add(new QueuedEmail
//{
// From = emailAccount.Email,
// To = message.To.First().Address,
// Subject = message.Subject,
// Body = message.Body,
// CreatedOnUtc = DateTime.UtcNow,
// EmailAccountId = emailAccount.Id,
// SendManually = true
//});
//await _db.SaveChangesAsync();
}
private void SetImportProfilePropertiesFromFormLayout(ASPxFormLayout formLayout, ImportProfile profile)
{
foreach (string fieldName in Utilities.GetPropertyNames<ImportProfile>())
{
ASPxEdit control = (ASPxEdit)formLayout.FindNestedControlByFieldName(fieldName);
// if control exists and part of this step's update process
if (control != null && control.Value != null)
{
PropertyInfo propertyInfo = profile.GetType().GetProperty(fieldName);
profile.SetMemberValue(fieldName, Convert.ChangeType(control.Value, propertyInfo.PropertyType));
}
}
}
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null)
{
if (profile == null)
{
if (model.Name.IsEmpty())
{
var defaultNames = T("Admin.DataExchange.Import.DefaultProfileNames").Text.SplitSafe(";");
model.Name = defaultNames.SafeGet((int)model.EntityType);
}
model.ExistingFileNames = new List <string>();
return;
}
model.Id = profile.Id;
model.Name = profile.Name;
model.EntityType = profile.EntityType;
model.Enabled = profile.Enabled;
model.Skip = profile.Skip;
model.Take = profile.Take;
model.UpdateOnly = profile.UpdateOnly;
model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray();
model.ScheduleTaskId = profile.SchedulingTaskId;
model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty();
model.IsTaskRunning = profile.ScheduleTask.IsRunning;
model.IsTaskEnabled = profile.ScheduleTask.Enabled;
model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath());
model.EntityTypeName = profile.EntityType.GetLocalizedEnum(_services.Localization, _services.WorkContext);
model.UnspecifiedString = T("Common.Unspecified");
model.AddNewString = T("Common.AddNew");
model.DeleteString = T("Common.Delete");
model.IgnoreString = T("Admin.Common.Ignore");
model.ExistingFileNames = profile.GetImportFiles()
.Select(x => Path.GetFileName(x))
.ToList();
if (profile.ResultInfo.HasValue())
{
model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo);
}
if (!forEdit)
{
return;
}
CsvConfiguration csvConfiguration = null;
if (profile.FileType == ImportFileType.CSV)
{
var csvConverter = new CsvConfigurationConverter();
csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration;
model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration);
}
else
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
// column mapping
model.AvailableSourceColumns = new List <ColumnMappingItemModel>();
model.AvailableEntityProperties = new List <SelectListItem>();
model.AvailableKeyFieldNames = new List <SelectListItem>();
model.ColumnMappings = new List <ColumnMappingItemModel>();
try
{
string[] availableKeyFieldNames = null;
var mapConverter = new ColumnMapConverter();
var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping);
var hasStoredMappings = (storedMap != null && storedMap.Mappings.Any());
var map = (invalidMap ?? storedMap) ?? new ColumnMap();
// property name to localized property name
var allProperties = _importService.GetImportableEntityProperties(profile.EntityType);
switch (profile.EntityType)
{
case ImportEntityType.Product:
availableKeyFieldNames = ProductImporter.SupportedKeyFields;
break;
case ImportEntityType.Category:
availableKeyFieldNames = CategoryImporter.SupportedKeyFields;
break;
case ImportEntityType.Customer:
availableKeyFieldNames = CustomerImporter.SupportedKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields;
break;
}
model.AvailableEntityProperties = allProperties
.Select(x => new SelectListItem {
Value = x.Key, Text = x.Value
})
.OrderBy(x => x.Text)
.ToList();
model.AvailableKeyFieldNames = availableKeyFieldNames
.Select(x =>
{
var item = new SelectListItem {
Value = x, Text = x
};
if (x == "Id")
{
item.Text = T("Admin.Common.Entity.Fields.Id");
}
else if (allProperties.ContainsKey(x))
{
item.Text = allProperties[x];
}
return(item);
})
.ToList();
model.ColumnMappings = map.Mappings
.Select(x =>
{
var mapping = new ColumnMappingItemModel
{
Column = (x.Value.Property.IsEmpty() ? null : x.Key),
Property = (x.Value.Property.IsEmpty() ? x.Key : x.Value.Property),
Default = x.Value.Default
};
// add localized to make mappings sortable
if (allProperties.ContainsKey(mapping.Property))
{
mapping.ColumnLocalized = allProperties[mapping.Property];
}
return(mapping);
})
.ToList();
var files = profile.GetImportFiles();
if (!files.Any())
{
return;
}
var filePath = files.First();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1);
foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue()))
{
string columnWithoutIndex, columnIndex;
ColumnMap.ParseSourceColumn(column.Name, out columnWithoutIndex, out columnIndex);
var mapModel = new ColumnMappingItemModel
{
Index = dataTable.Columns.IndexOf(column),
Column = column.Name,
ColumnWithoutIndex = columnWithoutIndex,
ColumnIndex = columnIndex,
ColumnLocalized = (allProperties.ContainsKey(column.Name) ? allProperties[column.Name] : column.Name)
};
model.AvailableSourceColumns.Add(mapModel);
// auto map where field equals property name
if (!hasStoredMappings && !model.ColumnMappings.Any(x => x.Column == column.Name))
{
var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name));
if (kvp.Key.HasValue())
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
ColumnLocalized = kvp.Value
});
}
else
{
var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name);
kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName));
if (kvp.Key.HasValue())
{
model.ColumnMappings.Add(new ColumnMappingItemModel
{
Column = column.Name,
Property = kvp.Key,
ColumnLocalized = kvp.Value
});
}
}
}
}
// sorting
model.AvailableSourceColumns = model.AvailableSourceColumns
.OrderBy(x => x.ColumnLocalized)
.ToList();
model.ColumnMappings = model.ColumnMappings
.OrderBy(x => x.ColumnLocalized)
.ToList();
}
}
catch (Exception exception)
{
NotifyError(exception, true, false);
}
}
public async Task <IQueryable <UploadImportFile> > ImportFiles()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw this.ExceptionUnsupportedMediaType();
}
ImportProfile profile = null;
string identifier = null;
var tempDir = FileSystemHelper.TempDir(Guid.NewGuid().ToString());
var provider = new MultipartFormDataStreamProvider(tempDir);
try
{
await Request.Content.ReadAsMultipartAsync(provider);
}
catch (Exception exception)
{
FileSystemHelper.ClearDirectory(tempDir, true);
throw this.ExceptionInternalServerError(exception);
}
// find import profile
if (provider.FormData.AllKeys.Contains("Id"))
{
identifier = provider.FormData.GetValues("Id").FirstOrDefault();
profile = _importProfileService.Value.GetImportProfileById(identifier.ToInt());
}
else if (provider.FormData.AllKeys.Contains("Name"))
{
identifier = provider.FormData.GetValues("Name").FirstOrDefault();
profile = _importProfileService.Value.GetImportProfileByName(identifier);
}
if (profile == null)
{
FileSystemHelper.ClearDirectory(tempDir, true);
throw this.ExceptionNotFound(WebApiGlobal.Error.EntityNotFound.FormatInvariant(identifier.NaIfEmpty()));
}
var deleteExisting = false;
var result = new List <UploadImportFile>();
var unzippedFiles = new List <MultipartFileData>();
var importFolder = profile.GetImportFolder(true, true);
var csvTypes = new string[] { ".csv", ".txt", ".tab" };
if (provider.FormData.AllKeys.Contains("deleteExisting"))
{
var strDeleteExisting = provider.FormData.GetValues("deleteExisting").FirstOrDefault();
deleteExisting = (strDeleteExisting.HasValue() && strDeleteExisting.ToBool());
}
// unzip files
foreach (var file in provider.FileData)
{
var import = new UploadImportFile(file.Headers);
if (import.FileExtension.IsCaseInsensitiveEqual(".zip"))
{
var subDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
ZipFile.ExtractToDirectory(file.LocalFileName, subDir);
FileSystemHelper.Delete(file.LocalFileName);
foreach (var unzippedFile in Directory.GetFiles(subDir, "*.*"))
{
var content = CloneHeaderContent(unzippedFile, file);
unzippedFiles.Add(new MultipartFileData(content.Headers, unzippedFile));
}
}
else
{
unzippedFiles.Add(new MultipartFileData(file.Headers, file.LocalFileName));
}
}
// copy files to import folder
if (unzippedFiles.Any())
{
using (_rwLock.GetWriteLock())
{
if (deleteExisting)
{
FileSystemHelper.ClearDirectory(importFolder, false);
}
foreach (var file in unzippedFiles)
{
var import = new UploadImportFile(file.Headers);
var destPath = Path.Combine(importFolder, import.FileName);
import.Exists = File.Exists(destPath);
switch (profile.FileType)
{
case ImportFileType.XLSX:
import.IsSupportedByProfile = import.FileExtension.IsCaseInsensitiveEqual(".xlsx");
break;
case ImportFileType.CSV:
import.IsSupportedByProfile = csvTypes.Contains(import.FileExtension, StringComparer.OrdinalIgnoreCase);
break;
}
import.Inserted = FileSystemHelper.Copy(file.LocalFileName, destPath);
result.Add(import);
}
}
}
FileSystemHelper.ClearDirectory(tempDir, true);
return(result.AsQueryable());
}
public virtual ImportProfile InsertImportProfile(string fileName, string name, ImportEntityType entityType)
{
Guard.ArgumentNotEmpty(() => fileName);
if (name.IsEmpty())
{
name = GetNewProfileName(entityType);
}
var task = new ScheduleTask
{
CronExpression = "0 */24 * * *",
Type = typeof(DataImportTask).AssemblyQualifiedNameWithoutVersion(),
Enabled = false,
StopOnError = false,
IsHidden = true
};
task.Name = string.Concat(name, " Task");
_scheduleTaskService.InsertTask(task);
var profile = new ImportProfile
{
Name = name,
EntityType = entityType,
Enabled = true,
SchedulingTaskId = task.Id
};
if (Path.GetExtension(fileName).IsCaseInsensitiveEqual(".xlsx"))
{
profile.FileType = ImportFileType.XLSX;
}
else
{
profile.FileType = ImportFileType.CSV;
}
string[] keyFieldNames = null;
switch (entityType)
{
case ImportEntityType.Product:
keyFieldNames = ProductImporter.DefaultKeyFields;
break;
case ImportEntityType.Category:
keyFieldNames = CategoryImporter.DefaultKeyFields;
break;
case ImportEntityType.Customer:
keyFieldNames = CustomerImporter.DefaultKeyFields;
break;
case ImportEntityType.NewsLetterSubscription:
keyFieldNames = NewsLetterSubscriptionImporter.DefaultKeyFields;
break;
}
profile.KeyFieldNames = string.Join(",", keyFieldNames);
profile.FolderName = SeoHelper.GetSeName(name, true, false)
.ToValidPath()
.Truncate(_dataExchangeSettings.MaxFileNameLength);
profile.FolderName = FileSystemHelper.CreateNonExistingDirectoryName(CommonHelper.MapPath("~/App_Data/ImportProfiles"), profile.FolderName);
_importProfileRepository.Insert(profile);
task.Alias = profile.Id.ToString();
_scheduleTaskService.UpdateTask(task);
_eventPublisher.EntityInserted(profile);
return(profile);
}
/// <summary>
/// Get log file path for an import profile
/// </summary>
/// <param name="profile">Import profile</param>
/// <returns>Log file path</returns>
public static string GetImportLogPath(this ImportProfile profile)
{
return(Path.Combine(profile.GetImportFolder(), "log.txt"));
}
/// <summary>
/// Read the XML file for a specific tree
/// </summary>
/// <param name="file">The path of the file to load</param>
/// <returns></returns>
public bool LoadXML(string file)
{
if (!System.IO.File.Exists(file))
{
return(false);
}
XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreComments = true;
XmlReader reader = XmlReader.Create(file, settings);
XmlDocument document = new XmlDocument();
document.Load(reader);
List <ImportProfile> avaliableCollections = new List <ImportProfile>();
float totalWeight = 0.0f;
// Read document
XmlElement root = document.DocumentElement;
foreach (XmlElement child in root.ChildNodes)
{
// Profile loading - TODO
if (child.Name == "Profile")
{
float weight = 1.0f;
if (child.HasAttribute("weight"))
{
float.TryParse(child.GetAttribute("weight"), out weight);
}
ImportProfile profile = new ImportProfile(weight);
totalWeight += weight;
foreach (XmlElement var in child.ChildNodes)
{
if (var.Name != "var")
{
Debug.LogError("Profile must only have child nodes of type 'var'");
}
if (!var.HasAttribute("name"))
{
Debug.LogError("Profile.var must have attribute 'name'");
}
float value = 0.0f;
float.TryParse(var.InnerXml, out value);
profile.vars.SetVar(var.GetAttribute("name"), value);
}
avaliableCollections.Add(profile);
}
// Load the actual tree
if (child.Name == "Decision")
{
if (child.ChildNodes.Count != 1)
{
Debug.LogError("Decision must have exactly 1 child");
}
rootDecision = ParseDecision((XmlElement)child.FirstChild);
}
}
Debug.Log("Read from '" + file + "'");
// Store all profiles for debug
agentProfiles = new VariableCollection[avaliableCollections.Count];
for (int i = 0; i < avaliableCollections.Count; ++i)
{
agentProfiles[i] = avaliableCollections[i].vars;
}
// Randomly assign a profile to this var
if (totalWeight == 0.0f)
{
agentProfile = new VariableCollection();
}
else
{
// Fetch a profile based on it's weight
float value = UnityEngine.Random.Range(0.0f, totalWeight);
for (int i = 0; i < avaliableCollections.Count; ++i)
{
if (value < avaliableCollections[i].weight)
{
agentProfile = avaliableCollections[i].vars;
break;
}
else
{
value -= avaliableCollections[i].weight;
}
}
}
return(true);
}
