protected override void Page_Load(object sender, EventArgs e) { RequireClientAuthentication(); importProfile = ObjectFromQueryString<ImportProfile>(); ImportProfileFormLayout.DataSourceID = null; if (importProfile != null && !IsPostBack) { ImportProfileFormLayout.DataSource = importProfile; ImportProfileFormLayout.DataBind(); } if (importProfile != null) { ImportProfileHeadline.HeaderText = "Edit"; DeleteImportProfileButton.ClientVisible = true; } else { ImportProfileHeadline.HeaderText = "Add new import profile"; } SetupRibbon(); base.Page_Load(sender, e); }
/// <summary> /// Gets import files for an import profile. /// </summary> /// <param name="profile">Import profile.</param> /// <param name="includeRelatedFiles">Whether to include related data files.</param> /// <returns>List of import files.</returns> public static List <ImportFile> GetImportFiles(this ImportProfile profile, bool includeRelatedFiles = true) { var result = new List <ImportFile>(); var importFolder = profile.GetImportFolder(true); if (System.IO.Directory.Exists(importFolder)) { var paths = System.IO.Directory.EnumerateFiles(importFolder, "*", SearchOption.TopDirectoryOnly); foreach (var path in paths) { var file = new ImportFile(path); if (!includeRelatedFiles && file.RelatedType.HasValue) { continue; } result.Add(file); } // Always main data files first. result = result.OrderBy(x => x.RelatedType).ThenBy(x => x.Path).ToList(); return(result); } return(result); }
/////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Internal execute. /// </summary> /// <param name="args">Commnad arguments.</param> protected override void _Execute(params object[] args) { _ClearGridSelectionOnParentPage(); ImportProfilesKeeper importProfileKeeper = _Application.ImportProfilesKeeper; ImportProfile profile = importProfileKeeper.GetDefaultProfile(Type); if (null != profile) { // start import with default profile _DoImport(profile); } else { // default profile absent - use on fly profile // create page to on fly creating profile Type type = typeof(FleetSetupWizardImportObjectsPage); _onFlyCreatingProfilePage = (FleetSetupWizardImportObjectsPage)Activator.CreateInstance(type); _onFlyCreatingProfilePage.Loaded += new RoutedEventHandler(_onFlyCreatingProfilePage_Loaded); // create empty profile _doesNewProfile = (null == importProfileKeeper.GetOneTimeProfile(Type)); ImportProfile onFlyProfile = CommonHelpers.GetOneTimeProfile(Type); // init page state _onFlyCreatingProfilePage.PostInit(onFlyProfile); // show page App.Current.MainWindow.PageFrame.Navigate(_onFlyCreatingProfilePage); } }
public IHttpActionResult Post([FromBody] ImportProfile profile) { var principal = (User)User.Identity; if (!principal.UserRights.Any(ur => ur.RoleId == (int)Roles.Administrator)) { return(new StatusCodeResult(HttpStatusCode.Forbidden, Request)); } if (Repository.FindBy <ImportProfile>(p => p.Name == profile.Name).Any()) { var warningMessage = "The ProfileName \"" + profile.Name + "\" already exists"; // Log.MonitoringLogger.Warn(warningMessage); ModelState.AddModelError("alreadyExists", warningMessage); return(BadRequest(ModelState)); } profile.CreatedBy = principal.Username; profile.CreatedDate = DateTime.Now; Repository.Add(profile); Repository.Save(); var message = "The Profile \"" + profile.Name + "\" has been added"; Log.MonitoringLogger.Info(message); return(Ok(message)); }
public virtual void UpdateImportProfile(ImportProfile profile) { if (profile == null) { throw new ArgumentNullException("profile"); } _importProfileRepository.Update(profile); }
public DataImportRequest(ImportProfile profile) { Guard.ArgumentNotNull(() => profile); Profile = profile; ProgressValueSetter = _voidProgressValueSetter; EntitiesToImport = new List <int>(); CustomData = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase); }
/// <summary> /// Do import process. /// </summary> /// <param name="profile">Profile to importing.</param> private void _DoImport(ImportProfile profile) { var manager = new ImportManager(); // Subscribe to order completed. If parent page is OptimizeAndEditPage - // call Ungeocoded orders page. manager.ImportCompleted += new ImportCompletedEventHandler(_ImportCompleted); manager.ImportAsync(ParentPage, profile, App.Current.CurrentDate); }
/// <summary> /// Get folder for import files /// </summary> /// <param name="profile">Import profile</param> /// <returns>Folder path</returns> public static string GetImportFolder(this ImportProfile profile, bool content = false, bool create = false) { var path = CommonHelper.MapPath(string.Concat("~/App_Data/ImportProfiles/", profile.FolderName, content ? "/Content" : "")); if (create && !System.IO.Directory.Exists(path)) { System.IO.Directory.CreateDirectory(path); } return(path); }
/// <summary> /// Get folder for import files /// </summary> /// <param name="profile">Import profile</param> /// <returns>Folder path</returns> public static string GetImportFolder(this ImportProfile profile, bool content = false, bool create = false) { var basePath = DataSettings.Current.TenantPath + "/ImportProfiles/"; var path = CommonHelper.MapPath(string.Concat(basePath, profile.FolderName, content ? "/Content" : "")); if (create && !System.IO.Directory.Exists(path)) { System.IO.Directory.CreateDirectory(path); } return(path); }
/// <summary> /// Gets import files for an import profile /// </summary> /// <param name="profile">Import profile</param> /// <returns>List of file paths</returns> public static List <string> GetImportFiles(this ImportProfile profile) { var importFolder = profile.GetImportFolder(true); if (System.IO.Directory.Exists(importFolder)) { return(System.IO.Directory.EnumerateFiles(importFolder, "*", SearchOption.TopDirectoryOnly) .OrderBy(x => x) .ToList()); } return(new List <string>()); }
protected void SaveButton_Click(object sender, EventArgs e) { if (importProfile == null) { importProfile = new ImportProfile(); SetImportProfilePropertiesFromFormLayout(ImportProfileFormLayout, importProfile); SaveOrShowErrors(importProfile, true); } else { SetImportProfilePropertiesFromFormLayout(ImportProfileFormLayout, importProfile); SaveOrShowErrors(importProfile, false); } }
/// <summary> /// Special initializing of wizard pages. /// </summary> protected override void PostInit() { base.PostInit(); foreach (WizardPageBase page in Pages) { FleetSetupWizardImportObjectsPage importObjectsPage = page as FleetSetupWizardImportObjectsPage; if (importObjectsPage != null) { ImportProfile profile = CommonHelpers.GetOneTimeProfile(ImportType.Orders); importObjectsPage.PostInit(profile); } } }
/// <summary> /// Get or create "one-time" import profile. /// </summary> /// <param name="type">Import object type.</param> /// <returns>Application "one-time" import profile.</returns> public static ImportProfile GetOneTimeProfile(ImportType type) { ImportProfile onFlyProfile = App.Current.ImportProfilesKeeper.GetOneTimeProfile(type); if (null == onFlyProfile) { // create new one time profile onFlyProfile = new ImportProfile(); onFlyProfile.Type = type; onFlyProfile.IsDefault = false; onFlyProfile.IsOnTime = true; string objectsName = GetImportObjectsName(onFlyProfile.Type); onFlyProfile.Name = _GetUniqueName(onFlyProfile.Type, objectsName); } return(onFlyProfile); }
private string[] GetDisabledDefaultFieldNames(ImportProfile profile) { switch (profile.EntityType) { case ImportEntityType.Product: return new string[] { "Name", "Sku", "ManufacturerPartNumber", "Gtin", "SeName" }; case ImportEntityType.Category: return new string[] { "Name", "SeName" }; case ImportEntityType.Customer: return new string[] { "CustomerGuid", "Email" }; case ImportEntityType.NewsLetterSubscription: return new string[] { "Email" }; default: return new string[0]; } }
public ImportProfile ConvertEnquiryToTableRow(DataRow dr) { ImportProfile importProfile = new ImportProfile(); importProfile.UserName = Convert.ToString(dr[0]); importProfile.Password = Convert.ToString(dr[1]); importProfile.ConsumerNo = Convert.ToString(dr[2]); importProfile.RegionCode = Convert.ToString(dr[3]); importProfile.Address = Convert.ToString(dr[4]); importProfile.Email = Convert.ToString(dr[5]); importProfile.City = Convert.ToString(dr[6]); importProfile.State = Convert.ToString(dr[7]); importProfile.ZipCode = Convert.ToString(dr[8]); importProfile.IsAdmin = Convert.ToString(dr[9]); return(importProfile); }
public ActionResult Create() { var importProfile = new ImportProfile { IsNew = true }; _importProfileRepository.InsertAndCommit(importProfile); //Create a new Directory with this profile var rootPath = ConfigurationManager.AppSettings["ImportFolder"]; var importProfileFolder = string.Format("Profile{0}", importProfile.Id); var importProfilePath = Path.Combine(rootPath, importProfileFolder); if (!Directory.Exists(importProfilePath)) { Directory.CreateDirectory(importProfilePath); } return(Json(new { Id = importProfile.Id })); }
private static void LogResults(ImportProfile profile, DataImporterContext ctx) { using var psb = StringBuilderPool.Instance.Get(out var sb); foreach (var item in ctx.Results) { var result = item.Value; var entityName = item.Key.HasValue() ? item.Key : profile.EntityType.ToString(); sb.Clear(); sb.AppendLine(); sb.AppendLine(new string('-', 40)); sb.AppendLine("Object: " + entityName); sb.AppendLine("Started: " + result.StartDateUtc.ToLocalTime()); sb.AppendLine("Finished: " + result.EndDateUtc.ToLocalTime()); sb.AppendLine("Duration: " + (result.EndDateUtc - result.StartDateUtc).ToString("g")); sb.AppendLine("Rows total: " + result.TotalRecords); sb.AppendLine("Rows processed: " + result.AffectedRecords); sb.AppendLine("Rows imported: " + result.NewRecords); sb.AppendLine("Rows updated: " + result.ModifiedRecords); sb.AppendLine("Warnings: " + result.Warnings); sb.Append("Errors: " + result.Errors); ctx.Log.Info(sb.ToString()); foreach (var message in result.Messages) { if (message.MessageType == ImportMessageType.Error) { ctx.Log.Error(new Exception(message.FullMessage), message.ToString()); } else if (message.MessageType == ImportMessageType.Warning) { ctx.Log.Warn(message.ToString()); } else { ctx.Log.Info(message.ToString()); } } } }
public virtual void DeleteImportProfile(ImportProfile profile) { if (profile == null) { throw new ArgumentNullException("profile"); } var scheduleTaskId = profile.SchedulingTaskId; var folder = profile.GetImportFolder(); _importProfileRepository.Delete(profile); var scheduleTask = _scheduleTaskService.GetTaskById(scheduleTaskId); _scheduleTaskService.DeleteTask(scheduleTask); if (System.IO.Directory.Exists(folder)) { FileSystemHelper.ClearDirectory(folder, true); } }
private string CreateLogHeader(ImportProfile profile, Multimap <string, ImportFile> files) { var executingCustomer = _services.WorkContext.CurrentCustomer; using var psb = StringBuilderPool.Instance.Get(out var sb); sb.AppendLine(); sb.AppendLine(new string('-', 40)); sb.AppendLine("Smartstore: v." + SmartstoreVersion.CurrentFullVersion); sb.AppendLine("Import profile: " + profile.Name); sb.AppendLine(profile.Id == 0 ? " (transient)" : $" (ID {profile.Id})"); foreach (var fileGroup in files) { var entityName = fileGroup.Key.NullEmpty() ?? profile.EntityType.ToString(); var fileNames = string.Join(", ", fileGroup.Value.Select(x => x.File.Name)); sb.AppendLine($"{entityName} files: {fileNames}"); } sb.Append("Executed by: " + (executingCustomer.Email.HasValue() ? executingCustomer.Email : executingCustomer.SystemName)); return(sb.ToString()); }
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null) { model.Id = profile.Id; model.Name = profile.Name; model.EntityType = profile.EntityType; model.Enabled = profile.Enabled; model.Skip = (profile.Skip == 0 ? (int?)null : profile.Skip); model.Take = (profile.Take == 0 ? (int?)null : profile.Take); model.UpdateOnly = profile.UpdateOnly; model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray(); model.ScheduleTaskId = profile.SchedulingTaskId; model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty(); model.IsTaskRunning = profile.ScheduleTask.IsRunning; model.IsTaskEnabled = profile.ScheduleTask.Enabled; model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath()); model.EntityTypeName = profile.EntityType.GetLocalizedEnum(Services.Localization, Services.WorkContext); model.ExistingFileNames = profile.GetImportFiles() .Select(x => Path.GetFileName(x)) .ToList(); if (profile.ResultInfo.HasValue()) { model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo); } if (!forEdit) { return; } CsvConfiguration csvConfiguration = null; if (profile.FileType == ImportFileType.CSV) { var csvConverter = new CsvConfigurationConverter(); csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration; model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration); } else { csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration; } // common configuration var extraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData); model.ExtraData.NumberOfPictures = extraData.NumberOfPictures; // column mapping model.AvailableSourceColumns = new List <ColumnMappingItemModel>(); model.AvailableEntityProperties = new List <ColumnMappingItemModel>(); model.AvailableKeyFieldNames = new List <SelectListItem>(); model.ColumnMappings = new List <ColumnMappingItemModel>(); try { string[] availableKeyFieldNames = null; string[] disabledDefaultFieldNames = GetDisabledDefaultFieldNames(profile); var mapConverter = new ColumnMapConverter(); var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping); var map = (invalidMap ?? storedMap) ?? new ColumnMap(); // property name to localized property name var allProperties = _importProfileService.GetImportableEntityProperties(profile.EntityType); switch (profile.EntityType) { case ImportEntityType.Product: availableKeyFieldNames = ProductImporter.SupportedKeyFields; break; case ImportEntityType.Category: availableKeyFieldNames = CategoryImporter.SupportedKeyFields; break; case ImportEntityType.Customer: availableKeyFieldNames = CustomerImporter.SupportedKeyFields; break; case ImportEntityType.NewsLetterSubscription: availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields; break; } model.AvailableEntityProperties = allProperties .Select(x => { var mapping = new ColumnMappingItemModel { Property = x.Key, PropertyDescription = x.Value, IsDefaultDisabled = IsDefaultValueDisabled(x.Key, x.Key, disabledDefaultFieldNames) }; return(mapping); }) .ToList(); model.AvailableKeyFieldNames = availableKeyFieldNames .Select(x => { var item = new SelectListItem { Value = x, Text = x }; if (x == "Id") { item.Text = T("Admin.Common.Entity.Fields.Id"); } else if (allProperties.ContainsKey(x)) { item.Text = allProperties[x]; } return(item); }) .ToList(); model.ColumnMappings = map.Mappings .Select(x => { var mapping = new ColumnMappingItemModel { Column = x.Value.MappedName, Property = x.Key, Default = x.Value.Default }; if (x.Value.IgnoreProperty) { // explicitly ignore the property mapping.Column = null; mapping.Default = null; } mapping.PropertyDescription = GetPropertyDescription(allProperties, mapping.Property); mapping.IsDefaultDisabled = IsDefaultValueDisabled(mapping.Column, mapping.Property, disabledDefaultFieldNames); return(mapping); }) .ToList(); var files = profile.GetImportFiles(); if (!files.Any()) { return; } var filePath = files.First(); using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1); foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue())) { string columnWithoutIndex, columnIndex; ColumnMap.ParseSourceName(column.Name, out columnWithoutIndex, out columnIndex); model.AvailableSourceColumns.Add(new ColumnMappingItemModel { Index = dataTable.Columns.IndexOf(column), Column = column.Name, ColumnWithoutIndex = columnWithoutIndex, ColumnIndex = columnIndex, PropertyDescription = GetPropertyDescription(allProperties, column.Name) }); // auto map where field equals property name if (!model.ColumnMappings.Any(x => x.Column == column.Name)) { var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name)); if (kvp.Key.IsEmpty()) { var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name); kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName)); } if (kvp.Key.HasValue() && !model.ColumnMappings.Any(x => x.Property == kvp.Key)) { model.ColumnMappings.Add(new ColumnMappingItemModel { Column = column.Name, Property = kvp.Key, PropertyDescription = kvp.Value, IsDefaultDisabled = IsDefaultValueDisabled(column.Name, kvp.Key, disabledDefaultFieldNames) }); } } } // sorting model.AvailableSourceColumns = model.AvailableSourceColumns .OrderBy(x => x.PropertyDescription) .ToList(); model.AvailableEntityProperties = model.AvailableEntityProperties .OrderBy(x => x.PropertyDescription) .ToList(); model.ColumnMappings = model.ColumnMappings .OrderBy(x => x.PropertyDescription) .ToList(); } } catch (Exception exception) { NotifyError(exception, true, false); } }
public async Task <IHttpActionResult> ImportFiles() { if (!Request.Content.IsMimeMultipartContent()) { return(StatusCode(HttpStatusCode.UnsupportedMediaType)); } ImportProfile profile = null; string identifier = null; var tempDir = FileSystemHelper.TempDirTenant(Guid.NewGuid().ToString()); var provider = new MultipartFormDataStreamProvider(tempDir); try { await Request.Content.ReadAsMultipartAsync(provider); } catch (Exception ex) { FileSystemHelper.ClearDirectory(tempDir, true); return(InternalServerError(ex)); } // Find import profile. if (provider.FormData.AllKeys.Contains("Id")) { identifier = provider.FormData.GetValues("Id").FirstOrDefault(); profile = _importProfileService.Value.GetImportProfileById(identifier.ToInt()); } else if (provider.FormData.AllKeys.Contains("Name")) { identifier = provider.FormData.GetValues("Name").FirstOrDefault(); profile = _importProfileService.Value.GetImportProfileByName(identifier); } if (profile == null) { FileSystemHelper.ClearDirectory(tempDir, true); throw Request.NotFoundException(WebApiGlobal.Error.EntityNotFound.FormatInvariant(identifier.NaIfEmpty())); } var startImport = false; var deleteExisting = false; var result = new List <UploadImportFile>(); var unzippedFiles = new List <MultipartFileData>(); var importFolder = profile.GetImportFolder(true, true); var csvTypes = new string[] { ".csv", ".txt", ".tab" }; if (provider.FormData.AllKeys.Contains("deleteExisting")) { var strDeleteExisting = provider.FormData.GetValues("deleteExisting").FirstOrDefault(); deleteExisting = strDeleteExisting.HasValue() && strDeleteExisting.ToBool(); } if (provider.FormData.AllKeys.Contains("startImport")) { var strStartImport = provider.FormData.GetValues("startImport").FirstOrDefault(); startImport = strStartImport.HasValue() && strStartImport.ToBool(); } // Unzip files. foreach (var file in provider.FileData) { var import = new UploadImportFile(file.Headers); if (import.FileExtension.IsCaseInsensitiveEqual(".zip")) { var subDir = Path.Combine(tempDir, Guid.NewGuid().ToString()); ZipFile.ExtractToDirectory(file.LocalFileName, subDir); FileSystemHelper.DeleteFile(file.LocalFileName); foreach (var unzippedFile in Directory.GetFiles(subDir, "*.*")) { var content = CloneHeaderContent(unzippedFile, file); unzippedFiles.Add(new MultipartFileData(content.Headers, unzippedFile)); } } else { unzippedFiles.Add(new MultipartFileData(file.Headers, file.LocalFileName)); } } // Copy files to import folder. if (unzippedFiles.Any()) { using (_rwLock.GetWriteLock()) { if (deleteExisting) { FileSystemHelper.ClearDirectory(importFolder, false); } foreach (var file in unzippedFiles) { var import = new UploadImportFile(file.Headers); var destPath = Path.Combine(importFolder, import.FileName); import.Exists = File.Exists(destPath); switch (profile.FileType) { case ImportFileType.XLSX: import.IsSupportedByProfile = import.FileExtension.IsCaseInsensitiveEqual(".xlsx"); break; case ImportFileType.CSV: import.IsSupportedByProfile = csvTypes.Contains(import.FileExtension, StringComparer.OrdinalIgnoreCase); break; } import.Inserted = FileSystemHelper.CopyFile(file.LocalFileName, destPath); result.Add(import); } } } FileSystemHelper.ClearDirectory(tempDir, true); if (startImport) { var customer = _workContext.Value.CurrentCustomer; if (_permissionService.Value.Authorize(Permissions.System.ScheduleTask.Execute, customer)) { _taskScheduler.Value.RunSingleTask(profile.SchedulingTaskId, new Dictionary <string, string> { { TaskExecutor.CurrentCustomerIdParamName, customer.Id.ToString() }, { TaskExecutor.CurrentStoreIdParamName, _storeContext.Value.CurrentStore.Id.ToString() } }); } } return(Ok(result.AsQueryable())); }
private void SaveOrShowErrors(ImportProfile importProfile, bool isNew) { ValidationResult validation = new ImportProfileValidator().Validate(importProfile); if (validation.IsValid == true) { if (isNew == true) { CustomerUtilities.CurrentCustomerFromXPODefaultSession().ImportProfiles.Add(importProfile); } importProfile.Save(); Response.Redirect(Request.RawUrl); } else { foreach (var error in validation.Errors) { ASPxTextEdit control = (ASPxTextEdit)ImportProfileFormLayout.FindNestedControlByFieldName(error.PropertyName); ShowValidationErrorForControl(control, error.ErrorMessage); } } }
private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken) { var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true); var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken) { Request = request, ProgressCallback = request.ProgressCallback, UpdateOnly = profile.UpdateOnly, KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(), ImportDirectory = dir, ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true), ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData), Languages = await _languageService.GetAllLanguagesAsync(true), Stores = _services.StoreContext.GetAllStores().AsReadOnly(), DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout)) }; // Relative paths for images always refer to the profile directory, not to its "Content" sub-directory. executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue() ? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false) : dir.Parent; var context = new DataImporterContext { Request = request, CancelToken = cancelToken, ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(), ExecuteContext = executeContext }; var logFile = await dir.Parent.GetFileAsync("log.txt"); return(context, logFile); }
private async Task SendCompletionEmail(ImportProfile profile, DataImporterContext ctx) { var emailAccount = _emailAccountService.GetDefaultEmailAccount(); var result = ctx.ExecuteContext.Result; var store = _services.StoreContext.CurrentStore; var storeInfo = $"{store.Name} ({store.Url})"; using var psb = StringBuilderPool.Instance.Get(out var body); body.Append(T("Admin.DataExchange.Import.CompletedEmail.Body", storeInfo)); if (result.LastError.HasValue()) { body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError); } body.Append("<p>"); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.TotalRows"), result.TotalRecords, T("Admin.Common.Skipped"), result.SkippedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.NewRecords"), result.NewRecords, T("Admin.Common.Updated"), result.ModifiedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.Errors"), result.Errors, T("Admin.Common.Warnings"), result.Warnings); body.Append("</p>"); var message = new MailMessage { From = new(emailAccount.Email, emailAccount.DisplayName), Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Value.FormatInvariant(profile.Name), Body = body.ToString() }; if (_contactDataSettings.WebmasterEmailAddress.HasValue()) { message.To.Add(new(_contactDataSettings.WebmasterEmailAddress)); } if (!message.To.Any() && _contactDataSettings.CompanyEmailAddress.HasValue()) { message.To.Add(new(_contactDataSettings.CompanyEmailAddress)); } if (!message.To.Any()) { message.To.Add(new(emailAccount.Email, emailAccount.DisplayName)); } await using var client = await _mailService.ConnectAsync(emailAccount); await client.SendAsync(message, ctx.CancelToken); //_db.QueuedEmails.Add(new QueuedEmail //{ // From = emailAccount.Email, // To = message.To.First().Address, // Subject = message.Subject, // Body = message.Body, // CreatedOnUtc = DateTime.UtcNow, // EmailAccountId = emailAccount.Id, // SendManually = true //}); //await _db.SaveChangesAsync(); }
private void SetImportProfilePropertiesFromFormLayout(ASPxFormLayout formLayout, ImportProfile profile) { foreach (string fieldName in Utilities.GetPropertyNames<ImportProfile>()) { ASPxEdit control = (ASPxEdit)formLayout.FindNestedControlByFieldName(fieldName); // if control exists and part of this step's update process if (control != null && control.Value != null) { PropertyInfo propertyInfo = profile.GetType().GetProperty(fieldName); profile.SetMemberValue(fieldName, Convert.ChangeType(control.Value, propertyInfo.PropertyType)); } } }
private void PrepareProfileModel(ImportProfileModel model, ImportProfile profile, bool forEdit, ColumnMap invalidMap = null) { if (profile == null) { if (model.Name.IsEmpty()) { var defaultNames = T("Admin.DataExchange.Import.DefaultProfileNames").Text.SplitSafe(";"); model.Name = defaultNames.SafeGet((int)model.EntityType); } model.ExistingFileNames = new List <string>(); return; } model.Id = profile.Id; model.Name = profile.Name; model.EntityType = profile.EntityType; model.Enabled = profile.Enabled; model.Skip = profile.Skip; model.Take = profile.Take; model.UpdateOnly = profile.UpdateOnly; model.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").Distinct().ToArray(); model.ScheduleTaskId = profile.SchedulingTaskId; model.ScheduleTaskName = profile.ScheduleTask.Name.NaIfEmpty(); model.IsTaskRunning = profile.ScheduleTask.IsRunning; model.IsTaskEnabled = profile.ScheduleTask.Enabled; model.LogFileExists = System.IO.File.Exists(profile.GetImportLogPath()); model.EntityTypeName = profile.EntityType.GetLocalizedEnum(_services.Localization, _services.WorkContext); model.UnspecifiedString = T("Common.Unspecified"); model.AddNewString = T("Common.AddNew"); model.DeleteString = T("Common.Delete"); model.IgnoreString = T("Admin.Common.Ignore"); model.ExistingFileNames = profile.GetImportFiles() .Select(x => Path.GetFileName(x)) .ToList(); if (profile.ResultInfo.HasValue()) { model.ImportResult = XmlHelper.Deserialize <SerializableImportResult>(profile.ResultInfo); } if (!forEdit) { return; } CsvConfiguration csvConfiguration = null; if (profile.FileType == ImportFileType.CSV) { var csvConverter = new CsvConfigurationConverter(); csvConfiguration = csvConverter.ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration; model.CsvConfiguration = new CsvConfigurationModel(csvConfiguration); } else { csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration; } // column mapping model.AvailableSourceColumns = new List <ColumnMappingItemModel>(); model.AvailableEntityProperties = new List <SelectListItem>(); model.AvailableKeyFieldNames = new List <SelectListItem>(); model.ColumnMappings = new List <ColumnMappingItemModel>(); try { string[] availableKeyFieldNames = null; var mapConverter = new ColumnMapConverter(); var storedMap = mapConverter.ConvertFrom <ColumnMap>(profile.ColumnMapping); var hasStoredMappings = (storedMap != null && storedMap.Mappings.Any()); var map = (invalidMap ?? storedMap) ?? new ColumnMap(); // property name to localized property name var allProperties = _importService.GetImportableEntityProperties(profile.EntityType); switch (profile.EntityType) { case ImportEntityType.Product: availableKeyFieldNames = ProductImporter.SupportedKeyFields; break; case ImportEntityType.Category: availableKeyFieldNames = CategoryImporter.SupportedKeyFields; break; case ImportEntityType.Customer: availableKeyFieldNames = CustomerImporter.SupportedKeyFields; break; case ImportEntityType.NewsLetterSubscription: availableKeyFieldNames = NewsLetterSubscriptionImporter.SupportedKeyFields; break; } model.AvailableEntityProperties = allProperties .Select(x => new SelectListItem { Value = x.Key, Text = x.Value }) .OrderBy(x => x.Text) .ToList(); model.AvailableKeyFieldNames = availableKeyFieldNames .Select(x => { var item = new SelectListItem { Value = x, Text = x }; if (x == "Id") { item.Text = T("Admin.Common.Entity.Fields.Id"); } else if (allProperties.ContainsKey(x)) { item.Text = allProperties[x]; } return(item); }) .ToList(); model.ColumnMappings = map.Mappings .Select(x => { var mapping = new ColumnMappingItemModel { Column = (x.Value.Property.IsEmpty() ? null : x.Key), Property = (x.Value.Property.IsEmpty() ? x.Key : x.Value.Property), Default = x.Value.Default }; // add localized to make mappings sortable if (allProperties.ContainsKey(mapping.Property)) { mapping.ColumnLocalized = allProperties[mapping.Property]; } return(mapping); }) .ToList(); var files = profile.GetImportFiles(); if (!files.Any()) { return; } var filePath = files.First(); using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var dataTable = LightweightDataTable.FromFile(Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, 0, 1); foreach (var column in dataTable.Columns.Where(x => x.Name.HasValue())) { string columnWithoutIndex, columnIndex; ColumnMap.ParseSourceColumn(column.Name, out columnWithoutIndex, out columnIndex); var mapModel = new ColumnMappingItemModel { Index = dataTable.Columns.IndexOf(column), Column = column.Name, ColumnWithoutIndex = columnWithoutIndex, ColumnIndex = columnIndex, ColumnLocalized = (allProperties.ContainsKey(column.Name) ? allProperties[column.Name] : column.Name) }; model.AvailableSourceColumns.Add(mapModel); // auto map where field equals property name if (!hasStoredMappings && !model.ColumnMappings.Any(x => x.Column == column.Name)) { var kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(column.Name)); if (kvp.Key.HasValue()) { model.ColumnMappings.Add(new ColumnMappingItemModel { Column = column.Name, Property = kvp.Key, ColumnLocalized = kvp.Value }); } else { var alternativeName = LightweightDataTable.GetAlternativeColumnNameFor(column.Name); kvp = allProperties.FirstOrDefault(x => x.Key.IsCaseInsensitiveEqual(alternativeName)); if (kvp.Key.HasValue()) { model.ColumnMappings.Add(new ColumnMappingItemModel { Column = column.Name, Property = kvp.Key, ColumnLocalized = kvp.Value }); } } } } // sorting model.AvailableSourceColumns = model.AvailableSourceColumns .OrderBy(x => x.ColumnLocalized) .ToList(); model.ColumnMappings = model.ColumnMappings .OrderBy(x => x.ColumnLocalized) .ToList(); } } catch (Exception exception) { NotifyError(exception, true, false); } }
public async Task <IQueryable <UploadImportFile> > ImportFiles() { if (!Request.Content.IsMimeMultipartContent()) { throw this.ExceptionUnsupportedMediaType(); } ImportProfile profile = null; string identifier = null; var tempDir = FileSystemHelper.TempDir(Guid.NewGuid().ToString()); var provider = new MultipartFormDataStreamProvider(tempDir); try { await Request.Content.ReadAsMultipartAsync(provider); } catch (Exception exception) { FileSystemHelper.ClearDirectory(tempDir, true); throw this.ExceptionInternalServerError(exception); } // find import profile if (provider.FormData.AllKeys.Contains("Id")) { identifier = provider.FormData.GetValues("Id").FirstOrDefault(); profile = _importProfileService.Value.GetImportProfileById(identifier.ToInt()); } else if (provider.FormData.AllKeys.Contains("Name")) { identifier = provider.FormData.GetValues("Name").FirstOrDefault(); profile = _importProfileService.Value.GetImportProfileByName(identifier); } if (profile == null) { FileSystemHelper.ClearDirectory(tempDir, true); throw this.ExceptionNotFound(WebApiGlobal.Error.EntityNotFound.FormatInvariant(identifier.NaIfEmpty())); } var deleteExisting = false; var result = new List <UploadImportFile>(); var unzippedFiles = new List <MultipartFileData>(); var importFolder = profile.GetImportFolder(true, true); var csvTypes = new string[] { ".csv", ".txt", ".tab" }; if (provider.FormData.AllKeys.Contains("deleteExisting")) { var strDeleteExisting = provider.FormData.GetValues("deleteExisting").FirstOrDefault(); deleteExisting = (strDeleteExisting.HasValue() && strDeleteExisting.ToBool()); } // unzip files foreach (var file in provider.FileData) { var import = new UploadImportFile(file.Headers); if (import.FileExtension.IsCaseInsensitiveEqual(".zip")) { var subDir = Path.Combine(tempDir, Guid.NewGuid().ToString()); ZipFile.ExtractToDirectory(file.LocalFileName, subDir); FileSystemHelper.Delete(file.LocalFileName); foreach (var unzippedFile in Directory.GetFiles(subDir, "*.*")) { var content = CloneHeaderContent(unzippedFile, file); unzippedFiles.Add(new MultipartFileData(content.Headers, unzippedFile)); } } else { unzippedFiles.Add(new MultipartFileData(file.Headers, file.LocalFileName)); } } // copy files to import folder if (unzippedFiles.Any()) { using (_rwLock.GetWriteLock()) { if (deleteExisting) { FileSystemHelper.ClearDirectory(importFolder, false); } foreach (var file in unzippedFiles) { var import = new UploadImportFile(file.Headers); var destPath = Path.Combine(importFolder, import.FileName); import.Exists = File.Exists(destPath); switch (profile.FileType) { case ImportFileType.XLSX: import.IsSupportedByProfile = import.FileExtension.IsCaseInsensitiveEqual(".xlsx"); break; case ImportFileType.CSV: import.IsSupportedByProfile = csvTypes.Contains(import.FileExtension, StringComparer.OrdinalIgnoreCase); break; } import.Inserted = FileSystemHelper.Copy(file.LocalFileName, destPath); result.Add(import); } } } FileSystemHelper.ClearDirectory(tempDir, true); return(result.AsQueryable()); }
public virtual ImportProfile InsertImportProfile(string fileName, string name, ImportEntityType entityType) { Guard.ArgumentNotEmpty(() => fileName); if (name.IsEmpty()) { name = GetNewProfileName(entityType); } var task = new ScheduleTask { CronExpression = "0 */24 * * *", Type = typeof(DataImportTask).AssemblyQualifiedNameWithoutVersion(), Enabled = false, StopOnError = false, IsHidden = true }; task.Name = string.Concat(name, " Task"); _scheduleTaskService.InsertTask(task); var profile = new ImportProfile { Name = name, EntityType = entityType, Enabled = true, SchedulingTaskId = task.Id }; if (Path.GetExtension(fileName).IsCaseInsensitiveEqual(".xlsx")) { profile.FileType = ImportFileType.XLSX; } else { profile.FileType = ImportFileType.CSV; } string[] keyFieldNames = null; switch (entityType) { case ImportEntityType.Product: keyFieldNames = ProductImporter.DefaultKeyFields; break; case ImportEntityType.Category: keyFieldNames = CategoryImporter.DefaultKeyFields; break; case ImportEntityType.Customer: keyFieldNames = CustomerImporter.DefaultKeyFields; break; case ImportEntityType.NewsLetterSubscription: keyFieldNames = NewsLetterSubscriptionImporter.DefaultKeyFields; break; } profile.KeyFieldNames = string.Join(",", keyFieldNames); profile.FolderName = SeoHelper.GetSeName(name, true, false) .ToValidPath() .Truncate(_dataExchangeSettings.MaxFileNameLength); profile.FolderName = FileSystemHelper.CreateNonExistingDirectoryName(CommonHelper.MapPath("~/App_Data/ImportProfiles"), profile.FolderName); _importProfileRepository.Insert(profile); task.Alias = profile.Id.ToString(); _scheduleTaskService.UpdateTask(task); _eventPublisher.EntityInserted(profile); return(profile); }
/// <summary> /// Get log file path for an import profile /// </summary> /// <param name="profile">Import profile</param> /// <returns>Log file path</returns> public static string GetImportLogPath(this ImportProfile profile) { return(Path.Combine(profile.GetImportFolder(), "log.txt")); }
/// <summary> /// Read the XML file for a specific tree /// </summary> /// <param name="file">The path of the file to load</param> /// <returns></returns> public bool LoadXML(string file) { if (!System.IO.File.Exists(file)) { return(false); } XmlReaderSettings settings = new XmlReaderSettings(); settings.IgnoreComments = true; XmlReader reader = XmlReader.Create(file, settings); XmlDocument document = new XmlDocument(); document.Load(reader); List <ImportProfile> avaliableCollections = new List <ImportProfile>(); float totalWeight = 0.0f; // Read document XmlElement root = document.DocumentElement; foreach (XmlElement child in root.ChildNodes) { // Profile loading - TODO if (child.Name == "Profile") { float weight = 1.0f; if (child.HasAttribute("weight")) { float.TryParse(child.GetAttribute("weight"), out weight); } ImportProfile profile = new ImportProfile(weight); totalWeight += weight; foreach (XmlElement var in child.ChildNodes) { if (var.Name != "var") { Debug.LogError("Profile must only have child nodes of type 'var'"); } if (!var.HasAttribute("name")) { Debug.LogError("Profile.var must have attribute 'name'"); } float value = 0.0f; float.TryParse(var.InnerXml, out value); profile.vars.SetVar(var.GetAttribute("name"), value); } avaliableCollections.Add(profile); } // Load the actual tree if (child.Name == "Decision") { if (child.ChildNodes.Count != 1) { Debug.LogError("Decision must have exactly 1 child"); } rootDecision = ParseDecision((XmlElement)child.FirstChild); } } Debug.Log("Read from '" + file + "'"); // Store all profiles for debug agentProfiles = new VariableCollection[avaliableCollections.Count]; for (int i = 0; i < avaliableCollections.Count; ++i) { agentProfiles[i] = avaliableCollections[i].vars; } // Randomly assign a profile to this var if (totalWeight == 0.0f) { agentProfile = new VariableCollection(); } else { // Fetch a profile based on it's weight float value = UnityEngine.Random.Range(0.0f, totalWeight); for (int i = 0; i < avaliableCollections.Count; ++i) { if (value < avaliableCollections[i].weight) { agentProfile = avaliableCollections[i].vars; break; } else { value -= avaliableCollections[i].weight; } } } return(true); }