private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken) { var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true); var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken) { Request = request, ProgressCallback = request.ProgressCallback, UpdateOnly = profile.UpdateOnly, KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(), ImportDirectory = dir, ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true), ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData), Languages = await _languageService.GetAllLanguagesAsync(true), Stores = _services.StoreContext.GetAllStores().AsReadOnly(), DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout)) }; // Relative paths for images always refer to the profile directory, not to its "Content" sub-directory. executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue() ? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false) : dir.Parent; var context = new DataImporterContext { Request = request, CancelToken = cancelToken, ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(), ExecuteContext = executeContext }; var logFile = await dir.Parent.GetFileAsync("log.txt"); return(context, logFile); }
private bool HasPermission(DataImporterContext ctx) { if (ctx.Request.HasPermission) { return(true); } var customer = _services.WorkContext.CurrentCustomer; if (customer.SystemName == SystemCustomerNames.BackgroundTask) { return(true); } if (ctx.Request.Profile.EntityType == ImportEntityType.Product || ctx.Request.Profile.EntityType == ImportEntityType.Category) { return(_services.Permissions.Authorize(StandardPermissionProvider.ManageCatalog, customer)); } if (ctx.Request.Profile.EntityType == ImportEntityType.Customer) { return(_services.Permissions.Authorize(StandardPermissionProvider.ManageCustomers, customer)); } if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription) { return(_services.Permissions.Authorize(StandardPermissionProvider.ManageNewsletterSubscribers, customer)); } return(true); }
private void ImportCoreInner(DataImporterContext ctx, ImportFile file) { var context = ctx.ExecuteContext; var profile = ctx.Request.Profile; if (context.Abort == DataExchangeAbortion.Hard) { return; } if (!File.Exists(file.Path)) { throw new SmartException($"File does not exist {file.Path}."); } using (var stream = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var csvConfiguration = file.IsCsv ? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration) : CsvConfiguration.ExcelFriendlyConfiguration; context.DataTable = LightweightDataTable.FromFile( file.Name, stream, stream.Length, csvConfiguration, profile.Skip, profile.Take > 0 ? profile.Take : int.MaxValue); context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap; context.File = file; try { ctx.Importer.Execute(context); } catch (Exception ex) { context.Abort = DataExchangeAbortion.Hard; context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}."); } finally { context.Result.EndDateUtc = DateTime.UtcNow; if (context.IsMaxFailures) { context.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancellationToken.IsCancellationRequested) { context.Result.AddWarning("Import aborted. A cancellation has been requested."); } } } }
public ProductController(ILogger <ProductController> logger, DataImporterService dataImporterService, ProductService productService, DataImporterContext dbContext) { _logger = logger; importerService = dataImporterService; ProdService = productService; context = dbContext; }
public void Import(DataImportRequest request, CancellationToken cancellationToken) { Guard.ArgumentNotNull(() => request); Guard.ArgumentNotNull(() => cancellationToken); var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo")); ImportCoreOuter(ctx); cancellationToken.ThrowIfCancellationRequested(); }
public void Import(DataImportRequest request, CancellationToken cancellationToken) { Guard.ArgumentNotNull(() => request); Guard.ArgumentNotNull(() => cancellationToken); var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo")); ImportCoreOuter(ctx); cancellationToken.ThrowIfCancellationRequested(); }
public void Import(DataImportRequest request, CancellationToken cancellationToken) { Guard.NotNull(request, nameof(request)); Guard.NotNull(cancellationToken, nameof(cancellationToken)); if (request.Profile != null && request.Profile.Enabled) { var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo")); ImportCoreOuter(ctx); } cancellationToken.ThrowIfCancellationRequested(); }
private bool HasPermission(DataImporterContext ctx) { if (ctx.Request.HasPermission) { return(true); } var customer = _services.WorkContext.CurrentCustomer; if (customer.SystemName == SystemCustomerNames.BackgroundTask) { return(true); } return(_services.Permissions.Authorize(Permissions.Configuration.Import.Execute)); }
private static void LogResults(ImportProfile profile, DataImporterContext ctx) { using var psb = StringBuilderPool.Instance.Get(out var sb); foreach (var item in ctx.Results) { var result = item.Value; var entityName = item.Key.HasValue() ? item.Key : profile.EntityType.ToString(); sb.Clear(); sb.AppendLine(); sb.AppendLine(new string('-', 40)); sb.AppendLine("Object: " + entityName); sb.AppendLine("Started: " + result.StartDateUtc.ToLocalTime()); sb.AppendLine("Finished: " + result.EndDateUtc.ToLocalTime()); sb.AppendLine("Duration: " + (result.EndDateUtc - result.StartDateUtc).ToString("g")); sb.AppendLine("Rows total: " + result.TotalRecords); sb.AppendLine("Rows processed: " + result.AffectedRecords); sb.AppendLine("Rows imported: " + result.NewRecords); sb.AppendLine("Rows updated: " + result.ModifiedRecords); sb.AppendLine("Warnings: " + result.Warnings); sb.Append("Errors: " + result.Errors); ctx.Log.Info(sb.ToString()); foreach (var message in result.Messages) { if (message.MessageType == ImportMessageType.Error) { ctx.Log.Error(new Exception(message.FullMessage), message.ToString()); } else if (message.MessageType == ImportMessageType.Warning) { ctx.Log.Warn(message.ToString()); } else { ctx.Log.Info(message.ToString()); } } } }
private void LogResult(DataImporterContext ctx) { var result = ctx.ExecuteContext.Result; var sb = new StringBuilder(); sb.AppendLine(); sb.AppendFormat("Started:\t\t{0}\r\n", result.StartDateUtc.ToLocalTime()); sb.AppendFormat("Finished:\t\t{0}\r\n", result.EndDateUtc.ToLocalTime()); sb.AppendFormat("Duration:\t\t{0}\r\n", (result.EndDateUtc - result.StartDateUtc).ToString("g")); sb.AppendLine(); sb.AppendFormat("Total rows:\t\t{0}\r\n", result.TotalRecords); sb.AppendFormat("Rows processed:\t\t{0}\r\n", result.AffectedRecords); sb.AppendFormat("Records imported:\t{0}\r\n", result.NewRecords); sb.AppendFormat("Records updated:\t{0}\r\n", result.ModifiedRecords); sb.AppendLine(); sb.AppendFormat("Warnings:\t\t{0}\r\n", result.Warnings); sb.AppendFormat("Errors:\t\t\t{0}", result.Errors); ctx.Log.Information(sb.ToString()); foreach (var message in result.Messages) { if (message.MessageType == ImportMessageType.Error) { ctx.Log.Error(message.ToString(), message.FullMessage); } else if (message.MessageType == ImportMessageType.Warning) { ctx.Log.Warning(message.ToString()); } else { ctx.Log.Information(message.ToString()); } } }
private bool HasPermission(DataImporterContext ctx) { if (ctx.Request.HasPermission) return true; var customer = _services.WorkContext.CurrentCustomer; if (customer.SystemName == SystemCustomerNames.BackgroundTask) return true; if (ctx.Request.Profile.EntityType == ImportEntityType.Product || ctx.Request.Profile.EntityType == ImportEntityType.Category) return _services.Permissions.Authorize(StandardPermissionProvider.ManageCatalog, customer); if (ctx.Request.Profile.EntityType == ImportEntityType.Customer) return _services.Permissions.Authorize(StandardPermissionProvider.ManageCustomers, customer); if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription) return _services.Permissions.Authorize(StandardPermissionProvider.ManageNewsletterSubscribers, customer); return true; }
private void SendCompletionEmail(DataImporterContext ctx) { var emailAccount = _emailAccountService.Value.GetDefaultEmailAccount(); var smtpContext = new SmtpContext(emailAccount); var message = new EmailMessage(); var store = _services.StoreContext.CurrentStore; var storeInfo = "{0} ({1})".FormatInvariant(store.Name, store.Url); var intro = _services.Localization.GetResource("Admin.DataExchange.Import.CompletedEmail.Body").FormatInvariant(storeInfo); var body = new StringBuilder(intro); var result = ctx.ExecuteContext.Result; if (result.LastError.HasValue()) { body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError); } body.Append("<p>"); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.TotalRows"), result.TotalRecords, T("Admin.Common.Skipped"), result.SkippedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.NewRecords"), result.NewRecords, T("Admin.Common.Updated"), result.ModifiedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.Errors"), result.Errors, T("Admin.Common.Warnings"), result.Warnings); body.Append("</p>"); message.From = new EmailAddress(emailAccount.Email, emailAccount.DisplayName); if (_contactDataSettings.Value.WebmasterEmailAddress.HasValue()) message.To.Add(new EmailAddress(_contactDataSettings.Value.WebmasterEmailAddress)); if (message.To.Count == 0 && _contactDataSettings.Value.CompanyEmailAddress.HasValue()) message.To.Add(new EmailAddress(_contactDataSettings.Value.CompanyEmailAddress)); if (message.To.Count == 0) message.To.Add(new EmailAddress(emailAccount.Email, emailAccount.DisplayName)); message.Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Text.FormatInvariant(ctx.Request.Profile.Name); message.Body = body.ToString(); _emailSender.Value.SendEmail(smtpContext, message); //Core.Infrastructure.EngineContext.Current.Resolve<IQueuedEmailService>().InsertQueuedEmail(new QueuedEmail //{ // From = emailAccount.Email, // FromName = emailAccount.DisplayName, // To = message.To.First().Address, // Subject = message.Subject, // Body = message.Body, // CreatedOnUtc = DateTime.UtcNow, // EmailAccountId = emailAccount.Id, // SendManually = true //}); //_services.DbContext.SaveChanges(); }
private void ImportCoreOuter(DataImporterContext ctx) { var customer = _services.WorkContext.CurrentCustomer; var profile = ctx.Request.Profile; var logPath = profile.GetImportLogPath(); FileSystemHelper.DeleteFile(logPath); using (var logger = new TraceLogger(logPath)) { var scopes = new List <IDisposable>(); try { var files = profile.GetImportFiles(profile.ImportRelatedData); var groupedFiles = files.GroupBy(x => x.RelatedType); if (!files.Any()) { throw new SmartException("No files to import."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } _dbCache.Enabled = false; _services.MediaService.ImagePostProcessingEnabled = false; scopes.Add(_localizedEntityService.BeginScope()); scopes.Add(_urlRecordService.BeginScope()); ctx.Log = logger; ctx.Importer = _importerFactory(profile.EntityType); ctx.ExecuteContext.Request = ctx.Request; ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value; ctx.ExecuteContext.Services = _services; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = profile.GetImportFolder(); ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData); var sb = new StringBuilder(); sb.AppendLine(); sb.AppendLine(new string('-', 40)); sb.AppendLine("Smartstore: v." + SmartStoreVersion.CurrentFullVersion); sb.AppendLine("Import profile: {0} {1}".FormatInvariant(profile.Name, profile.Id == 0 ? " (volatile)" : $" (Id {profile.Id})")); foreach (var fileGroup in groupedFiles) { var entityName = fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : profile.EntityType.ToString(); var fileNames = string.Join(", ", fileGroup.Select(x => x.Name)); sb.AppendLine("{0} files: {1}".FormatInvariant(entityName, fileNames)); } sb.Append("Executed by: " + customer.Email.NullEmpty() ?? customer.SystemName.NaIfEmpty()); ctx.Log.Info(sb.ToString()); _services.EventPublisher.Publish(new ImportExecutingEvent(ctx.ExecuteContext)); foreach (var fileGroup in groupedFiles) { ctx.ExecuteContext.Result = ctx.Results[fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : string.Empty] = new ImportResult(); fileGroup.Each(x => ImportCoreInner(ctx, x)); } } catch (Exception ex) { logger.ErrorsAll(ex); } finally { try { _dbCache.Enabled = true; _services.MediaService.ImagePostProcessingEnabled = true; scopes.Each(x => x.Dispose()); _services.EventPublisher.Publish(new ImportExecutedEvent(ctx.ExecuteContext)); } catch (Exception ex) { logger.ErrorsAll(ex); } try { // Database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all what the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception ex) { logger.ErrorsAll(ex); } try { SendCompletionEmail(ctx); } catch (Exception ex) { logger.ErrorsAll(ex); } try { LogResults(ctx); } catch (Exception ex) { logger.ErrorsAll(ex); } try { if (ctx.Results.TryGetValue(string.Empty, out var result)) { profile.ResultInfo = XmlHelper.Serialize(result.Clone()); _importProfileService.UpdateImportProfile(profile); } } catch (Exception ex) { logger.ErrorsAll(ex); } try { ctx.Request.CustomData.Clear(); ctx.Results.Clear(); ctx.Log = null; } catch (Exception ex) { logger.ErrorsAll(ex); } } } }
private void ImportCoreOuter(DataImporterContext ctx) { if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled) return; var logPath = ctx.Request.Profile.GetImportLogPath(); FileSystemHelper.Delete(logPath); using (var logger = new TraceLogger(logPath)) { try { ctx.Log = logger; ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value; ctx.ExecuteContext.Services = _services; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder(); ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize<ImportExtraData>(ctx.Request.Profile.ExtraData); { var mapConverter = new ColumnMapConverter(); ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom<ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap(); } var files = ctx.Request.Profile.GetImportFiles(); if (files.Count == 0) throw new SmartException("No files to import."); if (!HasPermission(ctx)) throw new SmartException("You do not have permission to perform the selected import."); ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType); files.ForEach(x => ImportCoreInner(ctx, x)); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } finally { try { // database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all what the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { SendCompletionEmail(ctx); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow; LogResult(ctx); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone()); _importProfileService.UpdateImportProfile(ctx.Request.Profile); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.CustomData.Clear(); ctx.Log = null; } catch (Exception exception) { logger.ErrorsAll(exception); } } } }
private void LogResult(DataImporterContext ctx) { var result = ctx.ExecuteContext.Result; var sb = new StringBuilder(); sb.AppendLine(); sb.AppendFormat("Started:\t\t{0}\r\n", result.StartDateUtc.ToLocalTime()); sb.AppendFormat("Finished:\t\t{0}\r\n", result.EndDateUtc.ToLocalTime()); sb.AppendFormat("Duration:\t\t{0}\r\n", (result.EndDateUtc - result.StartDateUtc).ToString("g")); sb.AppendLine(); sb.AppendFormat("Total rows:\t\t{0}\r\n", result.TotalRecords); sb.AppendFormat("Rows processed:\t\t{0}\r\n", result.AffectedRecords); sb.AppendFormat("Records imported:\t{0}\r\n", result.NewRecords); sb.AppendFormat("Records updated:\t{0}\r\n", result.ModifiedRecords); sb.AppendLine(); sb.AppendFormat("Warnings:\t\t{0}\r\n", result.Warnings); sb.AppendFormat("Errors:\t\t\t{0}", result.Errors); ctx.Log.Information(sb.ToString()); foreach (var message in result.Messages) { if (message.MessageType == ImportMessageType.Error) ctx.Log.Error(message.ToString(), message.FullMessage); else if (message.MessageType == ImportMessageType.Warning) ctx.Log.Warning(message.ToString()); else ctx.Log.Information(message.ToString()); } }
private void ImportCoreOuter(DataImporterContext ctx) { if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled) { return; } var logPath = ctx.Request.Profile.GetImportLogPath(); FileSystemHelper.Delete(logPath); using (var logger = new TraceLogger(logPath)) { try { ctx.Log = logger; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.Value.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder(); { var mapConverter = new ColumnMapConverter(); ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap(); } var files = ctx.Request.Profile.GetImportFiles(); if (files.Count == 0) { throw new SmartException("No files to import found."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } if (ctx.Request.Profile.EntityType == ImportEntityType.Product) { ctx.Importer = new ProductImporter( _productPictureRepository.Value, _productManufacturerRepository.Value, _productCategoryRepository.Value, _urlRecordRepository.Value, _productRepository.Value, _services, _localizedEntityService.Value, _pictureService.Value, _manufacturerService.Value, _categoryService.Value, _productService.Value, _urlRecordService.Value, _productTemplateService.Value, _storeMappingService.Value, _fileDownloadManager.Value, _seoSettings.Value, _dataExchangeSettings.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.Customer) { ctx.Importer = new CustomerImporter( _customerRepository.Value, _pictureRepository.Value, _services, _genericAttributeService.Value, _customerService, _pictureService.Value, _affiliateService.Value, _countryService.Value, _stateProvinceService.Value, _fileDownloadManager.Value, _customerSettings.Value, _dateTimeSettings.Value, _forumSettings.Value, _dataExchangeSettings.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription) { ctx.Importer = new NewsLetterSubscriptionImporter( _services, _subscriptionRepository.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.Category) { ctx.Importer = new CategoryImporter( _categoryRepository.Value, _urlRecordRepository.Value, _pictureRepository.Value, _services, _urlRecordService.Value, _categoryTemplateService.Value, _storeMappingService.Value, _pictureService.Value, _localizedEntityService.Value, _fileDownloadManager.Value, _seoSettings.Value, _dataExchangeSettings.Value); } else { throw new SmartException("Unsupported entity type {0}.".FormatInvariant(ctx.Request.Profile.EntityType.ToString())); } files.ForEach(x => ImportCoreInner(ctx, x)); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } finally { try { // database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all that the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { SendCompletionEmail(ctx); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow; LogResult(ctx); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone()); _importProfileService.UpdateImportProfile(ctx.Request.Profile); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.CustomData.Clear(); ctx.Log = null; } catch (Exception exception) { logger.ErrorsAll(exception); } } } }
private void ImportCoreInner(DataImporterContext ctx, string filePath) { if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard) return; { var logHead = new StringBuilder(); logHead.AppendLine(); logHead.AppendLine(new string('-', 40)); logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion); logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name); logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id)); logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString()); logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath)); var customer = _services.WorkContext.CurrentCustomer; logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName)); ctx.Log.Information(logHead.ToString()); } if (!File.Exists(filePath)) { throw new SmartException("File does not exist {0}.".FormatInvariant(filePath)); } CsvConfiguration csvConfiguration = null; var extension = Path.GetExtension(filePath); if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase)) { var converter = new CsvConfigurationConverter(); csvConfiguration = converter.ConvertFrom<CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration); } if (csvConfiguration == null) { csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration; } using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile( Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, ctx.Request.Profile.Skip, ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue ); try { ctx.Importer.Execute(ctx.ExecuteContext); } catch (Exception exception) { ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard; ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages())); } if (ctx.ExecuteContext.IsMaxFailures) ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); if (ctx.CancellationToken.IsCancellationRequested) ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested."); } }
private void SendCompletionEmail(DataImporterContext ctx) { var emailAccount = _emailAccountService.Value.GetDefaultEmailAccount(); var smtpContext = new SmtpContext(emailAccount); var message = new EmailMessage(); var store = _services.StoreContext.CurrentStore; var storeInfo = "{0} ({1})".FormatInvariant(store.Name, store.Url); var intro = _services.Localization.GetResource("Admin.DataExchange.Import.CompletedEmail.Body").FormatInvariant(storeInfo); var body = new StringBuilder(intro); var result = ctx.ExecuteContext.Result; if (result.LastError.HasValue()) { body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError); } body.Append("<p>"); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.TotalRows"), result.TotalRecords, T("Admin.Common.Skipped"), result.SkippedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.NewRecords"), result.NewRecords, T("Admin.Common.Updated"), result.ModifiedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.Errors"), result.Errors, T("Admin.Common.Warnings"), result.Warnings); body.Append("</p>"); message.From = new EmailAddress(emailAccount.Email, emailAccount.DisplayName); if (_contactDataSettings.Value.WebmasterEmailAddress.HasValue()) { message.To.Add(new EmailAddress(_contactDataSettings.Value.WebmasterEmailAddress)); } if (message.To.Count == 0 && _contactDataSettings.Value.CompanyEmailAddress.HasValue()) { message.To.Add(new EmailAddress(_contactDataSettings.Value.CompanyEmailAddress)); } if (message.To.Count == 0) { message.To.Add(new EmailAddress(emailAccount.Email, emailAccount.DisplayName)); } message.Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Text.FormatInvariant(ctx.Request.Profile.Name); message.Body = body.ToString(); _emailSender.Value.SendEmail(smtpContext, message); //Core.Infrastructure.EngineContext.Current.Resolve<IQueuedEmailService>().InsertQueuedEmail(new QueuedEmail //{ // From = emailAccount.Email, // FromName = emailAccount.DisplayName, // To = message.To.First().Address, // Subject = message.Subject, // Body = message.Body, // CreatedOnUtc = DateTime.UtcNow, // EmailAccountId = emailAccount.Id, // SendManually = true //}); //_services.DbContext.SaveChanges(); }
private async Task SendCompletionEmail(ImportProfile profile, DataImporterContext ctx) { var emailAccount = _emailAccountService.GetDefaultEmailAccount(); var result = ctx.ExecuteContext.Result; var store = _services.StoreContext.CurrentStore; var storeInfo = $"{store.Name} ({store.Url})"; using var psb = StringBuilderPool.Instance.Get(out var body); body.Append(T("Admin.DataExchange.Import.CompletedEmail.Body", storeInfo)); if (result.LastError.HasValue()) { body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError); } body.Append("<p>"); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.TotalRows"), result.TotalRecords, T("Admin.Common.Skipped"), result.SkippedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.NewRecords"), result.NewRecords, T("Admin.Common.Updated"), result.ModifiedRecords); body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>", T("Admin.Common.Errors"), result.Errors, T("Admin.Common.Warnings"), result.Warnings); body.Append("</p>"); var message = new MailMessage { From = new(emailAccount.Email, emailAccount.DisplayName), Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Value.FormatInvariant(profile.Name), Body = body.ToString() }; if (_contactDataSettings.WebmasterEmailAddress.HasValue()) { message.To.Add(new(_contactDataSettings.WebmasterEmailAddress)); } if (!message.To.Any() && _contactDataSettings.CompanyEmailAddress.HasValue()) { message.To.Add(new(_contactDataSettings.CompanyEmailAddress)); } if (!message.To.Any()) { message.To.Add(new(emailAccount.Email, emailAccount.DisplayName)); } await using var client = await _mailService.ConnectAsync(emailAccount); await client.SendAsync(message, ctx.CancelToken); //_db.QueuedEmails.Add(new QueuedEmail //{ // From = emailAccount.Email, // To = message.To.First().Address, // Subject = message.Subject, // Body = message.Body, // CreatedOnUtc = DateTime.UtcNow, // EmailAccountId = emailAccount.Id, // SendManually = true //}); //await _db.SaveChangesAsync(); }
private async Task Finalize(DataImporterContext ctx) { if (ctx == null) { return; } try { await _services.EventPublisher.PublishAsync(new ImportExecutedEvent(ctx.ExecuteContext), ctx.CancelToken); } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(ctx.Request.ProfileId, true, ctx.CancelToken); try { await SendCompletionEmail(profile, ctx); } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } try { LogResults(profile, ctx); } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } try { if (ctx.Results.TryGetValue(string.Empty, out var result)) { profile.ResultInfo = XmlHelper.Serialize(result.Clone()); await _services.DbContext.SaveChangesAsync(ctx.CancelToken); } } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } if (ctx.ExecuteContext?.ClearCache ?? false) { try { await _services.Cache.ClearAsync(); } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } } try { ctx.ExecuteContext.DownloadManager.Dispose(); ctx.Request.CustomData.Clear(); ctx.Results.Clear(); } catch (Exception ex) { ctx.Log?.ErrorsAll(ex); } }
private void ImportCoreInner(DataImporterContext ctx, string filePath) { if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard) { return; } { var logHead = new StringBuilder(); logHead.AppendLine(); logHead.AppendLine(new string('-', 40)); logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion); logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name); logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id)); logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString()); logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath)); var customer = _services.WorkContext.CurrentCustomer; logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName)); ctx.Log.Information(logHead.ToString()); } if (!File.Exists(filePath)) { throw new SmartException("File does not exist {0}.".FormatInvariant(filePath)); } CsvConfiguration csvConfiguration = null; var extension = Path.GetExtension(filePath); if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase)) { var converter = new CsvConfigurationConverter(); csvConfiguration = converter.ConvertFrom <CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration); } if (csvConfiguration == null) { csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration; } using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile( Path.GetFileName(filePath), stream, stream.Length, csvConfiguration, ctx.Request.Profile.Skip, ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue ); try { ctx.Importer.Execute(ctx.ExecuteContext); } catch (Exception exception) { ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard; ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages())); } if (ctx.ExecuteContext.IsMaxFailures) { ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancellationToken.IsCancellationRequested) { ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested."); } } }
private void ImportCoreOuter(DataImporterContext ctx) { if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled) { return; } var logPath = ctx.Request.Profile.GetImportLogPath(); FileSystemHelper.Delete(logPath); using (var logger = new TraceLogger(logPath)) { try { ctx.Log = logger; ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value; ctx.ExecuteContext.Services = _services; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder(); ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(ctx.Request.Profile.ExtraData); { var mapConverter = new ColumnMapConverter(); ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap(); } var files = ctx.Request.Profile.GetImportFiles(); if (files.Count == 0) { throw new SmartException("No files to import."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType); files.ForEach(x => ImportCoreInner(ctx, x)); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } finally { try { // database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all what the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { SendCompletionEmail(ctx); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow; LogResult(ctx); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone()); _importProfileService.UpdateImportProfile(ctx.Request.Profile); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.CustomData.Clear(); ctx.Log = null; } catch (Exception exception) { logger.ErrorsAll(exception); } } } }