private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken)
{
var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true);
var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken)
{
Request = request,
ProgressCallback = request.ProgressCallback,
UpdateOnly = profile.UpdateOnly,
KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(),
ImportDirectory = dir,
ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true),
ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData),
Languages = await _languageService.GetAllLanguagesAsync(true),
Stores = _services.StoreContext.GetAllStores().AsReadOnly(),
DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout))
};
// Relative paths for images always refer to the profile directory, not to its "Content" sub-directory.
executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue()
? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false)
: dir.Parent;
var context = new DataImporterContext
{
Request = request,
CancelToken = cancelToken,
ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(),
ExecuteContext = executeContext
};
var logFile = await dir.Parent.GetFileAsync("log.txt");
return(context, logFile);
}
private bool HasPermission(DataImporterContext ctx)
{
if (ctx.Request.HasPermission)
{
return(true);
}
var customer = _services.WorkContext.CurrentCustomer;
if (customer.SystemName == SystemCustomerNames.BackgroundTask)
{
return(true);
}
if (ctx.Request.Profile.EntityType == ImportEntityType.Product || ctx.Request.Profile.EntityType == ImportEntityType.Category)
{
return(_services.Permissions.Authorize(StandardPermissionProvider.ManageCatalog, customer));
}
if (ctx.Request.Profile.EntityType == ImportEntityType.Customer)
{
return(_services.Permissions.Authorize(StandardPermissionProvider.ManageCustomers, customer));
}
if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription)
{
return(_services.Permissions.Authorize(StandardPermissionProvider.ManageNewsletterSubscribers, customer));
}
return(true);
}
private void ImportCoreInner(DataImporterContext ctx, ImportFile file)
{
var context = ctx.ExecuteContext;
var profile = ctx.Request.Profile;
if (context.Abort == DataExchangeAbortion.Hard)
{
return;
}
if (!File.Exists(file.Path))
{
throw new SmartException($"File does not exist {file.Path}.");
}
using (var stream = new FileStream(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var csvConfiguration = file.IsCsv
? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
: CsvConfiguration.ExcelFriendlyConfiguration;
context.DataTable = LightweightDataTable.FromFile(
file.Name,
stream,
stream.Length,
csvConfiguration,
profile.Skip,
profile.Take > 0 ? profile.Take : int.MaxValue);
context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
context.File = file;
try
{
ctx.Importer.Execute(context);
}
catch (Exception ex)
{
context.Abort = DataExchangeAbortion.Hard;
context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
}
finally
{
context.Result.EndDateUtc = DateTime.UtcNow;
if (context.IsMaxFailures)
{
context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancellationToken.IsCancellationRequested)
{
context.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
}
public ProductController(ILogger <ProductController> logger,
DataImporterService dataImporterService,
ProductService productService,
DataImporterContext dbContext)
{
_logger = logger;
importerService = dataImporterService;
ProdService = productService;
context = dbContext;
}
public void Import(DataImportRequest request, CancellationToken cancellationToken)
{
Guard.ArgumentNotNull(() => request);
Guard.ArgumentNotNull(() => cancellationToken);
var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo"));
ImportCoreOuter(ctx);
cancellationToken.ThrowIfCancellationRequested();
}
public void Import(DataImportRequest request, CancellationToken cancellationToken)
{
Guard.ArgumentNotNull(() => request);
Guard.ArgumentNotNull(() => cancellationToken);
var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo"));
ImportCoreOuter(ctx);
cancellationToken.ThrowIfCancellationRequested();
}
public void Import(DataImportRequest request, CancellationToken cancellationToken)
{
Guard.NotNull(request, nameof(request));
Guard.NotNull(cancellationToken, nameof(cancellationToken));
if (request.Profile != null && request.Profile.Enabled)
{
var ctx = new DataImporterContext(request, cancellationToken, T("Admin.DataExchange.Import.ProgressInfo"));
ImportCoreOuter(ctx);
}
cancellationToken.ThrowIfCancellationRequested();
}
private bool HasPermission(DataImporterContext ctx)
{
if (ctx.Request.HasPermission)
{
return(true);
}
var customer = _services.WorkContext.CurrentCustomer;
if (customer.SystemName == SystemCustomerNames.BackgroundTask)
{
return(true);
}
return(_services.Permissions.Authorize(Permissions.Configuration.Import.Execute));
}
private static void LogResults(ImportProfile profile, DataImporterContext ctx)
{
using var psb = StringBuilderPool.Instance.Get(out var sb);
foreach (var item in ctx.Results)
{
var result = item.Value;
var entityName = item.Key.HasValue() ? item.Key : profile.EntityType.ToString();
sb.Clear();
sb.AppendLine();
sb.AppendLine(new string('-', 40));
sb.AppendLine("Object: " + entityName);
sb.AppendLine("Started: " + result.StartDateUtc.ToLocalTime());
sb.AppendLine("Finished: " + result.EndDateUtc.ToLocalTime());
sb.AppendLine("Duration: " + (result.EndDateUtc - result.StartDateUtc).ToString("g"));
sb.AppendLine("Rows total: " + result.TotalRecords);
sb.AppendLine("Rows processed: " + result.AffectedRecords);
sb.AppendLine("Rows imported: " + result.NewRecords);
sb.AppendLine("Rows updated: " + result.ModifiedRecords);
sb.AppendLine("Warnings: " + result.Warnings);
sb.Append("Errors: " + result.Errors);
ctx.Log.Info(sb.ToString());
foreach (var message in result.Messages)
{
if (message.MessageType == ImportMessageType.Error)
{
ctx.Log.Error(new Exception(message.FullMessage), message.ToString());
}
else if (message.MessageType == ImportMessageType.Warning)
{
ctx.Log.Warn(message.ToString());
}
else
{
ctx.Log.Info(message.ToString());
}
}
}
}
private void LogResult(DataImporterContext ctx)
{
var result = ctx.ExecuteContext.Result;
var sb = new StringBuilder();
sb.AppendLine();
sb.AppendFormat("Started:\t\t{0}\r\n", result.StartDateUtc.ToLocalTime());
sb.AppendFormat("Finished:\t\t{0}\r\n", result.EndDateUtc.ToLocalTime());
sb.AppendFormat("Duration:\t\t{0}\r\n", (result.EndDateUtc - result.StartDateUtc).ToString("g"));
sb.AppendLine();
sb.AppendFormat("Total rows:\t\t{0}\r\n", result.TotalRecords);
sb.AppendFormat("Rows processed:\t\t{0}\r\n", result.AffectedRecords);
sb.AppendFormat("Records imported:\t{0}\r\n", result.NewRecords);
sb.AppendFormat("Records updated:\t{0}\r\n", result.ModifiedRecords);
sb.AppendLine();
sb.AppendFormat("Warnings:\t\t{0}\r\n", result.Warnings);
sb.AppendFormat("Errors:\t\t\t{0}", result.Errors);
ctx.Log.Information(sb.ToString());
foreach (var message in result.Messages)
{
if (message.MessageType == ImportMessageType.Error)
{
ctx.Log.Error(message.ToString(), message.FullMessage);
}
else if (message.MessageType == ImportMessageType.Warning)
{
ctx.Log.Warning(message.ToString());
}
else
{
ctx.Log.Information(message.ToString());
}
}
}
private bool HasPermission(DataImporterContext ctx)
{
if (ctx.Request.HasPermission)
return true;
var customer = _services.WorkContext.CurrentCustomer;
if (customer.SystemName == SystemCustomerNames.BackgroundTask)
return true;
if (ctx.Request.Profile.EntityType == ImportEntityType.Product || ctx.Request.Profile.EntityType == ImportEntityType.Category)
return _services.Permissions.Authorize(StandardPermissionProvider.ManageCatalog, customer);
if (ctx.Request.Profile.EntityType == ImportEntityType.Customer)
return _services.Permissions.Authorize(StandardPermissionProvider.ManageCustomers, customer);
if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription)
return _services.Permissions.Authorize(StandardPermissionProvider.ManageNewsletterSubscribers, customer);
return true;
}
private void SendCompletionEmail(DataImporterContext ctx)
{
var emailAccount = _emailAccountService.Value.GetDefaultEmailAccount();
var smtpContext = new SmtpContext(emailAccount);
var message = new EmailMessage();
var store = _services.StoreContext.CurrentStore;
var storeInfo = "{0} ({1})".FormatInvariant(store.Name, store.Url);
var intro = _services.Localization.GetResource("Admin.DataExchange.Import.CompletedEmail.Body").FormatInvariant(storeInfo);
var body = new StringBuilder(intro);
var result = ctx.ExecuteContext.Result;
if (result.LastError.HasValue())
{
body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError);
}
body.Append("<p>");
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.TotalRows"), result.TotalRecords,
T("Admin.Common.Skipped"), result.SkippedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.NewRecords"), result.NewRecords,
T("Admin.Common.Updated"), result.ModifiedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.Errors"), result.Errors,
T("Admin.Common.Warnings"), result.Warnings);
body.Append("</p>");
message.From = new EmailAddress(emailAccount.Email, emailAccount.DisplayName);
if (_contactDataSettings.Value.WebmasterEmailAddress.HasValue())
message.To.Add(new EmailAddress(_contactDataSettings.Value.WebmasterEmailAddress));
if (message.To.Count == 0 && _contactDataSettings.Value.CompanyEmailAddress.HasValue())
message.To.Add(new EmailAddress(_contactDataSettings.Value.CompanyEmailAddress));
if (message.To.Count == 0)
message.To.Add(new EmailAddress(emailAccount.Email, emailAccount.DisplayName));
message.Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Text.FormatInvariant(ctx.Request.Profile.Name);
message.Body = body.ToString();
_emailSender.Value.SendEmail(smtpContext, message);
//Core.Infrastructure.EngineContext.Current.Resolve<IQueuedEmailService>().InsertQueuedEmail(new QueuedEmail
//{
// From = emailAccount.Email,
// FromName = emailAccount.DisplayName,
// To = message.To.First().Address,
// Subject = message.Subject,
// Body = message.Body,
// CreatedOnUtc = DateTime.UtcNow,
// EmailAccountId = emailAccount.Id,
// SendManually = true
//});
//_services.DbContext.SaveChanges();
}
private void ImportCoreOuter(DataImporterContext ctx)
{
var customer = _services.WorkContext.CurrentCustomer;
var profile = ctx.Request.Profile;
var logPath = profile.GetImportLogPath();
FileSystemHelper.DeleteFile(logPath);
using (var logger = new TraceLogger(logPath))
{
var scopes = new List <IDisposable>();
try
{
var files = profile.GetImportFiles(profile.ImportRelatedData);
var groupedFiles = files.GroupBy(x => x.RelatedType);
if (!files.Any())
{
throw new SmartException("No files to import.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
_dbCache.Enabled = false;
_services.MediaService.ImagePostProcessingEnabled = false;
scopes.Add(_localizedEntityService.BeginScope());
scopes.Add(_urlRecordService.BeginScope());
ctx.Log = logger;
ctx.Importer = _importerFactory(profile.EntityType);
ctx.ExecuteContext.Request = ctx.Request;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData);
var sb = new StringBuilder();
sb.AppendLine();
sb.AppendLine(new string('-', 40));
sb.AppendLine("Smartstore: v." + SmartStoreVersion.CurrentFullVersion);
sb.AppendLine("Import profile: {0} {1}".FormatInvariant(profile.Name, profile.Id == 0 ? " (volatile)" : $" (Id {profile.Id})"));
foreach (var fileGroup in groupedFiles)
{
var entityName = fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : profile.EntityType.ToString();
var fileNames = string.Join(", ", fileGroup.Select(x => x.Name));
sb.AppendLine("{0} files: {1}".FormatInvariant(entityName, fileNames));
}
sb.Append("Executed by: " + customer.Email.NullEmpty() ?? customer.SystemName.NaIfEmpty());
ctx.Log.Info(sb.ToString());
_services.EventPublisher.Publish(new ImportExecutingEvent(ctx.ExecuteContext));
foreach (var fileGroup in groupedFiles)
{
ctx.ExecuteContext.Result = ctx.Results[fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : string.Empty] = new ImportResult();
fileGroup.Each(x => ImportCoreInner(ctx, x));
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
finally
{
try
{
_dbCache.Enabled = true;
_services.MediaService.ImagePostProcessingEnabled = true;
scopes.Each(x => x.Dispose());
_services.EventPublisher.Publish(new ImportExecutedEvent(ctx.ExecuteContext));
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
// Database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
LogResults(ctx);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
if (ctx.Results.TryGetValue(string.Empty, out var result))
{
profile.ResultInfo = XmlHelper.Serialize(result.Clone());
_importProfileService.UpdateImportProfile(profile);
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Results.Clear();
ctx.Log = null;
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
}
}
}
private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
return;
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize<ImportExtraData>(ctx.Request.Profile.ExtraData);
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom<ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
throw new SmartException("No files to import.");
if (!HasPermission(ctx))
throw new SmartException("You do not have permission to perform the selected import.");
ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType);
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
private void LogResult(DataImporterContext ctx)
{
var result = ctx.ExecuteContext.Result;
var sb = new StringBuilder();
sb.AppendLine();
sb.AppendFormat("Started:\t\t{0}\r\n", result.StartDateUtc.ToLocalTime());
sb.AppendFormat("Finished:\t\t{0}\r\n", result.EndDateUtc.ToLocalTime());
sb.AppendFormat("Duration:\t\t{0}\r\n", (result.EndDateUtc - result.StartDateUtc).ToString("g"));
sb.AppendLine();
sb.AppendFormat("Total rows:\t\t{0}\r\n", result.TotalRecords);
sb.AppendFormat("Rows processed:\t\t{0}\r\n", result.AffectedRecords);
sb.AppendFormat("Records imported:\t{0}\r\n", result.NewRecords);
sb.AppendFormat("Records updated:\t{0}\r\n", result.ModifiedRecords);
sb.AppendLine();
sb.AppendFormat("Warnings:\t\t{0}\r\n", result.Warnings);
sb.AppendFormat("Errors:\t\t\t{0}", result.Errors);
ctx.Log.Information(sb.ToString());
foreach (var message in result.Messages)
{
if (message.MessageType == ImportMessageType.Error)
ctx.Log.Error(message.ToString(), message.FullMessage);
else if (message.MessageType == ImportMessageType.Warning)
ctx.Log.Warning(message.ToString());
else
ctx.Log.Information(message.ToString());
}
}
private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
{
return;
}
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.Value.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
{
throw new SmartException("No files to import found.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
if (ctx.Request.Profile.EntityType == ImportEntityType.Product)
{
ctx.Importer = new ProductImporter(
_productPictureRepository.Value,
_productManufacturerRepository.Value,
_productCategoryRepository.Value,
_urlRecordRepository.Value,
_productRepository.Value,
_services,
_localizedEntityService.Value,
_pictureService.Value,
_manufacturerService.Value,
_categoryService.Value,
_productService.Value,
_urlRecordService.Value,
_productTemplateService.Value,
_storeMappingService.Value,
_fileDownloadManager.Value,
_seoSettings.Value,
_dataExchangeSettings.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.Customer)
{
ctx.Importer = new CustomerImporter(
_customerRepository.Value,
_pictureRepository.Value,
_services,
_genericAttributeService.Value,
_customerService,
_pictureService.Value,
_affiliateService.Value,
_countryService.Value,
_stateProvinceService.Value,
_fileDownloadManager.Value,
_customerSettings.Value,
_dateTimeSettings.Value,
_forumSettings.Value,
_dataExchangeSettings.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription)
{
ctx.Importer = new NewsLetterSubscriptionImporter(
_services,
_subscriptionRepository.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.Category)
{
ctx.Importer = new CategoryImporter(
_categoryRepository.Value,
_urlRecordRepository.Value,
_pictureRepository.Value,
_services,
_urlRecordService.Value,
_categoryTemplateService.Value,
_storeMappingService.Value,
_pictureService.Value,
_localizedEntityService.Value,
_fileDownloadManager.Value,
_seoSettings.Value,
_dataExchangeSettings.Value);
}
else
{
throw new SmartException("Unsupported entity type {0}.".FormatInvariant(ctx.Request.Profile.EntityType.ToString()));
}
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all that the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
private void ImportCoreInner(DataImporterContext ctx, string filePath)
{
if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard)
return;
{
var logHead = new StringBuilder();
logHead.AppendLine();
logHead.AppendLine(new string('-', 40));
logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion);
logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name);
logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id));
logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString());
logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath));
var customer = _services.WorkContext.CurrentCustomer;
logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName));
ctx.Log.Information(logHead.ToString());
}
if (!File.Exists(filePath))
{
throw new SmartException("File does not exist {0}.".FormatInvariant(filePath));
}
CsvConfiguration csvConfiguration = null;
var extension = Path.GetExtension(filePath);
if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase))
{
var converter = new CsvConfigurationConverter();
csvConfiguration = converter.ConvertFrom<CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration);
}
if (csvConfiguration == null)
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile(
Path.GetFileName(filePath),
stream,
stream.Length,
csvConfiguration,
ctx.Request.Profile.Skip,
ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue
);
try
{
ctx.Importer.Execute(ctx.ExecuteContext);
}
catch (Exception exception)
{
ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard;
ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages()));
}
if (ctx.ExecuteContext.IsMaxFailures)
ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
if (ctx.CancellationToken.IsCancellationRequested)
ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
private void SendCompletionEmail(DataImporterContext ctx)
{
var emailAccount = _emailAccountService.Value.GetDefaultEmailAccount();
var smtpContext = new SmtpContext(emailAccount);
var message = new EmailMessage();
var store = _services.StoreContext.CurrentStore;
var storeInfo = "{0} ({1})".FormatInvariant(store.Name, store.Url);
var intro = _services.Localization.GetResource("Admin.DataExchange.Import.CompletedEmail.Body").FormatInvariant(storeInfo);
var body = new StringBuilder(intro);
var result = ctx.ExecuteContext.Result;
if (result.LastError.HasValue())
{
body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError);
}
body.Append("<p>");
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.TotalRows"), result.TotalRecords,
T("Admin.Common.Skipped"), result.SkippedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.NewRecords"), result.NewRecords,
T("Admin.Common.Updated"), result.ModifiedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.Errors"), result.Errors,
T("Admin.Common.Warnings"), result.Warnings);
body.Append("</p>");
message.From = new EmailAddress(emailAccount.Email, emailAccount.DisplayName);
if (_contactDataSettings.Value.WebmasterEmailAddress.HasValue())
{
message.To.Add(new EmailAddress(_contactDataSettings.Value.WebmasterEmailAddress));
}
if (message.To.Count == 0 && _contactDataSettings.Value.CompanyEmailAddress.HasValue())
{
message.To.Add(new EmailAddress(_contactDataSettings.Value.CompanyEmailAddress));
}
if (message.To.Count == 0)
{
message.To.Add(new EmailAddress(emailAccount.Email, emailAccount.DisplayName));
}
message.Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Text.FormatInvariant(ctx.Request.Profile.Name);
message.Body = body.ToString();
_emailSender.Value.SendEmail(smtpContext, message);
//Core.Infrastructure.EngineContext.Current.Resolve<IQueuedEmailService>().InsertQueuedEmail(new QueuedEmail
//{
// From = emailAccount.Email,
// FromName = emailAccount.DisplayName,
// To = message.To.First().Address,
// Subject = message.Subject,
// Body = message.Body,
// CreatedOnUtc = DateTime.UtcNow,
// EmailAccountId = emailAccount.Id,
// SendManually = true
//});
//_services.DbContext.SaveChanges();
}
private async Task SendCompletionEmail(ImportProfile profile, DataImporterContext ctx)
{
var emailAccount = _emailAccountService.GetDefaultEmailAccount();
var result = ctx.ExecuteContext.Result;
var store = _services.StoreContext.CurrentStore;
var storeInfo = $"{store.Name} ({store.Url})";
using var psb = StringBuilderPool.Instance.Get(out var body);
body.Append(T("Admin.DataExchange.Import.CompletedEmail.Body", storeInfo));
if (result.LastError.HasValue())
{
body.AppendFormat("<p style=\"color: #B94A48;\">{0}</p>", result.LastError);
}
body.Append("<p>");
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.TotalRows"), result.TotalRecords,
T("Admin.Common.Skipped"), result.SkippedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.NewRecords"), result.NewRecords,
T("Admin.Common.Updated"), result.ModifiedRecords);
body.AppendFormat("<div>{0}: {1} · {2}: {3}</div>",
T("Admin.Common.Errors"), result.Errors,
T("Admin.Common.Warnings"), result.Warnings);
body.Append("</p>");
var message = new MailMessage
{
From = new(emailAccount.Email, emailAccount.DisplayName),
Subject = T("Admin.DataExchange.Import.CompletedEmail.Subject").Value.FormatInvariant(profile.Name),
Body = body.ToString()
};
if (_contactDataSettings.WebmasterEmailAddress.HasValue())
{
message.To.Add(new(_contactDataSettings.WebmasterEmailAddress));
}
if (!message.To.Any() && _contactDataSettings.CompanyEmailAddress.HasValue())
{
message.To.Add(new(_contactDataSettings.CompanyEmailAddress));
}
if (!message.To.Any())
{
message.To.Add(new(emailAccount.Email, emailAccount.DisplayName));
}
await using var client = await _mailService.ConnectAsync(emailAccount);
await client.SendAsync(message, ctx.CancelToken);
//_db.QueuedEmails.Add(new QueuedEmail
//{
// From = emailAccount.Email,
// To = message.To.First().Address,
// Subject = message.Subject,
// Body = message.Body,
// CreatedOnUtc = DateTime.UtcNow,
// EmailAccountId = emailAccount.Id,
// SendManually = true
//});
//await _db.SaveChangesAsync();
}
private async Task Finalize(DataImporterContext ctx)
{
if (ctx == null)
{
return;
}
try
{
await _services.EventPublisher.PublishAsync(new ImportExecutedEvent(ctx.ExecuteContext), ctx.CancelToken);
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(ctx.Request.ProfileId, true, ctx.CancelToken);
try
{
await SendCompletionEmail(profile, ctx);
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
try
{
LogResults(profile, ctx);
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
try
{
if (ctx.Results.TryGetValue(string.Empty, out var result))
{
profile.ResultInfo = XmlHelper.Serialize(result.Clone());
await _services.DbContext.SaveChangesAsync(ctx.CancelToken);
}
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
if (ctx.ExecuteContext?.ClearCache ?? false)
{
try
{
await _services.Cache.ClearAsync();
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
}
try
{
ctx.ExecuteContext.DownloadManager.Dispose();
ctx.Request.CustomData.Clear();
ctx.Results.Clear();
}
catch (Exception ex)
{
ctx.Log?.ErrorsAll(ex);
}
}
private void ImportCoreInner(DataImporterContext ctx, string filePath)
{
if (ctx.ExecuteContext.Abort == DataExchangeAbortion.Hard)
{
return;
}
{
var logHead = new StringBuilder();
logHead.AppendLine();
logHead.AppendLine(new string('-', 40));
logHead.AppendLine("SmartStore.NET:\t\tv." + SmartStoreVersion.CurrentFullVersion);
logHead.Append("Import profile:\t\t" + ctx.Request.Profile.Name);
logHead.AppendLine(ctx.Request.Profile.Id == 0 ? " (volatile)" : " (Id {0})".FormatInvariant(ctx.Request.Profile.Id));
logHead.AppendLine("Entity:\t\t\t" + ctx.Request.Profile.EntityType.ToString());
logHead.AppendLine("File:\t\t\t" + Path.GetFileName(filePath));
var customer = _services.WorkContext.CurrentCustomer;
logHead.Append("Executed by:\t\t" + (customer.Email.HasValue() ? customer.Email : customer.SystemName));
ctx.Log.Information(logHead.ToString());
}
if (!File.Exists(filePath))
{
throw new SmartException("File does not exist {0}.".FormatInvariant(filePath));
}
CsvConfiguration csvConfiguration = null;
var extension = Path.GetExtension(filePath);
if ((new string[] { ".csv", ".txt", ".tab" }).Contains(extension, StringComparer.OrdinalIgnoreCase))
{
var converter = new CsvConfigurationConverter();
csvConfiguration = converter.ConvertFrom <CsvConfiguration>(ctx.Request.Profile.FileTypeConfiguration);
}
if (csvConfiguration == null)
{
csvConfiguration = CsvConfiguration.ExcelFriendlyConfiguration;
}
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
ctx.ExecuteContext.DataTable = LightweightDataTable.FromFile(
Path.GetFileName(filePath),
stream,
stream.Length,
csvConfiguration,
ctx.Request.Profile.Skip,
ctx.Request.Profile.Take > 0 ? ctx.Request.Profile.Take : int.MaxValue
);
try
{
ctx.Importer.Execute(ctx.ExecuteContext);
}
catch (Exception exception)
{
ctx.ExecuteContext.Abort = DataExchangeAbortion.Hard;
ctx.ExecuteContext.Result.AddError(exception, "The importer failed: {0}.".FormatInvariant(exception.ToAllMessages()));
}
if (ctx.ExecuteContext.IsMaxFailures)
{
ctx.ExecuteContext.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancellationToken.IsCancellationRequested)
{
ctx.ExecuteContext.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
{
return;
}
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(ctx.Request.Profile.ExtraData);
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
{
throw new SmartException("No files to import.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType);
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
