public async Task ImportAsync(DataImportRequest request, CancellationToken cancelToken = default)
{
Guard.NotNull(request, nameof(request));
Guard.NotNull(cancelToken, nameof(cancelToken));
var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(request.ProfileId, false, cancelToken);
if (!(profile?.Enabled ?? false))
{
return;
}
var(ctx, logFile) = await CreateImporterContext(request, profile, cancelToken);
using var logger = new TraceLogger(logFile, false);
ctx.Log = ctx.ExecuteContext.Log = logger;
try
{
if (!request.HasPermission && !await HasPermission())
{
throw new SmartException("You do not have permission to perform the selected import.");
}
var context = ctx.ExecuteContext;
var files = await _importProfileService.GetImportFilesAsync(profile, profile.ImportRelatedData);
var fileGroups = files.ToMultimap(x => x.RelatedType?.ToString() ?? string.Empty, x => x);
logger.Info(CreateLogHeader(profile, fileGroups));
await _services.EventPublisher.PublishAsync(new ImportExecutingEvent(context), cancelToken);
foreach (var fileGroup in fileGroups)
{
context.Result = ctx.Results[fileGroup.Key] = new();
foreach (var file in fileGroup.Value)
{
if (context.Abort == DataExchangeAbortion.Hard)
{
break;
}
if (!file.File.Exists)
{
throw new SmartException($"File does not exist {file.File.SubPath}.");
}
try
{
var csvConfiguration = file.IsCsv
? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
: CsvConfiguration.ExcelFriendlyConfiguration;
using var stream = file.File.OpenRead();
context.File = file;
context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
context.DataTable = LightweightDataTable.FromFile(
file.File.Name,
stream,
stream.Length,
csvConfiguration,
profile.Skip,
profile.Take > 0 ? profile.Take : int.MaxValue);
var segmenter = new ImportDataSegmenter(context.DataTable, context.ColumnMap);
context.DataSegmenter = segmenter;
context.Result.TotalRecords = segmenter.TotalRows;
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
using var batchScope = _scopeAccessor.LifetimeScope.BeginLifetimeScope();
// It would be nice if we could make all dependencies use our TraceLogger.
var importerFactory = batchScope.Resolve <Func <ImportEntityType, IEntityImporter> >();
var importer = importerFactory(profile.EntityType);
await importer.ExecuteAsync(context, cancelToken);
}
}
catch (Exception ex)
{
context.Abort = DataExchangeAbortion.Hard;
context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
}
finally
{
context.Result.EndDateUtc = DateTime.UtcNow;
if (context.IsMaxFailures)
{
context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancelToken.IsCancellationRequested)
{
context.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
finally
{
await Finalize(ctx);
}
cancelToken.ThrowIfCancellationRequested();
}
private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
{
return;
}
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.Value.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
{
throw new SmartException("No files to import found.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
if (ctx.Request.Profile.EntityType == ImportEntityType.Product)
{
ctx.Importer = new ProductImporter(
_productPictureRepository.Value,
_productManufacturerRepository.Value,
_productCategoryRepository.Value,
_urlRecordRepository.Value,
_productRepository.Value,
_services,
_localizedEntityService.Value,
_pictureService.Value,
_manufacturerService.Value,
_categoryService.Value,
_productService.Value,
_urlRecordService.Value,
_productTemplateService.Value,
_storeMappingService.Value,
_fileDownloadManager.Value,
_seoSettings.Value,
_dataExchangeSettings.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.Customer)
{
ctx.Importer = new CustomerImporter(
_customerRepository.Value,
_pictureRepository.Value,
_services,
_genericAttributeService.Value,
_customerService,
_pictureService.Value,
_affiliateService.Value,
_countryService.Value,
_stateProvinceService.Value,
_fileDownloadManager.Value,
_customerSettings.Value,
_dateTimeSettings.Value,
_forumSettings.Value,
_dataExchangeSettings.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription)
{
ctx.Importer = new NewsLetterSubscriptionImporter(
_services,
_subscriptionRepository.Value);
}
else if (ctx.Request.Profile.EntityType == ImportEntityType.Category)
{
ctx.Importer = new CategoryImporter(
_categoryRepository.Value,
_urlRecordRepository.Value,
_pictureRepository.Value,
_services,
_urlRecordService.Value,
_categoryTemplateService.Value,
_storeMappingService.Value,
_pictureService.Value,
_localizedEntityService.Value,
_fileDownloadManager.Value,
_seoSettings.Value,
_dataExchangeSettings.Value);
}
else
{
throw new SmartException("Unsupported entity type {0}.".FormatInvariant(ctx.Request.Profile.EntityType.ToString()));
}
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all that the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
private void ImportCoreOuter(DataImporterContext ctx)
{
if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled)
{
return;
}
var logPath = ctx.Request.Profile.GetImportLogPath();
FileSystemHelper.Delete(logPath);
using (var logger = new TraceLogger(logPath))
{
try
{
ctx.Log = logger;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(ctx.Request.Profile.ExtraData);
{
var mapConverter = new ColumnMapConverter();
ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap();
}
var files = ctx.Request.Profile.GetImportFiles();
if (files.Count == 0)
{
throw new SmartException("No files to import.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType);
files.ForEach(x => ImportCoreInner(ctx, x));
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
finally
{
try
{
// database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception exception)
{
ctx.ExecuteContext.Result.AddError(exception);
}
try
{
ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow;
LogResult(ctx);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone());
_importProfileService.UpdateImportProfile(ctx.Request.Profile);
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Log = null;
}
catch (Exception exception)
{
logger.ErrorsAll(exception);
}
}
}
}
private void ImportCoreOuter(DataImporterContext ctx)
{
var customer = _services.WorkContext.CurrentCustomer;
var profile = ctx.Request.Profile;
var logPath = profile.GetImportLogPath();
FileSystemHelper.DeleteFile(logPath);
using (var logger = new TraceLogger(logPath))
{
var scopes = new List <IDisposable>();
try
{
var files = profile.GetImportFiles(profile.ImportRelatedData);
var groupedFiles = files.GroupBy(x => x.RelatedType);
if (!files.Any())
{
throw new SmartException("No files to import.");
}
if (!HasPermission(ctx))
{
throw new SmartException("You do not have permission to perform the selected import.");
}
_dbCache.Enabled = false;
_services.MediaService.ImagePostProcessingEnabled = false;
scopes.Add(_localizedEntityService.BeginScope());
scopes.Add(_urlRecordService.BeginScope());
ctx.Log = logger;
ctx.Importer = _importerFactory(profile.EntityType);
ctx.ExecuteContext.Request = ctx.Request;
ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value;
ctx.ExecuteContext.Services = _services;
ctx.ExecuteContext.Log = logger;
ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true);
ctx.ExecuteContext.UpdateOnly = profile.UpdateOnly;
ctx.ExecuteContext.KeyFieldNames = profile.KeyFieldNames.SplitSafe(",");
ctx.ExecuteContext.ImportFolder = profile.GetImportFolder();
ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData);
var sb = new StringBuilder();
sb.AppendLine();
sb.AppendLine(new string('-', 40));
sb.AppendLine("Smartstore: v." + SmartStoreVersion.CurrentFullVersion);
sb.AppendLine("Import profile: {0} {1}".FormatInvariant(profile.Name, profile.Id == 0 ? " (volatile)" : $" (Id {profile.Id})"));
foreach (var fileGroup in groupedFiles)
{
var entityName = fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : profile.EntityType.ToString();
var fileNames = string.Join(", ", fileGroup.Select(x => x.Name));
sb.AppendLine("{0} files: {1}".FormatInvariant(entityName, fileNames));
}
sb.Append("Executed by: " + customer.Email.NullEmpty() ?? customer.SystemName.NaIfEmpty());
ctx.Log.Info(sb.ToString());
_services.EventPublisher.Publish(new ImportExecutingEvent(ctx.ExecuteContext));
foreach (var fileGroup in groupedFiles)
{
ctx.ExecuteContext.Result = ctx.Results[fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : string.Empty] = new ImportResult();
fileGroup.Each(x => ImportCoreInner(ctx, x));
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
finally
{
try
{
_dbCache.Enabled = true;
_services.MediaService.ImagePostProcessingEnabled = true;
scopes.Each(x => x.Dispose());
_services.EventPublisher.Publish(new ImportExecutedEvent(ctx.ExecuteContext));
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
// Database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure,
// then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...).
// so whatever it is, detach\dispose all what the tracker still has tracked.
_services.DbContext.DetachAll(false);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
SendCompletionEmail(ctx);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
LogResults(ctx);
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
if (ctx.Results.TryGetValue(string.Empty, out var result))
{
profile.ResultInfo = XmlHelper.Serialize(result.Clone());
_importProfileService.UpdateImportProfile(profile);
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
try
{
ctx.Request.CustomData.Clear();
ctx.Results.Clear();
ctx.Log = null;
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
}
}
}
