public async Task ImportAsync(DataImportRequest request, CancellationToken cancelToken = default) { Guard.NotNull(request, nameof(request)); Guard.NotNull(cancelToken, nameof(cancelToken)); var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(request.ProfileId, false, cancelToken); if (!(profile?.Enabled ?? false)) { return; } var(ctx, logFile) = await CreateImporterContext(request, profile, cancelToken); using var logger = new TraceLogger(logFile, false); ctx.Log = ctx.ExecuteContext.Log = logger; try { if (!request.HasPermission && !await HasPermission()) { throw new SmartException("You do not have permission to perform the selected import."); } var context = ctx.ExecuteContext; var files = await _importProfileService.GetImportFilesAsync(profile, profile.ImportRelatedData); var fileGroups = files.ToMultimap(x => x.RelatedType?.ToString() ?? string.Empty, x => x); logger.Info(CreateLogHeader(profile, fileGroups)); await _services.EventPublisher.PublishAsync(new ImportExecutingEvent(context), cancelToken); foreach (var fileGroup in fileGroups) { context.Result = ctx.Results[fileGroup.Key] = new(); foreach (var file in fileGroup.Value) { if (context.Abort == DataExchangeAbortion.Hard) { break; } if (!file.File.Exists) { throw new SmartException($"File does not exist {file.File.SubPath}."); } try { var csvConfiguration = file.IsCsv ? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration) : CsvConfiguration.ExcelFriendlyConfiguration; using var stream = file.File.OpenRead(); context.File = file; context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap; context.DataTable = LightweightDataTable.FromFile( file.File.Name, stream, stream.Length, csvConfiguration, profile.Skip, profile.Take > 0 ? profile.Take : int.MaxValue); var segmenter = new ImportDataSegmenter(context.DataTable, context.ColumnMap); context.DataSegmenter = segmenter; context.Result.TotalRecords = segmenter.TotalRows; while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch()) { using var batchScope = _scopeAccessor.LifetimeScope.BeginLifetimeScope(); // It would be nice if we could make all dependencies use our TraceLogger. var importerFactory = batchScope.Resolve <Func <ImportEntityType, IEntityImporter> >(); var importer = importerFactory(profile.EntityType); await importer.ExecuteAsync(context, cancelToken); } } catch (Exception ex) { context.Abort = DataExchangeAbortion.Hard; context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}."); } finally { context.Result.EndDateUtc = DateTime.UtcNow; if (context.IsMaxFailures) { context.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancelToken.IsCancellationRequested) { context.Result.AddWarning("Import aborted. A cancellation has been requested."); } } } } } catch (Exception ex) { logger.ErrorsAll(ex); } finally { await Finalize(ctx); } cancelToken.ThrowIfCancellationRequested(); }
private void ImportCoreOuter(DataImporterContext ctx) { if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled) { return; } var logPath = ctx.Request.Profile.GetImportLogPath(); FileSystemHelper.Delete(logPath); using (var logger = new TraceLogger(logPath)) { try { ctx.Log = logger; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.Value.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder(); { var mapConverter = new ColumnMapConverter(); ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap(); } var files = ctx.Request.Profile.GetImportFiles(); if (files.Count == 0) { throw new SmartException("No files to import found."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } if (ctx.Request.Profile.EntityType == ImportEntityType.Product) { ctx.Importer = new ProductImporter( _productPictureRepository.Value, _productManufacturerRepository.Value, _productCategoryRepository.Value, _urlRecordRepository.Value, _productRepository.Value, _services, _localizedEntityService.Value, _pictureService.Value, _manufacturerService.Value, _categoryService.Value, _productService.Value, _urlRecordService.Value, _productTemplateService.Value, _storeMappingService.Value, _fileDownloadManager.Value, _seoSettings.Value, _dataExchangeSettings.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.Customer) { ctx.Importer = new CustomerImporter( _customerRepository.Value, _pictureRepository.Value, _services, _genericAttributeService.Value, _customerService, _pictureService.Value, _affiliateService.Value, _countryService.Value, _stateProvinceService.Value, _fileDownloadManager.Value, _customerSettings.Value, _dateTimeSettings.Value, _forumSettings.Value, _dataExchangeSettings.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.NewsLetterSubscription) { ctx.Importer = new NewsLetterSubscriptionImporter( _services, _subscriptionRepository.Value); } else if (ctx.Request.Profile.EntityType == ImportEntityType.Category) { ctx.Importer = new CategoryImporter( _categoryRepository.Value, _urlRecordRepository.Value, _pictureRepository.Value, _services, _urlRecordService.Value, _categoryTemplateService.Value, _storeMappingService.Value, _pictureService.Value, _localizedEntityService.Value, _fileDownloadManager.Value, _seoSettings.Value, _dataExchangeSettings.Value); } else { throw new SmartException("Unsupported entity type {0}.".FormatInvariant(ctx.Request.Profile.EntityType.ToString())); } files.ForEach(x => ImportCoreInner(ctx, x)); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } finally { try { // database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all that the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { SendCompletionEmail(ctx); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow; LogResult(ctx); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone()); _importProfileService.UpdateImportProfile(ctx.Request.Profile); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.CustomData.Clear(); ctx.Log = null; } catch (Exception exception) { logger.ErrorsAll(exception); } } } }
private void ImportCoreOuter(DataImporterContext ctx) { if (ctx.Request.Profile == null || !ctx.Request.Profile.Enabled) { return; } var logPath = ctx.Request.Profile.GetImportLogPath(); FileSystemHelper.Delete(logPath); using (var logger = new TraceLogger(logPath)) { try { ctx.Log = logger; ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value; ctx.ExecuteContext.Services = _services; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = ctx.Request.Profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = ctx.Request.Profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = ctx.Request.Profile.GetImportFolder(); ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(ctx.Request.Profile.ExtraData); { var mapConverter = new ColumnMapConverter(); ctx.ExecuteContext.ColumnMap = mapConverter.ConvertFrom <ColumnMap>(ctx.Request.Profile.ColumnMapping) ?? new ColumnMap(); } var files = ctx.Request.Profile.GetImportFiles(); if (files.Count == 0) { throw new SmartException("No files to import."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } ctx.Importer = _importerFactory(ctx.Request.Profile.EntityType); files.ForEach(x => ImportCoreInner(ctx, x)); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } finally { try { // database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all what the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { SendCompletionEmail(ctx); } catch (Exception exception) { ctx.ExecuteContext.Result.AddError(exception); } try { ctx.ExecuteContext.Result.EndDateUtc = DateTime.UtcNow; LogResult(ctx); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.Profile.ResultInfo = XmlHelper.Serialize(ctx.ExecuteContext.Result.Clone()); _importProfileService.UpdateImportProfile(ctx.Request.Profile); } catch (Exception exception) { logger.ErrorsAll(exception); } try { ctx.Request.CustomData.Clear(); ctx.Log = null; } catch (Exception exception) { logger.ErrorsAll(exception); } } } }
private void ImportCoreOuter(DataImporterContext ctx) { var customer = _services.WorkContext.CurrentCustomer; var profile = ctx.Request.Profile; var logPath = profile.GetImportLogPath(); FileSystemHelper.DeleteFile(logPath); using (var logger = new TraceLogger(logPath)) { var scopes = new List <IDisposable>(); try { var files = profile.GetImportFiles(profile.ImportRelatedData); var groupedFiles = files.GroupBy(x => x.RelatedType); if (!files.Any()) { throw new SmartException("No files to import."); } if (!HasPermission(ctx)) { throw new SmartException("You do not have permission to perform the selected import."); } _dbCache.Enabled = false; _services.MediaService.ImagePostProcessingEnabled = false; scopes.Add(_localizedEntityService.BeginScope()); scopes.Add(_urlRecordService.BeginScope()); ctx.Log = logger; ctx.Importer = _importerFactory(profile.EntityType); ctx.ExecuteContext.Request = ctx.Request; ctx.ExecuteContext.DataExchangeSettings = _dataExchangeSettings.Value; ctx.ExecuteContext.Services = _services; ctx.ExecuteContext.Log = logger; ctx.ExecuteContext.Languages = _languageService.GetAllLanguages(true); ctx.ExecuteContext.UpdateOnly = profile.UpdateOnly; ctx.ExecuteContext.KeyFieldNames = profile.KeyFieldNames.SplitSafe(","); ctx.ExecuteContext.ImportFolder = profile.GetImportFolder(); ctx.ExecuteContext.ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData); var sb = new StringBuilder(); sb.AppendLine(); sb.AppendLine(new string('-', 40)); sb.AppendLine("Smartstore: v." + SmartStoreVersion.CurrentFullVersion); sb.AppendLine("Import profile: {0} {1}".FormatInvariant(profile.Name, profile.Id == 0 ? " (volatile)" : $" (Id {profile.Id})")); foreach (var fileGroup in groupedFiles) { var entityName = fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : profile.EntityType.ToString(); var fileNames = string.Join(", ", fileGroup.Select(x => x.Name)); sb.AppendLine("{0} files: {1}".FormatInvariant(entityName, fileNames)); } sb.Append("Executed by: " + customer.Email.NullEmpty() ?? customer.SystemName.NaIfEmpty()); ctx.Log.Info(sb.ToString()); _services.EventPublisher.Publish(new ImportExecutingEvent(ctx.ExecuteContext)); foreach (var fileGroup in groupedFiles) { ctx.ExecuteContext.Result = ctx.Results[fileGroup.Key.HasValue ? fileGroup.Key.Value.ToString() : string.Empty] = new ImportResult(); fileGroup.Each(x => ImportCoreInner(ctx, x)); } } catch (Exception ex) { logger.ErrorsAll(ex); } finally { try { _dbCache.Enabled = true; _services.MediaService.ImagePostProcessingEnabled = true; scopes.Each(x => x.Dispose()); _services.EventPublisher.Publish(new ImportExecutedEvent(ctx.ExecuteContext)); } catch (Exception ex) { logger.ErrorsAll(ex); } try { // Database context sharing problem: if there are entities in modified state left by the provider due to SaveChanges failure, // then all subsequent SaveChanges would fail too (e.g. IImportProfileService.UpdateImportProfile, IScheduledTaskService.UpdateTask...). // so whatever it is, detach\dispose all what the tracker still has tracked. _services.DbContext.DetachAll(false); } catch (Exception ex) { logger.ErrorsAll(ex); } try { SendCompletionEmail(ctx); } catch (Exception ex) { logger.ErrorsAll(ex); } try { LogResults(ctx); } catch (Exception ex) { logger.ErrorsAll(ex); } try { if (ctx.Results.TryGetValue(string.Empty, out var result)) { profile.ResultInfo = XmlHelper.Serialize(result.Clone()); _importProfileService.UpdateImportProfile(profile); } } catch (Exception ex) { logger.ErrorsAll(ex); } try { ctx.Request.CustomData.Clear(); ctx.Results.Clear(); ctx.Log = null; } catch (Exception ex) { logger.ErrorsAll(ex); } } } }