public async Task Run(TaskExecutionContext ctx, CancellationToken cancelToken = default) { var request = new DataImportRequest(ctx.ExecutionInfo.Task.Alias.ToInt()) { ProgressCallback = OnProgress }; // Process! await _importer.ImportAsync(request, cancelToken); Task OnProgress(int value, int max, string msg) { return(ctx.SetProgressAsync(value, max, msg, true)); } }
public async Task ImportAsync(DataImportRequest request, CancellationToken cancelToken = default) { Guard.NotNull(request, nameof(request)); Guard.NotNull(cancelToken, nameof(cancelToken)); var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(request.ProfileId, false, cancelToken); if (!(profile?.Enabled ?? false)) { return; } var(ctx, logFile) = await CreateImporterContext(request, profile, cancelToken); using var logger = new TraceLogger(logFile, false); ctx.Log = ctx.ExecuteContext.Log = logger; try { if (!request.HasPermission && !await HasPermission()) { throw new SmartException("You do not have permission to perform the selected import."); } var context = ctx.ExecuteContext; var files = await _importProfileService.GetImportFilesAsync(profile, profile.ImportRelatedData); var fileGroups = files.ToMultimap(x => x.RelatedType?.ToString() ?? string.Empty, x => x); logger.Info(CreateLogHeader(profile, fileGroups)); await _services.EventPublisher.PublishAsync(new ImportExecutingEvent(context), cancelToken); foreach (var fileGroup in fileGroups) { context.Result = ctx.Results[fileGroup.Key] = new(); foreach (var file in fileGroup.Value) { if (context.Abort == DataExchangeAbortion.Hard) { break; } if (!file.File.Exists) { throw new SmartException($"File does not exist {file.File.SubPath}."); } try { var csvConfiguration = file.IsCsv ? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration) : CsvConfiguration.ExcelFriendlyConfiguration; using var stream = file.File.OpenRead(); context.File = file; context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap; context.DataTable = LightweightDataTable.FromFile( file.File.Name, stream, stream.Length, csvConfiguration, profile.Skip, profile.Take > 0 ? profile.Take : int.MaxValue); var segmenter = new ImportDataSegmenter(context.DataTable, context.ColumnMap); context.DataSegmenter = segmenter; context.Result.TotalRecords = segmenter.TotalRows; while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch()) { using var batchScope = _scopeAccessor.LifetimeScope.BeginLifetimeScope(); // It would be nice if we could make all dependencies use our TraceLogger. var importerFactory = batchScope.Resolve <Func <ImportEntityType, IEntityImporter> >(); var importer = importerFactory(profile.EntityType); await importer.ExecuteAsync(context, cancelToken); } } catch (Exception ex) { context.Abort = DataExchangeAbortion.Hard; context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}."); } finally { context.Result.EndDateUtc = DateTime.UtcNow; if (context.IsMaxFailures) { context.Result.AddWarning("Import aborted. The maximum number of failures has been reached."); } if (ctx.CancelToken.IsCancellationRequested) { context.Result.AddWarning("Import aborted. A cancellation has been requested."); } } } } } catch (Exception ex) { logger.ErrorsAll(ex); } finally { await Finalize(ctx); } cancelToken.ThrowIfCancellationRequested(); }
private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken) { var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true); var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken) { Request = request, ProgressCallback = request.ProgressCallback, UpdateOnly = profile.UpdateOnly, KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(), ImportDirectory = dir, ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true), ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData), Languages = await _languageService.GetAllLanguagesAsync(true), Stores = _services.StoreContext.GetAllStores().AsReadOnly(), DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout)) }; // Relative paths for images always refer to the profile directory, not to its "Content" sub-directory. executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue() ? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false) : dir.Parent; var context = new DataImporterContext { Request = request, CancelToken = cancelToken, ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(), ExecuteContext = executeContext }; var logFile = await dir.Parent.GetFileAsync("log.txt"); return(context, logFile); }