public async Task Run(TaskExecutionContext ctx, CancellationToken cancelToken = default)
{
var request = new DataImportRequest(ctx.ExecutionInfo.Task.Alias.ToInt())
{
ProgressCallback = OnProgress
};
// Process!
await _importer.ImportAsync(request, cancelToken);
Task OnProgress(int value, int max, string msg)
{
return(ctx.SetProgressAsync(value, max, msg, true));
}
}
public async Task ImportAsync(DataImportRequest request, CancellationToken cancelToken = default)
{
Guard.NotNull(request, nameof(request));
Guard.NotNull(cancelToken, nameof(cancelToken));
var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(request.ProfileId, false, cancelToken);
if (!(profile?.Enabled ?? false))
{
return;
}
var(ctx, logFile) = await CreateImporterContext(request, profile, cancelToken);
using var logger = new TraceLogger(logFile, false);
ctx.Log = ctx.ExecuteContext.Log = logger;
try
{
if (!request.HasPermission && !await HasPermission())
{
throw new SmartException("You do not have permission to perform the selected import.");
}
var context = ctx.ExecuteContext;
var files = await _importProfileService.GetImportFilesAsync(profile, profile.ImportRelatedData);
var fileGroups = files.ToMultimap(x => x.RelatedType?.ToString() ?? string.Empty, x => x);
logger.Info(CreateLogHeader(profile, fileGroups));
await _services.EventPublisher.PublishAsync(new ImportExecutingEvent(context), cancelToken);
foreach (var fileGroup in fileGroups)
{
context.Result = ctx.Results[fileGroup.Key] = new();
foreach (var file in fileGroup.Value)
{
if (context.Abort == DataExchangeAbortion.Hard)
{
break;
}
if (!file.File.Exists)
{
throw new SmartException($"File does not exist {file.File.SubPath}.");
}
try
{
var csvConfiguration = file.IsCsv
? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
: CsvConfiguration.ExcelFriendlyConfiguration;
using var stream = file.File.OpenRead();
context.File = file;
context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
context.DataTable = LightweightDataTable.FromFile(
file.File.Name,
stream,
stream.Length,
csvConfiguration,
profile.Skip,
profile.Take > 0 ? profile.Take : int.MaxValue);
var segmenter = new ImportDataSegmenter(context.DataTable, context.ColumnMap);
context.DataSegmenter = segmenter;
context.Result.TotalRecords = segmenter.TotalRows;
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
using var batchScope = _scopeAccessor.LifetimeScope.BeginLifetimeScope();
// It would be nice if we could make all dependencies use our TraceLogger.
var importerFactory = batchScope.Resolve <Func <ImportEntityType, IEntityImporter> >();
var importer = importerFactory(profile.EntityType);
await importer.ExecuteAsync(context, cancelToken);
}
}
catch (Exception ex)
{
context.Abort = DataExchangeAbortion.Hard;
context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
}
finally
{
context.Result.EndDateUtc = DateTime.UtcNow;
if (context.IsMaxFailures)
{
context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancelToken.IsCancellationRequested)
{
context.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
finally
{
await Finalize(ctx);
}
cancelToken.ThrowIfCancellationRequested();
}
private async Task <(DataImporterContext Context, IFile LogFile)> CreateImporterContext(DataImportRequest request, ImportProfile profile, CancellationToken cancelToken)
{
var dir = await _importProfileService.GetImportDirectoryAsync(profile, "Content", true);
var executeContext = new ImportExecuteContext(T("Admin.DataExchange.Import.ProgressInfo"), cancelToken)
{
Request = request,
ProgressCallback = request.ProgressCallback,
UpdateOnly = profile.UpdateOnly,
KeyFieldNames = profile.KeyFieldNames.SplitSafe(",").ToArray(),
ImportDirectory = dir,
ImageDownloadDirectory = await _importProfileService.GetImportDirectoryAsync(profile, @"Content\DownloadedImages", true),
ExtraData = XmlHelper.Deserialize <ImportExtraData>(profile.ExtraData),
Languages = await _languageService.GetAllLanguagesAsync(true),
Stores = _services.StoreContext.GetAllStores().AsReadOnly(),
DownloadManager = new DownloadManager(TimeSpan.FromMinutes(_dataExchangeSettings.ImageDownloadTimeout))
};
// Relative paths for images always refer to the profile directory, not to its "Content" sub-directory.
executeContext.ImageDirectory = _dataExchangeSettings.ImageImportFolder.HasValue()
? await _importProfileService.GetImportDirectoryAsync(profile, _dataExchangeSettings.ImageImportFolder, false)
: dir.Parent;
var context = new DataImporterContext
{
Request = request,
CancelToken = cancelToken,
ColumnMap = new ColumnMapConverter().ConvertFrom <ColumnMap>(profile.ColumnMapping) ?? new ColumnMap(),
ExecuteContext = executeContext
};
var logFile = await dir.Parent.GetFileAsync("log.txt");
return(context, logFile);
}
