public ImportRow(ImportDataSegmenter parent, IDataRow row, int position)
{
_segmenter = parent;
_row = row;
_position = position;
}
public async Task ImportAsync(DataImportRequest request, CancellationToken cancelToken = default)
{
Guard.NotNull(request, nameof(request));
Guard.NotNull(cancelToken, nameof(cancelToken));
var profile = await _services.DbContext.ImportProfiles.FindByIdAsync(request.ProfileId, false, cancelToken);
if (!(profile?.Enabled ?? false))
{
return;
}
var(ctx, logFile) = await CreateImporterContext(request, profile, cancelToken);
using var logger = new TraceLogger(logFile, false);
ctx.Log = ctx.ExecuteContext.Log = logger;
try
{
if (!request.HasPermission && !await HasPermission())
{
throw new SmartException("You do not have permission to perform the selected import.");
}
var context = ctx.ExecuteContext;
var files = await _importProfileService.GetImportFilesAsync(profile, profile.ImportRelatedData);
var fileGroups = files.ToMultimap(x => x.RelatedType?.ToString() ?? string.Empty, x => x);
logger.Info(CreateLogHeader(profile, fileGroups));
await _services.EventPublisher.PublishAsync(new ImportExecutingEvent(context), cancelToken);
foreach (var fileGroup in fileGroups)
{
context.Result = ctx.Results[fileGroup.Key] = new();
foreach (var file in fileGroup.Value)
{
if (context.Abort == DataExchangeAbortion.Hard)
{
break;
}
if (!file.File.Exists)
{
throw new SmartException($"File does not exist {file.File.SubPath}.");
}
try
{
var csvConfiguration = file.IsCsv
? (new CsvConfigurationConverter().ConvertFrom <CsvConfiguration>(profile.FileTypeConfiguration) ?? CsvConfiguration.ExcelFriendlyConfiguration)
: CsvConfiguration.ExcelFriendlyConfiguration;
using var stream = file.File.OpenRead();
context.File = file;
context.ColumnMap = file.RelatedType.HasValue ? new ColumnMap() : ctx.ColumnMap;
context.DataTable = LightweightDataTable.FromFile(
file.File.Name,
stream,
stream.Length,
csvConfiguration,
profile.Skip,
profile.Take > 0 ? profile.Take : int.MaxValue);
var segmenter = new ImportDataSegmenter(context.DataTable, context.ColumnMap);
context.DataSegmenter = segmenter;
context.Result.TotalRecords = segmenter.TotalRows;
while (context.Abort == DataExchangeAbortion.None && segmenter.ReadNextBatch())
{
using var batchScope = _scopeAccessor.LifetimeScope.BeginLifetimeScope();
// It would be nice if we could make all dependencies use our TraceLogger.
var importerFactory = batchScope.Resolve <Func <ImportEntityType, IEntityImporter> >();
var importer = importerFactory(profile.EntityType);
await importer.ExecuteAsync(context, cancelToken);
}
}
catch (Exception ex)
{
context.Abort = DataExchangeAbortion.Hard;
context.Result.AddError(ex, $"The importer failed: {ex.ToAllMessages()}.");
}
finally
{
context.Result.EndDateUtc = DateTime.UtcNow;
if (context.IsMaxFailures)
{
context.Result.AddWarning("Import aborted. The maximum number of failures has been reached.");
}
if (ctx.CancelToken.IsCancellationRequested)
{
context.Result.AddWarning("Import aborted. A cancellation has been requested.");
}
}
}
}
}
catch (Exception ex)
{
logger.ErrorsAll(ex);
}
finally
{
await Finalize(ctx);
}
cancelToken.ThrowIfCancellationRequested();
}
