private static void SetupErrorHandlers(Action <ImportProgressInfo> progressCallback, ImportConfiguration configuration,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
{
configuration.ReadingExceptionOccurred = exception =>
{
var context = exception.ReadingContext;
if (!errorsContext.ErrorsRows.Contains(context.Row))
{
var fieldSourceValue = context.Record[context.CurrentIndex];
if (context.HeaderRecord.Length != context.Record.Length)
{
HandleNotClosedQuoteError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else if (fieldSourceValue == "")
{
HandleRequiredValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else
{
HandleWrongValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
}
return(false);
};
configuration.BadDataFound = async context =>
{
await HandleBadDataErrorAsync(progressCallback, importProgress, importReporter, context, errorsContext);
};
configuration.MissingFieldFound = null;
}
public static void Patch(this ImportPushNotification target, ImportProgressInfo source)
{
target.Description = source.Description;
target.ProcessedCount = source.ProcessedCount;
target.TotalCount = source.TotalCount;
target.ContactsCreated = source.ContactsCreated;
target.ContactsUpdated = source.ContactsUpdated;
target.ErrorCount = source.ErrorCount;
target.Errors = source.Errors;
target.ReportUrl = source.ReportUrl;
}
private static void SetupErrorHandlers(Action <ImportProgressInfo> progressCallback,
CsvConfiguration configuration,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress,
ICsvPriceImportReporter importReporter)
{
configuration.ReadingExceptionOccurred = args =>
{
var context = args.Exception.Context;
if (!errorsContext.ErrorsRows.Contains(context.Parser.Row))
{
var fieldSourceValue = context.Reader[context.Reader.CurrentIndex];
if (context.Reader.HeaderRecord.Length != context.Parser.Record.Length)
{
HandleNotClosedQuoteError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else if (fieldSourceValue == "")
{
HandleRequiredValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else
{
HandleWrongValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
}
return(false);
};
configuration.BadDataFound = args =>
{
HandleBadDataError(progressCallback, importProgress, importReporter, args.Context, errorsContext);
};
configuration.MissingFieldFound = args => HandleMissedColumnError(progressCallback, importProgress, importReporter, args.Context, errorsContext);
}
private static void HandleRequiredValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var fieldName = context.HeaderRecord[context.CurrentIndex];
var requiredFields = CsvCustomerImportHelper.GetImportCustomerRequiredColumns <ImportableContact>();
var missedValueColumns = new List <string>();
for (var i = 0; i < context.HeaderRecord.Length; i++)
{
if (requiredFields.Contains(context.HeaderRecord[i], StringComparer.InvariantCultureIgnoreCase) && context.Record[i].IsNullOrEmpty())
{
missedValueColumns.Add(context.HeaderRecord[i]);
}
}
var importError = new ImportError {
Error = $"The required value in column {fieldName} is missing.", RawRow = context.RawRecord
};
if (missedValueColumns.Count > 1)
{
importError.Error = $"The required values in columns: {string.Join(", ", missedValueColumns)} - are missing.";
}
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var invalidFieldName = context.HeaderRecord[context.CurrentIndex];
var importError = new ImportError {
Error = string.Format(ModuleConstants.ValidationMessages[ModuleConstants.ValidationErrors.InvalidValue], invalidFieldName), RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. Quotes should be closed.", RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static async Task HandleBadDataErrorAsync(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.RawRecord
};
await reporter.WriteAsync(importError);
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
protected static void HandleError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, string error = null)
{
if (error != null)
{
importProgress.Errors.Add(error);
}
progressCallback(importProgress);
}
protected override async Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <ImportableContact> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
{
var importContacts = dataSource.Items
// expect records that was parsed with errors
.Where(importContact => !errorsContext.ErrorsRows.Contains(importContact.Row))
.ToArray();
try
{
var internalIds = importContacts.Select(x => x.Record?.Id).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var outerIds = importContacts.Select(x => x.Record?.OuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var existedContacts =
(await SearchMembersByIdAndOuterIdAsync(internalIds, outerIds, new[] { nameof(Contact) }, true))
.OfType <Contact>().ToArray();
SetIdToNullForNotExisted(importContacts, existedContacts);
SetIdToRealForExistedOuterId(importContacts, existedContacts);
var validationResult = await ValidateAsync(importContacts, importReporter);
var invalidImportContacts = validationResult.Errors
.Select(x => (x.CustomState as ImportValidationState <ImportableContact>)?.InvalidRecord).Distinct().ToArray();
importContacts = importContacts.Except(invalidImportContacts).ToArray();
//reduce existed set after validation
existedContacts = existedContacts.Where(ec => importContacts.Any(ic =>
ec.Id.EqualsInvariant(ic.Record.Id) ||
!string.IsNullOrEmpty(ec.OuterId) && ec.OuterId.EqualsInvariant(ic.Record.OuterId)))
.ToArray();
var updateImportContacts = importContacts.Where(x => existedContacts.Any(ec =>
ec.Id.EqualsInvariant(x.Record.Id) ||
(!ec.OuterId.IsNullOrEmpty() && ec.OuterId.EqualsInvariant(x.Record.OuterId)))
).ToArray();
existedContacts = GetReducedExistedByWrongOuterId(updateImportContacts, existedContacts).OfType <Contact>().ToArray();
var createImportContacts = importContacts.Except(updateImportContacts).ToArray();
var internalOrgIds = importContacts.Select(x => x.Record?.OrganizationId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var outerOrgIds = importContacts.Select(x => x.Record?.OrganizationOuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var existedOrganizations =
(await SearchMembersByIdAndOuterIdAsync(internalOrgIds, outerOrgIds,
new[] { nameof(Organization) })).OfType <Organization>().ToArray();
var newContacts = CreateNewContacts(createImportContacts, existedOrganizations, request);
PatchExistedContacts(existedContacts, updateImportContacts, existedOrganizations, request);
var contactsForSave = newContacts.Union(existedContacts).ToArray();
await _memberService.SaveChangesAsync(contactsForSave);
await CreateAccountsForContacts(contactsForSave);
importProgress.ContactsCreated += newContacts.Length;
importProgress.ContactsUpdated += existedContacts.Length;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
importProgress.ErrorCount = importProgress.ProcessedCount - importProgress.ContactsCreated - importProgress.ContactsUpdated;
}
}
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var reportFilePath = GetReportFilePath(request.FilePath);
var configuration = _importConfigurationFactory.Create();
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started"
};
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
importReporter.WriteHeader(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists);
while (await dataSource.FetchAsync())
{
cancellationToken.ThrowIfCancellationRequested();
var importProductPrices = dataSource.Items
// expect records that was parsed with errors
.Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row))
.Select(importProductPrice =>
{
importProductPrice.Price.PricelistId = request.PricelistId;
return(importProductPrice);
}).ToArray();
try
{
var createdPrices = new List <Price>();
var updatedPrices = new List <Price>();
var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString()));
var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray();
var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
importProgress.ErrorCount += invalidImportProductPrices.Length;
importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray();
switch (request.ImportMode)
{
case ImportMode.CreateOnly:
createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price));
break;
case ImportMode.UpdateOnly:
var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices);
updatedPrices.AddRange(existingPrices);
break;
case ImportMode.CreateAndUpdate:
var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices);
var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors
.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray();
createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price));
var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate);
updatedPrices.AddRange(existingPricesToUpdate);
break;
default:
throw new ArgumentException("Import mode has invalid value", nameof(request));
}
var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray();
await _pricingService.SavePricesAsync(allChangingPrices);
importProgress.CreatedCount += createdPrices.Count;
importProgress.UpdatedCount += updatedPrices.Count;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
}
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}
private static void HandleMissedColumnError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var headerColumns = context.Reader.HeaderRecord;
var recordFields = context.Parser.Record;
var missedColumns = headerColumns.Skip(recordFields.Length).ToArray();
var error = $"This row has next missing columns: {string.Join(", ", missedColumns)}.";
var importError = new ImportError {
Error = error, RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var invalidFieldName = context.Reader.HeaderRecord[context.Reader.CurrentIndex];
var importError = new ImportError {
Error = $"This row has invalid value in the column {invalidFieldName}.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. Quotes should be closed.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleBadDataError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
if (context.Reader is VcCsvReader vcCsvReader)
{
vcCsvReader.IsFieldBadData = true;
}
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, string error = null)
{
if (error != null)
{
importProgress.Errors.Add(error);
}
importProgress.ErrorCount++;
progressCallback(importProgress);
}
public virtual async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _dataValidator.ValidateAsync <TCsvMember>(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var configuration = new ImportConfiguration();
var reportFilePath = GetReportFilePath(request.FilePath);
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
Description = "Import has started"
};
using var dataSource = await _dataSourceFactory.CreateAsync <TCsvMember, TMember>(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
await importReporter.WriteHeaderAsync(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
while (await dataSource.FetchAsync())
{
await ProcessChunkAsync(request, progressCallback, dataSource, errorsContext, importProgress, importReporter);
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}
protected abstract Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <TCsvMember> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter);
private void ProgressCallback(ImportProgressInfo x, ImportPushNotification pushNotification, PerformContext context)
{
pushNotification.Patch(x);
pushNotification.JobId = context.BackgroundJob.Id;
_pushNotificationManager.Send(pushNotification);
}
