private static void SetupErrorHandlers(Action <ImportProgressInfo> progressCallback, ImportConfiguration configuration,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
{
configuration.ReadingExceptionOccurred = exception =>
{
var context = exception.ReadingContext;
if (!errorsContext.ErrorsRows.Contains(context.Row))
{
var fieldSourceValue = context.Record[context.CurrentIndex];
if (context.HeaderRecord.Length != context.Record.Length)
{
HandleNotClosedQuoteError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else if (fieldSourceValue == "")
{
HandleRequiredValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
else
{
HandleWrongValueError(progressCallback, importProgress, importReporter, context, errorsContext);
}
}
return(false);
};
configuration.BadDataFound = async context =>
{
await HandleBadDataErrorAsync(progressCallback, importProgress, importReporter, context, errorsContext);
};
configuration.MissingFieldFound = null;
}
private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var invalidFieldName = context.HeaderRecord[context.CurrentIndex];
var importError = new ImportError {
Error = string.Format(ModuleConstants.ValidationMessages[ModuleConstants.ValidationErrors.InvalidValue], invalidFieldName), RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleRequiredValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var fieldName = context.HeaderRecord[context.CurrentIndex];
var requiredFields = CsvCustomerImportHelper.GetImportCustomerRequiredColumns <ImportableContact>();
var missedValueColumns = new List <string>();
for (var i = 0; i < context.HeaderRecord.Length; i++)
{
if (requiredFields.Contains(context.HeaderRecord[i], StringComparer.InvariantCultureIgnoreCase) && context.Record[i].IsNullOrEmpty())
{
missedValueColumns.Add(context.HeaderRecord[i]);
}
}
var importError = new ImportError {
Error = $"The required value in column {fieldName} is missing.", RawRow = context.RawRecord
};
if (missedValueColumns.Count > 1)
{
importError.Error = $"The required values in columns: {string.Join(", ", missedValueColumns)} - are missing.";
}
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. Quotes should be closed.", RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static async Task HandleBadDataErrorAsync(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.RawRecord
};
await reporter.WriteAsync(importError);
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
protected async Task <ValidationResult> ValidateAsync(ImportRecord <TCsvMember>[] importRecords, ICsvCustomerImportReporter importReporter)
{
var validationResult = await _importRecordsValidator.ValidateAsync(importRecords);
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, (x.CustomState as ImportValidationState <TCsvMember>)?.InvalidRecord }).ToArray();
// We need to order by row number because otherwise records will be written to report in random order
var errorsGroups = errorsInfos.OrderBy(x => x.InvalidRecord.Row).GroupBy(x => x.InvalidRecord);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
return(validationResult);
}
protected abstract Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <TCsvMember> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter);
protected override async Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <ImportableContact> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
{
var importContacts = dataSource.Items
// expect records that was parsed with errors
.Where(importContact => !errorsContext.ErrorsRows.Contains(importContact.Row))
.ToArray();
try
{
var internalIds = importContacts.Select(x => x.Record?.Id).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var outerIds = importContacts.Select(x => x.Record?.OuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var existedContacts =
(await SearchMembersByIdAndOuterIdAsync(internalIds, outerIds, new[] { nameof(Contact) }, true))
.OfType <Contact>().ToArray();
SetIdToNullForNotExisted(importContacts, existedContacts);
SetIdToRealForExistedOuterId(importContacts, existedContacts);
var validationResult = await ValidateAsync(importContacts, importReporter);
var invalidImportContacts = validationResult.Errors
.Select(x => (x.CustomState as ImportValidationState <ImportableContact>)?.InvalidRecord).Distinct().ToArray();
importContacts = importContacts.Except(invalidImportContacts).ToArray();
//reduce existed set after validation
existedContacts = existedContacts.Where(ec => importContacts.Any(ic =>
ec.Id.EqualsInvariant(ic.Record.Id) ||
!string.IsNullOrEmpty(ec.OuterId) && ec.OuterId.EqualsInvariant(ic.Record.OuterId)))
.ToArray();
var updateImportContacts = importContacts.Where(x => existedContacts.Any(ec =>
ec.Id.EqualsInvariant(x.Record.Id) ||
(!ec.OuterId.IsNullOrEmpty() && ec.OuterId.EqualsInvariant(x.Record.OuterId)))
).ToArray();
existedContacts = GetReducedExistedByWrongOuterId(updateImportContacts, existedContacts).OfType <Contact>().ToArray();
var createImportContacts = importContacts.Except(updateImportContacts).ToArray();
var internalOrgIds = importContacts.Select(x => x.Record?.OrganizationId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var outerOrgIds = importContacts.Select(x => x.Record?.OrganizationOuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var existedOrganizations =
(await SearchMembersByIdAndOuterIdAsync(internalOrgIds, outerOrgIds,
new[] { nameof(Organization) })).OfType <Organization>().ToArray();
var newContacts = CreateNewContacts(createImportContacts, existedOrganizations, request);
PatchExistedContacts(existedContacts, updateImportContacts, existedOrganizations, request);
var contactsForSave = newContacts.Union(existedContacts).ToArray();
await _memberService.SaveChangesAsync(contactsForSave);
await CreateAccountsForContacts(contactsForSave);
importProgress.ContactsCreated += newContacts.Length;
importProgress.ContactsUpdated += existedContacts.Length;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
importProgress.ErrorCount = importProgress.ProcessedCount - importProgress.ContactsCreated - importProgress.ContactsUpdated;
}
}
