private static void SetupErrorHandlers(Action <ImportProgressInfo> progressCallback, ImportConfiguration configuration,
                                               ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
        {
            configuration.ReadingExceptionOccurred = exception =>
            {
                var context = exception.ReadingContext;

                if (!errorsContext.ErrorsRows.Contains(context.Row))
                {
                    var fieldSourceValue = context.Record[context.CurrentIndex];

                    if (context.HeaderRecord.Length != context.Record.Length)
                    {
                        HandleNotClosedQuoteError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                    else if (fieldSourceValue == "")
                    {
                        HandleRequiredValueError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                    else
                    {
                        HandleWrongValueError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                }

                return(false);
            };

            configuration.BadDataFound = async context =>
            {
                await HandleBadDataErrorAsync(progressCallback, importProgress, importReporter, context, errorsContext);
            };

            configuration.MissingFieldFound = null;
        }
コード例 #2
0
        private static void SetupErrorHandlers(Action <ImportProgressInfo> progressCallback,
                                               CsvConfiguration configuration,
                                               ImportErrorsContext errorsContext, ImportProgressInfo importProgress,
                                               ICsvPriceImportReporter importReporter)
        {
            configuration.ReadingExceptionOccurred = args =>
            {
                var context = args.Exception.Context;

                if (!errorsContext.ErrorsRows.Contains(context.Parser.Row))
                {
                    var fieldSourceValue = context.Reader[context.Reader.CurrentIndex];

                    if (context.Reader.HeaderRecord.Length != context.Parser.Record.Length)
                    {
                        HandleNotClosedQuoteError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                    else if (fieldSourceValue == "")
                    {
                        HandleRequiredValueError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                    else
                    {
                        HandleWrongValueError(progressCallback, importProgress, importReporter, context, errorsContext);
                    }
                }

                return(false);
            };

            configuration.BadDataFound = args =>
            {
                HandleBadDataError(progressCallback, importProgress, importReporter, args.Context, errorsContext);
            };

            configuration.MissingFieldFound = args => HandleMissedColumnError(progressCallback, importProgress, importReporter, args.Context, errorsContext);
        }
        public virtual async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
        {
            ValidateParameters(request, progressCallback, cancellationToken);

            var errorsContext = new ImportErrorsContext();

            var csvPriceDataValidationResult = await _dataValidator.ValidateAsync <TCsvMember>(request.FilePath);

            if (csvPriceDataValidationResult.Errors.Any())
            {
                throw new InvalidDataException();
            }

            var configuration = new ImportConfiguration();

            var reportFilePath = GetReportFilePath(request.FilePath);

            await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);

            cancellationToken.ThrowIfCancellationRequested();

            var importProgress = new ImportProgressInfo {
                Description = "Import has started"
            };

            using var dataSource = await _dataSourceFactory.CreateAsync <TCsvMember, TMember>(request.FilePath, ModuleConstants.Settings.PageSize, configuration);

            var headerRaw = dataSource.GetHeaderRaw();

            if (!headerRaw.IsNullOrEmpty())
            {
                await importReporter.WriteHeaderAsync(headerRaw);
            }

            importProgress.TotalCount = dataSource.GetTotalCount();
            progressCallback(importProgress);

            const string importDescription = "{0} out of {1} have been imported.";

            SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);

            try
            {
                importProgress.Description = "Fetching...";
                progressCallback(importProgress);

                while (await dataSource.FetchAsync())
                {
                    await ProcessChunkAsync(request, progressCallback, dataSource, errorsContext, importProgress, importReporter);

                    if (importProgress.ProcessedCount != importProgress.TotalCount)
                    {
                        importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
                        progressCallback(importProgress);
                    }
                }
            }
            catch (Exception e)
            {
                HandleError(progressCallback, importProgress, e.Message);
            }
            finally
            {
                var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
                importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";

                if (importReporter.ReportIsNotEmpty)
                {
                    importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
                }

                progressCallback(importProgress);
            }
        }
        private static void HandleRequiredValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
        {
            var fieldName          = context.HeaderRecord[context.CurrentIndex];
            var requiredFields     = CsvCustomerImportHelper.GetImportCustomerRequiredColumns <ImportableContact>();
            var missedValueColumns = new List <string>();

            for (var i = 0; i < context.HeaderRecord.Length; i++)
            {
                if (requiredFields.Contains(context.HeaderRecord[i], StringComparer.InvariantCultureIgnoreCase) && context.Record[i].IsNullOrEmpty())
                {
                    missedValueColumns.Add(context.HeaderRecord[i]);
                }
            }

            var importError = new ImportError {
                Error = $"The required value in column {fieldName} is missing.", RawRow = context.RawRecord
            };

            if (missedValueColumns.Count > 1)
            {
                importError.Error = $"The required values in columns: {string.Join(", ", missedValueColumns)} - are missing.";
            }

            reporter.WriteAsync(importError).GetAwaiter().GetResult();

            errorsContext.ErrorsRows.Add(context.Row);
            HandleError(progressCallback, importProgress);
        }
        private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
        {
            var invalidFieldName = context.HeaderRecord[context.CurrentIndex];
            var importError      = new ImportError {
                Error = string.Format(ModuleConstants.ValidationMessages[ModuleConstants.ValidationErrors.InvalidValue], invalidFieldName), RawRow = context.RawRecord
            };

            reporter.WriteAsync(importError).GetAwaiter().GetResult();

            errorsContext.ErrorsRows.Add(context.Row);
            HandleError(progressCallback, importProgress);
        }
        private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
        {
            var importError = new ImportError {
                Error = "This row has invalid data. Quotes should be closed.", RawRow = context.RawRecord
            };

            reporter.WriteAsync(importError).GetAwaiter().GetResult();

            errorsContext.ErrorsRows.Add(context.Row);
            HandleError(progressCallback, importProgress);
        }
        private static async Task HandleBadDataErrorAsync(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
        {
            var importError = new ImportError {
                Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.RawRecord
            };

            await reporter.WriteAsync(importError);

            errorsContext.ErrorsRows.Add(context.Row);
            HandleError(progressCallback, importProgress);
        }
 protected abstract Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <TCsvMember> dataSource,
                                           ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter);
コード例 #9
0
        public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
        {
            ValidateParameters(request, progressCallback, cancellationToken);

            cancellationToken.ThrowIfCancellationRequested();

            var errorsContext = new ImportErrorsContext();

            var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath);

            if (csvPriceDataValidationResult.Errors.Any())
            {
                throw new InvalidDataException();
            }

            var reportFilePath = GetReportFilePath(request.FilePath);

            var configuration = _importConfigurationFactory.Create();

            await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);

            cancellationToken.ThrowIfCancellationRequested();

            var importProgress = new ImportProgressInfo {
                ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started"
            };

            SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);

            using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration);

            var headerRaw = dataSource.GetHeaderRaw();

            if (!headerRaw.IsNullOrEmpty())
            {
                importReporter.WriteHeader(headerRaw);
            }

            importProgress.TotalCount = dataSource.GetTotalCount();
            progressCallback(importProgress);

            const string importDescription = "{0} out of {1} have been imported.";

            try
            {
                importProgress.Description = "Fetching...";
                progressCallback(importProgress);

                var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists);

                while (await dataSource.FetchAsync())
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    var importProductPrices = dataSource.Items
                                              // expect records that was parsed with errors
                                              .Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row))
                                              .Select(importProductPrice =>
                    {
                        importProductPrice.Price.PricelistId = request.PricelistId;
                        return(importProductPrice);
                    }).ToArray();

                    try
                    {
                        var createdPrices = new List <Price>();
                        var updatedPrices = new List <Price>();

                        var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString()));

                        var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();

                        var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray();

                        var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice);

                        foreach (var group in errorsGroups)
                        {
                            var importPrice = group.Key;

                            var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());

                            var importError = new ImportError {
                                Error = errorMessages, RawRow = importPrice.RawRecord
                            };

                            await importReporter.WriteAsync(importError);
                        }

                        importProgress.ErrorCount += invalidImportProductPrices.Length;
                        importProductPrices        = importProductPrices.Except(invalidImportProductPrices).ToArray();

                        switch (request.ImportMode)
                        {
                        case ImportMode.CreateOnly:
                            createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price));
                            break;

                        case ImportMode.UpdateOnly:
                            var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices);

                            updatedPrices.AddRange(existingPrices);
                            break;

                        case ImportMode.CreateAndUpdate:
                            var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices);

                            var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors
                                                              .Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();

                            var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray();
                            createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price));
                            var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate);

                            updatedPrices.AddRange(existingPricesToUpdate);
                            break;

                        default:
                            throw new ArgumentException("Import mode has invalid value", nameof(request));
                        }

                        var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray();
                        await _pricingService.SavePricesAsync(allChangingPrices);

                        importProgress.CreatedCount += createdPrices.Count;
                        importProgress.UpdatedCount += updatedPrices.Count;
                    }
                    catch (Exception e)
                    {
                        HandleError(progressCallback, importProgress, e.Message);
                    }
                    finally
                    {
                        importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
                    }

                    if (importProgress.ProcessedCount != importProgress.TotalCount)
                    {
                        importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
                        progressCallback(importProgress);
                    }
                }
            }
            catch (Exception e)
            {
                HandleError(progressCallback, importProgress, e.Message);
            }
            finally
            {
                var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
                importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";

                if (importReporter.ReportIsNotEmpty)
                {
                    importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
                }

                progressCallback(importProgress);
            }
        }
コード例 #10
0
        private static void HandleMissedColumnError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
        {
            var headerColumns = context.Reader.HeaderRecord;
            var recordFields  = context.Parser.Record;
            var missedColumns = headerColumns.Skip(recordFields.Length).ToArray();
            var error         = $"This row has next missing columns: {string.Join(", ", missedColumns)}.";
            var importError   = new ImportError {
                Error = error, RawRow = context.Parser.RawRecord
            };

            reporter.Write(importError);

            errorsContext.ErrorsRows.Add(context.Parser.Row);
            HandleError(progressCallback, importProgress);
        }
コード例 #11
0
        private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
        {
            var invalidFieldName = context.Reader.HeaderRecord[context.Reader.CurrentIndex];
            var importError      = new ImportError {
                Error = $"This row has invalid value in the column {invalidFieldName}.", RawRow = context.Parser.RawRecord
            };

            reporter.Write(importError);

            errorsContext.ErrorsRows.Add(context.Parser.Row);
            HandleError(progressCallback, importProgress);
        }
コード例 #12
0
        private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
        {
            var importError = new ImportError {
                Error = "This row has invalid data. Quotes should be closed.", RawRow = context.Parser.RawRecord
            };

            reporter.Write(importError);

            errorsContext.ErrorsRows.Add(context.Parser.Row);
            HandleError(progressCallback, importProgress);
        }
コード例 #13
0
        private static void HandleBadDataError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
        {
            var importError = new ImportError {
                Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.Parser.RawRecord
            };

            reporter.Write(importError);

            if (context.Reader is VcCsvReader vcCsvReader)
            {
                vcCsvReader.IsFieldBadData = true;
            }

            errorsContext.ErrorsRows.Add(context.Parser.Row);
            HandleError(progressCallback, importProgress);
        }
コード例 #14
0
        protected override async Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <ImportableContact> dataSource,
                                                        ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
        {
            var importContacts = dataSource.Items
                                 // expect records that was parsed with errors
                                 .Where(importContact => !errorsContext.ErrorsRows.Contains(importContact.Row))
                                 .ToArray();

            try
            {
                var internalIds = importContacts.Select(x => x.Record?.Id).Distinct()
                                  .Where(x => !string.IsNullOrEmpty(x))
                                  .ToArray();

                var outerIds = importContacts.Select(x => x.Record?.OuterId).Distinct()
                               .Where(x => !string.IsNullOrEmpty(x))
                               .ToArray();

                var existedContacts =
                    (await SearchMembersByIdAndOuterIdAsync(internalIds, outerIds, new[] { nameof(Contact) }, true))
                    .OfType <Contact>().ToArray();

                SetIdToNullForNotExisted(importContacts, existedContacts);

                SetIdToRealForExistedOuterId(importContacts, existedContacts);

                var validationResult = await ValidateAsync(importContacts, importReporter);

                var invalidImportContacts = validationResult.Errors
                                            .Select(x => (x.CustomState as ImportValidationState <ImportableContact>)?.InvalidRecord).Distinct().ToArray();

                importContacts = importContacts.Except(invalidImportContacts).ToArray();

                //reduce existed set after validation
                existedContacts = existedContacts.Where(ec => importContacts.Any(ic =>
                                                                                 ec.Id.EqualsInvariant(ic.Record.Id) ||
                                                                                 !string.IsNullOrEmpty(ec.OuterId) && ec.OuterId.EqualsInvariant(ic.Record.OuterId)))
                                  .ToArray();

                var updateImportContacts = importContacts.Where(x => existedContacts.Any(ec =>
                                                                                         ec.Id.EqualsInvariant(x.Record.Id) ||
                                                                                         (!ec.OuterId.IsNullOrEmpty() && ec.OuterId.EqualsInvariant(x.Record.OuterId)))
                                                                ).ToArray();

                existedContacts = GetReducedExistedByWrongOuterId(updateImportContacts, existedContacts).OfType <Contact>().ToArray();

                var createImportContacts = importContacts.Except(updateImportContacts).ToArray();

                var internalOrgIds = importContacts.Select(x => x.Record?.OrganizationId).Distinct()
                                     .Where(x => !string.IsNullOrEmpty(x)).ToArray();

                var outerOrgIds = importContacts.Select(x => x.Record?.OrganizationOuterId).Distinct()
                                  .Where(x => !string.IsNullOrEmpty(x)).ToArray();

                var existedOrganizations =
                    (await SearchMembersByIdAndOuterIdAsync(internalOrgIds, outerOrgIds,
                                                            new[] { nameof(Organization) })).OfType <Organization>().ToArray();

                var newContacts = CreateNewContacts(createImportContacts, existedOrganizations, request);

                PatchExistedContacts(existedContacts, updateImportContacts, existedOrganizations, request);

                var contactsForSave = newContacts.Union(existedContacts).ToArray();

                await _memberService.SaveChangesAsync(contactsForSave);

                await CreateAccountsForContacts(contactsForSave);

                importProgress.ContactsCreated += newContacts.Length;
                importProgress.ContactsUpdated += existedContacts.Length;
            }
            catch (Exception e)
            {
                HandleError(progressCallback, importProgress, e.Message);
            }
            finally
            {
                importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
                importProgress.ErrorCount     = importProgress.ProcessedCount - importProgress.ContactsCreated - importProgress.ContactsUpdated;
            }
        }