public async Task <ImportDataValidationResult> ValidateAsync(string filePath) { var errorsList = new List <ImportDataValidationError>(); var fileMaxSize = _settingsManager.GetValue(ModuleConstants.Settings.General.ImportFileMaxSize.Name, (int)ModuleConstants.Settings.General.ImportFileMaxSize.DefaultValue) * ModuleConstants.MByte; var blobInfo = await _blobStorageProvider.GetBlobInfoAsync(filePath); if (blobInfo == null) { var error = new ImportDataValidationError() { ErrorCode = ModuleConstants.ValidationErrors.FileNotExisted }; errorsList.Add(error); } else if (blobInfo.Size > fileMaxSize) { var error = new ImportDataValidationError() { ErrorCode = ModuleConstants.ValidationErrors.ExceedingFileMaxSize }; error.Properties.Add(nameof(fileMaxSize), fileMaxSize.ToString()); error.Properties.Add(nameof(blobInfo.Size), blobInfo.Size.ToString()); errorsList.Add(error); } else { var stream = _blobStorageProvider.OpenRead(filePath); var csvConfiguration = _importConfigurationFactory.Create(); csvConfiguration.BadDataFound = null; csvConfiguration.ReadingExceptionOccurred = null; await ValidateDelimiterAndDataExists(stream, csvConfiguration, errorsList); ValidateRequiredColumns(stream, csvConfiguration, errorsList); ValidateLineLimit(stream, csvConfiguration, errorsList); await stream.DisposeAsync(); } var result = new ImportDataValidationResult { Errors = errorsList.ToArray() }; return(result); }
public ICsvPagedPriceDataSource Create(string filePath, int pageSize, CsvConfiguration configuration = null) { return(new CsvPagedPriceDataSource(filePath, _productSearchService, _blobStorageProvider, pageSize, configuration ?? _importConfigurationFactory.Create(), _csvReaderFactory)); }
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken) { ValidateParameters(request, progressCallback, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); var errorsContext = new ImportErrorsContext(); var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath); if (csvPriceDataValidationResult.Errors.Any()) { throw new InvalidDataException(); } var reportFilePath = GetReportFilePath(request.FilePath); var configuration = _importConfigurationFactory.Create(); await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter); cancellationToken.ThrowIfCancellationRequested(); var importProgress = new ImportProgressInfo { ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started" }; SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter); using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration); var headerRaw = dataSource.GetHeaderRaw(); if (!headerRaw.IsNullOrEmpty()) { importReporter.WriteHeader(headerRaw); } importProgress.TotalCount = dataSource.GetTotalCount(); progressCallback(importProgress); const string importDescription = "{0} out of {1} have been imported."; try { importProgress.Description = "Fetching..."; progressCallback(importProgress); var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists); while (await dataSource.FetchAsync()) { cancellationToken.ThrowIfCancellationRequested(); var importProductPrices = dataSource.Items // expect records that was parsed with errors .Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row)) .Select(importProductPrice => { importProductPrice.Price.PricelistId = request.PricelistId; return(importProductPrice); }).ToArray(); try { var createdPrices = new List <Price>(); var updatedPrices = new List <Price>(); var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString())); var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray(); var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray(); var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice); foreach (var group in errorsGroups) { var importPrice = group.Key; var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray()); var importError = new ImportError { Error = errorMessages, RawRow = importPrice.RawRecord }; await importReporter.WriteAsync(importError); } importProgress.ErrorCount += invalidImportProductPrices.Length; importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray(); switch (request.ImportMode) { case ImportMode.CreateOnly: createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price)); break; case ImportMode.UpdateOnly: var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices); updatedPrices.AddRange(existingPrices); break; case ImportMode.CreateAndUpdate: var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices); var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors .Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray(); var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray(); createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price)); var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate); updatedPrices.AddRange(existingPricesToUpdate); break; default: throw new ArgumentException("Import mode has invalid value", nameof(request)); } var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray(); await _pricingService.SavePricesAsync(allChangingPrices); importProgress.CreatedCount += createdPrices.Count; importProgress.UpdatedCount += updatedPrices.Count; } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount); } if (importProgress.ProcessedCount != importProgress.TotalCount) { importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount); progressCallback(importProgress); } } } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed"; importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}"; if (importReporter.ReportIsNotEmpty) { importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath); } progressCallback(importProgress); } }