public async Task <ImportDataValidationResult> ValidateAsync(string filePath)
{
var errorsList = new List <ImportDataValidationError>();
var fileMaxSize = _settingsManager.GetValue(ModuleConstants.Settings.General.ImportFileMaxSize.Name,
(int)ModuleConstants.Settings.General.ImportFileMaxSize.DefaultValue) * ModuleConstants.MByte;
var blobInfo = await _blobStorageProvider.GetBlobInfoAsync(filePath);
if (blobInfo == null)
{
var error = new ImportDataValidationError()
{
ErrorCode = ModuleConstants.ValidationErrors.FileNotExisted
};
errorsList.Add(error);
}
else if (blobInfo.Size > fileMaxSize)
{
var error = new ImportDataValidationError()
{
ErrorCode = ModuleConstants.ValidationErrors.ExceedingFileMaxSize
};
error.Properties.Add(nameof(fileMaxSize), fileMaxSize.ToString());
error.Properties.Add(nameof(blobInfo.Size), blobInfo.Size.ToString());
errorsList.Add(error);
}
else
{
var stream = _blobStorageProvider.OpenRead(filePath);
var csvConfiguration = _importConfigurationFactory.Create();
csvConfiguration.BadDataFound = null;
csvConfiguration.ReadingExceptionOccurred = null;
await ValidateDelimiterAndDataExists(stream, csvConfiguration, errorsList);
ValidateRequiredColumns(stream, csvConfiguration, errorsList);
ValidateLineLimit(stream, csvConfiguration, errorsList);
await stream.DisposeAsync();
}
var result = new ImportDataValidationResult {
Errors = errorsList.ToArray()
};
return(result);
}
public ICsvPagedPriceDataSource Create(string filePath, int pageSize, CsvConfiguration configuration = null)
{
return(new CsvPagedPriceDataSource(filePath, _productSearchService, _blobStorageProvider, pageSize, configuration ?? _importConfigurationFactory.Create(), _csvReaderFactory));
}
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var reportFilePath = GetReportFilePath(request.FilePath);
var configuration = _importConfigurationFactory.Create();
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started"
};
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
importReporter.WriteHeader(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists);
while (await dataSource.FetchAsync())
{
cancellationToken.ThrowIfCancellationRequested();
var importProductPrices = dataSource.Items
// expect records that was parsed with errors
.Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row))
.Select(importProductPrice =>
{
importProductPrice.Price.PricelistId = request.PricelistId;
return(importProductPrice);
}).ToArray();
try
{
var createdPrices = new List <Price>();
var updatedPrices = new List <Price>();
var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString()));
var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray();
var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
importProgress.ErrorCount += invalidImportProductPrices.Length;
importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray();
switch (request.ImportMode)
{
case ImportMode.CreateOnly:
createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price));
break;
case ImportMode.UpdateOnly:
var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices);
updatedPrices.AddRange(existingPrices);
break;
case ImportMode.CreateAndUpdate:
var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices);
var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors
.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray();
createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price));
var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate);
updatedPrices.AddRange(existingPricesToUpdate);
break;
default:
throw new ArgumentException("Import mode has invalid value", nameof(request));
}
var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray();
await _pricingService.SavePricesAsync(allChangingPrices);
importProgress.CreatedCount += createdPrices.Count;
importProgress.UpdatedCount += updatedPrices.Count;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
}
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}