private void Import(ImportJob job, CsvReader reader, ImportResult result) { var skip = job.ImportStep; var count = job.ImportCount; var startIndex = job.StartIndex; var csvNames = GetCsvNamesAndIndexes(reader); if (csvNames.Count > 0) { var importer = _entityImporters.SingleOrDefault(i => i.Name == job.EntityImporter); if (importer != null) { var processed = 0; while (true) { try { var csvValues = reader.ReadRow(); processed++; if (csvValues == null) break; if ((result.ProcessedRecordsCount < count || count <= 0) && processed >= startIndex && ((processed - startIndex) % skip == 0)) { var systemValues = MapColumns(job.PropertiesMap.Where(prop => prop.IsSystemProperty), csvNames, csvValues, result); var customValues = MapColumns(job.PropertiesMap.Where(prop => !prop.IsSystemProperty), csvNames, csvValues, result); if (systemValues != null && customValues != null) { var rep = IsTaxImport(importer.Name) ? _orderRepository : (IRepository)_catalogRepository; if (importer.Name == ImportEntityType.Localization.ToString() || importer.Name == ImportEntityType.Seo.ToString()) rep = _appConfigRepositoryFactory.GetRepositoryInstance(); var res = importer.Import(job.CatalogId, job.PropertySetId, systemValues, customValues, rep); result.CurrentProgress = reader.CurrentPosition; if (string.IsNullOrEmpty(res)) { rep.UnitOfWork.Commit(); result.ProcessedRecordsCount++; } else { result.ErrorsCount++; if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(string.Format("Row: {0}, Error: {1}", result.ProcessedRecordsCount + result.ErrorsCount, res)); //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } else { //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } } catch (Exception e) { result.ErrorsCount++; if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(e.Message+Environment.NewLine+e); //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } } } else { if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add("File contains no rows"); } if (result.ErrorsCount <= job.MaxErrorsCount) { try { _catalogRepository.UnitOfWork.Commit(); } catch (Exception e) { if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(e.Message); } } }
private static ImportItem[] MapColumns(IEnumerable<MappingItem> mappingItems, IReadOnlyDictionary<string, int> csvNamesAndIndexes, IList<string> csvValues, ImportResult result) { var csvNamesAndValues = new List<ImportItem>(); if (mappingItems != null) { foreach (var mappingItem in mappingItems) { var importItem = new ImportItem {Name = mappingItem.EntityColumnName, Locale = mappingItem.Locale}; if (mappingItem.CsvColumnName != null && csvNamesAndIndexes.ContainsKey(mappingItem.CsvColumnName)) { var columnIndex = csvNamesAndIndexes[mappingItem.CsvColumnName]; if (csvValues[columnIndex] != null) importItem.Value = csvValues[columnIndex]; if (!string.IsNullOrEmpty(mappingItem.StringFormat)) importItem.Value = string.Format(mappingItem.StringFormat, importItem.Value); if (mappingItem.IsRequired && string.IsNullOrEmpty(importItem.Value)) { if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(string.Format("Row: {0}, Property: {1}, Error: {2}", result.ProcessedRecordsCount + result.ErrorsCount, mappingItem.DisplayName, "The value for required property not provided")); result.ErrorsCount++; return null; } } else if (mappingItem.CustomValue != null) { importItem.Value = mappingItem.CustomValue; if (!string.IsNullOrEmpty(mappingItem.StringFormat)) importItem.Value = string.Format(mappingItem.StringFormat, importItem.Value); } csvNamesAndValues.Add(importItem); } } return csvNamesAndValues.ToArray(); }
public void RunImportJob(string jobId, string csvFileName) { var job = _importJobRepository.ImportJobs.Expand("PropertiesMap").Where(x => x.ImportJobId == jobId).SingleOrDefault(); if (job != null) { var result = new ImportResult(); _importResults[jobId] = result; using (var data = GetCsvContent(csvFileName)) { //result.Length = 100000; result.Length = data.BaseStream.CanSeek ? data.BaseStream.Length : 100000; var csvReader = new CsvReader(data, job.ColumnDelimiter); try { result.IsRunning = true; result.Started = DateTime.Now; Import(job, csvReader, result); } finally { result.Stopped = DateTime.Now; result.IsRunning = false; } } } }