public string GetCsvColumnsAutomatically(string csvFileName) { var _delimiter = string.Empty; string[] columns; using (var data = GetCsvContent(csvFileName)) { var csvReader = new CsvReader(data, Comma); columns = csvReader.ReadRow(); _delimiter = Comma; } using (var data = GetCsvContent(csvFileName)) { var csvReader = new CsvReader(data, Semicolon); var temp = csvReader.ReadRow(); if ((temp != null && columns!= null) && (temp.Count() > columns.Count())) { columns = temp; _delimiter = Semicolon; } } using (var data = GetCsvContent(csvFileName)) { var csvReader = new CsvReader(data, Tab); var temp = csvReader.ReadRow(); if ((temp != null && columns!= null) && (temp.Count() > columns.Count())) { _delimiter = Tab; } } return _delimiter; }
private void Import(ImportJob job, CsvReader reader, ImportResult result) { var skip = job.ImportStep; var count = job.ImportCount; var startIndex = job.StartIndex; var csvNames = GetCsvNamesAndIndexes(reader); if (csvNames.Count > 0) { var importer = _entityImporters.SingleOrDefault(i => i.Name == job.EntityImporter); if (importer != null) { var processed = 0; while (true) { try { var csvValues = reader.ReadRow(); processed++; if (csvValues == null) break; if ((result.ProcessedRecordsCount < count || count <= 0) && processed >= startIndex && ((processed - startIndex) % skip == 0)) { var systemValues = MapColumns(job.PropertiesMap.Where(prop => prop.IsSystemProperty), csvNames, csvValues, result); var customValues = MapColumns(job.PropertiesMap.Where(prop => !prop.IsSystemProperty), csvNames, csvValues, result); if (systemValues != null && customValues != null) { var rep = IsTaxImport(importer.Name) ? _orderRepository : (IRepository)_catalogRepository; if (importer.Name == ImportEntityType.Localization.ToString() || importer.Name == ImportEntityType.Seo.ToString()) rep = _appConfigRepositoryFactory.GetRepositoryInstance(); var res = importer.Import(job.CatalogId, job.PropertySetId, systemValues, customValues, rep); result.CurrentProgress = reader.CurrentPosition; if (string.IsNullOrEmpty(res)) { rep.UnitOfWork.Commit(); result.ProcessedRecordsCount++; } else { result.ErrorsCount++; if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(string.Format("Row: {0}, Error: {1}", result.ProcessedRecordsCount + result.ErrorsCount, res)); //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } else { //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } } catch (Exception e) { result.ErrorsCount++; if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(e.Message+Environment.NewLine+e); //check if errors amount reached the allowed errors limit if yes do not save made changes. if (result.ErrorsCount >= job.MaxErrorsCount) { break; } } } } } else { if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add("File contains no rows"); } if (result.ErrorsCount <= job.MaxErrorsCount) { try { _catalogRepository.UnitOfWork.Commit(); } catch (Exception e) { if (result.Errors == null) result.Errors = new List<string>(); result.Errors.Add(e.Message); } } }
private static Dictionary<string, int> GetCsvNamesAndIndexes(CsvReader reader) { var csvNames = reader.ReadRow(); var csvNamesAndIndexes = new Dictionary<string, int>(); if (csvNames != null) { for (var i = 0; i < csvNames.Length; i++) csvNamesAndIndexes.Add(csvNames[i], i); } return csvNamesAndIndexes; }
public void RunImportJob(string jobId, string csvFileName) { var job = _importJobRepository.ImportJobs.Expand("PropertiesMap").Where(x => x.ImportJobId == jobId).SingleOrDefault(); if (job != null) { var result = new ImportResult(); _importResults[jobId] = result; using (var data = GetCsvContent(csvFileName)) { //result.Length = 100000; result.Length = data.BaseStream.CanSeek ? data.BaseStream.Length : 100000; var csvReader = new CsvReader(data, job.ColumnDelimiter); try { result.IsRunning = true; result.Started = DateTime.Now; Import(job, csvReader, result); } finally { result.Stopped = DateTime.Now; result.IsRunning = false; } } } }
public string[] GetCsvColumns(string csvFileName, string columnDelimiter) { using (var data = GetCsvContent(csvFileName)) { var csvReader = new CsvReader(data, columnDelimiter); return csvReader.ReadRow(); } }
public string GetImportFileDelimiter(string assetPath) { var _delimiter = string.Empty; string[] columns; using (var data = GetCsvContent(assetPath)) { var csvReader = new CsvReader(data, Comma); columns = csvReader.ReadRow(); _delimiter = Comma; } using (var data = GetCsvContent(assetPath)) { var csvReader = new CsvReader(data, Semicolon); var temp = csvReader.ReadRow(); if ((temp != null && columns!= null) && (temp.Count() > columns.Count())) { columns = temp; _delimiter = Semicolon; } } using (var data = GetCsvContent(assetPath)) { var csvReader = new CsvReader(data, Tab); var temp = csvReader.ReadRow(); if ((temp != null && columns!= null) && (temp.Count() > columns.Count())) { _delimiter = Tab; } } return _delimiter; }
public string ImportData(string jobId, string assetPath) { var job = _importJobRepositoryFactory.GetRepositoryInstance().ImportJobs.Expand((x) => x.PropertiesMap).Where(x => x.ImportJobId == jobId).SingleOrDefault(); if (job != null) { var operationStatus = new OperationStatus(); _operationsList.Add(operationStatus); operationStatus.OperationState = OperationState.Initiated; using (var data = GetCsvContent(assetPath)) { operationStatus.Length = data.BaseStream.CanSeek ? data.BaseStream.Length : 100000; var csvReader = new CsvReader(data, job.ColumnDelimiter); try { operationStatus.OperationState = OperationState.InProgress; operationStatus.Started = DateTime.UtcNow; Task.Run(() => Import(job, csvReader, operationStatus)); } finally { operationStatus.Stopped = DateTime.UtcNow; operationStatus.OperationState = OperationState.Finished; } } return operationStatus.OperationId; } return null; }
public string[] GetImportFileColumns(string assetPath, string columnDelimiter = null) { if (string.IsNullOrEmpty(columnDelimiter)) columnDelimiter = GetImportFileDelimiter(assetPath); using (var data = GetCsvContent(assetPath)) { var csvReader = new CsvReader(data, columnDelimiter); return csvReader.ReadRow(); } }