public CsvImportManager ( IFileDownloadUtilities fileDownloadUtilities, IZipUtility zipUtility, IImporter <Cfda> cfdaImporter, CsvImportSettings importSettings, IImporter <General> generalImporter, IImporter <Agency> agencyImporter, IImporter <CapText> captextImporter, IImporter <Cpa> cpaImporter, IImporter <Dun> dunImporter, IImporter <Ein> einImporter, IImporter <Finding> findingImporter, IImporter <FindingText> findingTextImporter, IImporter <FormattedCapText> formattedCapTextImporter, IImporter <FormattedFindingsText> formattedFindingsTextImporter, IImporter <Passthrough> passthroughImporter ) { FileDownloadUtilities = fileDownloadUtilities; ZipUtility = zipUtility; CfdaImporter = cfdaImporter; CsvImportSettings = importSettings; GeneralImporter = generalImporter; AgencyImporter = agencyImporter; CapTextImporter = captextImporter; CpaImporter = cpaImporter; DunImporter = dunImporter; EinImporter = einImporter; FindingImporter = findingImporter; FindingTextImporter = findingTextImporter; FormattedCapTextImporter = formattedCapTextImporter; FormattedFindingsTextImporter = formattedFindingsTextImporter; PassthroughImporter = passthroughImporter; }
public PassthroughCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <Passthrough> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public GeneralCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <General> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public DunCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <Dun> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public AgencyCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <Agency> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public FormattedFindingTextCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <FormattedFindingsText> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public CfdaCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <Cfda> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public FindingCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <Finding> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
public CapTextCsvImporter ( CsvImportSettings csvImportSettings, IBulkImportRepository <CapText> dataRepository ) { CsvImportSettings = csvImportSettings; DataRepository = dataRepository; }
internal ImportCsvFileService(CsvImportSettings settings) { _settings = settings; if (_settings.HeaderRowSpaces != "Replace" && _settings.HeaderRowSpaces != "Remove") { throw new ArgumentOutOfRangeException(nameof(_settings.HeaderRowSpaces), _settings.HeaderRowSpaces, "Header Row Spaces value must be either 'Replace' or 'Remove'"); } }
public static CsvReader GetCsvReader(Stream fileStream, CsvImportSettings settings) { var streamReader = new StreamReader(fileStream); var csvReader = new CsvReader(streamReader); csvReader.Configuration.Encoding = Encoding.UTF8; csvReader.Configuration.HasHeaderRecord = true; csvReader.Configuration.DetectColumnCountChanges = true; csvReader.Configuration.Delimiter = settings.Delimiter; return(csvReader); }
public SalesRecordsController(ISalesRecordsService salesRecordsService, IHostingEnvironment environment, ICsvFileValidator csvFileValidator, ITemporaryFileNameGenerator temporaryFileNameGenerator, IOptions <CsvImportSettings> csvImportSettings, IMapper mapper) { _salesRecordsService = salesRecordsService; _environment = environment; _csvFileValidator = csvFileValidator; _temporaryFileNameGenerator = temporaryFileNameGenerator; _mapper = mapper; _csvImportSettings = csvImportSettings.Value; }
private static IImportService GetImporter(Properties properties, string method) { IImportService importer; var file = properties.SafeRead(PropertyConstants.File); var headerRowSpaces = properties.SafeRead(PropertyConstants.HeaderRowSpaces, "Remove"); switch (method) { case nameof(Method.ExcelBulkImport): case nameof(Method.ExcelBulkImportWithTransaction): case nameof(Method.ExcelImport): case nameof(Method.ExcelImportWithTransaction): var excelImportSettings = new ExcelImportSettings { File = file, SheetName = properties.SafeRead(PropertyConstants.SheetName), HeaderRowIndex = properties.SafeRead(PropertyConstants.HeaderRowIndex, 0), HeaderRowSpaces = headerRowSpaces, DuplicateDelimiter = properties.SafeRead(PropertyConstants.DuplicateColumnDelimiter, ";") }; importer = new ImportExcelFileService(excelImportSettings); break; case nameof(Method.CsvBulkImport): case nameof(Method.CsvBulkImportWithTransaction): case nameof(Method.CsvImport): case nameof(Method.CsvImportWithTransaction): var csvImportSettings = new CsvImportSettings { File = file, HeaderRowSpaces = headerRowSpaces, ColumnDelimiter = properties.SafeRead(PropertyConstants.ColumnDelimiter, ','), TextQualifier = properties.SafeRead(PropertyConstants.TextQualifier, '"') }; importer = new ImportCsvFileService(csvImportSettings); break; default: throw new Exception($"No importer found for '{method}'"); } return(importer); }
public static CsvImportSettings GetCsvImportSettings(string outputPath) { var settings = new CsvImportSettings(); var iConfig = GetIConfigurationRoot(outputPath); iConfig .GetSection("CsvImportSettings") .Bind(settings); //Custom map the local directory settings.LocalImportDirectory = Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.Personal), settings.LocalImportDirectory); return(settings); }
private static ImportStatus ImportCsv(string fileName) { var settings = new CsvImportSettings { File = GetK2FileString(fileName), HeaderRowSpaces = "Replace", ColumnDelimiter = ',', TextQualifier = '"' }; var importer = new ImportCsvFileService(settings); _importedData = importer.Results; Console.WriteLine(importer.Message); return(importer.Status); }
public static async Task <CsvImportResult> GetTokens(CsvImportSettings importSettings, bool unlimied = false) { return(await Task.Run(() => { var result = new CsvImportResult(); while (result.CanContinue && (unlimied || result.Tokens.Count + result.InvalidRows.Count < GlobalStore.SelectedEndpoint.BulkSize)) { try { result.CanContinue = importSettings.CsvReader.Read(); } catch (Exception exception) { result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow); if (!importSettings.Force) { result.CanContinue = false; } continue; } if (!result.CanContinue) { continue; } var tokenString = ""; try { tokenString = "{" + string.Join(",", importSettings.CsvReader.FieldHeaders.Select(h => $"\"{h}\":\"{(importSettings.CsvReader[h] == "NULL" ? "" : importSettings.CsvReader[h].Replace("\"","\\\""))}\"")) + "}"; result.Tokens.Add(JToken.Parse(tokenString)); } catch (Exception exception) { result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow); if (!importSettings.Force) { result.CanContinue = false; } } } return result; })); }
public static ISequence <string[]> Read( [InputPin(Name = "Path", Description = "Path to the imported csv file", PropertyMode = PropertyMode.Default, Editor = WellKnownEditors.SingleLineText, ResolvePath = true)] string path, [InputPin(Name = "Encoding", Description = "Encoding of the file", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.DropDown)] CsvReaderEncoding encoding = CsvReaderEncoding.Auto, [InputPin(Name = "SkipFirstLine", Description = "Skip the first line of the file", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.CheckBox)] bool skipFirstLine = false, [InputPin(Name = "SkipEmptyLines", Description = "Skip lines without text", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.CheckBox)] bool skipEmptyLines = true, [InputPin(Name = "Delimiters", Description = "Characters which are used as delimiters between two data fields", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.SingleLineText)] string delimiters = ";", [InputPin(Name = "CommentSymbol", Description = "Leading symbols for comments", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.SingleLineText)] string commentSymbol = "#", [InputPin(Name = "StringQuoting", Description = "Characters which are used to quote a string", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.SingleLineText)] string stringQuoting = "'", [InputPin(Name = "MultilineStrings", Description = "A quoted string contains line breaks", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.CheckBox)] bool multilineStrings = false, [InputPin(Name = "Escaping", Description = "Use character escaping", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.CheckBox)] bool escaping = false, [InputPin(Name = "TwoQuotationMarksEscape", Description = "Two quotation charactes marks an escaping", PropertyMode = PropertyMode.Always, Editor = WellKnownEditors.SingleLineText)] bool twoQuotationMarksEscape = false ) { var csvImportSettings = new CsvImportSettings { CommentSymbol = commentSymbol, Delimiters = delimiters, Encoding = EncodingsMapping[encoding] == null ? null : EncodingsMapping[encoding].WebName, Escaping = escaping, MultilineStrings = multilineStrings, SkipEmptyLines = skipEmptyLines, SkipFirstLine = skipFirstLine, StringQuoting = stringQuoting, TwoQuotationMarksEscape = twoQuotationMarksEscape }; return(Sequence.Using( () => File.Open(path, FileMode.Open, FileAccess.Read), fileStream => CsvImporter.ImportLines(fileStream, csvImportSettings).ToSequence() )); }
private async void ImportCsv <T>() { var ofd = new OpenFileDialog { Multiselect = false, Filter = "CSV|*.csv" }; if (ofd.ShowDialog() == true) { var importSettings = new CsvImportSettings { Delimiter = ",", Force = true }; var dialogResult = await _dialogHandler.Show( new CommonDialog { DataContext = new CommonDialogViewModel { Header = "Csv Import Settings", Content = importSettings, Buttons = ButtonsEnum.OkCancel } }, "RootDialog"); if ((CommonDialogResult)dialogResult == CommonDialogResult.Cancel) { return; } var invalidRows = new List <int>(); var errors = new ConcurrentBag <string>(); var cancellationToken = new CancellationTokenSource(); var status = new StatusDialogViewModel { ProgressValue = 0, Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken }; await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog", async (object sender, DialogOpenedEventArgs oa) => { FileStream stream = null; try { stream = new FileStream(ofd.FileName, FileMode.Open); stream.Position = 0; var importResult = new CsvImportResult(); importSettings.CsvReader = CsvProcesser.GetCsvReader(stream, importSettings); while (importResult.CanContinue && !cancellationToken.IsCancellationRequested) { importResult = await CsvProcesser.GetTokens(importSettings, typeof(T) == typeof(Tag)); invalidRows.AddRange(importResult.InvalidRows); status.ErrorCount += importResult.InvalidRows.Count; if (typeof(T) == typeof(Tag)) { var response = new ClientResponseWithObject <BulkResults>(); try { var settings = new TagBulkSettings(); settings.Tags = importResult.Tokens.Select(t => t.ToObject <Tag>()).ToList(); response = await new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings); ResponseValidator.Validate(response, false); } catch (Exception ex) { errors.Add(string.Format("Error during bulk process:{0}{1}: {2}", Environment.NewLine, ex.Message, string.Join(", ", response.Errors))); status.ErrorCount += importResult.Tokens.Count; } finally { if (response.IsSuccessful) { var bulkErrors = response.ResponseObject.Results.Where( br => br.StatusCode != (int)HttpStatusCode.OK).ToList(); if (bulkErrors.Any()) { foreach (var error in bulkErrors) { errors.Add(string.Format("Id: {0}, error: {1}", error.Id, error.Error)); status.ErrorCount++; } } } status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count); status.ProgressValue = (stream.Position / (double)stream.Length) * 100; importResult.Tokens = null; importResult.InvalidRows = null; } } else { var response = new ClientResponseWithObject <BulkResults>(); try { var settings = new DocumentBulkSettings(); var idFieldIsArray = false; if (SelectedDataSet.SampleDocument != null) { var docObject = JObject.FromObject(SelectedDataSet.SampleDocument); idFieldIsArray = docObject?[SelectedDataSet.TagField] is JArray; } else { var docObject = JObject.FromObject(SelectedDataSet.Schema); idFieldIsArray = docObject?["properties"][SelectedDataSet.TagField]["type"].ToString().ToLower() == "array"; } if (idFieldIsArray) { settings.Documents = importResult.Tokens.Select(t => { t[SelectedDataSet.TagField] = new JArray(t[SelectedDataSet.TagField]); return(t); }).Select(t => t.ToObject <object>()).ToList(); } else { settings.Documents = importResult.Tokens.Select(t => t.ToObject <object>()).ToList(); } response = await new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings); ResponseValidator.Validate(response, false); } catch (Exception ex) { Messenger.Default.Send(ex); status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize; } finally { if (response.IsSuccessful) { var bulkErrors = response.ResponseObject.Results.Where( br => br.StatusCode != (int)HttpStatusCode.OK).ToList(); if (bulkErrors.Any()) { foreach (var error in bulkErrors) { errors.Add(string.Format("Id: {0}, error: {1}", error.Id, error.Error)); status.ErrorCount++; } } } status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count); status.ProgressValue = (stream.Position / (double)stream.Length) * 100; importResult.Tokens = null; importResult.InvalidRows = null; } } } } catch (Exception exception) { DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception)); } finally { stream?.Close(); status.OperationIsFinished = true; } }); if (invalidRows.Any()) { Messenger.Default.Send(new Exception($"Invalid rows:{Environment.NewLine}{string.Join(Environment.NewLine, invalidRows)}")); } if (errors.Any()) { Messenger.Default.Send(new Exception($"Errors:{Environment.NewLine}{string.Join(Environment.NewLine, errors)}")); } } }
/// <summary> /// Convenience method for Csv import operations /// </summary> /// <param name="importRepository">Database repository for entity</param> /// <param name="map">CSV Class Map for entity</param> /// <param name="importSettings">CSV Import Settings</param> /// <param name="resultType">Import Result Type</param> /// <param name="importFilename">Target Csv File</param> /// <typeparam name="TE">Entity Type</typeparam> /// <returns>ImportResult</returns> protected async Task <ImportResult> ImportCsv <TE>(IBulkImportRepository <TE> importRepository, ClassMap <TE> map, CsvImportSettings importSettings, ImportResultType resultType, string importFilename ) { ImportResult result = new ImportResult(); bool isRecordBad = false; IList <TE> recordBuffer = new List <TE>(); result.ImportArea = resultType; result.ProcessStarted = DateTime.Now; //Clean the import repository await importRepository.Clean(); //Buffer Flush function Func <Task> flushBuffer = async() => { await importRepository.BulkImport(recordBuffer); recordBuffer.Clear(); }; //Read the csv file using (TextReader reader = File.OpenText( Path.Combine(importSettings.LocalImportDirectory, importFilename))) { CsvConfiguration conf = new CsvConfiguration(CultureInfo.CurrentCulture); conf.Delimiter = importSettings.Delimiter; conf.BufferSize = importSettings.ReadBufferSize; conf.MissingFieldFound = null; conf.HasHeaderRecord = true; conf.AllowComments = true; conf.IgnoreQuotes = true; conf.RegisterClassMap(map); conf.BadDataFound = context => { isRecordBad = true; result.BadData.Add(new Tuple <string, int>(context.RawRecord, (result.RecordsImported + 1))); }; //Instantiate our Csv reader CsvReader csv = new CsvReader(reader, conf); //Iterate over the records and perform import while (await csv.ReadAsync()) { //Parse the record var record = csv.GetRecord <TE>(); //Check to see if it isn't bad if (!isRecordBad) { isRecordBad = false; recordBuffer.Add(record); result.RecordsImported++; //Flush the buffer to the database if threshold is reached if (recordBuffer.Count == importSettings.ReadBufferSize) { await flushBuffer(); } } } //Flush any remaining records await flushBuffer(); //Set the result to true result.Success = true; result.ProcessEnded = DateTime.Now; return(result); } }
public async Task TestPerformImport() { IImportManager importManager = ServiceProvider.GetService <IImportManager>(); CsvImportSettings csvImportSettings = ServiceProvider.GetService <CsvImportSettings>(); if (importManager != null && csvImportSettings != null) { await importManager.CleanEnvironment(); IList <ImportResult> results = await importManager.PerformImport(); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.FacZipFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.AgencyImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.CfdaImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.CpaImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.DunsImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.EinsImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.FindingsImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.GeneralImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.NotesImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.PassthroughImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.RevisionImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.CapTextImportFilename))); Assert.True(File.Exists(Path.Combine(csvImportSettings.LocalImportDirectory, csvImportSettings.FindingsTextImportFilename))); //CFDA's ImportResult cfdaResult = results.FirstOrDefault(x => x.ImportArea == ImportResultType.Cfda); Assert.IsNotNull(cfdaResult); Assert.True(cfdaResult.Success); Assert.True(cfdaResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Cfda> >().CurrentRecords() == cfdaResult.RecordsImported); //General ImportResult generalResult = results.FirstOrDefault(x => x.ImportArea == ImportResultType.General); Assert.IsNotNull(generalResult); Assert.True(generalResult.Success); Assert.True(generalResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <General> >().CurrentRecords() == generalResult.RecordsImported); //Agency ImportResult agencyResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Agency); Assert.IsNotNull(agencyResult); Assert.True(agencyResult.Success); Assert.True(agencyResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Agency> >().CurrentRecords() == agencyResult.RecordsImported); //CapText ImportResult capTextResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.CapText); Assert.IsNotNull(capTextResult); Assert.True(capTextResult.Success); Assert.True(capTextResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <CapText> >().CurrentRecords() == capTextResult.RecordsImported); //Cpa ImportResult cpaResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Cpas); Assert.IsNotNull(cpaResult); Assert.True(cpaResult.Success); Assert.True(cpaResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Cpa> >().CurrentRecords() == cpaResult.RecordsImported); //Duns ImportResult dunResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Duns); Assert.IsNotNull(dunResult); Assert.True(dunResult.Success); Assert.True(dunResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Dun> >().CurrentRecords() == dunResult.RecordsImported); //Ein ImportResult einResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Eins); Assert.IsNotNull(einResult); Assert.True(einResult.Success); Assert.True(einResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Ein> >().CurrentRecords() == einResult.RecordsImported); //Findings ImportResult findingResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Findings); Assert.IsNotNull(findingResult); Assert.True(findingResult.Success); Assert.True(findingResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Finding> >().CurrentRecords() == findingResult.RecordsImported); //Findings Text ImportResult findingTextResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.FindingsText); Assert.IsNotNull(findingTextResult); Assert.True(findingTextResult.Success); Assert.True(findingTextResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <FindingText> >().CurrentRecords() == findingTextResult.RecordsImported); //Formatted Cap Text ImportResult formattedCapTextResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.FormattedCapText); Assert.IsNotNull(formattedCapTextResult); Assert.True(formattedCapTextResult.Success); Assert.True(formattedCapTextResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <FormattedCapText> >().CurrentRecords() == formattedCapTextResult.RecordsImported); //Formatted Findings Text ImportResult formattedFindingTextResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.FormattedFindingsText); Assert.IsNotNull(formattedFindingTextResult); Assert.True(formattedFindingTextResult.Success); Assert.True(formattedFindingTextResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <FormattedFindingsText> >().CurrentRecords() == formattedFindingTextResult.RecordsImported); //Passthough ImportResult passthroughResult = results.FirstOrDefault (x => x.ImportArea == ImportResultType.Passthrough); Assert.IsNotNull(passthroughResult); Assert.True(passthroughResult.Success); Assert.True(passthroughResult.RecordsImported > 0); Assert.True(await ServiceProvider.GetService <IBulkImportRepository <Passthrough> >().CurrentRecords() == passthroughResult.RecordsImported); } else { Assert.Fail(); } }