public void ExtractReadsTypedFile() { const string PATH = "test.csv"; const string CONTENT = "ABC"; try { File.WriteAllLines(PATH, new[] { "\"Test\"", $"\"{CONTENT}\"" }); var extractor = new CsvExtractor(); var extracted = extractor.Extract(new CsvExtractorParams { Files = new[] { new CsvExtractorParams.File(PATH, typeof(CsvClass1)) } }).ToArray(); Assert.NotNull(extracted); Assert.True(extracted.Length == 1); var extractedObj = extracted.First() as CsvClass1; Assert.NotNull(extractedObj); Assert.Equal(CONTENT, extractedObj.Test); } finally { if (File.Exists(PATH)) { File.Delete(PATH); } } }
/// <summary> /// Takes an XLSX or a CSV stream and unpackages its content into a 2-D table of strings. /// </summary> /// <param name="stream">The contents of the XLSX or CSV file.</param> /// <param name="fileName">The name of the file to extract if available.</param> /// <param name="contentType">The mime type of the file to extract if available.</param> /// <param name="localizer">To localize error messages.</param> /// <returns>A 2-D grid of strings representing the contents of the XLSX or the CSV file.</returns> public static IEnumerable <string[]> ExtractStringsFromFile(Stream stream, string fileName, string contentType, IStringLocalizer localizer) { IDataExtractor extracter; if (contentType == MimeTypes.Csv || (fileName?.ToLower()?.EndsWith(".csv") ?? false)) { extracter = new CsvExtractor(); } else if (contentType == MimeTypes.Excel || (fileName?.ToLower()?.EndsWith(".xlsx") ?? false)) { extracter = new ExcelExtractor(); } else { throw new FormatException(localizer["Error_OnlyCsvOrExcelAreSupported"]); } // Extrat and return try { return(extracter.Extract(stream).ToList()); } catch (Exception ex) { // Report any errors during extraction string msg = localizer["Error_FailedToParseFileError0", ex.Message]; throw new ServiceException(msg); } }
public void BadFilePathShouldFail() { // Arrange const string filePath = "randomfilepath"; // Act CsvExtractor.ExtractTestItemInput(filePath); }
public void CorrectFilePathWithIncorrectExtensionShouldFail() { // Arrange var filePath = Path.Combine(Directory.GetParent(Environment.CurrentDirectory).Parent.FullName, "Resources", "test.txt"); // Act CsvExtractor.ExtractTestItemInput(filePath); }
public void CorrectFilePathWithCorrectExtensionAndIncorrectHeadersShouldFail() { // Arrange var filePath = Path.Combine(Directory.GetParent(Environment.CurrentDirectory).Parent.FullName, "Resources", "BAD-FFP_Sample.csv"); // Act CsvExtractor.ExtractTestItemInput(filePath); }
public void ExtractBankMetadatasFromCsv_OnInvalidCsv_ThrowException() { //Arrange string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Invalid_1.CSV"); CsvExtractor repo = new CsvExtractor(); //Act, Assert List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList(); }
public void BadFilePathShouldFail() { // Arrange const string filePath = "randomfilepath"; //Act ActualValueDelegate <object> testDelegate = () => CsvExtractor.Extract <Item>(filePath); //Assert Assert.That(testDelegate, Throws.TypeOf <ArgumentException>()); }
public void CorrectFilePathWithIncorrectExtensionShouldFail() { // Arrange var filePath = Path.Combine(ResourcesDirectory, "test.txt"); //Act ActualValueDelegate <object> testDelegate = () => CsvExtractor.Extract <Item>(filePath); //Assert Assert.That(testDelegate, Throws.TypeOf <ArgumentException>()); }
public void ExtractExpensesFromCsv_OnValidCsv_ReturnExpensesList() { //Arrange string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Valid_1.CSV"); //Act CsvExtractor repo = new CsvExtractor(); List <Expense> output1 = repo.ExtractExpensesFromCsvString(input1).ToList(); //Assert CollectionAssert.AreEqual(_expList1, output1, new ExpenseComparer()); }
public void ExtractBankMetadataFromCsv_OnValidCsv_ReturnMetadataList() { //Arrange string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Valid_1.CSV"); //Act CsvExtractor repo = new CsvExtractor(); List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList(); //Assert CollectionAssert.AreEqual(_bankMetadataList1, output1, new BankMetadataComparer()); }
public async Task LoadFile( string fileName ) { var content = await CsvExtractor.ReadAllTextAsync(fileName); int count = 0; var parser = new Rfc4180Parser( field => { }, () => count += 1); var length = parser.Parse(content); Assert.That(length, Is.EqualTo(content.Length)); }
public void CorrectFilePathWithCorrectExtensionAndHeadersShouldSucceed() { // Arrange var filePath = Path.Combine(ResourcesDirectory, "FFP_Sample.csv"); // Act var result = CsvExtractor.Extract <Item>(filePath); // Assert Assert.IsTrue(result.Any()); Assert.AreEqual(result.Count, 5); Assert.AreEqual(result.First().ItemId, "187-300"); Assert.AreEqual(result.Last().FormPartitionId, "187-561"); Assert.IsTrue(result.Count(x => string.IsNullOrEmpty(x.AssociatedStimuliId)) == 1); }
static void Main(string[] args) { try { Console.WriteLine("Weather reports creation started.."); List <City> values = new CsvExtractor().CsvObject(); storeCityWeatherReportInDirectory(values); Console.WriteLine("Weather reports created successfully!!"); } catch (Exception ex) { Console.WriteLine("Weather report failed: " + ex.Message); } }
public void TestCsvExtractor() { var dt = CsvExtractor.GetDataTableFromCsv("./files/sampledata.csv", true); Assert.IsNotNull(dt); LogTableContents(dt); Assert.IsTrue(dt.Rows.Count == 4, string.Format("Expected 4 got {0}", dt.Rows.Count)); foreach (var c in dt.Columns) { Resolve <ILogger>().Info("co: {0}", (c as DataColumn).ColumnName); } Assert.IsTrue(dt.Columns[0].ColumnName == "Make", string.Format("Expected Make got {0}", dt.Columns[0].ColumnName)); }
public void ExtractBankMetadatasFromCsv_OnInvalidXml_ThrowCorrectExceptionMessage() { //Arrange string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Invalid_1.CSV"); CsvExtractor repo = new CsvExtractor(); //Act try { List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList(); } //Assert catch (Exception ex) { Assert.AreEqual("Badly formatted CSV string", ex.Message); } }
public void ExtractReadsMultipleTypesFromMultipleFiles() { var files = new CsvExtractorParams.File[] { new CsvExtractorParams.File("test.csv", typeof(CsvClass1)), new CsvExtractorParams.File("test2.csv", typeof(CsvClass2)) }; var contents = new Dictionary <string, string> { { "test.csv", "ABC" }, { "test2.csv", "123" } }; try { foreach (var file in files) { File.WriteAllLines(file.Path, new[] { "\"Test\"", $"\"{contents[file.Path]}\"" }); } var extractor = new CsvExtractor(); var extracted = extractor.Extract(new CsvExtractorParams { Files = files }).ToArray(); Assert.NotNull(extracted); Assert.True(extracted.Length == 2); var class1Obj = extracted[0] as CsvClass1; Assert.NotNull(class1Obj); Assert.Equal(contents["test.csv"], class1Obj.Test); var class2Obj = extracted[1] as CsvClass2; Assert.NotNull(class2Obj); Assert.Equal(Convert.ToInt32(contents["test2.csv"]), class2Obj.Test); } finally { foreach (var path in files.Select(x => x.Path)) { if (File.Exists(path)) { File.Delete(path); } } } }
private static void Main(string[] args) { Logger.Debug(string.Concat(Enumerable.Repeat("-", 60))); Logger.Debug("Fixed Form Packager Initialized"); try { var options = new Options(); if (Parser.Default.ParseArgumentsStrict(args, options, () => { Logger.Fatal( $"Incorrect parameters provided at the command line [{args.Aggregate((x, y) => $"{x},{y}")}]. Terminating."); })) { ExtractionSettings.GitLabInfo = new GitLabInfo { BaseUrl = options.GitLabBaseUrl, Group = options.GitLabGroup, Password = options.GitLabPassword, Username = options.GitLabUsername }; ExtractionSettings.AssessmentScoring = CsvExtractor.Extract <AssessmentScoringComputationRule>(options.AssessmentScoringInput).ToList(); ExtractionSettings.ItemInput = CsvExtractor.Extract <Item>(options.ItemInput).ToList(); ExtractionSettings.AssessmentInfo = CsvExtractor.Extract <Assessment>(options.AssessmentInput).First(); ExtractionSettings.ItemInput.ForEach( x => { ResourceGenerator.Retrieve(ExtractionSettings.GitLabInfo, $"Item-{x.ItemId}"); // If there is no scoring information provided in the input document, look for it in the item XML if (x.ItemScoringInformation.All(y => string.IsNullOrEmpty(y.MeasurementModel))) { ExtractionSettings.ItemInput[ExtractionSettings.ItemInput.FindIndex( y => y.ItemId.Equals(x.ItemId, StringComparison.OrdinalIgnoreCase))] .ItemScoringInformation = IrtMapper.RetrieveIrtParameters(x.ItemId); } }); var uniqueHash = HashGenerator.Hash(ExtractionSettings.AssessmentInfo.UniqueId.GetHashCode(), ExtractionSettings.ItemInput.First().SegmentId.GetHashCode(), ExtractionSettings.ItemInput.First().FormPartitionId.GetHashCode()); Logger.Debug($"Generated unique hash: {uniqueHash}"); ExtractionSettings.AssessmentInfo.UniqueId += $"{uniqueHash}"; ExtractionSettings.ItemInput = ExtractionSettings.ItemInput.Select(x => new Item { ItemId = x.ItemId, AssociatedStimuliId = x.AssociatedStimuliId, FormPartitionId = x.FormPartitionId + $"{uniqueHash}", FormPartitionPosition = x.FormPartitionPosition, FormPosition = x.FormPosition, ItemScoringInformation = x.ItemScoringInformation, SegmentId = x.SegmentId + $"{uniqueHash}", SegmentPosition = x.SegmentPosition }).ToList(); // Validate that the segment unique IDs and assessment IDs are either the same or different depending on # of segments var segmentIds = ExtractionSettings.ItemInput.Select(x => x.SegmentId).Distinct().ToList(); if (segmentIds.Count() > 1 && segmentIds.Any( x => x.Equals(ExtractionSettings.AssessmentInfo.UniqueId, StringComparison.OrdinalIgnoreCase))) { throw new Exception( "Identifiers of segments and assessments must not match in multi-segmented assessments. Please adjust the assessment and/or item inputs."); } if (segmentIds.Count() == 1 && !segmentIds.First() .Equals(ExtractionSettings.AssessmentInfo.UniqueId, StringComparison.OrdinalIgnoreCase)) { throw new Exception( "Identifiers of segments and assessments must match in single-segmented assessments. Please adjust the assessment and/or item inputs."); } var result = TestSpecification.Construct(); result.ToList().ForEach(x => x.Save( $"{x.XPathSelectElement("./identifier").Attribute("uniqueid")?.Value}_{x.Attribute("purpose")?.Value}.xml")); } } catch (Exception e) { Logger.Error(e); } finally { Console.Write("Press any key to exit."); Console.ReadKey(true); } }
public void TestSqlLoader() { var dt = CsvExtractor.GetDataTableFromCsv("./files/sampledata.csv", true); new SqlLoader(1).OnLoad(dt); }
public void ExtractThrowsExceptionIfWrongParameterType() { var extractor = new CsvExtractor(); Assert.Throws <ArgumentException>(() => extractor.Extract(new ParamsStub())); }
public async Task <IActionResult> UploadFile(IFormFile file) { if (file == null) { return(BadRequest("File required")); } var ext = Path.GetExtension(file.FileName).ToLower(); if (!(ext == ".csv" || ext == ".xml")) { return(BadRequest("Unknown format")); } if (file.Length > (MAX_FILE_SIZE)) { return(BadRequest($"Invalid File Size (Maximum: {MAX_FILE_SIZE / (1024 * 1024)} MB)")); } // Save file into Uploads folder var transFile = new TransactionFile(); string errormsg = ""; try { //throw new Exception(); transFile = await SaveFileAsync(file); Strategy strategy; if (ext == ".csv") { strategy = new CsvExtractor(); } else { strategy = new XmlExtractor(); } var entityExtractor = new TransactionExtractor(strategy); var entities = entityExtractor.ExtractTransaction(file); if (entities.Where(x => !string.IsNullOrWhiteSpace(x.Errors)).Count() == 0) { // Clean records, add to transaction var cleanEntities = entities.Select(x => new TransactionEntity { TransactionId = x.TransactionId, Amount = x.ValidAmount.Value, CurrencyCode = x.CurrencyCode, DateTime = x.ValidTransactionDate.Value.UtcDateTime, Status = x.ValidStatus }); _dbcontext.TransactionEntity.AddRange(cleanEntities); } else { // Dirty records, add to error log var errors = entities.Select(x => new TransactionErrorLog { TransactionId = x.TransactionId, Amount = x.Amount, CurrencyCode = x.CurrencyCode, DateTime = x.TransactionDate, Status = x.Status, Error = x.Errors, FileId = transFile.Id }); _dbcontext.TransactionErrorLog.AddRange(errors); errormsg = "Some records in uploaded file is not valid, please check error log for details."; } await _dbcontext.SaveChangesAsync(); if (string.IsNullOrWhiteSpace(errormsg)) { return(Ok("Success")); } else { return(BadRequest(errormsg)); } } catch (Exception ex) { _logger.LogError(ex, ex.Message); if (System.IO.File.Exists(Path.Combine(_hostingEnvironment.ContentRootPath, "Uploads", transFile.FileName ?? ""))) { GC.Collect(); GC.WaitForPendingFinalizers(); System.IO.File.Delete(Path.Combine(_hostingEnvironment.ContentRootPath, "Uploads", transFile.FileName ?? "")); } // TODO: If db exception in here if (transFile.Id != 0) { _dbcontext.TransactionFile.Remove(transFile); await _dbcontext.SaveChangesAsync(); } return(new StatusCodeResult(500)); } }