public void ExtractReadsTypedFile()
{
const string PATH = "test.csv";
const string CONTENT = "ABC";
try
{
File.WriteAllLines(PATH, new[] { "\"Test\"", $"\"{CONTENT}\"" });
var extractor = new CsvExtractor();
var extracted = extractor.Extract(new CsvExtractorParams {
Files = new[] {
new CsvExtractorParams.File(PATH, typeof(CsvClass1))
}
}).ToArray();
Assert.NotNull(extracted);
Assert.True(extracted.Length == 1);
var extractedObj = extracted.First() as CsvClass1;
Assert.NotNull(extractedObj);
Assert.Equal(CONTENT, extractedObj.Test);
}
finally
{
if (File.Exists(PATH))
{
File.Delete(PATH);
}
}
}
/// <summary>
/// Takes an XLSX or a CSV stream and unpackages its content into a 2-D table of strings.
/// </summary>
/// <param name="stream">The contents of the XLSX or CSV file.</param>
/// <param name="fileName">The name of the file to extract if available.</param>
/// <param name="contentType">The mime type of the file to extract if available.</param>
/// <param name="localizer">To localize error messages.</param>
/// <returns>A 2-D grid of strings representing the contents of the XLSX or the CSV file.</returns>
public static IEnumerable <string[]> ExtractStringsFromFile(Stream stream, string fileName, string contentType, IStringLocalizer localizer)
{
IDataExtractor extracter;
if (contentType == MimeTypes.Csv || (fileName?.ToLower()?.EndsWith(".csv") ?? false))
{
extracter = new CsvExtractor();
}
else if (contentType == MimeTypes.Excel || (fileName?.ToLower()?.EndsWith(".xlsx") ?? false))
{
extracter = new ExcelExtractor();
}
else
{
throw new FormatException(localizer["Error_OnlyCsvOrExcelAreSupported"]);
}
// Extrat and return
try
{
return(extracter.Extract(stream).ToList());
}
catch (Exception ex)
{
// Report any errors during extraction
string msg = localizer["Error_FailedToParseFileError0", ex.Message];
throw new ServiceException(msg);
}
}
public void BadFilePathShouldFail()
{
// Arrange
const string filePath = "randomfilepath";
// Act
CsvExtractor.ExtractTestItemInput(filePath);
}
public void CorrectFilePathWithIncorrectExtensionShouldFail()
{
// Arrange
var filePath = Path.Combine(Directory.GetParent(Environment.CurrentDirectory).Parent.FullName, "Resources",
"test.txt");
// Act
CsvExtractor.ExtractTestItemInput(filePath);
}
public void CorrectFilePathWithCorrectExtensionAndIncorrectHeadersShouldFail()
{
// Arrange
var filePath = Path.Combine(Directory.GetParent(Environment.CurrentDirectory).Parent.FullName, "Resources",
"BAD-FFP_Sample.csv");
// Act
CsvExtractor.ExtractTestItemInput(filePath);
}
public void ExtractBankMetadatasFromCsv_OnInvalidCsv_ThrowException()
{
//Arrange
string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Invalid_1.CSV");
CsvExtractor repo = new CsvExtractor();
//Act, Assert
List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList();
}
public void BadFilePathShouldFail()
{
// Arrange
const string filePath = "randomfilepath";
//Act
ActualValueDelegate <object> testDelegate = () => CsvExtractor.Extract <Item>(filePath);
//Assert
Assert.That(testDelegate, Throws.TypeOf <ArgumentException>());
}
public void CorrectFilePathWithIncorrectExtensionShouldFail()
{
// Arrange
var filePath = Path.Combine(ResourcesDirectory, "test.txt");
//Act
ActualValueDelegate <object> testDelegate = () => CsvExtractor.Extract <Item>(filePath);
//Assert
Assert.That(testDelegate, Throws.TypeOf <ArgumentException>());
}
public void ExtractExpensesFromCsv_OnValidCsv_ReturnExpensesList()
{
//Arrange
string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Valid_1.CSV");
//Act
CsvExtractor repo = new CsvExtractor();
List <Expense> output1 = repo.ExtractExpensesFromCsvString(input1).ToList();
//Assert
CollectionAssert.AreEqual(_expList1, output1, new ExpenseComparer());
}
public void ExtractBankMetadataFromCsv_OnValidCsv_ReturnMetadataList()
{
//Arrange
string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Valid_1.CSV");
//Act
CsvExtractor repo = new CsvExtractor();
List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList();
//Assert
CollectionAssert.AreEqual(_bankMetadataList1, output1, new BankMetadataComparer());
}
public async Task LoadFile(
string fileName
)
{
var content = await CsvExtractor.ReadAllTextAsync(fileName);
int count = 0;
var parser = new Rfc4180Parser(
field => { },
() => count += 1);
var length = parser.Parse(content);
Assert.That(length, Is.EqualTo(content.Length));
}
public void CorrectFilePathWithCorrectExtensionAndHeadersShouldSucceed()
{
// Arrange
var filePath = Path.Combine(ResourcesDirectory, "FFP_Sample.csv");
// Act
var result = CsvExtractor.Extract <Item>(filePath);
// Assert
Assert.IsTrue(result.Any());
Assert.AreEqual(result.Count, 5);
Assert.AreEqual(result.First().ItemId, "187-300");
Assert.AreEqual(result.Last().FormPartitionId, "187-561");
Assert.IsTrue(result.Count(x => string.IsNullOrEmpty(x.AssociatedStimuliId)) == 1);
}
static void Main(string[] args)
{
try
{
Console.WriteLine("Weather reports creation started..");
List <City> values = new CsvExtractor().CsvObject();
storeCityWeatherReportInDirectory(values);
Console.WriteLine("Weather reports created successfully!!");
}
catch (Exception ex)
{
Console.WriteLine("Weather report failed: " + ex.Message);
}
}
public void TestCsvExtractor()
{
var dt = CsvExtractor.GetDataTableFromCsv("./files/sampledata.csv", true);
Assert.IsNotNull(dt);
LogTableContents(dt);
Assert.IsTrue(dt.Rows.Count == 4, string.Format("Expected 4 got {0}", dt.Rows.Count));
foreach (var c in dt.Columns)
{
Resolve <ILogger>().Info("co: {0}", (c as DataColumn).ColumnName);
}
Assert.IsTrue(dt.Columns[0].ColumnName == "Make", string.Format("Expected Make got {0}", dt.Columns[0].ColumnName));
}
public void ExtractBankMetadatasFromCsv_OnInvalidXml_ThrowCorrectExceptionMessage()
{
//Arrange
string input1 = File.ReadAllText(@"..\..\..\TestFiles1\Csv_Invalid_1.CSV");
CsvExtractor repo = new CsvExtractor();
//Act
try
{
List <BankMetadata> output1 = repo.ExtractBankMetadataFromCsvString(input1).ToList();
}
//Assert
catch (Exception ex)
{
Assert.AreEqual("Badly formatted CSV string", ex.Message);
}
}
public void ExtractReadsMultipleTypesFromMultipleFiles()
{
var files = new CsvExtractorParams.File[] {
new CsvExtractorParams.File("test.csv", typeof(CsvClass1)),
new CsvExtractorParams.File("test2.csv", typeof(CsvClass2))
};
var contents = new Dictionary <string, string> {
{ "test.csv", "ABC" },
{ "test2.csv", "123" }
};
try
{
foreach (var file in files)
{
File.WriteAllLines(file.Path, new[] { "\"Test\"", $"\"{contents[file.Path]}\"" });
}
var extractor = new CsvExtractor();
var extracted = extractor.Extract(new CsvExtractorParams {
Files = files
}).ToArray();
Assert.NotNull(extracted);
Assert.True(extracted.Length == 2);
var class1Obj = extracted[0] as CsvClass1;
Assert.NotNull(class1Obj);
Assert.Equal(contents["test.csv"], class1Obj.Test);
var class2Obj = extracted[1] as CsvClass2;
Assert.NotNull(class2Obj);
Assert.Equal(Convert.ToInt32(contents["test2.csv"]), class2Obj.Test);
}
finally
{
foreach (var path in files.Select(x => x.Path))
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}
private static void Main(string[] args)
{
Logger.Debug(string.Concat(Enumerable.Repeat("-", 60)));
Logger.Debug("Fixed Form Packager Initialized");
try
{
var options = new Options();
if (Parser.Default.ParseArgumentsStrict(args, options,
() =>
{
Logger.Fatal(
$"Incorrect parameters provided at the command line [{args.Aggregate((x, y) => $"{x},{y}")}]. Terminating.");
}))
{
ExtractionSettings.GitLabInfo = new GitLabInfo
{
BaseUrl = options.GitLabBaseUrl,
Group = options.GitLabGroup,
Password = options.GitLabPassword,
Username = options.GitLabUsername
};
ExtractionSettings.AssessmentScoring =
CsvExtractor.Extract <AssessmentScoringComputationRule>(options.AssessmentScoringInput).ToList();
ExtractionSettings.ItemInput = CsvExtractor.Extract <Item>(options.ItemInput).ToList();
ExtractionSettings.AssessmentInfo =
CsvExtractor.Extract <Assessment>(options.AssessmentInput).First();
ExtractionSettings.ItemInput.ForEach(
x =>
{
ResourceGenerator.Retrieve(ExtractionSettings.GitLabInfo, $"Item-{x.ItemId}");
// If there is no scoring information provided in the input document, look for it in the item XML
if (x.ItemScoringInformation.All(y => string.IsNullOrEmpty(y.MeasurementModel)))
{
ExtractionSettings.ItemInput[ExtractionSettings.ItemInput.FindIndex(
y => y.ItemId.Equals(x.ItemId, StringComparison.OrdinalIgnoreCase))]
.ItemScoringInformation = IrtMapper.RetrieveIrtParameters(x.ItemId);
}
});
var uniqueHash = HashGenerator.Hash(ExtractionSettings.AssessmentInfo.UniqueId.GetHashCode(),
ExtractionSettings.ItemInput.First().SegmentId.GetHashCode(),
ExtractionSettings.ItemInput.First().FormPartitionId.GetHashCode());
Logger.Debug($"Generated unique hash: {uniqueHash}");
ExtractionSettings.AssessmentInfo.UniqueId += $"{uniqueHash}";
ExtractionSettings.ItemInput = ExtractionSettings.ItemInput.Select(x => new Item
{
ItemId = x.ItemId,
AssociatedStimuliId = x.AssociatedStimuliId,
FormPartitionId = x.FormPartitionId + $"{uniqueHash}",
FormPartitionPosition = x.FormPartitionPosition,
FormPosition = x.FormPosition,
ItemScoringInformation = x.ItemScoringInformation,
SegmentId = x.SegmentId + $"{uniqueHash}",
SegmentPosition = x.SegmentPosition
}).ToList();
// Validate that the segment unique IDs and assessment IDs are either the same or different depending on # of segments
var segmentIds = ExtractionSettings.ItemInput.Select(x => x.SegmentId).Distinct().ToList();
if (segmentIds.Count() > 1 &&
segmentIds.Any(
x =>
x.Equals(ExtractionSettings.AssessmentInfo.UniqueId, StringComparison.OrdinalIgnoreCase)))
{
throw new Exception(
"Identifiers of segments and assessments must not match in multi-segmented assessments. Please adjust the assessment and/or item inputs.");
}
if (segmentIds.Count() == 1 &&
!segmentIds.First()
.Equals(ExtractionSettings.AssessmentInfo.UniqueId, StringComparison.OrdinalIgnoreCase))
{
throw new Exception(
"Identifiers of segments and assessments must match in single-segmented assessments. Please adjust the assessment and/or item inputs.");
}
var result = TestSpecification.Construct();
result.ToList().ForEach(x =>
x.Save(
$"{x.XPathSelectElement("./identifier").Attribute("uniqueid")?.Value}_{x.Attribute("purpose")?.Value}.xml"));
}
}
catch (Exception e)
{
Logger.Error(e);
}
finally
{
Console.Write("Press any key to exit.");
Console.ReadKey(true);
}
}
public void TestSqlLoader()
{
var dt = CsvExtractor.GetDataTableFromCsv("./files/sampledata.csv", true);
new SqlLoader(1).OnLoad(dt);
}
public void ExtractThrowsExceptionIfWrongParameterType()
{
var extractor = new CsvExtractor();
Assert.Throws <ArgumentException>(() => extractor.Extract(new ParamsStub()));
}
public async Task <IActionResult> UploadFile(IFormFile file)
{
if (file == null)
{
return(BadRequest("File required"));
}
var ext = Path.GetExtension(file.FileName).ToLower();
if (!(ext == ".csv" || ext == ".xml"))
{
return(BadRequest("Unknown format"));
}
if (file.Length > (MAX_FILE_SIZE))
{
return(BadRequest($"Invalid File Size (Maximum: {MAX_FILE_SIZE / (1024 * 1024)} MB)"));
}
// Save file into Uploads folder
var transFile = new TransactionFile();
string errormsg = "";
try
{
//throw new Exception();
transFile = await SaveFileAsync(file);
Strategy strategy;
if (ext == ".csv")
{
strategy = new CsvExtractor();
}
else
{
strategy = new XmlExtractor();
}
var entityExtractor = new TransactionExtractor(strategy);
var entities = entityExtractor.ExtractTransaction(file);
if (entities.Where(x => !string.IsNullOrWhiteSpace(x.Errors)).Count() == 0)
{
// Clean records, add to transaction
var cleanEntities = entities.Select(x => new TransactionEntity
{
TransactionId = x.TransactionId,
Amount = x.ValidAmount.Value,
CurrencyCode = x.CurrencyCode,
DateTime = x.ValidTransactionDate.Value.UtcDateTime,
Status = x.ValidStatus
});
_dbcontext.TransactionEntity.AddRange(cleanEntities);
}
else
{
// Dirty records, add to error log
var errors = entities.Select(x => new TransactionErrorLog
{
TransactionId = x.TransactionId,
Amount = x.Amount,
CurrencyCode = x.CurrencyCode,
DateTime = x.TransactionDate,
Status = x.Status,
Error = x.Errors,
FileId = transFile.Id
});
_dbcontext.TransactionErrorLog.AddRange(errors);
errormsg = "Some records in uploaded file is not valid, please check error log for details.";
}
await _dbcontext.SaveChangesAsync();
if (string.IsNullOrWhiteSpace(errormsg))
{
return(Ok("Success"));
}
else
{
return(BadRequest(errormsg));
}
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
if (System.IO.File.Exists(Path.Combine(_hostingEnvironment.ContentRootPath, "Uploads", transFile.FileName ?? "")))
{
GC.Collect();
GC.WaitForPendingFinalizers();
System.IO.File.Delete(Path.Combine(_hostingEnvironment.ContentRootPath, "Uploads", transFile.FileName ?? ""));
}
// TODO: If db exception in here
if (transFile.Id != 0)
{
_dbcontext.TransactionFile.Remove(transFile);
await _dbcontext.SaveChangesAsync();
}
return(new StatusCodeResult(500));
}
}
