public ActionResult ChangeAsciiFileInfo(string name, string value)
{
TaskManager TaskManager = (TaskManager)Session["TaskManager"];
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv"))
{
AsciiFileReaderInfo info = (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO];
switch (name)
{
case "Separator": { info.Seperator = AsciiFileReaderInfo.GetSeperator(value); break; }
case "Decimal": { info.Decimal = AsciiFileReaderInfo.GetDecimalCharacter(value); break; }
case "Orientation": { info.Orientation = AsciiFileReaderInfo.GetOrientation(value); break; }
case "TextMarker": { info.TextMarker = AsciiFileReaderInfo.GetTextMarker(value); break; }
case "Variables": { info.Variables = Convert.ToInt32(value); break; }
case "Data": { info.Data = Convert.ToInt32(value); break; }
case "Offset": { info.Offset = Convert.ToInt32(value); break; }
}
TaskManager.Bus[TaskManager.FILE_READER_INFO] = info;
Session["TaskManager"] = TaskManager;
}
return(Content(""));
}
public void textMarkerHandling_NotTextMarkerInRow_ReturnExpectedListOfStrings()
{
//Arrange
string row = "V1;V2;V3;V4";
List <string> expectedOutcome = new List <string> {
"V1", "V2", "V3", "V4"
};
AsciiFileReaderInfo info = new AsciiFileReaderInfo();
info.Seperator = TextSeperator.semicolon;
AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
//Act
List <string> values = reader.TextMarkerHandling(row,
AsciiFileReaderInfo.GetSeperator(TextSeperator.semicolon),
AsciiFileReaderInfo.GetTextMarker(TextMarker.quotes));
//Assert
Assert.That(values.Count, Is.EqualTo(expectedOutcome.Count));
Assert.That(values, Is.EquivalentTo(expectedOutcome));
}
public ActionResult SaveAsciiFileInfos(FileInfoModel info)
{
TaskManager TaskManager = (TaskManager)Session["TaskManager"];
AsciiFileReaderInfo asciiFileReaderInfo = new AsciiFileReaderInfo();
asciiFileReaderInfo.Data = info.Data;
asciiFileReaderInfo.Dateformat = "";//info.Dateformat;
asciiFileReaderInfo.Decimal = info.Decimal;
asciiFileReaderInfo.Offset = info.Offset;
asciiFileReaderInfo.Orientation = info.Orientation;
asciiFileReaderInfo.Seperator = info.Separator;
asciiFileReaderInfo.Variables = info.Variables;
asciiFileReaderInfo.TextMarker = info.TextMarker;
TaskManager.AddToBus(TaskManager.FILE_READER_INFO, asciiFileReaderInfo);
GetFileInformationModel model = new GetFileInformationModel();
model.StepInfo = TaskManager.Current();
model.StepInfo.SetValid(true);
model.Extention = TaskManager.Bus[TaskManager.EXTENTION].ToString();
model.FileInfoModel = info;
model.FileInfoModel.Extention = TaskManager.Bus[TaskManager.EXTENTION].ToString();
model.IsSaved = true;
return(RedirectToAction("ReloadUploadWizard", new { restart = false }));
}
public void ValidateRow_runNotValid_LimitErrors()
{
//Arrange
DataGeneratorHelper dgh = new DataGeneratorHelper();
var errors = new List <Error>();
var testData = dgh.GenerateRowsWithRandomValuesBasedOnDatastructureWithErrors(dataStructure, ",", 1000000, true);
IEnumerable <string> vairableNames = dataStructure.Variables.Select(v => v.Label);
//generate file to read
Encoding encoding = Encoding.Default;
string path = Path.Combine(AppConfiguration.DataPath, "testdataforvalidation.txt");
if (File.Exists(path))
{
File.Delete(path);
}
using (StreamWriter sw = new StreamWriter(path))
{
string header = string.Join(",", vairableNames.ToArray());
sw.WriteLine(header);
foreach (var r in testData)
{
sw.WriteLine(r);
}
}
//Mock IOUtility
var ioUtilityMock = new Mock <IOUtility>();
ioUtilityMock.Setup(i => i.ConvertDateToCulture("2018")).Returns("2018");
try
{
AsciiFileReaderInfo afr = new AsciiFileReaderInfo();
afr.TextMarker = TextMarker.doubleQuotes;
afr.Seperator = TextSeperator.comma;
DataReader reader = new AsciiReader(dataStructure, afr, ioUtilityMock.Object);
var asciireader = (AsciiReader)reader;
//Act
var row = new List <string>();
using (Stream stream = reader.Open(path))
{
asciireader.ValidateFile(stream, "", 1);
}
//Assert
Assert.That(asciireader.ErrorMessages.Count, Is.EqualTo(1000));
}
catch (Exception ex)
{
throw ex;
}
}
private FileInfoModel GetFileInfoModel(AsciiFileReaderInfo info, string extention)
{
FileInfoModel FileReaderInfo = new FileInfoModel();
FileReaderInfo.Data = info.Data;
FileReaderInfo.Dateformat = info.Dateformat;
FileReaderInfo.Decimal = info.Decimal;
FileReaderInfo.Offset = info.Offset;
FileReaderInfo.Orientation = info.Orientation;
FileReaderInfo.Variables = info.Variables;
FileReaderInfo.Separator = info.Seperator;
FileReaderInfo.TextMarker = info.TextMarker;
return(FileReaderInfo);
}
public void rowToList_RowAsQuotesAndSeperatorInQuotes_ReturnExpectedListOfStrings()
{
//Arrange
string row = "'V1';'V2';'V3;V4'";
List <string> expectedOutcome = new List <string> {
"V1", "V2", "V3;V4"
};
AsciiFileReaderInfo info = new AsciiFileReaderInfo();
info.Seperator = TextSeperator.semicolon;
AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
//Act
List <string> values = reader.rowToList(row,
AsciiFileReaderInfo.GetSeperator(TextSeperator.semicolon));
//Assert
Assert.That(values.Count, Is.EqualTo(expectedOutcome.Count));
Assert.That(values, Is.EquivalentTo(expectedOutcome));
}
public void ValidateRow_runValid_noErrors()
{
//Arrange
DataGeneratorHelper dgh = new DataGeneratorHelper();
var errors = new List <Error>();
var testData = dgh.GenerateRowsWithRandomValuesBasedOnDatastructure(dataStructure, ",", 1000, true);
//generate file to read
Encoding encoding = Encoding.Default;
string path = Path.Combine(AppConfiguration.DataPath, "testdataforvalidation.txt");
if (File.Exists(path))
{
File.Delete(path);
}
using (StreamWriter sw = new StreamWriter(path))
{
foreach (var r in testData)
{
sw.WriteLine(r);
}
}
//Mock IOUtility
var ioUtilityMock = new Mock <IOUtility>();
ioUtilityMock.Setup(i => i.ConvertDateToCulture("2018")).Returns("2018");
try
{
AsciiFileReaderInfo afr = new AsciiFileReaderInfo();
afr.TextMarker = TextMarker.doubleQuotes;
afr.Seperator = TextSeperator.comma;
DataReader reader = new AsciiReader(dataStructure, new AsciiFileReaderInfo(), ioUtilityMock.Object);
IEnumerable <string> vairableNames = dataStructure.Variables.Select(v => v.Label);
List <VariableIdentifier> variableIdentifiers = reader.SetSubmitedVariableIdentifiers(vairableNames.ToList());
reader.ValidateComparisonWithDatatsructure(variableIdentifiers);
var asciireader = (AsciiReader)reader;
//Act
var row = new List <string>();
using (StreamReader streamReader = new StreamReader(path, encoding))
{
string line;
int index = 1;
char seperator = AsciiFileReaderInfo.GetSeperator(afr.Seperator);
while ((line = streamReader.ReadLine()) != null)
{
row = asciireader.rowToList(line, ',');
errors = asciireader.ValidateRow(row, index);
index++;
}
}
//Assert
Assert.That(errors.Count, Is.EqualTo(0));
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// test unique of primary keys in a FileStream
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="taskManager"></param>
/// <param name="datasetId"></param>
/// <param name="primaryKeys"></param>
/// <param name="ext"></param>
/// <param name="filename"></param>
/// <returns></returns>
public bool IsUnique(TaskManager taskManager, long datasetId, List <long> primaryKeys, string ext, string filename)
{
Hashtable hashtable = new Hashtable();
Hashtable test = new Hashtable();
List <string> testString = new List <string>();
List <string> primaryValuesAsOneString = new List <string>();
TaskManager TaskManager = taskManager;
int packageSize = 1000;
int position = 1;
if (ext.Equals(".txt") || ext.Equals(".csv"))
{
#region csv
do
{
primaryValuesAsOneString = new List <string>();
AsciiReader reader = new AsciiReader();
reader.Position = position;
Stream stream = reader.Open(TaskManager.Bus["FilePath"].ToString());
AsciiFileReaderInfo afri = (AsciiFileReaderInfo)TaskManager.Bus["FileReaderInfo"];
DataStructureManager datastructureManager = new DataStructureManager();
StructuredDataStructure sds = datastructureManager.StructuredDataStructureRepo.Get(Convert.ToInt64(TaskManager.Bus["DataStructureId"].ToString()));
// get a list of values for each row
// e.g.
// primarky keys id, name
// 1 [1][David]
// 2 [2][Javad]
List <List <string> > tempList = reader.ReadValuesFromFile(stream, filename, afri, sds, datasetId, primaryKeys, packageSize);
// convert List of Lists to list of strings
// 1 [1][David] = 1David
// 2 [2][Javad] = 2Javad
foreach (List <string> l in tempList)
{
string tempString = "";
foreach (string s in l)
{
tempString += s;
}
if (!String.IsNullOrEmpty(tempString))
{
primaryValuesAsOneString.Add(tempString);
}
}
// add all primary keys pair into the hasttable
foreach (string pKey in primaryValuesAsOneString)
{
if (pKey != "")
{
try
{
hashtable.Add(Utility.ComputeKey(pKey), "pKey");
}
catch
{
return(false);
}
}
}
position = reader.Position + 1;
stream.Close();
} while (primaryValuesAsOneString.Count > 0);
#endregion
}
if (ext.Equals(".xlsm"))
{
#region excel template
do
{
//reset
primaryValuesAsOneString = new List <string>();
ExcelReader reader = new ExcelReader();
reader.Position = position;
Stream stream = reader.Open(TaskManager.Bus["FilePath"].ToString());
DataStructureManager datastructureManager = new DataStructureManager();
StructuredDataStructure sds = datastructureManager.StructuredDataStructureRepo.Get(Convert.ToInt64(TaskManager.Bus["DataStructureId"].ToString()));
// get a list of values for each row
// e.g.
// primarky keys id, name
// 1 [1][David]
// 2 [2][Javad]
List <List <string> > tempList = reader.ReadValuesFromFile(stream, filename, sds, datasetId, primaryKeys, packageSize);
// convert List of Lists to list of strings
// 1 [1][David] = 1David
// 2 [2][Javad] = 2Javad
foreach (List <string> l in tempList)
{
string tempString = "";
foreach (string s in l)
{
tempString += s;
}
if (!String.IsNullOrEmpty(tempString))
{
primaryValuesAsOneString.Add(tempString);
}
}
// add all primary keys pair into the hasttable
foreach (string pKey in primaryValuesAsOneString)
{
if (pKey != "")
{
try
{
hashtable.Add(Utility.ComputeKey(pKey), pKey);
}
catch
{
stream.Close();
return(false);
}
}
}
position = reader.Position + 1;
stream.Close();
} while (primaryValuesAsOneString.Count > 0);
#endregion
}
return(true);
}
