/// <summary> /// Validate the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks>A list of errorMessages is filled when the fil is not valid </remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> public void ValidateFile(Stream file, string fileName, long datasetId) { this.FileStream = file; this.FileName = fileName; this.DatasetId = datasetId; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = 1; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); bool dsdIsOk = false; while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { dsdIsOk = ValidateDatastructure(line, seperator); // if data is not in the correct order, create a dictionary with the new position } if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line) && !isEmpty(line, seperator)) { var r = rowToList(line, seperator); var e = ValidateRow(r, index); this.ErrorMessages = this.ErrorMessages.Union(e).ToList(); if (this.ErrorMessages.Count >= 1000) { break; } } index++; } } } }
/// <summary> /// Read the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, long datasetId) { this.FileStream = file; this.FileName = fileName; this.DatasetId = datasetId; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { ValidateDatastructure(line, seperator); } if (index >= this.Info.Data) { // return List of VariablesValues, and error messages if (!isEmpty(line, seperator)) { this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } } index++; } } } return(this.DataTuples); }
/// <summary> /// Read the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(line, seperator)); convertAndAddToSubmitedVariableIdentifier(); } if (index >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } index++; } } } return(this.DataTuples); }
/// <summary> /// Validate the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks>A list of errorMessages is filled when the fil is not valid </remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> public void ValidateFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); bool dsdIsOk = false; while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { dsdIsOk = ValidateDatastructure(line, seperator); } if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line)) { this.ErrorMessages = this.ErrorMessages.Union(ValidateRow(rowToList(line, seperator), index)).ToList(); } index++; } } } }
public List <List <string> > ReadFile(Stream file) { List <List <string> > tmp = new List <List <string> >(); this.FileStream = file; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index >= Info.Data) { // return List of VariablesValues, and error messages if (!isEmpty(line, seperator)) { tmp.Add(rowToList(line, seperator)); } } index++; } } } return(tmp); }
public List<List<string>> ReadFile(Stream file, AsciiFileReaderInfo fri) { List<List<string>> tmp = new List<List<string>>(); this.FileStream = file; this.Info = fri; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index >= this.Info.Data) { // return List of VariablesValues, and error messages tmp.Add(rowToList(line, seperator)); } index++; } } } return tmp; }
/// <summary> /// Convert a row as a string to a list of strings /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="line">Row as a string</param> /// <param name="seperator">Character used as TextSeparator</param> /// <returns>List of values</returns> public List <string> rowToList(string line, char seperator) { if (this.Info != null) { if (fileReaderInfo == null) { fileReaderInfo = (AsciiFileReaderInfo)this.Info; } if (fileReaderInfo != null) { tempRow = new List <string>(); tempRow = TextMarkerHandling(line, seperator, AsciiFileReaderInfo.GetTextMarker(fileReaderInfo.TextMarker)); return(tempRow); } else { return(line.Split(seperator).ToList()); } } return(line.Split(seperator).ToList()); }
/// <summary> /// Get all values from the FileStream of each variable in variable list /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="variableList">List of variables</param> /// <param name="packageSize">size of a package</param> /// <returns></returns> public List<List<string>> ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List<long> variableList, int packageSize) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; List<List<string>> listOfSelectedvalues = new List<List<string>>(); // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { Stopwatch totalTime = Stopwatch.StartNew(); using (StreamReader streamReader = new StreamReader(file)) { string line; //int index = fri.Variables; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } // go to every line while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1) { //// is position of datastructure? //if (index == this.info.Variables) //{ // variableIdentifierRows.Add(RowToList(line, seperator)); // ConvertAndAddToSubmitedVariableIdentifier(); //} // is position = or over startposition of data? if (Position >= this.Info.Data) { Stopwatch rowTime = Stopwatch.StartNew(); // return List of VariablesValues, and error messages listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList)); rowTime.Stop(); //Debug.WriteLine("index : "+index+" ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString()); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } totalTime.Stop(); Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString()); } return listOfSelectedvalues; }
/// <summary> /// Get all values from the FileStream of each variable in variable list /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="variableList">List of variables</param> /// <param name="packageSize">size of a package</param> /// <returns></returns> public List <List <string> > ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List <long> variableList, int packageSize) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; List <List <string> > listOfSelectedvalues = new List <List <string> >(); // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { Stopwatch totalTime = Stopwatch.StartNew(); using (StreamReader streamReader = new StreamReader(file)) { string line; //int index = fri.Variables; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } // go to every line while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1) { //// is position of datastructure? //if (index == this.info.Variables) //{ // variableIdentifierRows.Add(RowToList(line, seperator)); // ConvertAndAddToSubmitedVariableIdentifier(); //} // is position = or over startposition of data? if (Position >= this.Info.Data) { Stopwatch rowTime = Stopwatch.StartNew(); // return List of VariablesValues, and error messages listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList)); rowTime.Stop(); //Debug.WriteLine("index : "+index+" ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString()); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } totalTime.Stop(); Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString()); } return(listOfSelectedvalues); }
/// <summary> /// Read line by line based on a packageSize. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="packageSize"></param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize) { // clear list of datatuples this.DataTuples = new List <DataTuple>(); this.VariableIdentifierRows = new List <List <string> >(); this.SubmitedVariableIdentifiers = new List <VariableIdentifier>(); this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } _timer.Stop(); Debug.WriteLine(" "); Debug.WriteLine("*****************************************************************"); Debug.WriteLine(" position : " + Position + " --> Timer: " + _timer.Elapsed.TotalSeconds.ToString()); _timer = Stopwatch.StartNew(); /// <summary> /// read each line as long as the packet size is not reached /// generating a datatuple from the line /// </summary> /// <remarks></remarks> while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1) { if (Position >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } } return(this.DataTuples); }
private FileInfoModel GetFileInfoModel(AsciiFileReaderInfo info, string extention) { FileInfoModel FileReaderInfo = new FileInfoModel(); FileReaderInfo.Data = info.Data; FileReaderInfo.Dateformat = info.Dateformat; FileReaderInfo.Decimal = info.Decimal; FileReaderInfo.Offset = info.Offset; FileReaderInfo.Orientation = info.Orientation; FileReaderInfo.Variables = info.Variables; FileReaderInfo.Separator = info.Seperator; FileReaderInfo.TextMarker = info.TextMarker; return FileReaderInfo; }
public ActionResult SaveAsciiFileInfos(FileInfoModel info) { TaskManager TaskManager = (TaskManager)Session["TaskManager"]; AsciiFileReaderInfo asciiFileReaderInfo = new AsciiFileReaderInfo(); asciiFileReaderInfo.Data = info.Data; asciiFileReaderInfo.Dateformat = "";//info.Dateformat; asciiFileReaderInfo.Decimal = info.Decimal; asciiFileReaderInfo.Offset = info.Offset; asciiFileReaderInfo.Orientation = info.Orientation; asciiFileReaderInfo.Seperator = info.Separator; asciiFileReaderInfo.Variables = info.Variables; asciiFileReaderInfo.TextMarker = info.TextMarker; TaskManager.AddToBus(TaskManager.FILE_READER_INFO, asciiFileReaderInfo); GetFileInformationModel model = new GetFileInformationModel(); model.StepInfo = TaskManager.Current(); model.StepInfo.SetValid(true); model.Extention = TaskManager.Bus[TaskManager.EXTENTION].ToString(); model.FileInfoModel = info; model.FileInfoModel.Extention = TaskManager.Bus[TaskManager.EXTENTION].ToString(); model.IsSaved = true; return RedirectToAction("ReloadUploadWizard", new { restart = false }); }
/// <summary> /// Read the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <returns>List of datatuples</returns> public List<DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(line, seperator)); convertAndAddToSubmitedVariableIdentifier(); } if (index >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } index++; } } } return this.DataTuples; }
/// <summary> /// Validate the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks>A list of errorMessages is filled when the fil is not valid </remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> public void ValidateFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); bool dsdIsOk = false; while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { dsdIsOk = ValidateDatastructure(line, seperator); } if (dsdIsOk && index >= this.Info.Data) { this.ErrorMessages = this.ErrorMessages.Union(ValidateRow(rowToList(line, seperator), index)).ToList(); } index++; } } } }
public AsciiReader(StructuredDataStructure structuredDatastructure, AsciiFileReaderInfo fileReaderInfo) : base(structuredDatastructure, fileReaderInfo) { fileReaderInfo = (AsciiFileReaderInfo)this.Info; }
/// <summary> /// Read line by line based on a packageSize. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="packageSize"></param> /// <returns>List of datatuples</returns> public List<DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize) { // clear list of datatuples this.DataTuples = new List<DataTuple>(); this.VariableIdentifierRows = new List<List<string>>(); this.SubmitedVariableIdentifiers = new List<VariableIdentifier>(); this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } _timer.Stop(); Debug.WriteLine(" "); Debug.WriteLine("*****************************************************************"); Debug.WriteLine(" position : " + Position+" --> Timer: "+ _timer.Elapsed.TotalSeconds.ToString() ); _timer = Stopwatch.StartNew(); /// <summary> /// read each line as long as the packet size is not reached /// generating a datatuple from the line /// </summary> /// <remarks></remarks> while ((line = streamReader.ReadLine()) != null && items <= packageSize-1) { if (Position >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } } return this.DataTuples; }
public AsciiReader(StructuredDataStructure structuredDatastructure, AsciiFileReaderInfo fileReaderInfo, IOUtility iOUtility, DatasetManager datasetManager) : base(structuredDatastructure, fileReaderInfo, iOUtility, datasetManager) { fileReaderInfo = (AsciiFileReaderInfo)this.Info; }