public static GetSeperator ( string seperator ) : TextSeperator | ||
seperator | string | Name of TextSeperator |
return | TextSeperator |
/// <summary> /// Validate the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks>A list of errorMessages is filled when the fil is not valid </remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> public void ValidateFile(Stream file, string fileName, long datasetId) { this.FileStream = file; this.FileName = fileName; this.DatasetId = datasetId; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = 1; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); bool dsdIsOk = false; while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { dsdIsOk = ValidateDatastructure(line, seperator); // if data is not in the correct order, create a dictionary with the new position } if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line) && !isEmpty(line, seperator)) { var r = rowToList(line, seperator); var e = ValidateRow(r, index); this.ErrorMessages = this.ErrorMessages.Union(e).ToList(); if (this.ErrorMessages.Count >= 1000) { break; } } index++; } } } }
/// <summary> /// Read the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, long datasetId) { this.FileStream = file; this.FileName = fileName; this.DatasetId = datasetId; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { ValidateDatastructure(line, seperator); } if (index >= this.Info.Data) { // return List of VariablesValues, and error messages if (!isEmpty(line, seperator)) { this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } } index++; } } } return(this.DataTuples); }
/// <summary> /// Read the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(line, seperator)); convertAndAddToSubmitedVariableIdentifier(); } if (index >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } index++; } } } return(this.DataTuples); }
/// <summary> /// Validate the whole FileStream line by line until no more come. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks>A list of errorMessages is filled when the fil is not valid </remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> public void ValidateFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); bool dsdIsOk = false; while ((line = streamReader.ReadLine()) != null) { if (index == this.Info.Variables) { dsdIsOk = ValidateDatastructure(line, seperator); } if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line)) { this.ErrorMessages = this.ErrorMessages.Union(ValidateRow(rowToList(line, seperator), index)).ToList(); } index++; } } } }
public List <List <string> > ReadFile(Stream file) { List <List <string> > tmp = new List <List <string> >(); this.FileStream = file; AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file, encoding)) { string line; int index = fri.Variables; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); while ((line = streamReader.ReadLine()) != null) { if (index >= Info.Data) { // return List of VariablesValues, and error messages if (!isEmpty(line, seperator)) { tmp.Add(rowToList(line, seperator)); } } index++; } } } return(tmp); }
/// <summary> /// Get all values from the FileStream of each variable in variable list /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="variableList">List of variables</param> /// <param name="packageSize">size of a package</param> /// <returns></returns> public List <List <string> > ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List <long> variableList, int packageSize) { this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; List <List <string> > listOfSelectedvalues = new List <List <string> >(); // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { Stopwatch totalTime = Stopwatch.StartNew(); using (StreamReader streamReader = new StreamReader(file)) { string line; //int index = fri.Variables; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } // go to every line while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1) { //// is position of datastructure? //if (index == this.info.Variables) //{ // variableIdentifierRows.Add(RowToList(line, seperator)); // ConvertAndAddToSubmitedVariableIdentifier(); //} // is position = or over startposition of data? if (Position >= this.Info.Data) { Stopwatch rowTime = Stopwatch.StartNew(); // return List of VariablesValues, and error messages listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList)); rowTime.Stop(); //Debug.WriteLine("index : "+index+" ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString()); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } totalTime.Stop(); Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString()); } return(listOfSelectedvalues); }
/// <summary> /// Read line by line based on a packageSize. /// Convert the lines into a datatuple based on the datastructure. /// Return value is a list of datatuples /// </summary> /// <remarks></remarks> /// <seealso cref="AsciiFileReaderInfo"/> /// <seealso cref="DataTuple"/> /// <seealso cref="StructuredDataStructure"/> /// <param name="FileStream">Stream of the FileStream</param> /// <param name="fileName">name of the FileStream</param> /// <param name="fri">AsciiFileReaderInfo needed</param> /// <param name="sds">StructuredDataStructure</param> /// <param name="datasetId">Id of the dataset</param> /// <param name="packageSize"></param> /// <returns>List of datatuples</returns> public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize) { // clear list of datatuples this.DataTuples = new List <DataTuple>(); this.VariableIdentifierRows = new List <List <string> >(); this.SubmitedVariableIdentifiers = new List <VariableIdentifier>(); this.FileStream = file; this.FileName = fileName; this.Info = fri; this.StructuredDataStructure = sds; this.DatasetId = datasetId; // Check params if (this.FileStream == null) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist")); } if (!this.FileStream.CanRead) { this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable")); } if (this.Info.Variables <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0")); } if (this.Info.Data <= 0) { this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0")); } if (this.ErrorMessages.Count == 0) { using (StreamReader streamReader = new StreamReader(file)) { string line; int index = 1; int items = 0; char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator); //int end = packageSize; //int start = 1; // read to position if (Position == 1) { Position = this.Info.Data; } Stopwatch _timer = Stopwatch.StartNew(); /// <summary> /// go to current position is reached at the line /// </summary> /// <remarks></remarks> for (int i = 1; i < Position; i++) { string l = streamReader.ReadLine(); if (i == this.Info.Variables) { VariableIdentifierRows.Add(rowToList(l, seperator)); convertAndAddToSubmitedVariableIdentifier(); } } _timer.Stop(); Debug.WriteLine(" "); Debug.WriteLine("*****************************************************************"); Debug.WriteLine(" position : " + Position + " --> Timer: " + _timer.Elapsed.TotalSeconds.ToString()); _timer = Stopwatch.StartNew(); /// <summary> /// read each line as long as the packet size is not reached /// generating a datatuple from the line /// </summary> /// <remarks></remarks> while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1) { if (Position >= this.Info.Data) { // return List of VariablesValues, and error messages this.DataTuples.Add(ReadRow(rowToList(line, seperator), index)); } Position++; index++; items++; } _timer.Stop(); Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString()); } } return(this.DataTuples); }