| public static GetSeperator ( string seperator ) : TextSeperator | ||
| seperator | string | Name of TextSeperator |
| return | TextSeperator | |
/// <summary>
/// Validate the whole FileStream line by line until no more come.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks>A list of errorMessages is filled when the fil is not valid </remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
public void ValidateFile(Stream file, string fileName, long datasetId)
{
this.FileStream = file;
this.FileName = fileName;
this.DatasetId = datasetId;
AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file, encoding))
{
string line;
int index = 1;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
bool dsdIsOk = false;
while ((line = streamReader.ReadLine()) != null)
{
if (index == this.Info.Variables)
{
dsdIsOk = ValidateDatastructure(line, seperator);
// if data is not in the correct order, create a dictionary with the new position
}
if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line) && !isEmpty(line, seperator))
{
var r = rowToList(line, seperator);
var e = ValidateRow(r, index);
this.ErrorMessages = this.ErrorMessages.Union(e).ToList();
if (this.ErrorMessages.Count >= 1000)
{
break;
}
}
index++;
}
}
}
}
/// <summary>
/// Read the whole FileStream line by line until no more come.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <returns>List of datatuples</returns>
public List <DataTuple> ReadFile(Stream file, string fileName, long datasetId)
{
this.FileStream = file;
this.FileName = fileName;
this.DatasetId = datasetId;
AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file, encoding))
{
string line;
int index = fri.Variables;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
while ((line = streamReader.ReadLine()) != null)
{
if (index == this.Info.Variables)
{
ValidateDatastructure(line, seperator);
}
if (index >= this.Info.Data)
{
// return List of VariablesValues, and error messages
if (!isEmpty(line, seperator))
{
this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
}
}
index++;
}
}
}
return(this.DataTuples);
}
/// <summary>
/// Read the whole FileStream line by line until no more come.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <returns>List of datatuples</returns>
public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId)
{
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file))
{
string line;
int index = fri.Variables;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
while ((line = streamReader.ReadLine()) != null)
{
if (index == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(line, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
if (index >= this.Info.Data)
{
// return List of VariablesValues, and error messages
this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
}
index++;
}
}
}
return(this.DataTuples);
}
/// <summary>
/// Validate the whole FileStream line by line until no more come.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks>A list of errorMessages is filled when the fil is not valid </remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
public void ValidateFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId)
{
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file))
{
string line;
int index = 1;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
bool dsdIsOk = false;
while ((line = streamReader.ReadLine()) != null)
{
if (index == this.Info.Variables)
{
dsdIsOk = ValidateDatastructure(line, seperator);
}
if (dsdIsOk && index >= this.Info.Data && !string.IsNullOrEmpty(line))
{
this.ErrorMessages = this.ErrorMessages.Union(ValidateRow(rowToList(line, seperator), index)).ToList();
}
index++;
}
}
}
}
public List <List <string> > ReadFile(Stream file)
{
List <List <string> > tmp = new List <List <string> >();
this.FileStream = file;
AsciiFileReaderInfo fri = (AsciiFileReaderInfo)Info;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file, encoding))
{
string line;
int index = fri.Variables;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
while ((line = streamReader.ReadLine()) != null)
{
if (index >= Info.Data)
{
// return List of VariablesValues, and error messages
if (!isEmpty(line, seperator))
{
tmp.Add(rowToList(line, seperator));
}
}
index++;
}
}
}
return(tmp);
}
/// <summary>
/// Get all values from the FileStream of each variable in variable list
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <param name="variableList">List of variables</param>
/// <param name="packageSize">size of a package</param>
/// <returns></returns>
public List <List <string> > ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List <long> variableList, int packageSize)
{
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
List <List <string> > listOfSelectedvalues = new List <List <string> >();
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
Stopwatch totalTime = Stopwatch.StartNew();
using (StreamReader streamReader = new StreamReader(file))
{
string line;
//int index = fri.Variables;
int index = 1;
int items = 0;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
//int end = packageSize;
//int start = 1;
// read to position
if (Position == 1)
{
Position = this.Info.Data;
}
Stopwatch _timer = Stopwatch.StartNew();
/// <summary>
/// go to current position is reached at the line
/// </summary>
/// <remarks></remarks>
for (int i = 1; i < Position; i++)
{
string l = streamReader.ReadLine();
if (i == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(l, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
}
// go to every line
while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
{
//// is position of datastructure?
//if (index == this.info.Variables)
//{
// variableIdentifierRows.Add(RowToList(line, seperator));
// ConvertAndAddToSubmitedVariableIdentifier();
//}
// is position = or over startposition of data?
if (Position >= this.Info.Data)
{
Stopwatch rowTime = Stopwatch.StartNew();
// return List of VariablesValues, and error messages
listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList));
rowTime.Stop();
//Debug.WriteLine("index : "+index+" ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString());
}
Position++;
index++;
items++;
}
_timer.Stop();
Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
}
totalTime.Stop();
Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString());
}
return(listOfSelectedvalues);
}
/// <summary>
/// Read line by line based on a packageSize.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <param name="packageSize"></param>
/// <returns>List of datatuples</returns>
public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize)
{
// clear list of datatuples
this.DataTuples = new List <DataTuple>();
this.VariableIdentifierRows = new List <List <string> >();
this.SubmitedVariableIdentifiers = new List <VariableIdentifier>();
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file))
{
string line;
int index = 1;
int items = 0;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
//int end = packageSize;
//int start = 1;
// read to position
if (Position == 1)
{
Position = this.Info.Data;
}
Stopwatch _timer = Stopwatch.StartNew();
/// <summary>
/// go to current position is reached at the line
/// </summary>
/// <remarks></remarks>
for (int i = 1; i < Position; i++)
{
string l = streamReader.ReadLine();
if (i == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(l, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
}
_timer.Stop();
Debug.WriteLine(" ");
Debug.WriteLine("*****************************************************************");
Debug.WriteLine(" position : " + Position + " --> Timer: " + _timer.Elapsed.TotalSeconds.ToString());
_timer = Stopwatch.StartNew();
/// <summary>
/// read each line as long as the packet size is not reached
/// generating a datatuple from the line
/// </summary>
/// <remarks></remarks>
while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
{
if (Position >= this.Info.Data)
{
// return List of VariablesValues, and error messages
this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
}
Position++;
index++;
items++;
}
_timer.Stop();
Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
}
}
return(this.DataTuples);
}