/// <summary>
/// Read the whole FileStream line by line until no more come.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <returns>List of datatuples</returns>
public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId)
{
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file))
{
string line;
int index = fri.Variables;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
while ((line = streamReader.ReadLine()) != null)
{
if (index == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(line, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
if (index >= this.Info.Data)
{
// return List of VariablesValues, and error messages
this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
}
index++;
}
}
}
return(this.DataTuples);
}
/// <summary>
/// Validate the datastructure
/// </summary>
/// <remarks></remarks>
/// <seealso cref="TextSeparator"/>
/// <param name="line">line which include the datastructure as names</param>
/// <param name="sep">TextSeparator as Character</param>
protected bool ValidateDatastructure(string line, char sep)
{
/// <summary>
/// the incoming line should be the datastructure line from a FileStream
/// this line converted into a list of strings which incluide the variable names
/// </summary>
/// <remarks></remarks>
VariableIdentifierRows.Add(rowToList(line, sep));
/// <summary>
/// Convert a list of variable names to
/// VariableIdentifiers
/// </summary>
/// <remarks>SubmitedVariableIdentifiers is the list</remarks>
convertAndAddToSubmitedVariableIdentifier();
/// <summary>
/// Validate datastructure with the SubmitedVariableIdentifiers from FileStream
/// </summary>
/// <remarks></remarks>
List <Error> ecList = ValidateComparisonWithDatatsructure(SubmitedVariableIdentifiers);
if (ecList != null)
{
if (ecList.Count > 0)
{
this.ErrorMessages = this.ErrorMessages.Concat(ecList).ToList();
return(false);
}
else
{
return(true);
}
}
else
{
return(true);
}
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="worksheetPart"></param>
/// <param name="startRow"></param>
/// <param name="endRow"></param>
/// <returns></returns>
private List <VariableIdentifier> getVariableIdentifiers(WorksheetPart worksheetPart, int startRow, int endRow)
{
//NEW OPENXMLREADER
if (this.VariableIdentifiers == null || this.VariableIdentifiers.Count == 0)
{
OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
int rowNum = 0;
// read variable rows to get name and id from area variable
while (reader.Read())
{
if (reader.ElementType == typeof(Row))
{
do
{
if (reader.HasAttributes)
{
rowNum = Convert.ToInt32(reader.Attributes.First(a => a.LocalName == "r").Value);
}
if (rowNum >= startRow && rowNum <= endRow)
{
Row row = (Row)reader.LoadCurrentElement();
if (row.Hidden == null)
{
VariableIdentifierRows.Add(rowToList(row));
}
else if (row.Hidden != true)
{
VariableIdentifierRows.Add(rowToList(row));
}
}
} while (reader.ReadNextSibling() && rowNum < endRow); // Skip to the next row
break;
}
}
// convert variable rows to VariableIdentifiers
if (VariableIdentifierRows != null)
{
foreach (List <string> l in VariableIdentifierRows)
{
//create headerVariables
if (VariableIdentifiers.Count == 0)
{
foreach (string s in l)
{
VariableIdentifier hv = new VariableIdentifier();
hv.name = s;
VariableIdentifiers.Add(hv);
}
}
else
{
foreach (string s in l)
{
int id = Convert.ToInt32(s);
int index = l.IndexOf(s);
VariableIdentifiers.ElementAt(index).id = id;
}
}
}
}
}
if (this.VariableIdentifiers != null)
{
return(this.VariableIdentifiers);
}
else
{
return(null);
}
}
/// <summary>
/// Get all values from the FileStream of each variable in variable list
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <param name="variableList">List of variables</param>
/// <param name="packageSize">size of a package</param>
/// <returns></returns>
public List <List <string> > ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List <long> variableList, int packageSize)
{
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
List <List <string> > listOfSelectedvalues = new List <List <string> >();
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
Stopwatch totalTime = Stopwatch.StartNew();
using (StreamReader streamReader = new StreamReader(file))
{
string line;
//int index = fri.Variables;
int index = 1;
int items = 0;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
//int end = packageSize;
//int start = 1;
// read to position
if (Position == 1)
{
Position = this.Info.Data;
}
Stopwatch _timer = Stopwatch.StartNew();
/// <summary>
/// go to current position is reached at the line
/// </summary>
/// <remarks></remarks>
for (int i = 1; i < Position; i++)
{
string l = streamReader.ReadLine();
if (i == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(l, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
}
// go to every line
while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
{
//// is position of datastructure?
//if (index == this.info.Variables)
//{
// variableIdentifierRows.Add(RowToList(line, seperator));
// ConvertAndAddToSubmitedVariableIdentifier();
//}
// is position = or over startposition of data?
if (Position >= this.Info.Data)
{
Stopwatch rowTime = Stopwatch.StartNew();
// return List of VariablesValues, and error messages
listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList));
rowTime.Stop();
//Debug.WriteLine("index : "+index+" ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString());
}
Position++;
index++;
items++;
}
_timer.Stop();
Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
}
totalTime.Stop();
Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString());
}
return(listOfSelectedvalues);
}
/// <summary>
/// Read line by line based on a packageSize.
/// Convert the lines into a datatuple based on the datastructure.
/// Return value is a list of datatuples
/// </summary>
/// <remarks></remarks>
/// <seealso cref="AsciiFileReaderInfo"/>
/// <seealso cref="DataTuple"/>
/// <seealso cref="StructuredDataStructure"/>
/// <param name="FileStream">Stream of the FileStream</param>
/// <param name="fileName">name of the FileStream</param>
/// <param name="fri">AsciiFileReaderInfo needed</param>
/// <param name="sds">StructuredDataStructure</param>
/// <param name="datasetId">Id of the dataset</param>
/// <param name="packageSize"></param>
/// <returns>List of datatuples</returns>
public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize)
{
// clear list of datatuples
this.DataTuples = new List <DataTuple>();
this.VariableIdentifierRows = new List <List <string> >();
this.SubmitedVariableIdentifiers = new List <VariableIdentifier>();
this.FileStream = file;
this.FileName = fileName;
this.Info = fri;
this.StructuredDataStructure = sds;
this.DatasetId = datasetId;
// Check params
if (this.FileStream == null)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
}
if (!this.FileStream.CanRead)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
}
if (this.Info.Variables <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
}
if (this.Info.Data <= 0)
{
this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
}
if (this.ErrorMessages.Count == 0)
{
using (StreamReader streamReader = new StreamReader(file))
{
string line;
int index = 1;
int items = 0;
char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);
//int end = packageSize;
//int start = 1;
// read to position
if (Position == 1)
{
Position = this.Info.Data;
}
Stopwatch _timer = Stopwatch.StartNew();
/// <summary>
/// go to current position is reached at the line
/// </summary>
/// <remarks></remarks>
for (int i = 1; i < Position; i++)
{
string l = streamReader.ReadLine();
if (i == this.Info.Variables)
{
VariableIdentifierRows.Add(rowToList(l, seperator));
convertAndAddToSubmitedVariableIdentifier();
}
}
_timer.Stop();
Debug.WriteLine(" ");
Debug.WriteLine("*****************************************************************");
Debug.WriteLine(" position : " + Position + " --> Timer: " + _timer.Elapsed.TotalSeconds.ToString());
_timer = Stopwatch.StartNew();
/// <summary>
/// read each line as long as the packet size is not reached
/// generating a datatuple from the line
/// </summary>
/// <remarks></remarks>
while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
{
if (Position >= this.Info.Data)
{
// return List of VariablesValues, and error messages
this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
}
Position++;
index++;
items++;
}
_timer.Stop();
Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
}
}
return(this.DataTuples);
}