Exemplo n.º 1
0
        /// <summary>
        /// Read the whole FileStream line by line until no more come.
        /// Convert the lines into a datatuple based on the datastructure.
        /// Return value is a list of datatuples
        /// </summary>
        /// <remarks></remarks>
        /// <seealso cref="AsciiFileReaderInfo"/>
        /// <seealso cref="DataTuple"/>
        /// <seealso cref="StructuredDataStructure"/>
        /// <param name="FileStream">Stream of the FileStream</param>
        /// <param name="fileName">name of the FileStream</param>
        /// <param name="fri">AsciiFileReaderInfo needed</param>
        /// <param name="sds">StructuredDataStructure</param>
        /// <param name="datasetId">Id of the dataset</param>
        /// <returns>List of datatuples</returns>
        public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId)
        {
            this.FileStream = file;
            this.FileName   = fileName;
            this.Info       = fri;
            this.StructuredDataStructure = sds;
            this.DatasetId = datasetId;

            // Check params
            if (this.FileStream == null)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
            }
            if (!this.FileStream.CanRead)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
            }
            if (this.Info.Variables <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
            }
            if (this.Info.Data <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
            }

            if (this.ErrorMessages.Count == 0)
            {
                using (StreamReader streamReader = new StreamReader(file))
                {
                    string line;
                    int    index     = fri.Variables;
                    char   seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);

                    while ((line = streamReader.ReadLine()) != null)
                    {
                        if (index == this.Info.Variables)
                        {
                            VariableIdentifierRows.Add(rowToList(line, seperator));
                            convertAndAddToSubmitedVariableIdentifier();
                        }

                        if (index >= this.Info.Data)
                        {
                            // return List of VariablesValues, and error messages
                            this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
                        }

                        index++;
                    }
                }
            }

            return(this.DataTuples);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Validate the datastructure
        /// </summary>
        /// <remarks></remarks>
        /// <seealso cref="TextSeparator"/>
        /// <param name="line">line which include the datastructure as names</param>
        /// <param name="sep">TextSeparator as Character</param>
        protected bool ValidateDatastructure(string line, char sep)
        {
            /// <summary>
            /// the incoming line should be the datastructure line from a FileStream
            /// this line converted into a list of strings which incluide the variable names
            /// </summary>
            /// <remarks></remarks>
            VariableIdentifierRows.Add(rowToList(line, sep));

            /// <summary>
            /// Convert a list of variable names to
            /// VariableIdentifiers
            /// </summary>
            /// <remarks>SubmitedVariableIdentifiers is the list</remarks>
            convertAndAddToSubmitedVariableIdentifier();


            /// <summary>
            /// Validate datastructure with the SubmitedVariableIdentifiers from FileStream
            /// </summary>
            /// <remarks></remarks>
            List <Error> ecList = ValidateComparisonWithDatatsructure(SubmitedVariableIdentifiers);


            if (ecList != null)
            {
                if (ecList.Count > 0)
                {
                    this.ErrorMessages = this.ErrorMessages.Concat(ecList).ToList();
                    return(false);
                }
                else
                {
                    return(true);
                }
            }
            else
            {
                return(true);
            }
        }
Exemplo n.º 3
0
        /// <summary>
        ///
        /// </summary>
        /// <remarks></remarks>
        /// <seealso cref=""/>
        /// <param name="worksheetPart"></param>
        /// <param name="startRow"></param>
        /// <param name="endRow"></param>
        /// <returns></returns>
        private List <VariableIdentifier> getVariableIdentifiers(WorksheetPart worksheetPart, int startRow, int endRow)
        {
            //NEW OPENXMLREADER
            if (this.VariableIdentifiers == null || this.VariableIdentifiers.Count == 0)
            {
                OpenXmlReader reader = OpenXmlReader.Create(worksheetPart);
                int           rowNum = 0;

                // read variable rows to get name and id from area variable
                while (reader.Read())
                {
                    if (reader.ElementType == typeof(Row))
                    {
                        do
                        {
                            if (reader.HasAttributes)
                            {
                                rowNum = Convert.ToInt32(reader.Attributes.First(a => a.LocalName == "r").Value);
                            }

                            if (rowNum >= startRow && rowNum <= endRow)
                            {
                                Row row = (Row)reader.LoadCurrentElement();

                                if (row.Hidden == null)
                                {
                                    VariableIdentifierRows.Add(rowToList(row));
                                }
                                else if (row.Hidden != true)
                                {
                                    VariableIdentifierRows.Add(rowToList(row));
                                }
                            }
                        } while (reader.ReadNextSibling() && rowNum < endRow); // Skip to the next row
                        break;
                    }
                }

                // convert variable rows to VariableIdentifiers
                if (VariableIdentifierRows != null)
                {
                    foreach (List <string> l in VariableIdentifierRows)
                    {
                        //create headerVariables
                        if (VariableIdentifiers.Count == 0)
                        {
                            foreach (string s in l)
                            {
                                VariableIdentifier hv = new VariableIdentifier();
                                hv.name = s;
                                VariableIdentifiers.Add(hv);
                            }
                        }
                        else
                        {
                            foreach (string s in l)
                            {
                                int id    = Convert.ToInt32(s);
                                int index = l.IndexOf(s);
                                VariableIdentifiers.ElementAt(index).id = id;
                            }
                        }
                    }
                }
            }

            if (this.VariableIdentifiers != null)
            {
                return(this.VariableIdentifiers);
            }
            else
            {
                return(null);
            }
        }
Exemplo n.º 4
0
        /// <summary>
        /// Get all values from the FileStream of each variable in variable list
        /// </summary>
        /// <remarks></remarks>
        /// <seealso cref="AsciiFileReaderInfo"/>
        /// <seealso cref="DataTuple"/>
        /// <seealso cref="StructuredDataStructure"/>
        /// <param name="FileStream">Stream of the FileStream</param>
        /// <param name="fileName">name of the FileStream</param>
        /// <param name="fri">AsciiFileReaderInfo needed</param>
        /// <param name="sds">StructuredDataStructure</param>
        /// <param name="datasetId">Id of the dataset</param>
        /// <param name="variableList">List of variables</param>
        /// <param name="packageSize">size of a package</param>
        /// <returns></returns>
        public List <List <string> > ReadValuesFromFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, List <long> variableList, int packageSize)
        {
            this.FileStream = file;
            this.FileName   = fileName;
            this.Info       = fri;
            this.StructuredDataStructure = sds;
            this.DatasetId = datasetId;

            List <List <string> > listOfSelectedvalues = new List <List <string> >();

            // Check params
            if (this.FileStream == null)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
            }
            if (!this.FileStream.CanRead)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
            }
            if (this.Info.Variables <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
            }
            if (this.Info.Data <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
            }

            if (this.ErrorMessages.Count == 0)
            {
                Stopwatch totalTime = Stopwatch.StartNew();

                using (StreamReader streamReader = new StreamReader(file))
                {
                    string line;
                    //int index = fri.Variables;
                    int  index     = 1;
                    int  items     = 0;
                    char seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);

                    //int end = packageSize;
                    //int start = 1;
                    // read to position
                    if (Position == 1)
                    {
                        Position = this.Info.Data;
                    }

                    Stopwatch _timer = Stopwatch.StartNew();


                    /// <summary>
                    /// go to current position is reached at the line
                    /// </summary>
                    /// <remarks></remarks>
                    for (int i = 1; i < Position; i++)
                    {
                        string l = streamReader.ReadLine();

                        if (i == this.Info.Variables)
                        {
                            VariableIdentifierRows.Add(rowToList(l, seperator));
                            convertAndAddToSubmitedVariableIdentifier();
                        }
                    }


                    // go to every line
                    while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
                    {
                        //// is position of datastructure?
                        //if (index == this.info.Variables)
                        //{
                        //    variableIdentifierRows.Add(RowToList(line, seperator));
                        //    ConvertAndAddToSubmitedVariableIdentifier();
                        //}

                        // is position = or over startposition of data?
                        if (Position >= this.Info.Data)
                        {
                            Stopwatch rowTime = Stopwatch.StartNew();

                            // return List of VariablesValues, and error messages
                            listOfSelectedvalues.Add(GetValuesFromRow(rowToList(line, seperator), index, variableList));

                            rowTime.Stop();
                            //Debug.WriteLine("index : "+index+"   ---- Total Time of primary key check " + rowTime.Elapsed.TotalSeconds.ToString());
                        }

                        Position++;
                        index++;
                        items++;
                    }

                    _timer.Stop();

                    Debug.WriteLine(" get values for primary key check datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
                }

                totalTime.Stop();
                Debug.WriteLine(" Total Time of primary key check " + totalTime.Elapsed.TotalSeconds.ToString());
            }

            return(listOfSelectedvalues);
        }
Exemplo n.º 5
0
        /// <summary>
        /// Read line by line based on a packageSize.
        /// Convert the lines into a datatuple based on the datastructure.
        /// Return value is a list of datatuples
        /// </summary>
        /// <remarks></remarks>
        /// <seealso cref="AsciiFileReaderInfo"/>
        /// <seealso cref="DataTuple"/>
        /// <seealso cref="StructuredDataStructure"/>
        /// <param name="FileStream">Stream of the FileStream</param>
        /// <param name="fileName">name of the FileStream</param>
        /// <param name="fri">AsciiFileReaderInfo needed</param>
        /// <param name="sds">StructuredDataStructure</param>
        /// <param name="datasetId">Id of the dataset</param>
        /// <param name="packageSize"></param>
        /// <returns>List of datatuples</returns>
        public List <DataTuple> ReadFile(Stream file, string fileName, AsciiFileReaderInfo fri, StructuredDataStructure sds, long datasetId, int packageSize)
        {
            // clear list of datatuples
            this.DataTuples                  = new List <DataTuple>();
            this.VariableIdentifierRows      = new List <List <string> >();
            this.SubmitedVariableIdentifiers = new List <VariableIdentifier>();

            this.FileStream = file;
            this.FileName   = fileName;
            this.Info       = fri;
            this.StructuredDataStructure = sds;
            this.DatasetId = datasetId;

            // Check params
            if (this.FileStream == null)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File not exist"));
            }
            if (!this.FileStream.CanRead)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "File is not readable"));
            }
            if (this.Info.Variables <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Variable can´t be 0"));
            }
            if (this.Info.Data <= 0)
            {
                this.ErrorMessages.Add(new Error(ErrorType.Other, "Startrow of Data can´t be 0"));
            }

            if (this.ErrorMessages.Count == 0)
            {
                using (StreamReader streamReader = new StreamReader(file))
                {
                    string line;
                    int    index     = 1;
                    int    items     = 0;
                    char   seperator = AsciiFileReaderInfo.GetSeperator(fri.Seperator);

                    //int end = packageSize;
                    //int start = 1;
                    // read to position
                    if (Position == 1)
                    {
                        Position = this.Info.Data;
                    }

                    Stopwatch _timer = Stopwatch.StartNew();


                    /// <summary>
                    /// go to current position is reached at the line
                    /// </summary>
                    /// <remarks></remarks>
                    for (int i = 1; i < Position; i++)
                    {
                        string l = streamReader.ReadLine();

                        if (i == this.Info.Variables)
                        {
                            VariableIdentifierRows.Add(rowToList(l, seperator));
                            convertAndAddToSubmitedVariableIdentifier();
                        }
                    }

                    _timer.Stop();
                    Debug.WriteLine(" ");
                    Debug.WriteLine("*****************************************************************");
                    Debug.WriteLine(" position : " + Position + "    -->  Timer: " + _timer.Elapsed.TotalSeconds.ToString());

                    _timer = Stopwatch.StartNew();

                    /// <summary>
                    /// read each line as long as the packet size is not reached
                    /// generating a datatuple from the line
                    /// </summary>
                    /// <remarks></remarks>
                    while ((line = streamReader.ReadLine()) != null && items <= packageSize - 1)
                    {
                        if (Position >= this.Info.Data)
                        {
                            // return List of VariablesValues, and error messages
                            this.DataTuples.Add(ReadRow(rowToList(line, seperator), index));
                        }

                        Position++;
                        index++;
                        items++;
                    }

                    _timer.Stop();

                    Debug.WriteLine(" created datatuples : " + _timer.Elapsed.TotalSeconds.ToString());
                }
            }


            return(this.DataTuples);
        }