public static bool GetHasAddtlMatrices(StreamReader reader, IList <int> expressionColIndices, char separator)
        {
            if (expressionColIndices.Count == 0)
            {
                return(false);
            }
            int expressionColIndex = expressionColIndices[0];

            reader.ReadLine();
            string line;
            bool   hasAddtl = false;

            while ((line = reader.ReadLine()) != null)
            {
                if (TabSep.IsCommentLine(line, commentPrefix, commentPrefixExceptions))
                {
                    continue;
                }
                string[] w = SplitLine(line, separator);
                if (expressionColIndex < w.Length)
                {
                    string s = StringUtils.RemoveWhitespace(w[expressionColIndex]);
                    hasAddtl = s.Contains(";");
                    break;
                }
            }
            reader.Close();
            return(hasAddtl);
        }
        public static int GetRowCount(StreamReader reader, StreamReader auxReader, int[] mainColIndices,
                                      List <Tuple <Relation[], int[], bool> > filters, char separator)
        {
            reader.BaseStream.Seek(0, SeekOrigin.Begin);
            reader.ReadLine();
            int    count            = 0;
            bool   hasAddtlMatrices = auxReader != null && GetHasAddtlMatrices(auxReader, mainColIndices, separator);
            string line;

            while ((line = reader.ReadLine()) != null)
            {
                while (TabSep.IsCommentLine(line, commentPrefix, commentPrefixExceptions))
                {
                    line = reader.ReadLine();
                }
                if (IsValidLine(line, separator, filters, hasAddtlMatrices))
                {
                    count++;
                }
            }
            return(count);
        }
        private static void LoadMatrixData(IList <string> colNames, IList <string> colDescriptions, IList <int> mainColIndices,
                                           IList <int> catColIndices, IList <int> numColIndices, IList <int> textColIndices, IList <int> multiNumColIndices,
                                           string origin, IMatrixData matrixData, IDictionary <string, string[]> annotationRows, Action <int> progress,
                                           Action <string> status, char separator, TextReader reader, StreamReader auxReader, int nrows,
                                           bool shortenExpressionNames, List <Tuple <Relation[], int[], bool> > filters)
        {
            Dictionary <string, string[]> catAnnotatRows;
            Dictionary <string, string[]> numAnnotatRows;

            status("Reading data");
            SplitAnnotRows(annotationRows, out catAnnotatRows, out numAnnotatRows);
            List <string[][]> categoryAnnotation = new List <string[][]>();

            for (int i = 0; i < catColIndices.Count; i++)
            {
                categoryAnnotation.Add(new string[nrows][]);
            }
            List <double[]> numericAnnotation = new List <double[]>();

            for (int i = 0; i < numColIndices.Count; i++)
            {
                numericAnnotation.Add(new double[nrows]);
            }
            List <double[][]> multiNumericAnnotation = new List <double[][]>();

            for (int i = 0; i < multiNumColIndices.Count; i++)
            {
                multiNumericAnnotation.Add(new double[nrows][]);
            }
            List <string[]> stringAnnotation = new List <string[]>();

            for (int i = 0; i < textColIndices.Count; i++)
            {
                stringAnnotation.Add(new string[nrows]);
            }
            float[,] mainValues     = new float[nrows, mainColIndices.Count];
            float[,] qualityValues  = null;
            bool[,] isImputedValues = null;
            bool hasAddtlMatrices = auxReader != null && GetHasAddtlMatrices(auxReader, mainColIndices, separator);

            if (hasAddtlMatrices)
            {
                qualityValues   = new float[nrows, mainColIndices.Count];
                isImputedValues = new bool[nrows, mainColIndices.Count];
            }
            reader.ReadLine();
            int    count = 0;
            string line;

            while ((line = reader.ReadLine()) != null)
            {
                progress(100 * (count + 1) / nrows);
                if (TabSep.IsCommentLine(line, commentPrefix, commentPrefixExceptions))
                {
                    continue;
                }
                string[] w;
                if (!IsValidLine(line, separator, filters, out w, hasAddtlMatrices))
                {
                    continue;
                }
                for (int i = 0; i < mainColIndices.Count; i++)
                {
                    if (mainColIndices[i] >= w.Length)
                    {
                        mainValues[count, i] = float.NaN;
                    }
                    else
                    {
                        string s = StringUtils.RemoveWhitespace(w[mainColIndices[i]]);
                        if (hasAddtlMatrices)
                        {
                            ParseExp(s, out mainValues[count, i], out isImputedValues[count, i], out qualityValues[count, i]);
                        }
                        else
                        {
                            if (count < mainValues.GetLength(0))
                            {
                                bool success = float.TryParse(s, out mainValues[count, i]);
                                if (!success)
                                {
                                    mainValues[count, i] = float.NaN;
                                }
                            }
                        }
                    }
                }
                for (int i = 0; i < numColIndices.Count; i++)
                {
                    if (numColIndices[i] >= w.Length)
                    {
                        numericAnnotation[i][count] = double.NaN;
                    }
                    else
                    {
                        double q;
                        bool   success = double.TryParse(w[numColIndices[i]].Trim(), out q);
                        if (numericAnnotation[i].Length > count)
                        {
                            numericAnnotation[i][count] = success ? q : double.NaN;
                        }
                    }
                }
                for (int i = 0; i < multiNumColIndices.Count; i++)
                {
                    if (multiNumColIndices[i] >= w.Length)
                    {
                        multiNumericAnnotation[i][count] = new double[0];
                    }
                    else
                    {
                        string q = w[multiNumColIndices[i]].Trim();
                        if (q.Length >= 2 && q[0] == '\"' && q[q.Length - 1] == '\"')
                        {
                            q = q.Substring(1, q.Length - 2);
                        }
                        if (q.Length >= 2 && q[0] == '\'' && q[q.Length - 1] == '\'')
                        {
                            q = q.Substring(1, q.Length - 2);
                        }
                        string[] ww = q.Length == 0 ? new string[0] : q.Split(';');
                        multiNumericAnnotation[i][count] = new double[ww.Length];
                        for (int j = 0; j < ww.Length; j++)
                        {
                            double q1;
                            bool   success = double.TryParse(ww[j], out q1);
                            multiNumericAnnotation[i][count][j] = success ? q1 : double.NaN;
                        }
                    }
                }
                for (int i = 0; i < catColIndices.Count; i++)
                {
                    if (catColIndices[i] >= w.Length)
                    {
                        categoryAnnotation[i][count] = new string[0];
                    }
                    else
                    {
                        string q = w[catColIndices[i]].Trim();
                        if (q.Length >= 2 && q[0] == '\"' && q[q.Length - 1] == '\"')
                        {
                            q = q.Substring(1, q.Length - 2);
                        }
                        if (q.Length >= 2 && q[0] == '\'' && q[q.Length - 1] == '\'')
                        {
                            q = q.Substring(1, q.Length - 2);
                        }
                        string[]   ww     = q.Length == 0 ? new string[0] : q.Split(';');
                        List <int> valids = new List <int>();
                        for (int j = 0; j < ww.Length; j++)
                        {
                            ww[j] = ww[j].Trim();
                            if (ww[j].Length > 0)
                            {
                                valids.Add(j);
                            }
                        }
                        ww = ArrayUtils.SubArray(ww, valids);
                        Array.Sort(ww);
                        if (categoryAnnotation[i].Length > count)
                        {
                            categoryAnnotation[i][count] = ww;
                        }
                    }
                }
                for (int i = 0; i < textColIndices.Count; i++)
                {
                    if (textColIndices[i] >= w.Length)
                    {
                        stringAnnotation[i][count] = "";
                    }
                    else
                    {
                        string q = w[textColIndices[i]].Trim();
                        if (stringAnnotation[i].Length > count)
                        {
                            stringAnnotation[i][count] = RemoveSplitWhitespace(RemoveQuotes(q));
                        }
                    }
                }
                count++;
            }
            reader.Close();
            string[] columnNames = ArrayUtils.SubArray(colNames, mainColIndices);
            if (shortenExpressionNames)
            {
                columnNames = StringUtils.RemoveCommonSubstrings(columnNames, true);
            }
            string[] catColnames      = ArrayUtils.SubArray(colNames, catColIndices);
            string[] numColnames      = ArrayUtils.SubArray(colNames, numColIndices);
            string[] multiNumColnames = ArrayUtils.SubArray(colNames, multiNumColIndices);
            string[] textColnames     = ArrayUtils.SubArray(colNames, textColIndices);
            matrixData.Name        = origin;
            matrixData.ColumnNames = RemoveQuotes(columnNames);
            matrixData.Values.Set(mainValues);
            if (hasAddtlMatrices)
            {
                matrixData.Quality.Set(qualityValues);
                matrixData.IsImputed.Set(isImputedValues);
            }
            else
            {
                matrixData.Quality.Set(new float[mainValues.GetLength(0), mainValues.GetLength(1)]);
                matrixData.IsImputed.Set(new bool[mainValues.GetLength(0), mainValues.GetLength(1)]);
            }
            matrixData.SetAnnotationColumns(RemoveQuotes(textColnames), stringAnnotation, RemoveQuotes(catColnames),
                                            categoryAnnotation, RemoveQuotes(numColnames), numericAnnotation, RemoveQuotes(multiNumColnames),
                                            multiNumericAnnotation);
            if (colDescriptions != null)
            {
                string[] columnDesc      = ArrayUtils.SubArray(colDescriptions, mainColIndices);
                string[] catColDesc      = ArrayUtils.SubArray(colDescriptions, catColIndices);
                string[] numColDesc      = ArrayUtils.SubArray(colDescriptions, numColIndices);
                string[] multiNumColDesc = ArrayUtils.SubArray(colDescriptions, multiNumColIndices);
                string[] textColDesc     = ArrayUtils.SubArray(colDescriptions, textColIndices);
                matrixData.ColumnDescriptions             = new List <string>(columnDesc);
                matrixData.NumericColumnDescriptions      = new List <string>(numColDesc);
                matrixData.CategoryColumnDescriptions     = new List <string>(catColDesc);
                matrixData.StringColumnDescriptions       = new List <string>(textColDesc);
                matrixData.MultiNumericColumnDescriptions = new List <string>(multiNumColDesc);
            }
            foreach (string key in catAnnotatRows.Keys)
            {
                string     name  = key;
                string[]   svals = ArrayUtils.SubArray(catAnnotatRows[key], mainColIndices);
                string[][] cat   = new string[svals.Length][];
                for (int i = 0; i < cat.Length; i++)
                {
                    string s = svals[i].Trim();
                    cat[i] = s.Length > 0 ? s.Split(';') : new string[0];
                    List <int> valids = new List <int>();
                    for (int j = 0; j < cat[i].Length; j++)
                    {
                        cat[i][j] = cat[i][j].Trim();
                        if (cat[i][j].Length > 0)
                        {
                            valids.Add(j);
                        }
                    }
                    cat[i] = ArrayUtils.SubArray(cat[i], valids);
                    Array.Sort(cat[i]);
                }
                matrixData.AddCategoryRow(name, name, cat);
            }
            foreach (string key in numAnnotatRows.Keys)
            {
                string   name  = key;
                string[] svals = ArrayUtils.SubArray(numAnnotatRows[key], mainColIndices);
                double[] num   = new double[svals.Length];
                for (int i = 0; i < num.Length; i++)
                {
                    string s = svals[i].Trim();
                    num[i] = double.NaN;
                    double.TryParse(s, out num[i]);
                }
                matrixData.AddNumericRow(name, name, num);
            }
            matrixData.Origin = origin;
            progress(0);
            status("");
        }